初始化提交

tools/python/ampy/cli.py

@@ -0,0 +1,534 @@
+# Adafruit MicroPython Tool - Command Line Interface
+# Author: Tony DiCola
+# Copyright (c) 2016 Adafruit Industries
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+from __future__ import print_function
+import os
+import platform
+import posixpath
+import re
+import serial.serialutil
+
+import click
+import dotenv
+
+import sys
+
+# Load AMPY_PORT et al from .ampy file
+# Performed here because we need to beat click's decorators.
+config = dotenv.find_dotenv(filename=".ampy", usecwd=True)
+if config:
+    dotenv.load_dotenv(dotenv_path=config)
+
+import files as files
+import pyboard as pyboard
+
+
+_board = None
+
+
+def windows_full_port_name(portname):
+    # Helper function to generate proper Windows COM port paths.  Apparently
+    # Windows requires COM ports above 9 to have a special path, where ports below
+    # 9 are just referred to by COM1, COM2, etc. (wacky!)  See this post for
+    # more info and where this code came from:
+    # http://eli.thegreenplace.net/2009/07/31/listing-all-serial-ports-on-windows-with-python/
+    m = re.match("^COM(\d+)$", portname)
+    if m and int(m.group(1)) < 10:
+        return portname
+    else:
+        return "\\\\.\\{0}".format(portname)
+
+
+@click.group()
+@click.option(
+    "--port",
+    "-p",
+    envvar="AMPY_PORT",
+    required=True,
+    type=click.STRING,
+    help="Name of serial port for connected board.  Can optionally specify with AMPY_PORT environment variable.",
+    metavar="PORT",
+)
+@click.option(
+    "--baud",
+    "-b",
+    envvar="AMPY_BAUD",
+    default=115200,
+    type=click.INT,
+    help="Baud rate for the serial connection (default 115200).  Can optionally specify with AMPY_BAUD environment variable.",
+    metavar="BAUD",
+)
+@click.option(
+    "--delay",
+    "-d",
+    envvar="AMPY_DELAY",
+    default=0,
+    type=click.FLOAT,
+    help="Delay in seconds before entering RAW MODE (default 0). Can optionally specify with AMPY_DELAY environment variable.",
+    metavar="DELAY",
+)
+@click.option(
+    "--reset",
+    "-r",
+    envvar="AMPY_RESET",
+    default="{}",
+    type=click.STRING,
+    help="default={}",
+    metavar="RESET",
+)
+@click.option(
+    "--empty",
+    "-e",
+    envvar="AMPY_EMPTY",
+    default="main.py",
+    type=click.STRING,
+    help="default=main.py",
+    metavar="EMPTY",
+)
+@click.option(
+    "--info",
+    "-i",
+    envvar="AMPY_INFO",
+    default=True,
+    type=click.BOOL,
+    help="default=True",
+    metavar="INFO",
+)
+@click.version_option()
+def cli(port, baud, delay, reset="{}", empty="main.py", info=True):
+    """ampy - Adafruit MicroPython Tool
+
+    Ampy is a tool to control MicroPython boards over a serial connection.  Using
+    ampy you can manipulate files on the board's internal filesystem and even run
+    scripts.
+    """
+    global _board
+    # On Windows fix the COM port path name for ports above 9 (see comment in
+    # windows_full_port_name function).
+    #sys.stdout.write(reset + "\n")
+    #sys.stdout.flush()
+    if platform.system() == "Windows":
+        port = windows_full_port_name(port)
+    _board = pyboard.Pyboard(port, baudrate=baud, rawdelay=delay, boardreset=reset, file_empty=empty, info=info)
+
+
+@cli.command()
+@click.argument("remote_file")
+@click.argument("local_file", type=click.File("wb"), required=False)
+def get(remote_file, local_file):
+    """
+    Retrieve a file from the board.
+
+    Get will download a file from the board and print its contents or save it
+    locally.  You must pass at least one argument which is the path to the file
+    to download from the board.  If you don't specify a second argument then
+    the file contents will be printed to standard output.  However if you pass
+    a file name as the second argument then the contents of the downloaded file
+    will be saved to that file (overwriting anything inside it!).
+
+    For example to retrieve the boot.py and print it out run:
+
+      ampy --port /board/serial/port get boot.py
+
+    Or to get main.py and save it as main.py locally run:
+
+      ampy --port /board/serial/port get main.py main.py
+    """
+    # Get the file contents.
+    board_files = files.Files(_board)
+    contents = board_files.get(remote_file)
+    # Print the file out if no local file was provided, otherwise save it.
+    if local_file is None:
+        print(contents.decode("utf-8"))
+    else:
+        local_file.write(contents)
+
+
+@cli.command()
+@click.option(
+    "--exists-okay", is_flag=True, help="Ignore if the directory already exists."
+)
+@click.argument("directory")
+def mkdir(directory, exists_okay):
+    """
+    Create a directory on the board.
+
+    Mkdir will create the specified directory on the board.  One argument is
+    required, the full path of the directory to create.
+
+    Note that you cannot recursively create a hierarchy of directories with one
+    mkdir command, instead you must create each parent directory with separate
+    mkdir command calls.
+
+    For example to make a directory under the root called 'code':
+
+      ampy --port /board/serial/port mkdir /code
+    """
+    # Run the mkdir command.
+    board_files = files.Files(_board)
+    board_files.mkdir(directory, exists_okay=exists_okay)
+
+
+@cli.command()
+@click.argument("file")
+def mkfile(file):
+    board_files = files.Files(_board)
+    board_files.mkfile(file)
+
+
+@cli.command()
+@click.argument("directory", default="/")
+@click.option(
+    "--long_format",
+    "-l",
+    is_flag=True,
+    help="Print long format info including size of files.  Note the size of directories is not supported and will show 0 values.",
+)
+@click.option(
+    "--recursive",
+    "-r",
+    is_flag=True,
+    help="recursively list all files and (empty) directories.",
+)
+def ls(directory, long_format, recursive):
+    """List contents of a directory on the board.
+
+    Can pass an optional argument which is the path to the directory.  The
+    default is to list the contents of the root, /, path.
+
+    For example to list the contents of the root run:
+
+      ampy --port /board/serial/port ls
+
+    Or to list the contents of the /foo/bar directory on the board run:
+
+      ampy --port /board/serial/port ls /foo/bar
+
+    Add the -l or --long_format flag to print the size of files (however note
+    MicroPython does not calculate the size of folders and will show 0 bytes):
+
+      ampy --port /board/serial/port ls -l /foo/bar
+    """
+    # List each file/directory on a separate line.
+    board_files = files.Files(_board)
+    for f in board_files.ls(directory, long_format=long_format, recursive=recursive):
+        print(f)
+
+
+@cli.command()
+@click.argument("local", type=click.Path(exists=True))
+@click.argument("remote", required=False)
+def put(local, remote):
+    """Put a file or folder and its contents on the board.
+
+    Put will upload a local file or folder  to the board.  If the file already
+    exists on the board it will be overwritten with no warning!  You must pass
+    at least one argument which is the path to the local file/folder to
+    upload.  If the item to upload is a folder then it will be copied to the
+    board recursively with its entire child structure.  You can pass a second
+    optional argument which is the path and name of the file/folder to put to
+    on the connected board.
+
+    For example to upload a main.py from the current directory to the board's
+    root run:
+
+      ampy --port /board/serial/port put main.py
+
+    Or to upload a board_boot.py from a ./foo subdirectory and save it as boot.py
+    in the board's root run:
+
+      ampy --port /board/serial/port put ./foo/board_boot.py boot.py
+
+    To upload a local folder adafruit_library and all of its child files/folders
+    as an item under the board's root run:
+
+      ampy --port /board/serial/port put adafruit_library
+
+    Or to put a local folder adafruit_library on the board under the path
+    /lib/adafruit_library on the board run:
+
+      ampy --port /board/serial/port put adafruit_library /lib/adafruit_library
+    """
+    # Use the local filename if no remote filename is provided.
+    
+    # Check if path is a folder and do recursive copy of everything inside it.
+    # Otherwise it's a file and should simply be copied over.
+    
+    if os.path.isdir(local):
+        if remote is None:
+            remote = ""
+        # Directory copy, create the directory and walk all children to copy
+        # over the files.
+        #print("true")
+        #print(remote)
+        board_files = files.Files(_board)
+        board_files._pyboard.enter_raw_repl()
+        file_empty = board_files._pyboard.file_empty
+        # sys.stdout.write("Empty ./{}...\n".format(file_empty))
+        # sys.stdout.flush()
+        # board_files.put('./{}'.format(file_empty), '', False, False)
+        # sys.stdout.write("Empty ./{} Done!\n".format(file_empty))
+        # sys.stdout.flush()
+        files_info = board_files.getFilesInfo('')
+        files_dict = {}
+        files_info_len = len(files_info)
+        for i in range(0, files_info_len, 1):
+            if files_info[i][0][0] == '/':
+                files_info[i][0] = files_info[i][0][1 : len(files_info[i][0])]
+            files_dict[files_info[i][0]] = files_info[i][1]
+        # sys.stdout.write(str(files_dict))
+        # sys.stdout.flush()
+        for parent, child_dirs, child_files in os.walk(local, followlinks=True):
+            # Create board filesystem absolute path to parent directory.
+            
+            remote_parent = posixpath.normpath(
+                posixpath.join(remote, os.path.relpath(parent, local))
+            )
+            #print(remote_parent)
+            '''
+            try:
+                # Create remote parent directory.
+                print(remote_parent)
+                board_files.mkdir(remote_parent)
+            except files.DirectoryExistsError:
+                # Ignore errors for directories that already exist.
+                pass
+            # Loop through all the files and put them on the board too.
+            '''
+
+            
+            file_name_list = []
+            data_list = []
+            for filename in child_files:
+                file_path = os.path.join(parent, filename)
+                if remote_parent == '.':
+                    remote_filename = filename
+                else:
+                    remote_filename = posixpath.join(remote_parent, filename)
+                file_size = os.path.getsize(file_path)
+                board_file_size = files_dict.get(remote_filename, -1)
+                # sys.stdout.write('name {0} size {1} {2}\n'.format(remote_filename, board_file_size, file_size))
+                # sys.stdout.flush()
+                if board_file_size != file_size:
+                    with open(file_path, "rb") as infile:
+                        file_name_list.append(remote_filename)
+                        data_list.append(infile.read())
+                        #board_files.put(remote_filename, infile.read())
+                else:
+                    sys.stdout.write("Skip " + filename + "\n")
+                    sys.stdout.flush()
+            board_files.putDir(file_name_list, data_list, False)
+
+            '''
+            for filename in child_files:
+                with open(os.path.join(parent, filename), "rb") as infile:
+                    remote_filename = posixpath.join(remote_parent, filename)
+                    board_files.put(remote_filename, infile.read())
+            '''
+
+    else:
+        if remote is None:
+            remote = os.path.basename(os.path.abspath(local))
+        # File copy, open the file and copy its contents to the board.
+        # Put the file on the board.
+        with open(local, "rb") as infile:
+            board_files = files.Files(_board)
+            board_files.put(remote, infile.read())
+
+
+@cli.command()
+@click.argument("remote_file")
+def rm(remote_file):
+    """Remove a file from the board.
+
+    Remove the specified file from the board's filesystem.  Must specify one
+    argument which is the path to the file to delete.  Note that this can't
+    delete directories which have files inside them, but can delete empty
+    directories.
+
+    For example to delete main.py from the root of a board run:
+
+      ampy --port /board/serial/port rm main.py
+    """
+    # Delete the provided file/directory on the board.
+    board_files = files.Files(_board)
+    board_files.rm(remote_file)
+
+
+@cli.command()
+@click.option(
+    "--missing-okay", is_flag=True, help="Ignore if the directory does not exist."
+)
+@click.argument("remote_folder")
+def rmdir(remote_folder, missing_okay):
+    """Forcefully remove a folder and all its children from the board.
+
+    Remove the specified folder from the board's filesystem.  Must specify one
+    argument which is the path to the folder to delete.  This will delete the
+    directory and ALL of its children recursively, use with caution!
+
+    For example to delete everything under /adafruit_library from the root of a
+    board run:
+
+      ampy --port /board/serial/port rmdir adafruit_library
+    """
+    # Delete the provided file/directory on the board.
+    board_files = files.Files(_board)
+    board_files.rmdir(remote_folder, missing_okay=missing_okay)
+
+@cli.command()
+@click.argument("oldname")
+@click.argument("newname")
+def rename(oldname, newname):
+    board_files = files.Files(_board)
+    board_files.rename(oldname, newname)
+
+
+@cli.command()
+@click.argument("local_file")
+@click.option(
+    "--no-output",
+    "-n",
+    is_flag=True,
+    help="Run the code without waiting for it to finish and print output.  Use this when running code with main loops that never return.",
+)
+def run(local_file, no_output):
+    """Run a script and print its output.
+
+    Run will send the specified file to the board and execute it immediately.
+    Any output from the board will be printed to the console (note that this is
+    not a 'shell' and you can't send input to the program).
+
+    Note that if your code has a main or infinite loop you should add the --no-output
+    option.  This will run the script and immediately exit without waiting for
+    the script to finish and print output.
+
+    For example to run a test.py script and print any output until it finishes:
+
+      ampy --port /board/serial/port run test.py
+
+    Or to run test.py and not wait for it to finish:
+
+      ampy --port /board/serial/port run --no-output test.py
+    """
+    # Run the provided file and print its output.
+    board_files = files.Files(_board)
+    try:
+        output = board_files.run(local_file, not no_output, not no_output)
+        if output is not None:
+            print(output.decode("utf-8"), end="")
+    except IOError:
+        click.echo(
+            "Failed to find or read input file: {0}".format(local_file), err=True
+        )
+
+
+@cli.command()
+@click.option(
+    "--bootloader", "mode", flag_value="BOOTLOADER", help="Reboot into the bootloader"
+)
+@click.option(
+    "--hard",
+    "mode",
+    flag_value="NORMAL",
+    help="Perform a hard reboot, including running init.py",
+)
+@click.option(
+    "--repl",
+    "mode",
+    flag_value="SOFT",
+    default=True,
+    help="Perform a soft reboot, entering the REPL  [default]",
+)
+@click.option(
+    "--safe",
+    "mode",
+    flag_value="SAFE_MODE",
+    help="Perform a safe-mode reboot.  User code will not be run and the filesystem will be writeable over USB",
+)
+def reset(mode):
+    """Perform soft reset/reboot of the board.
+
+    Will connect to the board and perform a reset.  Depending on the board
+    and firmware, several different types of reset may be supported.
+
+      ampy --port /board/serial/port reset
+    """
+    _board.enter_raw_repl()
+    if mode == "SOFT":
+        _board.exit_raw_repl()
+        return
+
+    _board.exec_(
+        """if 1:
+        def on_next_reset(x):
+            try:
+                import microcontroller
+            except:
+                if x == 'NORMAL': return ''
+                return 'Reset mode only supported on CircuitPython'
+            try:
+                microcontroller.on_next_reset(getattr(microcontroller.RunMode, x))
+            except ValueError as e:
+                return str(e)
+            return ''
+        def reset():
+            try:
+                import microcontroller
+            except:
+                import machine as microcontroller
+            microcontroller.reset()
+    """
+    )
+    r = _board.eval("on_next_reset({})".format(repr(mode)))
+    print("here we are", repr(r))
+    if r:
+        click.echo(r, err=True)
+        return
+
+    try:
+        _board.exec_raw_no_follow("reset()")
+    except serial.serialutil.SerialException as e:
+        # An error is expected to occur, as the board should disconnect from
+        # serial when restarted via microcontroller.reset()
+        pass
+
+
+if __name__ == "__main__":
+    error_exit = False
+    try:
+        cli()
+    except BaseException as e:
+        if getattr(e, 'code', True):
+            print('Error: {}'.format(e))
+            error_exit = True
+    finally:
+        # Try to ensure the board serial connection is always gracefully closed.
+        if _board is not None:
+            try:
+                _board.close()
+            except:
+                # Swallow errors when attempting to close as it's just a best effort
+                # and shouldn't cause a new error or problem if the connection can't
+                # be closed.
+                pass
+        if error_exit:
+            sys.exit(1)

tools/python/ampy/click/__init__.py

@@ -0,0 +1,79 @@
+"""
+Click is a simple Python module inspired by the stdlib optparse to make
+writing command line scripts fun. Unlike other modules, it's based
+around a simple API that does not come with too much magic and is
+composable.
+"""
+from .core import Argument
+from .core import BaseCommand
+from .core import Command
+from .core import CommandCollection
+from .core import Context
+from .core import Group
+from .core import MultiCommand
+from .core import Option
+from .core import Parameter
+from .decorators import argument
+from .decorators import command
+from .decorators import confirmation_option
+from .decorators import group
+from .decorators import help_option
+from .decorators import make_pass_decorator
+from .decorators import option
+from .decorators import pass_context
+from .decorators import pass_obj
+from .decorators import password_option
+from .decorators import version_option
+from .exceptions import Abort
+from .exceptions import BadArgumentUsage
+from .exceptions import BadOptionUsage
+from .exceptions import BadParameter
+from .exceptions import ClickException
+from .exceptions import FileError
+from .exceptions import MissingParameter
+from .exceptions import NoSuchOption
+from .exceptions import UsageError
+from .formatting import HelpFormatter
+from .formatting import wrap_text
+from .globals import get_current_context
+from .parser import OptionParser
+from .termui import clear
+from .termui import confirm
+from .termui import echo_via_pager
+from .termui import edit
+from .termui import get_terminal_size
+from .termui import getchar
+from .termui import launch
+from .termui import pause
+from .termui import progressbar
+from .termui import prompt
+from .termui import secho
+from .termui import style
+from .termui import unstyle
+from .types import BOOL
+from .types import Choice
+from .types import DateTime
+from .types import File
+from .types import FLOAT
+from .types import FloatRange
+from .types import INT
+from .types import IntRange
+from .types import ParamType
+from .types import Path
+from .types import STRING
+from .types import Tuple
+from .types import UNPROCESSED
+from .types import UUID
+from .utils import echo
+from .utils import format_filename
+from .utils import get_app_dir
+from .utils import get_binary_stream
+from .utils import get_os_args
+from .utils import get_text_stream
+from .utils import open_file
+
+# Controls if click should emit the warning about the use of unicode
+# literals.
+disable_unicode_literals_warning = False
+
+__version__ = "7.1.2"

tools/python/ampy/click/_bashcomplete.py

@@ -0,0 +1,375 @@
+import copy
+import os
+import re
+
+from .core import Argument
+from .core import MultiCommand
+from .core import Option
+from .parser import split_arg_string
+from .types import Choice
+from .utils import echo
+
+try:
+    from collections import abc
+except ImportError:
+    import collections as abc
+
+WORDBREAK = "="
+
+# Note, only BASH version 4.4 and later have the nosort option.
+COMPLETION_SCRIPT_BASH = """
+%(complete_func)s() {
+    local IFS=$'\n'
+    COMPREPLY=( $( env COMP_WORDS="${COMP_WORDS[*]}" \\
+                   COMP_CWORD=$COMP_CWORD \\
+                   %(autocomplete_var)s=complete $1 ) )
+    return 0
+}
+
+%(complete_func)setup() {
+    local COMPLETION_OPTIONS=""
+    local BASH_VERSION_ARR=(${BASH_VERSION//./ })
+    # Only BASH version 4.4 and later have the nosort option.
+    if [ ${BASH_VERSION_ARR[0]} -gt 4 ] || ([ ${BASH_VERSION_ARR[0]} -eq 4 ] \
+&& [ ${BASH_VERSION_ARR[1]} -ge 4 ]); then
+        COMPLETION_OPTIONS="-o nosort"
+    fi
+
+    complete $COMPLETION_OPTIONS -F %(complete_func)s %(script_names)s
+}
+
+%(complete_func)setup
+"""
+
+COMPLETION_SCRIPT_ZSH = """
+#compdef %(script_names)s
+
+%(complete_func)s() {
+    local -a completions
+    local -a completions_with_descriptions
+    local -a response
+    (( ! $+commands[%(script_names)s] )) && return 1
+
+    response=("${(@f)$( env COMP_WORDS=\"${words[*]}\" \\
+                        COMP_CWORD=$((CURRENT-1)) \\
+                        %(autocomplete_var)s=\"complete_zsh\" \\
+                        %(script_names)s )}")
+
+    for key descr in ${(kv)response}; do
+      if [[ "$descr" == "_" ]]; then
+          completions+=("$key")
+      else
+          completions_with_descriptions+=("$key":"$descr")
+      fi
+    done
+
+    if [ -n "$completions_with_descriptions" ]; then
+        _describe -V unsorted completions_with_descriptions -U
+    fi
+
+    if [ -n "$completions" ]; then
+        compadd -U -V unsorted -a completions
+    fi
+    compstate[insert]="automenu"
+}
+
+compdef %(complete_func)s %(script_names)s
+"""
+
+COMPLETION_SCRIPT_FISH = (
+    "complete --no-files --command %(script_names)s --arguments"
+    ' "(env %(autocomplete_var)s=complete_fish'
+    " COMP_WORDS=(commandline -cp) COMP_CWORD=(commandline -t)"
+    ' %(script_names)s)"'
+)
+
+_completion_scripts = {
+    "bash": COMPLETION_SCRIPT_BASH,
+    "zsh": COMPLETION_SCRIPT_ZSH,
+    "fish": COMPLETION_SCRIPT_FISH,
+}
+
+_invalid_ident_char_re = re.compile(r"[^a-zA-Z0-9_]")
+
+
+def get_completion_script(prog_name, complete_var, shell):
+    cf_name = _invalid_ident_char_re.sub("", prog_name.replace("-", "_"))
+    script = _completion_scripts.get(shell, COMPLETION_SCRIPT_BASH)
+    return (
+        script
+        % {
+            "complete_func": "_{}_completion".format(cf_name),
+            "script_names": prog_name,
+            "autocomplete_var": complete_var,
+        }
+    ).strip() + ";"
+
+
+def resolve_ctx(cli, prog_name, args):
+    """Parse into a hierarchy of contexts. Contexts are connected
+    through the parent variable.
+
+    :param cli: command definition
+    :param prog_name: the program that is running
+    :param args: full list of args
+    :return: the final context/command parsed
+    """
+    ctx = cli.make_context(prog_name, args, resilient_parsing=True)
+    args = ctx.protected_args + ctx.args
+    while args:
+        if isinstance(ctx.command, MultiCommand):
+            if not ctx.command.chain:
+                cmd_name, cmd, args = ctx.command.resolve_command(ctx, args)
+                if cmd is None:
+                    return ctx
+                ctx = cmd.make_context(
+                    cmd_name, args, parent=ctx, resilient_parsing=True
+                )
+                args = ctx.protected_args + ctx.args
+            else:
+                # Walk chained subcommand contexts saving the last one.
+                while args:
+                    cmd_name, cmd, args = ctx.command.resolve_command(ctx, args)
+                    if cmd is None:
+                        return ctx
+                    sub_ctx = cmd.make_context(
+                        cmd_name,
+                        args,
+                        parent=ctx,
+                        allow_extra_args=True,
+                        allow_interspersed_args=False,
+                        resilient_parsing=True,
+                    )
+                    args = sub_ctx.args
+                ctx = sub_ctx
+                args = sub_ctx.protected_args + sub_ctx.args
+        else:
+            break
+    return ctx
+
+
+def start_of_option(param_str):
+    """
+    :param param_str: param_str to check
+    :return: whether or not this is the start of an option declaration
+        (i.e. starts "-" or "--")
+    """
+    return param_str and param_str[:1] == "-"
+
+
+def is_incomplete_option(all_args, cmd_param):
+    """
+    :param all_args: the full original list of args supplied
+    :param cmd_param: the current command paramter
+    :return: whether or not the last option declaration (i.e. starts
+        "-" or "--") is incomplete and corresponds to this cmd_param. In
+        other words whether this cmd_param option can still accept
+        values
+    """
+    if not isinstance(cmd_param, Option):
+        return False
+    if cmd_param.is_flag:
+        return False
+    last_option = None
+    for index, arg_str in enumerate(
+        reversed([arg for arg in all_args if arg != WORDBREAK])
+    ):
+        if index + 1 > cmd_param.nargs:
+            break
+        if start_of_option(arg_str):
+            last_option = arg_str
+
+    return True if last_option and last_option in cmd_param.opts else False
+
+
+def is_incomplete_argument(current_params, cmd_param):
+    """
+    :param current_params: the current params and values for this
+        argument as already entered
+    :param cmd_param: the current command parameter
+    :return: whether or not the last argument is incomplete and
+        corresponds to this cmd_param. In other words whether or not the
+        this cmd_param argument can still accept values
+    """
+    if not isinstance(cmd_param, Argument):
+        return False
+    current_param_values = current_params[cmd_param.name]
+    if current_param_values is None:
+        return True
+    if cmd_param.nargs == -1:
+        return True
+    if (
+        isinstance(current_param_values, abc.Iterable)
+        and cmd_param.nargs > 1
+        and len(current_param_values) < cmd_param.nargs
+    ):
+        return True
+    return False
+
+
+def get_user_autocompletions(ctx, args, incomplete, cmd_param):
+    """
+    :param ctx: context associated with the parsed command
+    :param args: full list of args
+    :param incomplete: the incomplete text to autocomplete
+    :param cmd_param: command definition
+    :return: all the possible user-specified completions for the param
+    """
+    results = []
+    if isinstance(cmd_param.type, Choice):
+        # Choices don't support descriptions.
+        results = [
+            (c, None) for c in cmd_param.type.choices if str(c).startswith(incomplete)
+        ]
+    elif cmd_param.autocompletion is not None:
+        dynamic_completions = cmd_param.autocompletion(
+            ctx=ctx, args=args, incomplete=incomplete
+        )
+        results = [
+            c if isinstance(c, tuple) else (c, None) for c in dynamic_completions
+        ]
+    return results
+
+
+def get_visible_commands_starting_with(ctx, starts_with):
+    """
+    :param ctx: context associated with the parsed command
+    :starts_with: string that visible commands must start with.
+    :return: all visible (not hidden) commands that start with starts_with.
+    """
+    for c in ctx.command.list_commands(ctx):
+        if c.startswith(starts_with):
+            command = ctx.command.get_command(ctx, c)
+            if not command.hidden:
+                yield command
+
+
+def add_subcommand_completions(ctx, incomplete, completions_out):
+    # Add subcommand completions.
+    if isinstance(ctx.command, MultiCommand):
+        completions_out.extend(
+            [
+                (c.name, c.get_short_help_str())
+                for c in get_visible_commands_starting_with(ctx, incomplete)
+            ]
+        )
+
+    # Walk up the context list and add any other completion
+    # possibilities from chained commands
+    while ctx.parent is not None:
+        ctx = ctx.parent
+        if isinstance(ctx.command, MultiCommand) and ctx.command.chain:
+            remaining_commands = [
+                c
+                for c in get_visible_commands_starting_with(ctx, incomplete)
+                if c.name not in ctx.protected_args
+            ]
+            completions_out.extend(
+                [(c.name, c.get_short_help_str()) for c in remaining_commands]
+            )
+
+
+def get_choices(cli, prog_name, args, incomplete):
+    """
+    :param cli: command definition
+    :param prog_name: the program that is running
+    :param args: full list of args
+    :param incomplete: the incomplete text to autocomplete
+    :return: all the possible completions for the incomplete
+    """
+    all_args = copy.deepcopy(args)
+
+    ctx = resolve_ctx(cli, prog_name, args)
+    if ctx is None:
+        return []
+
+    has_double_dash = "--" in all_args
+
+    # In newer versions of bash long opts with '='s are partitioned, but
+    # it's easier to parse without the '='
+    if start_of_option(incomplete) and WORDBREAK in incomplete:
+        partition_incomplete = incomplete.partition(WORDBREAK)
+        all_args.append(partition_incomplete[0])
+        incomplete = partition_incomplete[2]
+    elif incomplete == WORDBREAK:
+        incomplete = ""
+
+    completions = []
+    if not has_double_dash and start_of_option(incomplete):
+        # completions for partial options
+        for param in ctx.command.params:
+            if isinstance(param, Option) and not param.hidden:
+                param_opts = [
+                    param_opt
+                    for param_opt in param.opts + param.secondary_opts
+                    if param_opt not in all_args or param.multiple
+                ]
+                completions.extend(
+                    [(o, param.help) for o in param_opts if o.startswith(incomplete)]
+                )
+        return completions
+    # completion for option values from user supplied values
+    for param in ctx.command.params:
+        if is_incomplete_option(all_args, param):
+            return get_user_autocompletions(ctx, all_args, incomplete, param)
+    # completion for argument values from user supplied values
+    for param in ctx.command.params:
+        if is_incomplete_argument(ctx.params, param):
+            return get_user_autocompletions(ctx, all_args, incomplete, param)
+
+    add_subcommand_completions(ctx, incomplete, completions)
+    # Sort before returning so that proper ordering can be enforced in custom types.
+    return sorted(completions)
+
+
+def do_complete(cli, prog_name, include_descriptions):
+    cwords = split_arg_string(os.environ["COMP_WORDS"])
+    cword = int(os.environ["COMP_CWORD"])
+    args = cwords[1:cword]
+    try:
+        incomplete = cwords[cword]
+    except IndexError:
+        incomplete = ""
+
+    for item in get_choices(cli, prog_name, args, incomplete):
+        echo(item[0])
+        if include_descriptions:
+            # ZSH has trouble dealing with empty array parameters when
+            # returned from commands, use '_' to indicate no description
+            # is present.
+            echo(item[1] if item[1] else "_")
+
+    return True
+
+
+def do_complete_fish(cli, prog_name):
+    cwords = split_arg_string(os.environ["COMP_WORDS"])
+    incomplete = os.environ["COMP_CWORD"]
+    args = cwords[1:]
+
+    for item in get_choices(cli, prog_name, args, incomplete):
+        if item[1]:
+            echo("{arg}\t{desc}".format(arg=item[0], desc=item[1]))
+        else:
+            echo(item[0])
+
+    return True
+
+
+def bashcomplete(cli, prog_name, complete_var, complete_instr):
+    if "_" in complete_instr:
+        command, shell = complete_instr.split("_", 1)
+    else:
+        command = complete_instr
+        shell = "bash"
+
+    if command == "source":
+        echo(get_completion_script(prog_name, complete_var, shell))
+        return True
+    elif command == "complete":
+        if shell == "fish":
+            return do_complete_fish(cli, prog_name)
+        elif shell in {"bash", "zsh"}:
+            return do_complete(cli, prog_name, shell == "zsh")
+
+    return False

tools/python/ampy/click/_compat.py

@@ -0,0 +1,786 @@
+# flake8: noqa
+import codecs
+import io
+import os
+import re
+import sys
+from weakref import WeakKeyDictionary
+
+PY2 = sys.version_info[0] == 2
+CYGWIN = sys.platform.startswith("cygwin")
+MSYS2 = sys.platform.startswith("win") and ("GCC" in sys.version)
+# Determine local App Engine environment, per Google's own suggestion
+APP_ENGINE = "APPENGINE_RUNTIME" in os.environ and "Development/" in os.environ.get(
+    "SERVER_SOFTWARE", ""
+)
+WIN = sys.platform.startswith("win") and not APP_ENGINE and not MSYS2
+DEFAULT_COLUMNS = 80
+
+
+_ansi_re = re.compile(r"\033\[[;?0-9]*[a-zA-Z]")
+
+
+def get_filesystem_encoding():
+    return sys.getfilesystemencoding() or sys.getdefaultencoding()
+
+
+def _make_text_stream(
+    stream, encoding, errors, force_readable=False, force_writable=False
+):
+    if encoding is None:
+        encoding = get_best_encoding(stream)
+    if errors is None:
+        errors = "replace"
+    return _NonClosingTextIOWrapper(
+        stream,
+        encoding,
+        errors,
+        line_buffering=True,
+        force_readable=force_readable,
+        force_writable=force_writable,
+    )
+
+
+def is_ascii_encoding(encoding):
+    """Checks if a given encoding is ascii."""
+    try:
+        return codecs.lookup(encoding).name == "ascii"
+    except LookupError:
+        return False
+
+
+def get_best_encoding(stream):
+    """Returns the default stream encoding if not found."""
+    rv = getattr(stream, "encoding", None) or sys.getdefaultencoding()
+    if is_ascii_encoding(rv):
+        return "utf-8"
+    return rv
+
+
+class _NonClosingTextIOWrapper(io.TextIOWrapper):
+    def __init__(
+        self,
+        stream,
+        encoding,
+        errors,
+        force_readable=False,
+        force_writable=False,
+        **extra
+    ):
+        self._stream = stream = _FixupStream(stream, force_readable, force_writable)
+        io.TextIOWrapper.__init__(self, stream, encoding, errors, **extra)
+
+    # The io module is a place where the Python 3 text behavior
+    # was forced upon Python 2, so we need to unbreak
+    # it to look like Python 2.
+    if PY2:
+
+        def write(self, x):
+            if isinstance(x, str) or is_bytes(x):
+                try:
+                    self.flush()
+                except Exception:
+                    pass
+                return self.buffer.write(str(x))
+            return io.TextIOWrapper.write(self, x)
+
+        def writelines(self, lines):
+            for line in lines:
+                self.write(line)
+
+    def __del__(self):
+        try:
+            self.detach()
+        except Exception:
+            pass
+
+    def isatty(self):
+        # https://bitbucket.org/pypy/pypy/issue/1803
+        return self._stream.isatty()
+
+
+class _FixupStream(object):
+    """The new io interface needs more from streams than streams
+    traditionally implement.  As such, this fix-up code is necessary in
+    some circumstances.
+
+    The forcing of readable and writable flags are there because some tools
+    put badly patched objects on sys (one such offender are certain version
+    of jupyter notebook).
+    """
+
+    def __init__(self, stream, force_readable=False, force_writable=False):
+        self._stream = stream
+        self._force_readable = force_readable
+        self._force_writable = force_writable
+
+    def __getattr__(self, name):
+        return getattr(self._stream, name)
+
+    def read1(self, size):
+        f = getattr(self._stream, "read1", None)
+        if f is not None:
+            return f(size)
+        # We only dispatch to readline instead of read in Python 2 as we
+        # do not want cause problems with the different implementation
+        # of line buffering.
+        if PY2:
+            return self._stream.readline(size)
+        return self._stream.read(size)
+
+    def readable(self):
+        if self._force_readable:
+            return True
+        x = getattr(self._stream, "readable", None)
+        if x is not None:
+            return x()
+        try:
+            self._stream.read(0)
+        except Exception:
+            return False
+        return True
+
+    def writable(self):
+        if self._force_writable:
+            return True
+        x = getattr(self._stream, "writable", None)
+        if x is not None:
+            return x()
+        try:
+            self._stream.write("")
+        except Exception:
+            try:
+                self._stream.write(b"")
+            except Exception:
+                return False
+        return True
+
+    def seekable(self):
+        x = getattr(self._stream, "seekable", None)
+        if x is not None:
+            return x()
+        try:
+            self._stream.seek(self._stream.tell())
+        except Exception:
+            return False
+        return True
+
+
+if PY2:
+    text_type = unicode
+    raw_input = raw_input
+    string_types = (str, unicode)
+    int_types = (int, long)
+    iteritems = lambda x: x.iteritems()
+    range_type = xrange
+
+    def is_bytes(x):
+        return isinstance(x, (buffer, bytearray))
+
+    _identifier_re = re.compile(r"^[a-zA-Z_][a-zA-Z0-9_]*$")
+
+    # For Windows, we need to force stdout/stdin/stderr to binary if it's
+    # fetched for that.  This obviously is not the most correct way to do
+    # it as it changes global state.  Unfortunately, there does not seem to
+    # be a clear better way to do it as just reopening the file in binary
+    # mode does not change anything.
+    #
+    # An option would be to do what Python 3 does and to open the file as
+    # binary only, patch it back to the system, and then use a wrapper
+    # stream that converts newlines.  It's not quite clear what's the
+    # correct option here.
+    #
+    # This code also lives in _winconsole for the fallback to the console
+    # emulation stream.
+    #
+    # There are also Windows environments where the `msvcrt` module is not
+    # available (which is why we use try-catch instead of the WIN variable
+    # here), such as the Google App Engine development server on Windows. In
+    # those cases there is just nothing we can do.
+    def set_binary_mode(f):
+        return f
+
+    try:
+        import msvcrt
+    except ImportError:
+        pass
+    else:
+
+        def set_binary_mode(f):
+            try:
+                fileno = f.fileno()
+            except Exception:
+                pass
+            else:
+                msvcrt.setmode(fileno, os.O_BINARY)
+            return f
+
+    try:
+        import fcntl
+    except ImportError:
+        pass
+    else:
+
+        def set_binary_mode(f):
+            try:
+                fileno = f.fileno()
+            except Exception:
+                pass
+            else:
+                flags = fcntl.fcntl(fileno, fcntl.F_GETFL)
+                fcntl.fcntl(fileno, fcntl.F_SETFL, flags & ~os.O_NONBLOCK)
+            return f
+
+    def isidentifier(x):
+        return _identifier_re.search(x) is not None
+
+    def get_binary_stdin():
+        return set_binary_mode(sys.stdin)
+
+    def get_binary_stdout():
+        _wrap_std_stream("stdout")
+        return set_binary_mode(sys.stdout)
+
+    def get_binary_stderr():
+        _wrap_std_stream("stderr")
+        return set_binary_mode(sys.stderr)
+
+    def get_text_stdin(encoding=None, errors=None):
+        rv = _get_windows_console_stream(sys.stdin, encoding, errors)
+        if rv is not None:
+            return rv
+        return _make_text_stream(sys.stdin, encoding, errors, force_readable=True)
+
+    def get_text_stdout(encoding=None, errors=None):
+        _wrap_std_stream("stdout")
+        rv = _get_windows_console_stream(sys.stdout, encoding, errors)
+        if rv is not None:
+            return rv
+        return _make_text_stream(sys.stdout, encoding, errors, force_writable=True)
+
+    def get_text_stderr(encoding=None, errors=None):
+        _wrap_std_stream("stderr")
+        rv = _get_windows_console_stream(sys.stderr, encoding, errors)
+        if rv is not None:
+            return rv
+        return _make_text_stream(sys.stderr, encoding, errors, force_writable=True)
+
+    def filename_to_ui(value):
+        if isinstance(value, bytes):
+            value = value.decode(get_filesystem_encoding(), "replace")
+        return value
+
+
+else:
+    import io
+
+    text_type = str
+    raw_input = input
+    string_types = (str,)
+    int_types = (int,)
+    range_type = range
+    isidentifier = lambda x: x.isidentifier()
+    iteritems = lambda x: iter(x.items())
+
+    def is_bytes(x):
+        return isinstance(x, (bytes, memoryview, bytearray))
+
+    def _is_binary_reader(stream, default=False):
+        try:
+            return isinstance(stream.read(0), bytes)
+        except Exception:
+            return default
+            # This happens in some cases where the stream was already
+            # closed.  In this case, we assume the default.
+
+    def _is_binary_writer(stream, default=False):
+        try:
+            stream.write(b"")
+        except Exception:
+            try:
+                stream.write("")
+                return False
+            except Exception:
+                pass
+            return default
+        return True
+
+    def _find_binary_reader(stream):
+        # We need to figure out if the given stream is already binary.
+        # This can happen because the official docs recommend detaching
+        # the streams to get binary streams.  Some code might do this, so
+        # we need to deal with this case explicitly.
+        if _is_binary_reader(stream, False):
+            return stream
+
+        buf = getattr(stream, "buffer", None)
+
+        # Same situation here; this time we assume that the buffer is
+        # actually binary in case it's closed.
+        if buf is not None and _is_binary_reader(buf, True):
+            return buf
+
+    def _find_binary_writer(stream):
+        # We need to figure out if the given stream is already binary.
+        # This can happen because the official docs recommend detatching
+        # the streams to get binary streams.  Some code might do this, so
+        # we need to deal with this case explicitly.
+        if _is_binary_writer(stream, False):
+            return stream
+
+        buf = getattr(stream, "buffer", None)
+
+        # Same situation here; this time we assume that the buffer is
+        # actually binary in case it's closed.
+        if buf is not None and _is_binary_writer(buf, True):
+            return buf
+
+    def _stream_is_misconfigured(stream):
+        """A stream is misconfigured if its encoding is ASCII."""
+        # If the stream does not have an encoding set, we assume it's set
+        # to ASCII.  This appears to happen in certain unittest
+        # environments.  It's not quite clear what the correct behavior is
+        # but this at least will force Click to recover somehow.
+        return is_ascii_encoding(getattr(stream, "encoding", None) or "ascii")
+
+    def _is_compat_stream_attr(stream, attr, value):
+        """A stream attribute is compatible if it is equal to the
+        desired value or the desired value is unset and the attribute
+        has a value.
+        """
+        stream_value = getattr(stream, attr, None)
+        return stream_value == value or (value is None and stream_value is not None)
+
+    def _is_compatible_text_stream(stream, encoding, errors):
+        """Check if a stream's encoding and errors attributes are
+        compatible with the desired values.
+        """
+        return _is_compat_stream_attr(
+            stream, "encoding", encoding
+        ) and _is_compat_stream_attr(stream, "errors", errors)
+
+    def _force_correct_text_stream(
+        text_stream,
+        encoding,
+        errors,
+        is_binary,
+        find_binary,
+        force_readable=False,
+        force_writable=False,
+    ):
+        if is_binary(text_stream, False):
+            binary_reader = text_stream
+        else:
+            # If the stream looks compatible, and won't default to a
+            # misconfigured ascii encoding, return it as-is.
+            if _is_compatible_text_stream(text_stream, encoding, errors) and not (
+                encoding is None and _stream_is_misconfigured(text_stream)
+            ):
+                return text_stream
+
+            # Otherwise, get the underlying binary reader.
+            binary_reader = find_binary(text_stream)
+
+            # If that's not possible, silently use the original reader
+            # and get mojibake instead of exceptions.
+            if binary_reader is None:
+                return text_stream
+
+        # Default errors to replace instead of strict in order to get
+        # something that works.
+        if errors is None:
+            errors = "replace"
+
+        # Wrap the binary stream in a text stream with the correct
+        # encoding parameters.
+        return _make_text_stream(
+            binary_reader,
+            encoding,
+            errors,
+            force_readable=force_readable,
+            force_writable=force_writable,
+        )
+
+    def _force_correct_text_reader(text_reader, encoding, errors, force_readable=False):
+        return _force_correct_text_stream(
+            text_reader,
+            encoding,
+            errors,
+            _is_binary_reader,
+            _find_binary_reader,
+            force_readable=force_readable,
+        )
+
+    def _force_correct_text_writer(text_writer, encoding, errors, force_writable=False):
+        return _force_correct_text_stream(
+            text_writer,
+            encoding,
+            errors,
+            _is_binary_writer,
+            _find_binary_writer,
+            force_writable=force_writable,
+        )
+
+    def get_binary_stdin():
+        reader = _find_binary_reader(sys.stdin)
+        if reader is None:
+            raise RuntimeError("Was not able to determine binary stream for sys.stdin.")
+        return reader
+
+    def get_binary_stdout():
+        writer = _find_binary_writer(sys.stdout)
+        if writer is None:
+            raise RuntimeError(
+                "Was not able to determine binary stream for sys.stdout."
+            )
+        return writer
+
+    def get_binary_stderr():
+        writer = _find_binary_writer(sys.stderr)
+        if writer is None:
+            raise RuntimeError(
+                "Was not able to determine binary stream for sys.stderr."
+            )
+        return writer
+
+    def get_text_stdin(encoding=None, errors=None):
+        rv = _get_windows_console_stream(sys.stdin, encoding, errors)
+        if rv is not None:
+            return rv
+        return _force_correct_text_reader(
+            sys.stdin, encoding, errors, force_readable=True
+        )
+
+    def get_text_stdout(encoding=None, errors=None):
+        rv = _get_windows_console_stream(sys.stdout, encoding, errors)
+        if rv is not None:
+            return rv
+        return _force_correct_text_writer(
+            sys.stdout, encoding, errors, force_writable=True
+        )
+
+    def get_text_stderr(encoding=None, errors=None):
+        rv = _get_windows_console_stream(sys.stderr, encoding, errors)
+        if rv is not None:
+            return rv
+        return _force_correct_text_writer(
+            sys.stderr, encoding, errors, force_writable=True
+        )
+
+    def filename_to_ui(value):
+        if isinstance(value, bytes):
+            value = value.decode(get_filesystem_encoding(), "replace")
+        else:
+            value = value.encode("utf-8", "surrogateescape").decode("utf-8", "replace")
+        return value
+
+
+def get_streerror(e, default=None):
+    if hasattr(e, "strerror"):
+        msg = e.strerror
+    else:
+        if default is not None:
+            msg = default
+        else:
+            msg = str(e)
+    if isinstance(msg, bytes):
+        msg = msg.decode("utf-8", "replace")
+    return msg
+
+
+def _wrap_io_open(file, mode, encoding, errors):
+    """On Python 2, :func:`io.open` returns a text file wrapper that
+    requires passing ``unicode`` to ``write``. Need to open the file in
+    binary mode then wrap it in a subclass that can write ``str`` and
+    ``unicode``.
+
+    Also handles not passing ``encoding`` and ``errors`` in binary mode.
+    """
+    binary = "b" in mode
+
+    if binary:
+        kwargs = {}
+    else:
+        kwargs = {"encoding": encoding, "errors": errors}
+
+    if not PY2 or binary:
+        return io.open(file, mode, **kwargs)
+
+    f = io.open(file, "{}b".format(mode.replace("t", "")))
+    return _make_text_stream(f, **kwargs)
+
+
+def open_stream(filename, mode="r", encoding=None, errors="strict", atomic=False):
+    binary = "b" in mode
+
+    # Standard streams first.  These are simple because they don't need
+    # special handling for the atomic flag.  It's entirely ignored.
+    if filename == "-":
+        if any(m in mode for m in ["w", "a", "x"]):
+            if binary:
+                return get_binary_stdout(), False
+            return get_text_stdout(encoding=encoding, errors=errors), False
+        if binary:
+            return get_binary_stdin(), False
+        return get_text_stdin(encoding=encoding, errors=errors), False
+
+    # Non-atomic writes directly go out through the regular open functions.
+    if not atomic:
+        return _wrap_io_open(filename, mode, encoding, errors), True
+
+    # Some usability stuff for atomic writes
+    if "a" in mode:
+        raise ValueError(
+            "Appending to an existing file is not supported, because that"
+            " would involve an expensive `copy`-operation to a temporary"
+            " file. Open the file in normal `w`-mode and copy explicitly"
+            " if that's what you're after."
+        )
+    if "x" in mode:
+        raise ValueError("Use the `overwrite`-parameter instead.")
+    if "w" not in mode:
+        raise ValueError("Atomic writes only make sense with `w`-mode.")
+
+    # Atomic writes are more complicated.  They work by opening a file
+    # as a proxy in the same folder and then using the fdopen
+    # functionality to wrap it in a Python file.  Then we wrap it in an
+    # atomic file that moves the file over on close.
+    import errno
+    import random
+
+    try:
+        perm = os.stat(filename).st_mode
+    except OSError:
+        perm = None
+
+    flags = os.O_RDWR | os.O_CREAT | os.O_EXCL
+
+    if binary:
+        flags |= getattr(os, "O_BINARY", 0)
+
+    while True:
+        tmp_filename = os.path.join(
+            os.path.dirname(filename),
+            ".__atomic-write{:08x}".format(random.randrange(1 << 32)),
+        )
+        try:
+            fd = os.open(tmp_filename, flags, 0o666 if perm is None else perm)
+            break
+        except OSError as e:
+            if e.errno == errno.EEXIST or (
+                os.name == "nt"
+                and e.errno == errno.EACCES
+                and os.path.isdir(e.filename)
+                and os.access(e.filename, os.W_OK)
+            ):
+                continue
+            raise
+
+    if perm is not None:
+        os.chmod(tmp_filename, perm)  # in case perm includes bits in umask
+
+    f = _wrap_io_open(fd, mode, encoding, errors)
+    return _AtomicFile(f, tmp_filename, os.path.realpath(filename)), True
+
+
+# Used in a destructor call, needs extra protection from interpreter cleanup.
+if hasattr(os, "replace"):
+    _replace = os.replace
+    _can_replace = True
+else:
+    _replace = os.rename
+    _can_replace = not WIN
+
+
+class _AtomicFile(object):
+    def __init__(self, f, tmp_filename, real_filename):
+        self._f = f
+        self._tmp_filename = tmp_filename
+        self._real_filename = real_filename
+        self.closed = False
+
+    @property
+    def name(self):
+        return self._real_filename
+
+    def close(self, delete=False):
+        if self.closed:
+            return
+        self._f.close()
+        if not _can_replace:
+            try:
+                os.remove(self._real_filename)
+            except OSError:
+                pass
+        _replace(self._tmp_filename, self._real_filename)
+        self.closed = True
+
+    def __getattr__(self, name):
+        return getattr(self._f, name)
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_value, tb):
+        self.close(delete=exc_type is not None)
+
+    def __repr__(self):
+        return repr(self._f)
+
+
+auto_wrap_for_ansi = None
+colorama = None
+get_winterm_size = None
+
+
+def strip_ansi(value):
+    return _ansi_re.sub("", value)
+
+
+def _is_jupyter_kernel_output(stream):
+    if WIN:
+        # TODO: Couldn't test on Windows, should't try to support until
+        # someone tests the details wrt colorama.
+        return
+
+    while isinstance(stream, (_FixupStream, _NonClosingTextIOWrapper)):
+        stream = stream._stream
+
+    return stream.__class__.__module__.startswith("ipykernel.")
+
+
+def should_strip_ansi(stream=None, color=None):
+    if color is None:
+        if stream is None:
+            stream = sys.stdin
+        return not isatty(stream) and not _is_jupyter_kernel_output(stream)
+    return not color
+
+
+# If we're on Windows, we provide transparent integration through
+# colorama.  This will make ANSI colors through the echo function
+# work automatically.
+if WIN:
+    # Windows has a smaller terminal
+    DEFAULT_COLUMNS = 79
+
+    from ._winconsole import _get_windows_console_stream, _wrap_std_stream
+
+    def _get_argv_encoding():
+        import locale
+
+        return locale.getpreferredencoding()
+
+    if PY2:
+
+        def raw_input(prompt=""):
+            sys.stderr.flush()
+            if prompt:
+                stdout = _default_text_stdout()
+                stdout.write(prompt)
+            stdin = _default_text_stdin()
+            return stdin.readline().rstrip("\r\n")
+
+    try:
+        import colorama
+    except ImportError:
+        pass
+    else:
+        _ansi_stream_wrappers = WeakKeyDictionary()
+
+        def auto_wrap_for_ansi(stream, color=None):
+            """This function wraps a stream so that calls through colorama
+            are issued to the win32 console API to recolor on demand.  It
+            also ensures to reset the colors if a write call is interrupted
+            to not destroy the console afterwards.
+            """
+            try:
+                cached = _ansi_stream_wrappers.get(stream)
+            except Exception:
+                cached = None
+            if cached is not None:
+                return cached
+            strip = should_strip_ansi(stream, color)
+            ansi_wrapper = colorama.AnsiToWin32(stream, strip=strip)
+            rv = ansi_wrapper.stream
+            _write = rv.write
+
+            def _safe_write(s):
+                try:
+                    return _write(s)
+                except:
+                    ansi_wrapper.reset_all()
+                    raise
+
+            rv.write = _safe_write
+            try:
+                _ansi_stream_wrappers[stream] = rv
+            except Exception:
+                pass
+            return rv
+
+        def get_winterm_size():
+            win = colorama.win32.GetConsoleScreenBufferInfo(
+                colorama.win32.STDOUT
+            ).srWindow
+            return win.Right - win.Left, win.Bottom - win.Top
+
+
+else:
+
+    def _get_argv_encoding():
+        return getattr(sys.stdin, "encoding", None) or get_filesystem_encoding()
+
+    _get_windows_console_stream = lambda *x: None
+    _wrap_std_stream = lambda *x: None
+
+
+def term_len(x):
+    return len(strip_ansi(x))
+
+
+def isatty(stream):
+    try:
+        return stream.isatty()
+    except Exception:
+        return False
+
+
+def _make_cached_stream_func(src_func, wrapper_func):
+    cache = WeakKeyDictionary()
+
+    def func():
+        stream = src_func()
+        try:
+            rv = cache.get(stream)
+        except Exception:
+            rv = None
+        if rv is not None:
+            return rv
+        rv = wrapper_func()
+        try:
+            stream = src_func()  # In case wrapper_func() modified the stream
+            cache[stream] = rv
+        except Exception:
+            pass
+        return rv
+
+    return func
+
+
+_default_text_stdin = _make_cached_stream_func(lambda: sys.stdin, get_text_stdin)
+_default_text_stdout = _make_cached_stream_func(lambda: sys.stdout, get_text_stdout)
+_default_text_stderr = _make_cached_stream_func(lambda: sys.stderr, get_text_stderr)
+
+
+binary_streams = {
+    "stdin": get_binary_stdin,
+    "stdout": get_binary_stdout,
+    "stderr": get_binary_stderr,
+}
+
+text_streams = {
+    "stdin": get_text_stdin,
+    "stdout": get_text_stdout,
+    "stderr": get_text_stderr,
+}

tools/python/ampy/click/_termui_impl.py

@@ -0,0 +1,657 @@
+# -*- coding: utf-8 -*-
+"""
+This module contains implementations for the termui module. To keep the
+import time of Click down, some infrequently used functionality is
+placed in this module and only imported as needed.
+"""
+import contextlib
+import math
+import os
+import sys
+import time
+
+from ._compat import _default_text_stdout
+from ._compat import CYGWIN
+from ._compat import get_best_encoding
+from ._compat import int_types
+from ._compat import isatty
+from ._compat import open_stream
+from ._compat import range_type
+from ._compat import strip_ansi
+from ._compat import term_len
+from ._compat import WIN
+from .exceptions import ClickException
+from .utils import echo
+
+if os.name == "nt":
+    BEFORE_BAR = "\r"
+    AFTER_BAR = "\n"
+else:
+    BEFORE_BAR = "\r\033[?25l"
+    AFTER_BAR = "\033[?25h\n"
+
+
+def _length_hint(obj):
+    """Returns the length hint of an object."""
+    try:
+        return len(obj)
+    except (AttributeError, TypeError):
+        try:
+            get_hint = type(obj).__length_hint__
+        except AttributeError:
+            return None
+        try:
+            hint = get_hint(obj)
+        except TypeError:
+            return None
+        if hint is NotImplemented or not isinstance(hint, int_types) or hint < 0:
+            return None
+        return hint
+
+
+class ProgressBar(object):
+    def __init__(
+        self,
+        iterable,
+        length=None,
+        fill_char="#",
+        empty_char=" ",
+        bar_template="%(bar)s",
+        info_sep="  ",
+        show_eta=True,
+        show_percent=None,
+        show_pos=False,
+        item_show_func=None,
+        label=None,
+        file=None,
+        color=None,
+        width=30,
+    ):
+        self.fill_char = fill_char
+        self.empty_char = empty_char
+        self.bar_template = bar_template
+        self.info_sep = info_sep
+        self.show_eta = show_eta
+        self.show_percent = show_percent
+        self.show_pos = show_pos
+        self.item_show_func = item_show_func
+        self.label = label or ""
+        if file is None:
+            file = _default_text_stdout()
+        self.file = file
+        self.color = color
+        self.width = width
+        self.autowidth = width == 0
+
+        if length is None:
+            length = _length_hint(iterable)
+        if iterable is None:
+            if length is None:
+                raise TypeError("iterable or length is required")
+            iterable = range_type(length)
+        self.iter = iter(iterable)
+        self.length = length
+        self.length_known = length is not None
+        self.pos = 0
+        self.avg = []
+        self.start = self.last_eta = time.time()
+        self.eta_known = False
+        self.finished = False
+        self.max_width = None
+        self.entered = False
+        self.current_item = None
+        self.is_hidden = not isatty(self.file)
+        self._last_line = None
+        self.short_limit = 0.5
+
+    def __enter__(self):
+        self.entered = True
+        self.render_progress()
+        return self
+
+    def __exit__(self, exc_type, exc_value, tb):
+        self.render_finish()
+
+    def __iter__(self):
+        if not self.entered:
+            raise RuntimeError("You need to use progress bars in a with block.")
+        self.render_progress()
+        return self.generator()
+
+    def __next__(self):
+        # Iteration is defined in terms of a generator function,
+        # returned by iter(self); use that to define next(). This works
+        # because `self.iter` is an iterable consumed by that generator,
+        # so it is re-entry safe. Calling `next(self.generator())`
+        # twice works and does "what you want".
+        return next(iter(self))
+
+    # Python 2 compat
+    next = __next__
+
+    def is_fast(self):
+        return time.time() - self.start <= self.short_limit
+
+    def render_finish(self):
+        if self.is_hidden or self.is_fast():
+            return
+        self.file.write(AFTER_BAR)
+        self.file.flush()
+
+    @property
+    def pct(self):
+        if self.finished:
+            return 1.0
+        return min(self.pos / (float(self.length) or 1), 1.0)
+
+    @property
+    def time_per_iteration(self):
+        if not self.avg:
+            return 0.0
+        return sum(self.avg) / float(len(self.avg))
+
+    @property
+    def eta(self):
+        if self.length_known and not self.finished:
+            return self.time_per_iteration * (self.length - self.pos)
+        return 0.0
+
+    def format_eta(self):
+        if self.eta_known:
+            t = int(self.eta)
+            seconds = t % 60
+            t //= 60
+            minutes = t % 60
+            t //= 60
+            hours = t % 24
+            t //= 24
+            if t > 0:
+                return "{}d {:02}:{:02}:{:02}".format(t, hours, minutes, seconds)
+            else:
+                return "{:02}:{:02}:{:02}".format(hours, minutes, seconds)
+        return ""
+
+    def format_pos(self):
+        pos = str(self.pos)
+        if self.length_known:
+            pos += "/{}".format(self.length)
+        return pos
+
+    def format_pct(self):
+        return "{: 4}%".format(int(self.pct * 100))[1:]
+
+    def format_bar(self):
+        if self.length_known:
+            bar_length = int(self.pct * self.width)
+            bar = self.fill_char * bar_length
+            bar += self.empty_char * (self.width - bar_length)
+        elif self.finished:
+            bar = self.fill_char * self.width
+        else:
+            bar = list(self.empty_char * (self.width or 1))
+            if self.time_per_iteration != 0:
+                bar[
+                    int(
+                        (math.cos(self.pos * self.time_per_iteration) / 2.0 + 0.5)
+                        * self.width
+                    )
+                ] = self.fill_char
+            bar = "".join(bar)
+        return bar
+
+    def format_progress_line(self):
+        show_percent = self.show_percent
+
+        info_bits = []
+        if self.length_known and show_percent is None:
+            show_percent = not self.show_pos
+
+        if self.show_pos:
+            info_bits.append(self.format_pos())
+        if show_percent:
+            info_bits.append(self.format_pct())
+        if self.show_eta and self.eta_known and not self.finished:
+            info_bits.append(self.format_eta())
+        if self.item_show_func is not None:
+            item_info = self.item_show_func(self.current_item)
+            if item_info is not None:
+                info_bits.append(item_info)
+
+        return (
+            self.bar_template
+            % {
+                "label": self.label,
+                "bar": self.format_bar(),
+                "info": self.info_sep.join(info_bits),
+            }
+        ).rstrip()
+
+    def render_progress(self):
+        from .termui import get_terminal_size
+
+        if self.is_hidden:
+            return
+
+        buf = []
+        # Update width in case the terminal has been resized
+        if self.autowidth:
+            old_width = self.width
+            self.width = 0
+            clutter_length = term_len(self.format_progress_line())
+            new_width = max(0, get_terminal_size()[0] - clutter_length)
+            if new_width < old_width:
+                buf.append(BEFORE_BAR)
+                buf.append(" " * self.max_width)
+                self.max_width = new_width
+            self.width = new_width
+
+        clear_width = self.width
+        if self.max_width is not None:
+            clear_width = self.max_width
+
+        buf.append(BEFORE_BAR)
+        line = self.format_progress_line()
+        line_len = term_len(line)
+        if self.max_width is None or self.max_width < line_len:
+            self.max_width = line_len
+
+        buf.append(line)
+        buf.append(" " * (clear_width - line_len))
+        line = "".join(buf)
+        # Render the line only if it changed.
+
+        if line != self._last_line and not self.is_fast():
+            self._last_line = line
+            echo(line, file=self.file, color=self.color, nl=False)
+            self.file.flush()
+
+    def make_step(self, n_steps):
+        self.pos += n_steps
+        if self.length_known and self.pos >= self.length:
+            self.finished = True
+
+        if (time.time() - self.last_eta) < 1.0:
+            return
+
+        self.last_eta = time.time()
+
+        # self.avg is a rolling list of length <= 7 of steps where steps are
+        # defined as time elapsed divided by the total progress through
+        # self.length.
+        if self.pos:
+            step = (time.time() - self.start) / self.pos
+        else:
+            step = time.time() - self.start
+
+        self.avg = self.avg[-6:] + [step]
+
+        self.eta_known = self.length_known
+
+    def update(self, n_steps):
+        self.make_step(n_steps)
+        self.render_progress()
+
+    def finish(self):
+        self.eta_known = 0
+        self.current_item = None
+        self.finished = True
+
+    def generator(self):
+        """Return a generator which yields the items added to the bar
+        during construction, and updates the progress bar *after* the
+        yielded block returns.
+        """
+        # WARNING: the iterator interface for `ProgressBar` relies on
+        # this and only works because this is a simple generator which
+        # doesn't create or manage additional state. If this function
+        # changes, the impact should be evaluated both against
+        # `iter(bar)` and `next(bar)`. `next()` in particular may call
+        # `self.generator()` repeatedly, and this must remain safe in
+        # order for that interface to work.
+        if not self.entered:
+            raise RuntimeError("You need to use progress bars in a with block.")
+
+        if self.is_hidden:
+            for rv in self.iter:
+                yield rv
+        else:
+            for rv in self.iter:
+                self.current_item = rv
+                yield rv
+                self.update(1)
+            self.finish()
+            self.render_progress()
+
+
+def pager(generator, color=None):
+    """Decide what method to use for paging through text."""
+    stdout = _default_text_stdout()
+    if not isatty(sys.stdin) or not isatty(stdout):
+        return _nullpager(stdout, generator, color)
+    pager_cmd = (os.environ.get("PAGER", None) or "").strip()
+    if pager_cmd:
+        if WIN:
+            return _tempfilepager(generator, pager_cmd, color)
+        return _pipepager(generator, pager_cmd, color)
+    if os.environ.get("TERM") in ("dumb", "emacs"):
+        return _nullpager(stdout, generator, color)
+    if WIN or sys.platform.startswith("os2"):
+        return _tempfilepager(generator, "more <", color)
+    if hasattr(os, "system") and os.system("(less) 2>/dev/null") == 0:
+        return _pipepager(generator, "less", color)
+
+    import tempfile
+
+    fd, filename = tempfile.mkstemp()
+    os.close(fd)
+    try:
+        if hasattr(os, "system") and os.system('more "{}"'.format(filename)) == 0:
+            return _pipepager(generator, "more", color)
+        return _nullpager(stdout, generator, color)
+    finally:
+        os.unlink(filename)
+
+
+def _pipepager(generator, cmd, color):
+    """Page through text by feeding it to another program.  Invoking a
+    pager through this might support colors.
+    """
+    import subprocess
+
+    env = dict(os.environ)
+
+    # If we're piping to less we might support colors under the
+    # condition that
+    cmd_detail = cmd.rsplit("/", 1)[-1].split()
+    if color is None and cmd_detail[0] == "less":
+        less_flags = "{}{}".format(os.environ.get("LESS", ""), " ".join(cmd_detail[1:]))
+        if not less_flags:
+            env["LESS"] = "-R"
+            color = True
+        elif "r" in less_flags or "R" in less_flags:
+            color = True
+
+    c = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE, env=env)
+    encoding = get_best_encoding(c.stdin)
+    try:
+        for text in generator:
+            if not color:
+                text = strip_ansi(text)
+
+            c.stdin.write(text.encode(encoding, "replace"))
+    except (IOError, KeyboardInterrupt):
+        pass
+    else:
+        c.stdin.close()
+
+    # Less doesn't respect ^C, but catches it for its own UI purposes (aborting
+    # search or other commands inside less).
+    #
+    # That means when the user hits ^C, the parent process (click) terminates,
+    # but less is still alive, paging the output and messing up the terminal.
+    #
+    # If the user wants to make the pager exit on ^C, they should set
+    # `LESS='-K'`. It's not our decision to make.
+    while True:
+        try:
+            c.wait()
+        except KeyboardInterrupt:
+            pass
+        else:
+            break
+
+
+def _tempfilepager(generator, cmd, color):
+    """Page through text by invoking a program on a temporary file."""
+    import tempfile
+
+    filename = tempfile.mktemp()
+    # TODO: This never terminates if the passed generator never terminates.
+    text = "".join(generator)
+    if not color:
+        text = strip_ansi(text)
+    encoding = get_best_encoding(sys.stdout)
+    with open_stream(filename, "wb")[0] as f:
+        f.write(text.encode(encoding))
+    try:
+        os.system('{} "{}"'.format(cmd, filename))
+    finally:
+        os.unlink(filename)
+
+
+def _nullpager(stream, generator, color):
+    """Simply print unformatted text.  This is the ultimate fallback."""
+    for text in generator:
+        if not color:
+            text = strip_ansi(text)
+        stream.write(text)
+
+
+class Editor(object):
+    def __init__(self, editor=None, env=None, require_save=True, extension=".txt"):
+        self.editor = editor
+        self.env = env
+        self.require_save = require_save
+        self.extension = extension
+
+    def get_editor(self):
+        if self.editor is not None:
+            return self.editor
+        for key in "VISUAL", "EDITOR":
+            rv = os.environ.get(key)
+            if rv:
+                return rv
+        if WIN:
+            return "notepad"
+        for editor in "sensible-editor", "vim", "nano":
+            if os.system("which {} >/dev/null 2>&1".format(editor)) == 0:
+                return editor
+        return "vi"
+
+    def edit_file(self, filename):
+        import subprocess
+
+        editor = self.get_editor()
+        if self.env:
+            environ = os.environ.copy()
+            environ.update(self.env)
+        else:
+            environ = None
+        try:
+            c = subprocess.Popen(
+                '{} "{}"'.format(editor, filename), env=environ, shell=True,
+            )
+            exit_code = c.wait()
+            if exit_code != 0:
+                raise ClickException("{}: Editing failed!".format(editor))
+        except OSError as e:
+            raise ClickException("{}: Editing failed: {}".format(editor, e))
+
+    def edit(self, text):
+        import tempfile
+
+        text = text or ""
+        if text and not text.endswith("\n"):
+            text += "\n"
+
+        fd, name = tempfile.mkstemp(prefix="editor-", suffix=self.extension)
+        try:
+            if WIN:
+                encoding = "utf-8-sig"
+                text = text.replace("\n", "\r\n")
+            else:
+                encoding = "utf-8"
+            text = text.encode(encoding)
+
+            f = os.fdopen(fd, "wb")
+            f.write(text)
+            f.close()
+            timestamp = os.path.getmtime(name)
+
+            self.edit_file(name)
+
+            if self.require_save and os.path.getmtime(name) == timestamp:
+                return None
+
+            f = open(name, "rb")
+            try:
+                rv = f.read()
+            finally:
+                f.close()
+            return rv.decode("utf-8-sig").replace("\r\n", "\n")
+        finally:
+            os.unlink(name)
+
+
+def open_url(url, wait=False, locate=False):
+    import subprocess
+
+    def _unquote_file(url):
+        try:
+            import urllib
+        except ImportError:
+            import urllib
+        if url.startswith("file://"):
+            url = urllib.unquote(url[7:])
+        return url
+
+    if sys.platform == "darwin":
+        args = ["open"]
+        if wait:
+            args.append("-W")
+        if locate:
+            args.append("-R")
+        args.append(_unquote_file(url))
+        null = open("/dev/null", "w")
+        try:
+            return subprocess.Popen(args, stderr=null).wait()
+        finally:
+            null.close()
+    elif WIN:
+        if locate:
+            url = _unquote_file(url)
+            args = 'explorer /select,"{}"'.format(_unquote_file(url.replace('"', "")))
+        else:
+            args = 'start {} "" "{}"'.format(
+                "/WAIT" if wait else "", url.replace('"', "")
+            )
+        return os.system(args)
+    elif CYGWIN:
+        if locate:
+            url = _unquote_file(url)
+            args = 'cygstart "{}"'.format(os.path.dirname(url).replace('"', ""))
+        else:
+            args = 'cygstart {} "{}"'.format("-w" if wait else "", url.replace('"', ""))
+        return os.system(args)
+
+    try:
+        if locate:
+            url = os.path.dirname(_unquote_file(url)) or "."
+        else:
+            url = _unquote_file(url)
+        c = subprocess.Popen(["xdg-open", url])
+        if wait:
+            return c.wait()
+        return 0
+    except OSError:
+        if url.startswith(("http://", "https://")) and not locate and not wait:
+            import webbrowser
+
+            webbrowser.open(url)
+            return 0
+        return 1
+
+
+def _translate_ch_to_exc(ch):
+    if ch == u"\x03":
+        raise KeyboardInterrupt()
+    if ch == u"\x04" and not WIN:  # Unix-like, Ctrl+D
+        raise EOFError()
+    if ch == u"\x1a" and WIN:  # Windows, Ctrl+Z
+        raise EOFError()
+
+
+if WIN:
+    import msvcrt
+
+    @contextlib.contextmanager
+    def raw_terminal():
+        yield
+
+    def getchar(echo):
+        # The function `getch` will return a bytes object corresponding to
+        # the pressed character. Since Windows 10 build 1803, it will also
+        # return \x00 when called a second time after pressing a regular key.
+        #
+        # `getwch` does not share this probably-bugged behavior. Moreover, it
+        # returns a Unicode object by default, which is what we want.
+        #
+        # Either of these functions will return \x00 or \xe0 to indicate
+        # a special key, and you need to call the same function again to get
+        # the "rest" of the code. The fun part is that \u00e0 is
+        # "latin small letter a with grave", so if you type that on a French
+        # keyboard, you _also_ get a \xe0.
+        # E.g., consider the Up arrow. This returns \xe0 and then \x48. The
+        # resulting Unicode string reads as "a with grave" + "capital H".
+        # This is indistinguishable from when the user actually types
+        # "a with grave" and then "capital H".
+        #
+        # When \xe0 is returned, we assume it's part of a special-key sequence
+        # and call `getwch` again, but that means that when the user types
+        # the \u00e0 character, `getchar` doesn't return until a second
+        # character is typed.
+        # The alternative is returning immediately, but that would mess up
+        # cross-platform handling of arrow keys and others that start with
+        # \xe0. Another option is using `getch`, but then we can't reliably
+        # read non-ASCII characters, because return values of `getch` are
+        # limited to the current 8-bit codepage.
+        #
+        # Anyway, Click doesn't claim to do this Right(tm), and using `getwch`
+        # is doing the right thing in more situations than with `getch`.
+        if echo:
+            func = msvcrt.getwche
+        else:
+            func = msvcrt.getwch
+
+        rv = func()
+        if rv in (u"\x00", u"\xe0"):
+            # \x00 and \xe0 are control characters that indicate special key,
+            # see above.
+            rv += func()
+        _translate_ch_to_exc(rv)
+        return rv
+
+
+else:
+    import tty
+    import termios
+
+    @contextlib.contextmanager
+    def raw_terminal():
+        if not isatty(sys.stdin):
+            f = open("/dev/tty")
+            fd = f.fileno()
+        else:
+            fd = sys.stdin.fileno()
+            f = None
+        try:
+            old_settings = termios.tcgetattr(fd)
+            try:
+                tty.setraw(fd)
+                yield fd
+            finally:
+                termios.tcsetattr(fd, termios.TCSADRAIN, old_settings)
+                sys.stdout.flush()
+                if f is not None:
+                    f.close()
+        except termios.error:
+            pass
+
+    def getchar(echo):
+        with raw_terminal() as fd:
+            ch = os.read(fd, 32)
+            ch = ch.decode(get_best_encoding(sys.stdin), "replace")
+            if echo and isatty(sys.stdout):
+                sys.stdout.write(ch)
+            _translate_ch_to_exc(ch)
+            return ch

tools/python/ampy/click/_textwrap.py

@@ -0,0 +1,37 @@
+import textwrap
+from contextlib import contextmanager
+
+
+class TextWrapper(textwrap.TextWrapper):
+    def _handle_long_word(self, reversed_chunks, cur_line, cur_len, width):
+        space_left = max(width - cur_len, 1)
+
+        if self.break_long_words:
+            last = reversed_chunks[-1]
+            cut = last[:space_left]
+            res = last[space_left:]
+            cur_line.append(cut)
+            reversed_chunks[-1] = res
+        elif not cur_line:
+            cur_line.append(reversed_chunks.pop())
+
+    @contextmanager
+    def extra_indent(self, indent):
+        old_initial_indent = self.initial_indent
+        old_subsequent_indent = self.subsequent_indent
+        self.initial_indent += indent
+        self.subsequent_indent += indent
+        try:
+            yield
+        finally:
+            self.initial_indent = old_initial_indent
+            self.subsequent_indent = old_subsequent_indent
+
+    def indent_only(self, text):
+        rv = []
+        for idx, line in enumerate(text.splitlines()):
+            indent = self.initial_indent
+            if idx > 0:
+                indent = self.subsequent_indent
+            rv.append(indent + line)
+        return "\n".join(rv)

tools/python/ampy/click/_unicodefun.py

@@ -0,0 +1,131 @@
+import codecs
+import os
+import sys
+
+from ._compat import PY2
+
+
+def _find_unicode_literals_frame():
+    import __future__
+
+    if not hasattr(sys, "_getframe"):  # not all Python implementations have it
+        return 0
+    frm = sys._getframe(1)
+    idx = 1
+    while frm is not None:
+        if frm.f_globals.get("__name__", "").startswith("click."):
+            frm = frm.f_back
+            idx += 1
+        elif frm.f_code.co_flags & __future__.unicode_literals.compiler_flag:
+            return idx
+        else:
+            break
+    return 0
+
+
+def _check_for_unicode_literals():
+    if not __debug__:
+        return
+
+    from . import disable_unicode_literals_warning
+
+    if not PY2 or disable_unicode_literals_warning:
+        return
+    bad_frame = _find_unicode_literals_frame()
+    if bad_frame <= 0:
+        return
+    from warnings import warn
+
+    warn(
+        Warning(
+            "Click detected the use of the unicode_literals __future__"
+            " import. This is heavily discouraged because it can"
+            " introduce subtle bugs in your code. You should instead"
+            ' use explicit u"" literals for your unicode strings. For'
+            " more information see"
+            " https://click.palletsprojects.com/python3/"
+        ),
+        stacklevel=bad_frame,
+    )
+
+
+def _verify_python3_env():
+    """Ensures that the environment is good for unicode on Python 3."""
+    if PY2:
+        return
+    try:
+        import locale
+
+        fs_enc = codecs.lookup(locale.getpreferredencoding()).name
+    except Exception:
+        fs_enc = "ascii"
+    if fs_enc != "ascii":
+        return
+
+    extra = ""
+    if os.name == "posix":
+        import subprocess
+
+        try:
+            rv = subprocess.Popen(
+                ["locale", "-a"], stdout=subprocess.PIPE, stderr=subprocess.PIPE
+            ).communicate()[0]
+        except OSError:
+            rv = b""
+        good_locales = set()
+        has_c_utf8 = False
+
+        # Make sure we're operating on text here.
+        if isinstance(rv, bytes):
+            rv = rv.decode("ascii", "replace")
+
+        for line in rv.splitlines():
+            locale = line.strip()
+            if locale.lower().endswith((".utf-8", ".utf8")):
+                good_locales.add(locale)
+                if locale.lower() in ("c.utf8", "c.utf-8"):
+                    has_c_utf8 = True
+
+        extra += "\n\n"
+        if not good_locales:
+            extra += (
+                "Additional information: on this system no suitable"
+                " UTF-8 locales were discovered. This most likely"
+                " requires resolving by reconfiguring the locale"
+                " system."
+            )
+        elif has_c_utf8:
+            extra += (
+                "This system supports the C.UTF-8 locale which is"
+                " recommended. You might be able to resolve your issue"
+                " by exporting the following environment variables:\n\n"
+                "    export LC_ALL=C.UTF-8\n"
+                "    export LANG=C.UTF-8"
+            )
+        else:
+            extra += (
+                "This system lists a couple of UTF-8 supporting locales"
+                " that you can pick from. The following suitable"
+                " locales were discovered: {}".format(", ".join(sorted(good_locales)))
+            )
+
+        bad_locale = None
+        for locale in os.environ.get("LC_ALL"), os.environ.get("LANG"):
+            if locale and locale.lower().endswith((".utf-8", ".utf8")):
+                bad_locale = locale
+            if locale is not None:
+                break
+        if bad_locale is not None:
+            extra += (
+                "\n\nClick discovered that you exported a UTF-8 locale"
+                " but the locale system could not pick up from it"
+                " because it does not exist. The exported locale is"
+                " '{}' but it is not supported".format(bad_locale)
+            )
+
+    raise RuntimeError(
+        "Click will abort further execution because Python 3 was"
+        " configured to use ASCII as encoding for the environment."
+        " Consult https://click.palletsprojects.com/python3/ for"
+        " mitigation steps.{}".format(extra)
+    )

tools/python/ampy/click/_winconsole.py

@@ -0,0 +1,370 @@
+# -*- coding: utf-8 -*-
+# This module is based on the excellent work by Adam Bartoš who
+# provided a lot of what went into the implementation here in
+# the discussion to issue1602 in the Python bug tracker.
+#
+# There are some general differences in regards to how this works
+# compared to the original patches as we do not need to patch
+# the entire interpreter but just work in our little world of
+# echo and prmopt.
+import ctypes
+import io
+import os
+import sys
+import time
+import zlib
+from ctypes import byref
+from ctypes import c_char
+from ctypes import c_char_p
+from ctypes import c_int
+from ctypes import c_ssize_t
+from ctypes import c_ulong
+from ctypes import c_void_p
+from ctypes import POINTER
+from ctypes import py_object
+from ctypes import windll
+from ctypes import WinError
+from ctypes import WINFUNCTYPE
+from ctypes.wintypes import DWORD
+from ctypes.wintypes import HANDLE
+from ctypes.wintypes import LPCWSTR
+from ctypes.wintypes import LPWSTR
+
+import msvcrt
+
+from ._compat import _NonClosingTextIOWrapper
+from ._compat import PY2
+from ._compat import text_type
+
+try:
+    from ctypes import pythonapi
+
+    PyObject_GetBuffer = pythonapi.PyObject_GetBuffer
+    PyBuffer_Release = pythonapi.PyBuffer_Release
+except ImportError:
+    pythonapi = None
+
+
+c_ssize_p = POINTER(c_ssize_t)
+
+kernel32 = windll.kernel32
+GetStdHandle = kernel32.GetStdHandle
+ReadConsoleW = kernel32.ReadConsoleW
+WriteConsoleW = kernel32.WriteConsoleW
+GetConsoleMode = kernel32.GetConsoleMode
+GetLastError = kernel32.GetLastError
+GetCommandLineW = WINFUNCTYPE(LPWSTR)(("GetCommandLineW", windll.kernel32))
+CommandLineToArgvW = WINFUNCTYPE(POINTER(LPWSTR), LPCWSTR, POINTER(c_int))(
+    ("CommandLineToArgvW", windll.shell32)
+)
+LocalFree = WINFUNCTYPE(ctypes.c_void_p, ctypes.c_void_p)(
+    ("LocalFree", windll.kernel32)
+)
+
+
+STDIN_HANDLE = GetStdHandle(-10)
+STDOUT_HANDLE = GetStdHandle(-11)
+STDERR_HANDLE = GetStdHandle(-12)
+
+
+PyBUF_SIMPLE = 0
+PyBUF_WRITABLE = 1
+
+ERROR_SUCCESS = 0
+ERROR_NOT_ENOUGH_MEMORY = 8
+ERROR_OPERATION_ABORTED = 995
+
+STDIN_FILENO = 0
+STDOUT_FILENO = 1
+STDERR_FILENO = 2
+
+EOF = b"\x1a"
+MAX_BYTES_WRITTEN = 32767
+
+
+class Py_buffer(ctypes.Structure):
+    _fields_ = [
+        ("buf", c_void_p),
+        ("obj", py_object),
+        ("len", c_ssize_t),
+        ("itemsize", c_ssize_t),
+        ("readonly", c_int),
+        ("ndim", c_int),
+        ("format", c_char_p),
+        ("shape", c_ssize_p),
+        ("strides", c_ssize_p),
+        ("suboffsets", c_ssize_p),
+        ("internal", c_void_p),
+    ]
+
+    if PY2:
+        _fields_.insert(-1, ("smalltable", c_ssize_t * 2))
+
+
+# On PyPy we cannot get buffers so our ability to operate here is
+# serverly limited.
+if pythonapi is None:
+    get_buffer = None
+else:
+
+    def get_buffer(obj, writable=False):
+        buf = Py_buffer()
+        flags = PyBUF_WRITABLE if writable else PyBUF_SIMPLE
+        PyObject_GetBuffer(py_object(obj), byref(buf), flags)
+        try:
+            buffer_type = c_char * buf.len
+            return buffer_type.from_address(buf.buf)
+        finally:
+            PyBuffer_Release(byref(buf))
+
+
+class _WindowsConsoleRawIOBase(io.RawIOBase):
+    def __init__(self, handle):
+        self.handle = handle
+
+    def isatty(self):
+        io.RawIOBase.isatty(self)
+        return True
+
+
+class _WindowsConsoleReader(_WindowsConsoleRawIOBase):
+    def readable(self):
+        return True
+
+    def readinto(self, b):
+        bytes_to_be_read = len(b)
+        if not bytes_to_be_read:
+            return 0
+        elif bytes_to_be_read % 2:
+            raise ValueError(
+                "cannot read odd number of bytes from UTF-16-LE encoded console"
+            )
+
+        buffer = get_buffer(b, writable=True)
+        code_units_to_be_read = bytes_to_be_read // 2
+        code_units_read = c_ulong()
+
+        rv = ReadConsoleW(
+            HANDLE(self.handle),
+            buffer,
+            code_units_to_be_read,
+            byref(code_units_read),
+            None,
+        )
+        if GetLastError() == ERROR_OPERATION_ABORTED:
+            # wait for KeyboardInterrupt
+            time.sleep(0.1)
+        if not rv:
+            raise OSError("Windows error: {}".format(GetLastError()))
+
+        if buffer[0] == EOF:
+            return 0
+        return 2 * code_units_read.value
+
+
+class _WindowsConsoleWriter(_WindowsConsoleRawIOBase):
+    def writable(self):
+        return True
+
+    @staticmethod
+    def _get_error_message(errno):
+        if errno == ERROR_SUCCESS:
+            return "ERROR_SUCCESS"
+        elif errno == ERROR_NOT_ENOUGH_MEMORY:
+            return "ERROR_NOT_ENOUGH_MEMORY"
+        return "Windows error {}".format(errno)
+
+    def write(self, b):
+        bytes_to_be_written = len(b)
+        buf = get_buffer(b)
+        code_units_to_be_written = min(bytes_to_be_written, MAX_BYTES_WRITTEN) // 2
+        code_units_written = c_ulong()
+
+        WriteConsoleW(
+            HANDLE(self.handle),
+            buf,
+            code_units_to_be_written,
+            byref(code_units_written),
+            None,
+        )
+        bytes_written = 2 * code_units_written.value
+
+        if bytes_written == 0 and bytes_to_be_written > 0:
+            raise OSError(self._get_error_message(GetLastError()))
+        return bytes_written
+
+
+class ConsoleStream(object):
+    def __init__(self, text_stream, byte_stream):
+        self._text_stream = text_stream
+        self.buffer = byte_stream
+
+    @property
+    def name(self):
+        return self.buffer.name
+
+    def write(self, x):
+        if isinstance(x, text_type):
+            return self._text_stream.write(x)
+        try:
+            self.flush()
+        except Exception:
+            pass
+        return self.buffer.write(x)
+
+    def writelines(self, lines):
+        for line in lines:
+            self.write(line)
+
+    def __getattr__(self, name):
+        return getattr(self._text_stream, name)
+
+    def isatty(self):
+        return self.buffer.isatty()
+
+    def __repr__(self):
+        return "<ConsoleStream name={!r} encoding={!r}>".format(
+            self.name, self.encoding
+        )
+
+
+class WindowsChunkedWriter(object):
+    """
+    Wraps a stream (such as stdout), acting as a transparent proxy for all
+    attribute access apart from method 'write()' which we wrap to write in
+    limited chunks due to a Windows limitation on binary console streams.
+    """
+
+    def __init__(self, wrapped):
+        # double-underscore everything to prevent clashes with names of
+        # attributes on the wrapped stream object.
+        self.__wrapped = wrapped
+
+    def __getattr__(self, name):
+        return getattr(self.__wrapped, name)
+
+    def write(self, text):
+        total_to_write = len(text)
+        written = 0
+
+        while written < total_to_write:
+            to_write = min(total_to_write - written, MAX_BYTES_WRITTEN)
+            self.__wrapped.write(text[written : written + to_write])
+            written += to_write
+
+
+_wrapped_std_streams = set()
+
+
+def _wrap_std_stream(name):
+    # Python 2 & Windows 7 and below
+    if (
+        PY2
+        and sys.getwindowsversion()[:2] <= (6, 1)
+        and name not in _wrapped_std_streams
+    ):
+        setattr(sys, name, WindowsChunkedWriter(getattr(sys, name)))
+        _wrapped_std_streams.add(name)
+
+
+def _get_text_stdin(buffer_stream):
+    text_stream = _NonClosingTextIOWrapper(
+        io.BufferedReader(_WindowsConsoleReader(STDIN_HANDLE)),
+        "utf-16-le",
+        "strict",
+        line_buffering=True,
+    )
+    return ConsoleStream(text_stream, buffer_stream)
+
+
+def _get_text_stdout(buffer_stream):
+    text_stream = _NonClosingTextIOWrapper(
+        io.BufferedWriter(_WindowsConsoleWriter(STDOUT_HANDLE)),
+        "utf-16-le",
+        "strict",
+        line_buffering=True,
+    )
+    return ConsoleStream(text_stream, buffer_stream)
+
+
+def _get_text_stderr(buffer_stream):
+    text_stream = _NonClosingTextIOWrapper(
+        io.BufferedWriter(_WindowsConsoleWriter(STDERR_HANDLE)),
+        "utf-16-le",
+        "strict",
+        line_buffering=True,
+    )
+    return ConsoleStream(text_stream, buffer_stream)
+
+
+if PY2:
+
+    def _hash_py_argv():
+        return zlib.crc32("\x00".join(sys.argv[1:]))
+
+    _initial_argv_hash = _hash_py_argv()
+
+    def _get_windows_argv():
+        argc = c_int(0)
+        argv_unicode = CommandLineToArgvW(GetCommandLineW(), byref(argc))
+        if not argv_unicode:
+            raise WinError()
+        try:
+            argv = [argv_unicode[i] for i in range(0, argc.value)]
+        finally:
+            LocalFree(argv_unicode)
+            del argv_unicode
+
+        if not hasattr(sys, "frozen"):
+            argv = argv[1:]
+            while len(argv) > 0:
+                arg = argv[0]
+                if not arg.startswith("-") or arg == "-":
+                    break
+                argv = argv[1:]
+                if arg.startswith(("-c", "-m")):
+                    break
+
+        return argv[1:]
+
+
+_stream_factories = {
+    0: _get_text_stdin,
+    1: _get_text_stdout,
+    2: _get_text_stderr,
+}
+
+
+def _is_console(f):
+    if not hasattr(f, "fileno"):
+        return False
+
+    try:
+        fileno = f.fileno()
+    except OSError:
+        return False
+
+    handle = msvcrt.get_osfhandle(fileno)
+    return bool(GetConsoleMode(handle, byref(DWORD())))
+
+
+def _get_windows_console_stream(f, encoding, errors):
+    if (
+        get_buffer is not None
+        and encoding in ("utf-16-le", None)
+        and errors in ("strict", None)
+        and _is_console(f)
+    ):
+        func = _stream_factories.get(f.fileno())
+        if func is not None:
+            if not PY2:
+                f = getattr(f, "buffer", None)
+                if f is None:
+                    return None
+            else:
+                # If we are on Python 2 we need to set the stream that we
+                # deal with to binary mode as otherwise the exercise if a
+                # bit moot.  The same problems apply as for
+                # get_binary_stdin and friends from _compat.
+                msvcrt.setmode(f.fileno(), os.O_BINARY)
+            return func(f)

tools/python/ampy/click/decorators.py

@@ -0,0 +1,333 @@
+import inspect
+import sys
+from functools import update_wrapper
+
+from ._compat import iteritems
+from ._unicodefun import _check_for_unicode_literals
+from .core import Argument
+from .core import Command
+from .core import Group
+from .core import Option
+from .globals import get_current_context
+from .utils import echo
+
+
+def pass_context(f):
+    """Marks a callback as wanting to receive the current context
+    object as first argument.
+    """
+
+    def new_func(*args, **kwargs):
+        return f(get_current_context(), *args, **kwargs)
+
+    return update_wrapper(new_func, f)
+
+
+def pass_obj(f):
+    """Similar to :func:`pass_context`, but only pass the object on the
+    context onwards (:attr:`Context.obj`).  This is useful if that object
+    represents the state of a nested system.
+    """
+
+    def new_func(*args, **kwargs):
+        return f(get_current_context().obj, *args, **kwargs)
+
+    return update_wrapper(new_func, f)
+
+
+def make_pass_decorator(object_type, ensure=False):
+    """Given an object type this creates a decorator that will work
+    similar to :func:`pass_obj` but instead of passing the object of the
+    current context, it will find the innermost context of type
+    :func:`object_type`.
+
+    This generates a decorator that works roughly like this::
+
+        from functools import update_wrapper
+
+        def decorator(f):
+            @pass_context
+            def new_func(ctx, *args, **kwargs):
+                obj = ctx.find_object(object_type)
+                return ctx.invoke(f, obj, *args, **kwargs)
+            return update_wrapper(new_func, f)
+        return decorator
+
+    :param object_type: the type of the object to pass.
+    :param ensure: if set to `True`, a new object will be created and
+                   remembered on the context if it's not there yet.
+    """
+
+    def decorator(f):
+        def new_func(*args, **kwargs):
+            ctx = get_current_context()
+            if ensure:
+                obj = ctx.ensure_object(object_type)
+            else:
+                obj = ctx.find_object(object_type)
+            if obj is None:
+                raise RuntimeError(
+                    "Managed to invoke callback without a context"
+                    " object of type '{}' existing".format(object_type.__name__)
+                )
+            return ctx.invoke(f, obj, *args, **kwargs)
+
+        return update_wrapper(new_func, f)
+
+    return decorator
+
+
+def _make_command(f, name, attrs, cls):
+    if isinstance(f, Command):
+        raise TypeError("Attempted to convert a callback into a command twice.")
+    try:
+        params = f.__click_params__
+        params.reverse()
+        del f.__click_params__
+    except AttributeError:
+        params = []
+    help = attrs.get("help")
+    if help is None:
+        help = inspect.getdoc(f)
+        if isinstance(help, bytes):
+            help = help.decode("utf-8")
+    else:
+        help = inspect.cleandoc(help)
+    attrs["help"] = help
+    _check_for_unicode_literals()
+    return cls(
+        name=name or f.__name__.lower().replace("_", "-"),
+        callback=f,
+        params=params,
+        **attrs
+    )
+
+
+def command(name=None, cls=None, **attrs):
+    r"""Creates a new :class:`Command` and uses the decorated function as
+    callback.  This will also automatically attach all decorated
+    :func:`option`\s and :func:`argument`\s as parameters to the command.
+
+    The name of the command defaults to the name of the function with
+    underscores replaced by dashes.  If you want to change that, you can
+    pass the intended name as the first argument.
+
+    All keyword arguments are forwarded to the underlying command class.
+
+    Once decorated the function turns into a :class:`Command` instance
+    that can be invoked as a command line utility or be attached to a
+    command :class:`Group`.
+
+    :param name: the name of the command.  This defaults to the function
+                 name with underscores replaced by dashes.
+    :param cls: the command class to instantiate.  This defaults to
+                :class:`Command`.
+    """
+    if cls is None:
+        cls = Command
+
+    def decorator(f):
+        cmd = _make_command(f, name, attrs, cls)
+        cmd.__doc__ = f.__doc__
+        return cmd
+
+    return decorator
+
+
+def group(name=None, **attrs):
+    """Creates a new :class:`Group` with a function as callback.  This
+    works otherwise the same as :func:`command` just that the `cls`
+    parameter is set to :class:`Group`.
+    """
+    attrs.setdefault("cls", Group)
+    return command(name, **attrs)
+
+
+def _param_memo(f, param):
+    if isinstance(f, Command):
+        f.params.append(param)
+    else:
+        if not hasattr(f, "__click_params__"):
+            f.__click_params__ = []
+        f.__click_params__.append(param)
+
+
+def argument(*param_decls, **attrs):
+    """Attaches an argument to the command.  All positional arguments are
+    passed as parameter declarations to :class:`Argument`; all keyword
+    arguments are forwarded unchanged (except ``cls``).
+    This is equivalent to creating an :class:`Argument` instance manually
+    and attaching it to the :attr:`Command.params` list.
+
+    :param cls: the argument class to instantiate.  This defaults to
+                :class:`Argument`.
+    """
+
+    def decorator(f):
+        ArgumentClass = attrs.pop("cls", Argument)
+        _param_memo(f, ArgumentClass(param_decls, **attrs))
+        return f
+
+    return decorator
+
+
+def option(*param_decls, **attrs):
+    """Attaches an option to the command.  All positional arguments are
+    passed as parameter declarations to :class:`Option`; all keyword
+    arguments are forwarded unchanged (except ``cls``).
+    This is equivalent to creating an :class:`Option` instance manually
+    and attaching it to the :attr:`Command.params` list.
+
+    :param cls: the option class to instantiate.  This defaults to
+                :class:`Option`.
+    """
+
+    def decorator(f):
+        # Issue 926, copy attrs, so pre-defined options can re-use the same cls=
+        option_attrs = attrs.copy()
+
+        if "help" in option_attrs:
+            option_attrs["help"] = inspect.cleandoc(option_attrs["help"])
+        OptionClass = option_attrs.pop("cls", Option)
+        _param_memo(f, OptionClass(param_decls, **option_attrs))
+        return f
+
+    return decorator
+
+
+def confirmation_option(*param_decls, **attrs):
+    """Shortcut for confirmation prompts that can be ignored by passing
+    ``--yes`` as parameter.
+
+    This is equivalent to decorating a function with :func:`option` with
+    the following parameters::
+
+        def callback(ctx, param, value):
+            if not value:
+                ctx.abort()
+
+        @click.command()
+        @click.option('--yes', is_flag=True, callback=callback,
+                      expose_value=False, prompt='Do you want to continue?')
+        def dropdb():
+            pass
+    """
+
+    def decorator(f):
+        def callback(ctx, param, value):
+            if not value:
+                ctx.abort()
+
+        attrs.setdefault("is_flag", True)
+        attrs.setdefault("callback", callback)
+        attrs.setdefault("expose_value", False)
+        attrs.setdefault("prompt", "Do you want to continue?")
+        attrs.setdefault("help", "Confirm the action without prompting.")
+        return option(*(param_decls or ("--yes",)), **attrs)(f)
+
+    return decorator
+
+
+def password_option(*param_decls, **attrs):
+    """Shortcut for password prompts.
+
+    This is equivalent to decorating a function with :func:`option` with
+    the following parameters::
+
+        @click.command()
+        @click.option('--password', prompt=True, confirmation_prompt=True,
+                      hide_input=True)
+        def changeadmin(password):
+            pass
+    """
+
+    def decorator(f):
+        attrs.setdefault("prompt", True)
+        attrs.setdefault("confirmation_prompt", True)
+        attrs.setdefault("hide_input", True)
+        return option(*(param_decls or ("--password",)), **attrs)(f)
+
+    return decorator
+
+
+def version_option(version=None, *param_decls, **attrs):
+    """Adds a ``--version`` option which immediately ends the program
+    printing out the version number.  This is implemented as an eager
+    option that prints the version and exits the program in the callback.
+
+    :param version: the version number to show.  If not provided Click
+                    attempts an auto discovery via setuptools.
+    :param prog_name: the name of the program (defaults to autodetection)
+    :param message: custom message to show instead of the default
+                    (``'%(prog)s, version %(version)s'``)
+    :param others: everything else is forwarded to :func:`option`.
+    """
+    if version is None:
+        if hasattr(sys, "_getframe"):
+            module = sys._getframe(1).f_globals.get("__name__")
+        else:
+            module = ""
+
+    def decorator(f):
+        prog_name = attrs.pop("prog_name", None)
+        message = attrs.pop("message", "%(prog)s, version %(version)s")
+
+        def callback(ctx, param, value):
+            if not value or ctx.resilient_parsing:
+                return
+            prog = prog_name
+            if prog is None:
+                prog = ctx.find_root().info_name
+            ver = version
+            if ver is None:
+                try:
+                    import pkg_resources
+                except ImportError:
+                    pass
+                else:
+                    for dist in pkg_resources.working_set:
+                        scripts = dist.get_entry_map().get("console_scripts") or {}
+                        for _, entry_point in iteritems(scripts):
+                            if entry_point.module_name == module:
+                                ver = dist.version
+                                break
+                if ver is None:
+                    raise RuntimeError("Could not determine version")
+            echo(message % {"prog": prog, "version": ver}, color=ctx.color)
+            ctx.exit()
+
+        attrs.setdefault("is_flag", True)
+        attrs.setdefault("expose_value", False)
+        attrs.setdefault("is_eager", True)
+        attrs.setdefault("help", "Show the version and exit.")
+        attrs["callback"] = callback
+        return option(*(param_decls or ("--version",)), **attrs)(f)
+
+    return decorator
+
+
+def help_option(*param_decls, **attrs):
+    """Adds a ``--help`` option which immediately ends the program
+    printing out the help page.  This is usually unnecessary to add as
+    this is added by default to all commands unless suppressed.
+
+    Like :func:`version_option`, this is implemented as eager option that
+    prints in the callback and exits.
+
+    All arguments are forwarded to :func:`option`.
+    """
+
+    def decorator(f):
+        def callback(ctx, param, value):
+            if value and not ctx.resilient_parsing:
+                echo(ctx.get_help(), color=ctx.color)
+                ctx.exit()
+
+        attrs.setdefault("is_flag", True)
+        attrs.setdefault("expose_value", False)
+        attrs.setdefault("help", "Show this message and exit.")
+        attrs.setdefault("is_eager", True)
+        attrs["callback"] = callback
+        return option(*(param_decls or ("--help",)), **attrs)(f)
+
+    return decorator

tools/python/ampy/click/exceptions.py

@@ -0,0 +1,253 @@
+from ._compat import filename_to_ui
+from ._compat import get_text_stderr
+from ._compat import PY2
+from .utils import echo
+
+
+def _join_param_hints(param_hint):
+    if isinstance(param_hint, (tuple, list)):
+        return " / ".join(repr(x) for x in param_hint)
+    return param_hint
+
+
+class ClickException(Exception):
+    """An exception that Click can handle and show to the user."""
+
+    #: The exit code for this exception
+    exit_code = 1
+
+    def __init__(self, message):
+        ctor_msg = message
+        if PY2:
+            if ctor_msg is not None:
+                ctor_msg = ctor_msg.encode("utf-8")
+        Exception.__init__(self, ctor_msg)
+        self.message = message
+
+    def format_message(self):
+        return self.message
+
+    def __str__(self):
+        return self.message
+
+    if PY2:
+        __unicode__ = __str__
+
+        def __str__(self):
+            return self.message.encode("utf-8")
+
+    def show(self, file=None):
+        if file is None:
+            file = get_text_stderr()
+        echo("Error: {}".format(self.format_message()), file=file)
+
+
+class UsageError(ClickException):
+    """An internal exception that signals a usage error.  This typically
+    aborts any further handling.
+
+    :param message: the error message to display.
+    :param ctx: optionally the context that caused this error.  Click will
+                fill in the context automatically in some situations.
+    """
+
+    exit_code = 2
+
+    def __init__(self, message, ctx=None):
+        ClickException.__init__(self, message)
+        self.ctx = ctx
+        self.cmd = self.ctx.command if self.ctx else None
+
+    def show(self, file=None):
+        if file is None:
+            file = get_text_stderr()
+        color = None
+        hint = ""
+        if self.cmd is not None and self.cmd.get_help_option(self.ctx) is not None:
+            hint = "Try '{} {}' for help.\n".format(
+                self.ctx.command_path, self.ctx.help_option_names[0]
+            )
+        if self.ctx is not None:
+            color = self.ctx.color
+            echo("{}\n{}".format(self.ctx.get_usage(), hint), file=file, color=color)
+        echo("Error: {}".format(self.format_message()), file=file, color=color)
+
+
+class BadParameter(UsageError):
+    """An exception that formats out a standardized error message for a
+    bad parameter.  This is useful when thrown from a callback or type as
+    Click will attach contextual information to it (for instance, which
+    parameter it is).
+
+    .. versionadded:: 2.0
+
+    :param param: the parameter object that caused this error.  This can
+                  be left out, and Click will attach this info itself
+                  if possible.
+    :param param_hint: a string that shows up as parameter name.  This
+                       can be used as alternative to `param` in cases
+                       where custom validation should happen.  If it is
+                       a string it's used as such, if it's a list then
+                       each item is quoted and separated.
+    """
+
+    def __init__(self, message, ctx=None, param=None, param_hint=None):
+        UsageError.__init__(self, message, ctx)
+        self.param = param
+        self.param_hint = param_hint
+
+    def format_message(self):
+        if self.param_hint is not None:
+            param_hint = self.param_hint
+        elif self.param is not None:
+            param_hint = self.param.get_error_hint(self.ctx)
+        else:
+            return "Invalid value: {}".format(self.message)
+        param_hint = _join_param_hints(param_hint)
+
+        return "Invalid value for {}: {}".format(param_hint, self.message)
+
+
+class MissingParameter(BadParameter):
+    """Raised if click required an option or argument but it was not
+    provided when invoking the script.
+
+    .. versionadded:: 4.0
+
+    :param param_type: a string that indicates the type of the parameter.
+                       The default is to inherit the parameter type from
+                       the given `param`.  Valid values are ``'parameter'``,
+                       ``'option'`` or ``'argument'``.
+    """
+
+    def __init__(
+        self, message=None, ctx=None, param=None, param_hint=None, param_type=None
+    ):
+        BadParameter.__init__(self, message, ctx, param, param_hint)
+        self.param_type = param_type
+
+    def format_message(self):
+        if self.param_hint is not None:
+            param_hint = self.param_hint
+        elif self.param is not None:
+            param_hint = self.param.get_error_hint(self.ctx)
+        else:
+            param_hint = None
+        param_hint = _join_param_hints(param_hint)
+
+        param_type = self.param_type
+        if param_type is None and self.param is not None:
+            param_type = self.param.param_type_name
+
+        msg = self.message
+        if self.param is not None:
+            msg_extra = self.param.type.get_missing_message(self.param)
+            if msg_extra:
+                if msg:
+                    msg += ".  {}".format(msg_extra)
+                else:
+                    msg = msg_extra
+
+        return "Missing {}{}{}{}".format(
+            param_type,
+            " {}".format(param_hint) if param_hint else "",
+            ".  " if msg else ".",
+            msg or "",
+        )
+
+    def __str__(self):
+        if self.message is None:
+            param_name = self.param.name if self.param else None
+            return "missing parameter: {}".format(param_name)
+        else:
+            return self.message
+
+    if PY2:
+        __unicode__ = __str__
+
+        def __str__(self):
+            return self.__unicode__().encode("utf-8")
+
+
+class NoSuchOption(UsageError):
+    """Raised if click attempted to handle an option that does not
+    exist.
+
+    .. versionadded:: 4.0
+    """
+
+    def __init__(self, option_name, message=None, possibilities=None, ctx=None):
+        if message is None:
+            message = "no such option: {}".format(option_name)
+        UsageError.__init__(self, message, ctx)
+        self.option_name = option_name
+        self.possibilities = possibilities
+
+    def format_message(self):
+        bits = [self.message]
+        if self.possibilities:
+            if len(self.possibilities) == 1:
+                bits.append("Did you mean {}?".format(self.possibilities[0]))
+            else:
+                possibilities = sorted(self.possibilities)
+                bits.append("(Possible options: {})".format(", ".join(possibilities)))
+        return "  ".join(bits)
+
+
+class BadOptionUsage(UsageError):
+    """Raised if an option is generally supplied but the use of the option
+    was incorrect.  This is for instance raised if the number of arguments
+    for an option is not correct.
+
+    .. versionadded:: 4.0
+
+    :param option_name: the name of the option being used incorrectly.
+    """
+
+    def __init__(self, option_name, message, ctx=None):
+        UsageError.__init__(self, message, ctx)
+        self.option_name = option_name
+
+
+class BadArgumentUsage(UsageError):
+    """Raised if an argument is generally supplied but the use of the argument
+    was incorrect.  This is for instance raised if the number of values
+    for an argument is not correct.
+
+    .. versionadded:: 6.0
+    """
+
+    def __init__(self, message, ctx=None):
+        UsageError.__init__(self, message, ctx)
+
+
+class FileError(ClickException):
+    """Raised if a file cannot be opened."""
+
+    def __init__(self, filename, hint=None):
+        ui_filename = filename_to_ui(filename)
+        if hint is None:
+            hint = "unknown error"
+        ClickException.__init__(self, hint)
+        self.ui_filename = ui_filename
+        self.filename = filename
+
+    def format_message(self):
+        return "Could not open file {}: {}".format(self.ui_filename, self.message)
+
+
+class Abort(RuntimeError):
+    """An internal signalling exception that signals Click to abort."""
+
+
+class Exit(RuntimeError):
+    """An exception that indicates that the application should exit with some
+    status code.
+
+    :param code: the status code to exit with.
+    """
+
+    __slots__ = ("exit_code",)
+
+    def __init__(self, code=0):
+        self.exit_code = code

tools/python/ampy/click/formatting.py

@@ -0,0 +1,283 @@
+from contextlib import contextmanager
+
+from ._compat import term_len
+from .parser import split_opt
+from .termui import get_terminal_size
+
+# Can force a width.  This is used by the test system
+FORCED_WIDTH = None
+
+
+def measure_table(rows):
+    widths = {}
+    for row in rows:
+        for idx, col in enumerate(row):
+            widths[idx] = max(widths.get(idx, 0), term_len(col))
+    return tuple(y for x, y in sorted(widths.items()))
+
+
+def iter_rows(rows, col_count):
+    for row in rows:
+        row = tuple(row)
+        yield row + ("",) * (col_count - len(row))
+
+
+def wrap_text(
+    text, width=78, initial_indent="", subsequent_indent="", preserve_paragraphs=False
+):
+    """A helper function that intelligently wraps text.  By default, it
+    assumes that it operates on a single paragraph of text but if the
+    `preserve_paragraphs` parameter is provided it will intelligently
+    handle paragraphs (defined by two empty lines).
+
+    If paragraphs are handled, a paragraph can be prefixed with an empty
+    line containing the ``\\b`` character (``\\x08``) to indicate that
+    no rewrapping should happen in that block.
+
+    :param text: the text that should be rewrapped.
+    :param width: the maximum width for the text.
+    :param initial_indent: the initial indent that should be placed on the
+                           first line as a string.
+    :param subsequent_indent: the indent string that should be placed on
+                              each consecutive line.
+    :param preserve_paragraphs: if this flag is set then the wrapping will
+                                intelligently handle paragraphs.
+    """
+    from ._textwrap import TextWrapper
+
+    text = text.expandtabs()
+    wrapper = TextWrapper(
+        width,
+        initial_indent=initial_indent,
+        subsequent_indent=subsequent_indent,
+        replace_whitespace=False,
+    )
+    if not preserve_paragraphs:
+        return wrapper.fill(text)
+
+    p = []
+    buf = []
+    indent = None
+
+    def _flush_par():
+        if not buf:
+            return
+        if buf[0].strip() == "\b":
+            p.append((indent or 0, True, "\n".join(buf[1:])))
+        else:
+            p.append((indent or 0, False, " ".join(buf)))
+        del buf[:]
+
+    for line in text.splitlines():
+        if not line:
+            _flush_par()
+            indent = None
+        else:
+            if indent is None:
+                orig_len = term_len(line)
+                line = line.lstrip()
+                indent = orig_len - term_len(line)
+            buf.append(line)
+    _flush_par()
+
+    rv = []
+    for indent, raw, text in p:
+        with wrapper.extra_indent(" " * indent):
+            if raw:
+                rv.append(wrapper.indent_only(text))
+            else:
+                rv.append(wrapper.fill(text))
+
+    return "\n\n".join(rv)
+
+
+class HelpFormatter(object):
+    """This class helps with formatting text-based help pages.  It's
+    usually just needed for very special internal cases, but it's also
+    exposed so that developers can write their own fancy outputs.
+
+    At present, it always writes into memory.
+
+    :param indent_increment: the additional increment for each level.
+    :param width: the width for the text.  This defaults to the terminal
+                  width clamped to a maximum of 78.
+    """
+
+    def __init__(self, indent_increment=2, width=None, max_width=None):
+        self.indent_increment = indent_increment
+        if max_width is None:
+            max_width = 80
+        if width is None:
+            width = FORCED_WIDTH
+            if width is None:
+                width = max(min(get_terminal_size()[0], max_width) - 2, 50)
+        self.width = width
+        self.current_indent = 0
+        self.buffer = []
+
+    def write(self, string):
+        """Writes a unicode string into the internal buffer."""
+        self.buffer.append(string)
+
+    def indent(self):
+        """Increases the indentation."""
+        self.current_indent += self.indent_increment
+
+    def dedent(self):
+        """Decreases the indentation."""
+        self.current_indent -= self.indent_increment
+
+    def write_usage(self, prog, args="", prefix="Usage: "):
+        """Writes a usage line into the buffer.
+
+        :param prog: the program name.
+        :param args: whitespace separated list of arguments.
+        :param prefix: the prefix for the first line.
+        """
+        usage_prefix = "{:>{w}}{} ".format(prefix, prog, w=self.current_indent)
+        text_width = self.width - self.current_indent
+
+        if text_width >= (term_len(usage_prefix) + 20):
+            # The arguments will fit to the right of the prefix.
+            indent = " " * term_len(usage_prefix)
+            self.write(
+                wrap_text(
+                    args,
+                    text_width,
+                    initial_indent=usage_prefix,
+                    subsequent_indent=indent,
+                )
+            )
+        else:
+            # The prefix is too long, put the arguments on the next line.
+            self.write(usage_prefix)
+            self.write("\n")
+            indent = " " * (max(self.current_indent, term_len(prefix)) + 4)
+            self.write(
+                wrap_text(
+                    args, text_width, initial_indent=indent, subsequent_indent=indent
+                )
+            )
+
+        self.write("\n")
+
+    def write_heading(self, heading):
+        """Writes a heading into the buffer."""
+        self.write("{:>{w}}{}:\n".format("", heading, w=self.current_indent))
+
+    def write_paragraph(self):
+        """Writes a paragraph into the buffer."""
+        if self.buffer:
+            self.write("\n")
+
+    def write_text(self, text):
+        """Writes re-indented text into the buffer.  This rewraps and
+        preserves paragraphs.
+        """
+        text_width = max(self.width - self.current_indent, 11)
+        indent = " " * self.current_indent
+        self.write(
+            wrap_text(
+                text,
+                text_width,
+                initial_indent=indent,
+                subsequent_indent=indent,
+                preserve_paragraphs=True,
+            )
+        )
+        self.write("\n")
+
+    def write_dl(self, rows, col_max=30, col_spacing=2):
+        """Writes a definition list into the buffer.  This is how options
+        and commands are usually formatted.
+
+        :param rows: a list of two item tuples for the terms and values.
+        :param col_max: the maximum width of the first column.
+        :param col_spacing: the number of spaces between the first and
+                            second column.
+        """
+        rows = list(rows)
+        widths = measure_table(rows)
+        if len(widths) != 2:
+            raise TypeError("Expected two columns for definition list")
+
+        first_col = min(widths[0], col_max) + col_spacing
+
+        for first, second in iter_rows(rows, len(widths)):
+            self.write("{:>{w}}{}".format("", first, w=self.current_indent))
+            if not second:
+                self.write("\n")
+                continue
+            if term_len(first) <= first_col - col_spacing:
+                self.write(" " * (first_col - term_len(first)))
+            else:
+                self.write("\n")
+                self.write(" " * (first_col + self.current_indent))
+
+            text_width = max(self.width - first_col - 2, 10)
+            wrapped_text = wrap_text(second, text_width, preserve_paragraphs=True)
+            lines = wrapped_text.splitlines()
+
+            if lines:
+                self.write("{}\n".format(lines[0]))
+
+                for line in lines[1:]:
+                    self.write(
+                        "{:>{w}}{}\n".format(
+                            "", line, w=first_col + self.current_indent
+                        )
+                    )
+
+                if len(lines) > 1:
+                    # separate long help from next option
+                    self.write("\n")
+            else:
+                self.write("\n")
+
+    @contextmanager
+    def section(self, name):
+        """Helpful context manager that writes a paragraph, a heading,
+        and the indents.
+
+        :param name: the section name that is written as heading.
+        """
+        self.write_paragraph()
+        self.write_heading(name)
+        self.indent()
+        try:
+            yield
+        finally:
+            self.dedent()
+
+    @contextmanager
+    def indentation(self):
+        """A context manager that increases the indentation."""
+        self.indent()
+        try:
+            yield
+        finally:
+            self.dedent()
+
+    def getvalue(self):
+        """Returns the buffer contents."""
+        return "".join(self.buffer)
+
+
+def join_options(options):
+    """Given a list of option strings this joins them in the most appropriate
+    way and returns them in the form ``(formatted_string,
+    any_prefix_is_slash)`` where the second item in the tuple is a flag that
+    indicates if any of the option prefixes was a slash.
+    """
+    rv = []
+    any_prefix_is_slash = False
+    for opt in options:
+        prefix = split_opt(opt)[0]
+        if prefix == "/":
+            any_prefix_is_slash = True
+        rv.append((len(prefix), opt))
+
+    rv.sort(key=lambda x: x[0])
+
+    rv = ", ".join(x[1] for x in rv)
+    return rv, any_prefix_is_slash

tools/python/ampy/click/globals.py

@@ -0,0 +1,47 @@
+from threading import local
+
+_local = local()
+
+
+def get_current_context(silent=False):
+    """Returns the current click context.  This can be used as a way to
+    access the current context object from anywhere.  This is a more implicit
+    alternative to the :func:`pass_context` decorator.  This function is
+    primarily useful for helpers such as :func:`echo` which might be
+    interested in changing its behavior based on the current context.
+
+    To push the current context, :meth:`Context.scope` can be used.
+
+    .. versionadded:: 5.0
+
+    :param silent: if set to `True` the return value is `None` if no context
+                   is available.  The default behavior is to raise a
+                   :exc:`RuntimeError`.
+    """
+    try:
+        return _local.stack[-1]
+    except (AttributeError, IndexError):
+        if not silent:
+            raise RuntimeError("There is no active click context.")
+
+
+def push_context(ctx):
+    """Pushes a new context to the current stack."""
+    _local.__dict__.setdefault("stack", []).append(ctx)
+
+
+def pop_context():
+    """Removes the top level from the stack."""
+    _local.stack.pop()
+
+
+def resolve_color_default(color=None):
+    """"Internal helper to get the default value of the color flag.  If a
+    value is passed it's returned unchanged, otherwise it's looked up from
+    the current context.
+    """
+    if color is not None:
+        return color
+    ctx = get_current_context(silent=True)
+    if ctx is not None:
+        return ctx.color

tools/python/ampy/click/parser.py

@@ -0,0 +1,428 @@
+# -*- coding: utf-8 -*-
+"""
+This module started out as largely a copy paste from the stdlib's
+optparse module with the features removed that we do not need from
+optparse because we implement them in Click on a higher level (for
+instance type handling, help formatting and a lot more).
+
+The plan is to remove more and more from here over time.
+
+The reason this is a different module and not optparse from the stdlib
+is that there are differences in 2.x and 3.x about the error messages
+generated and optparse in the stdlib uses gettext for no good reason
+and might cause us issues.
+
+Click uses parts of optparse written by Gregory P. Ward and maintained
+by the Python Software Foundation. This is limited to code in parser.py.
+
+Copyright 2001-2006 Gregory P. Ward. All rights reserved.
+Copyright 2002-2006 Python Software Foundation. All rights reserved.
+"""
+import re
+from collections import deque
+
+from .exceptions import BadArgumentUsage
+from .exceptions import BadOptionUsage
+from .exceptions import NoSuchOption
+from .exceptions import UsageError
+
+
+def _unpack_args(args, nargs_spec):
+    """Given an iterable of arguments and an iterable of nargs specifications,
+    it returns a tuple with all the unpacked arguments at the first index
+    and all remaining arguments as the second.
+
+    The nargs specification is the number of arguments that should be consumed
+    or `-1` to indicate that this position should eat up all the remainders.
+
+    Missing items are filled with `None`.
+    """
+    args = deque(args)
+    nargs_spec = deque(nargs_spec)
+    rv = []
+    spos = None
+
+    def _fetch(c):
+        try:
+            if spos is None:
+                return c.popleft()
+            else:
+                return c.pop()
+        except IndexError:
+            return None
+
+    while nargs_spec:
+        nargs = _fetch(nargs_spec)
+        if nargs == 1:
+            rv.append(_fetch(args))
+        elif nargs > 1:
+            x = [_fetch(args) for _ in range(nargs)]
+            # If we're reversed, we're pulling in the arguments in reverse,
+            # so we need to turn them around.
+            if spos is not None:
+                x.reverse()
+            rv.append(tuple(x))
+        elif nargs < 0:
+            if spos is not None:
+                raise TypeError("Cannot have two nargs < 0")
+            spos = len(rv)
+            rv.append(None)
+
+    # spos is the position of the wildcard (star).  If it's not `None`,
+    # we fill it with the remainder.
+    if spos is not None:
+        rv[spos] = tuple(args)
+        args = []
+        rv[spos + 1 :] = reversed(rv[spos + 1 :])
+
+    return tuple(rv), list(args)
+
+
+def _error_opt_args(nargs, opt):
+    if nargs == 1:
+        raise BadOptionUsage(opt, "{} option requires an argument".format(opt))
+    raise BadOptionUsage(opt, "{} option requires {} arguments".format(opt, nargs))
+
+
+def split_opt(opt):
+    first = opt[:1]
+    if first.isalnum():
+        return "", opt
+    if opt[1:2] == first:
+        return opt[:2], opt[2:]
+    return first, opt[1:]
+
+
+def normalize_opt(opt, ctx):
+    if ctx is None or ctx.token_normalize_func is None:
+        return opt
+    prefix, opt = split_opt(opt)
+    return prefix + ctx.token_normalize_func(opt)
+
+
+def split_arg_string(string):
+    """Given an argument string this attempts to split it into small parts."""
+    rv = []
+    for match in re.finditer(
+        r"('([^'\\]*(?:\\.[^'\\]*)*)'|\"([^\"\\]*(?:\\.[^\"\\]*)*)\"|\S+)\s*",
+        string,
+        re.S,
+    ):
+        arg = match.group().strip()
+        if arg[:1] == arg[-1:] and arg[:1] in "\"'":
+            arg = arg[1:-1].encode("ascii", "backslashreplace").decode("unicode-escape")
+        try:
+            arg = type(string)(arg)
+        except UnicodeError:
+            pass
+        rv.append(arg)
+    return rv
+
+
+class Option(object):
+    def __init__(self, opts, dest, action=None, nargs=1, const=None, obj=None):
+        self._short_opts = []
+        self._long_opts = []
+        self.prefixes = set()
+
+        for opt in opts:
+            prefix, value = split_opt(opt)
+            if not prefix:
+                raise ValueError("Invalid start character for option ({})".format(opt))
+            self.prefixes.add(prefix[0])
+            if len(prefix) == 1 and len(value) == 1:
+                self._short_opts.append(opt)
+            else:
+                self._long_opts.append(opt)
+                self.prefixes.add(prefix)
+
+        if action is None:
+            action = "store"
+
+        self.dest = dest
+        self.action = action
+        self.nargs = nargs
+        self.const = const
+        self.obj = obj
+
+    @property
+    def takes_value(self):
+        return self.action in ("store", "append")
+
+    def process(self, value, state):
+        if self.action == "store":
+            state.opts[self.dest] = value
+        elif self.action == "store_const":
+            state.opts[self.dest] = self.const
+        elif self.action == "append":
+            state.opts.setdefault(self.dest, []).append(value)
+        elif self.action == "append_const":
+            state.opts.setdefault(self.dest, []).append(self.const)
+        elif self.action == "count":
+            state.opts[self.dest] = state.opts.get(self.dest, 0) + 1
+        else:
+            raise ValueError("unknown action '{}'".format(self.action))
+        state.order.append(self.obj)
+
+
+class Argument(object):
+    def __init__(self, dest, nargs=1, obj=None):
+        self.dest = dest
+        self.nargs = nargs
+        self.obj = obj
+
+    def process(self, value, state):
+        if self.nargs > 1:
+            holes = sum(1 for x in value if x is None)
+            if holes == len(value):
+                value = None
+            elif holes != 0:
+                raise BadArgumentUsage(
+                    "argument {} takes {} values".format(self.dest, self.nargs)
+                )
+        state.opts[self.dest] = value
+        state.order.append(self.obj)
+
+
+class ParsingState(object):
+    def __init__(self, rargs):
+        self.opts = {}
+        self.largs = []
+        self.rargs = rargs
+        self.order = []
+
+
+class OptionParser(object):
+    """The option parser is an internal class that is ultimately used to
+    parse options and arguments.  It's modelled after optparse and brings
+    a similar but vastly simplified API.  It should generally not be used
+    directly as the high level Click classes wrap it for you.
+
+    It's not nearly as extensible as optparse or argparse as it does not
+    implement features that are implemented on a higher level (such as
+    types or defaults).
+
+    :param ctx: optionally the :class:`~click.Context` where this parser
+                should go with.
+    """
+
+    def __init__(self, ctx=None):
+        #: The :class:`~click.Context` for this parser.  This might be
+        #: `None` for some advanced use cases.
+        self.ctx = ctx
+        #: This controls how the parser deals with interspersed arguments.
+        #: If this is set to `False`, the parser will stop on the first
+        #: non-option.  Click uses this to implement nested subcommands
+        #: safely.
+        self.allow_interspersed_args = True
+        #: This tells the parser how to deal with unknown options.  By
+        #: default it will error out (which is sensible), but there is a
+        #: second mode where it will ignore it and continue processing
+        #: after shifting all the unknown options into the resulting args.
+        self.ignore_unknown_options = False
+        if ctx is not None:
+            self.allow_interspersed_args = ctx.allow_interspersed_args
+            self.ignore_unknown_options = ctx.ignore_unknown_options
+        self._short_opt = {}
+        self._long_opt = {}
+        self._opt_prefixes = {"-", "--"}
+        self._args = []
+
+    def add_option(self, opts, dest, action=None, nargs=1, const=None, obj=None):
+        """Adds a new option named `dest` to the parser.  The destination
+        is not inferred (unlike with optparse) and needs to be explicitly
+        provided.  Action can be any of ``store``, ``store_const``,
+        ``append``, ``appnd_const`` or ``count``.
+
+        The `obj` can be used to identify the option in the order list
+        that is returned from the parser.
+        """
+        if obj is None:
+            obj = dest
+        opts = [normalize_opt(opt, self.ctx) for opt in opts]
+        option = Option(opts, dest, action=action, nargs=nargs, const=const, obj=obj)
+        self._opt_prefixes.update(option.prefixes)
+        for opt in option._short_opts:
+            self._short_opt[opt] = option
+        for opt in option._long_opts:
+            self._long_opt[opt] = option
+
+    def add_argument(self, dest, nargs=1, obj=None):
+        """Adds a positional argument named `dest` to the parser.
+
+        The `obj` can be used to identify the option in the order list
+        that is returned from the parser.
+        """
+        if obj is None:
+            obj = dest
+        self._args.append(Argument(dest=dest, nargs=nargs, obj=obj))
+
+    def parse_args(self, args):
+        """Parses positional arguments and returns ``(values, args, order)``
+        for the parsed options and arguments as well as the leftover
+        arguments if there are any.  The order is a list of objects as they
+        appear on the command line.  If arguments appear multiple times they
+        will be memorized multiple times as well.
+        """
+        state = ParsingState(args)
+        try:
+            self._process_args_for_options(state)
+            self._process_args_for_args(state)
+        except UsageError:
+            if self.ctx is None or not self.ctx.resilient_parsing:
+                raise
+        return state.opts, state.largs, state.order
+
+    def _process_args_for_args(self, state):
+        pargs, args = _unpack_args(
+            state.largs + state.rargs, [x.nargs for x in self._args]
+        )
+
+        for idx, arg in enumerate(self._args):
+            arg.process(pargs[idx], state)
+
+        state.largs = args
+        state.rargs = []
+
+    def _process_args_for_options(self, state):
+        while state.rargs:
+            arg = state.rargs.pop(0)
+            arglen = len(arg)
+            # Double dashes always handled explicitly regardless of what
+            # prefixes are valid.
+            if arg == "--":
+                return
+            elif arg[:1] in self._opt_prefixes and arglen > 1:
+                self._process_opts(arg, state)
+            elif self.allow_interspersed_args:
+                state.largs.append(arg)
+            else:
+                state.rargs.insert(0, arg)
+                return
+
+        # Say this is the original argument list:
+        # [arg0, arg1, ..., arg(i-1), arg(i), arg(i+1), ..., arg(N-1)]
+        #                            ^
+        # (we are about to process arg(i)).
+        #
+        # Then rargs is [arg(i), ..., arg(N-1)] and largs is a *subset* of
+        # [arg0, ..., arg(i-1)] (any options and their arguments will have
+        # been removed from largs).
+        #
+        # The while loop will usually consume 1 or more arguments per pass.
+        # If it consumes 1 (eg. arg is an option that takes no arguments),
+        # then after _process_arg() is done the situation is:
+        #
+        #   largs = subset of [arg0, ..., arg(i)]
+        #   rargs = [arg(i+1), ..., arg(N-1)]
+        #
+        # If allow_interspersed_args is false, largs will always be
+        # *empty* -- still a subset of [arg0, ..., arg(i-1)], but
+        # not a very interesting subset!
+
+    def _match_long_opt(self, opt, explicit_value, state):
+        if opt not in self._long_opt:
+            possibilities = [word for word in self._long_opt if word.startswith(opt)]
+            raise NoSuchOption(opt, possibilities=possibilities, ctx=self.ctx)
+
+        option = self._long_opt[opt]
+        if option.takes_value:
+            # At this point it's safe to modify rargs by injecting the
+            # explicit value, because no exception is raised in this
+            # branch.  This means that the inserted value will be fully
+            # consumed.
+            if explicit_value is not None:
+                state.rargs.insert(0, explicit_value)
+
+            nargs = option.nargs
+            if len(state.rargs) < nargs:
+                _error_opt_args(nargs, opt)
+            elif nargs == 1:
+                value = state.rargs.pop(0)
+            else:
+                value = tuple(state.rargs[:nargs])
+                del state.rargs[:nargs]
+
+        elif explicit_value is not None:
+            raise BadOptionUsage(opt, "{} option does not take a value".format(opt))
+
+        else:
+            value = None
+
+        option.process(value, state)
+
+    def _match_short_opt(self, arg, state):
+        stop = False
+        i = 1
+        prefix = arg[0]
+        unknown_options = []
+
+        for ch in arg[1:]:
+            opt = normalize_opt(prefix + ch, self.ctx)
+            option = self._short_opt.get(opt)
+            i += 1
+
+            if not option:
+                if self.ignore_unknown_options:
+                    unknown_options.append(ch)
+                    continue
+                raise NoSuchOption(opt, ctx=self.ctx)
+            if option.takes_value:
+                # Any characters left in arg?  Pretend they're the
+                # next arg, and stop consuming characters of arg.
+                if i < len(arg):
+                    state.rargs.insert(0, arg[i:])
+                    stop = True
+
+                nargs = option.nargs
+                if len(state.rargs) < nargs:
+                    _error_opt_args(nargs, opt)
+                elif nargs == 1:
+                    value = state.rargs.pop(0)
+                else:
+                    value = tuple(state.rargs[:nargs])
+                    del state.rargs[:nargs]
+
+            else:
+                value = None
+
+            option.process(value, state)
+
+            if stop:
+                break
+
+        # If we got any unknown options we re-combinate the string of the
+        # remaining options and re-attach the prefix, then report that
+        # to the state as new larg.  This way there is basic combinatorics
+        # that can be achieved while still ignoring unknown arguments.
+        if self.ignore_unknown_options and unknown_options:
+            state.largs.append("{}{}".format(prefix, "".join(unknown_options)))
+
+    def _process_opts(self, arg, state):
+        explicit_value = None
+        # Long option handling happens in two parts.  The first part is
+        # supporting explicitly attached values.  In any case, we will try
+        # to long match the option first.
+        if "=" in arg:
+            long_opt, explicit_value = arg.split("=", 1)
+        else:
+            long_opt = arg
+        norm_long_opt = normalize_opt(long_opt, self.ctx)
+
+        # At this point we will match the (assumed) long option through
+        # the long option matching code.  Note that this allows options
+        # like "-foo" to be matched as long options.
+        try:
+            self._match_long_opt(norm_long_opt, explicit_value, state)
+        except NoSuchOption:
+            # At this point the long option matching failed, and we need
+            # to try with short options.  However there is a special rule
+            # which says, that if we have a two character options prefix
+            # (applies to "--foo" for instance), we do not dispatch to the
+            # short option code and will instead raise the no option
+            # error.
+            if arg[:2] not in self._opt_prefixes:
+                return self._match_short_opt(arg, state)
+            if not self.ignore_unknown_options:
+                raise
+            state.largs.append(arg)

tools/python/ampy/click/termui.py

@@ -0,0 +1,681 @@
+import inspect
+import io
+import itertools
+import os
+import struct
+import sys
+
+from ._compat import DEFAULT_COLUMNS
+from ._compat import get_winterm_size
+from ._compat import isatty
+from ._compat import raw_input
+from ._compat import string_types
+from ._compat import strip_ansi
+from ._compat import text_type
+from ._compat import WIN
+from .exceptions import Abort
+from .exceptions import UsageError
+from .globals import resolve_color_default
+from .types import Choice
+from .types import convert_type
+from .types import Path
+from .utils import echo
+from .utils import LazyFile
+
+# The prompt functions to use.  The doc tools currently override these
+# functions to customize how they work.
+visible_prompt_func = raw_input
+
+_ansi_colors = {
+    "black": 30,
+    "red": 31,
+    "green": 32,
+    "yellow": 33,
+    "blue": 34,
+    "magenta": 35,
+    "cyan": 36,
+    "white": 37,
+    "reset": 39,
+    "bright_black": 90,
+    "bright_red": 91,
+    "bright_green": 92,
+    "bright_yellow": 93,
+    "bright_blue": 94,
+    "bright_magenta": 95,
+    "bright_cyan": 96,
+    "bright_white": 97,
+}
+_ansi_reset_all = "\033[0m"
+
+
+def hidden_prompt_func(prompt):
+    import getpass
+
+    return getpass.getpass(prompt)
+
+
+def _build_prompt(
+    text, suffix, show_default=False, default=None, show_choices=True, type=None
+):
+    prompt = text
+    if type is not None and show_choices and isinstance(type, Choice):
+        prompt += " ({})".format(", ".join(map(str, type.choices)))
+    if default is not None and show_default:
+        prompt = "{} [{}]".format(prompt, _format_default(default))
+    return prompt + suffix
+
+
+def _format_default(default):
+    if isinstance(default, (io.IOBase, LazyFile)) and hasattr(default, "name"):
+        return default.name
+
+    return default
+
+
+def prompt(
+    text,
+    default=None,
+    hide_input=False,
+    confirmation_prompt=False,
+    type=None,
+    value_proc=None,
+    prompt_suffix=": ",
+    show_default=True,
+    err=False,
+    show_choices=True,
+):
+    """Prompts a user for input.  This is a convenience function that can
+    be used to prompt a user for input later.
+
+    If the user aborts the input by sending a interrupt signal, this
+    function will catch it and raise a :exc:`Abort` exception.
+
+    .. versionadded:: 7.0
+       Added the show_choices parameter.
+
+    .. versionadded:: 6.0
+       Added unicode support for cmd.exe on Windows.
+
+    .. versionadded:: 4.0
+       Added the `err` parameter.
+
+    :param text: the text to show for the prompt.
+    :param default: the default value to use if no input happens.  If this
+                    is not given it will prompt until it's aborted.
+    :param hide_input: if this is set to true then the input value will
+                       be hidden.
+    :param confirmation_prompt: asks for confirmation for the value.
+    :param type: the type to use to check the value against.
+    :param value_proc: if this parameter is provided it's a function that
+                       is invoked instead of the type conversion to
+                       convert a value.
+    :param prompt_suffix: a suffix that should be added to the prompt.
+    :param show_default: shows or hides the default value in the prompt.
+    :param err: if set to true the file defaults to ``stderr`` instead of
+                ``stdout``, the same as with echo.
+    :param show_choices: Show or hide choices if the passed type is a Choice.
+                         For example if type is a Choice of either day or week,
+                         show_choices is true and text is "Group by" then the
+                         prompt will be "Group by (day, week): ".
+    """
+    result = None
+
+    def prompt_func(text):
+        f = hidden_prompt_func if hide_input else visible_prompt_func
+        try:
+            # Write the prompt separately so that we get nice
+            # coloring through colorama on Windows
+            echo(text, nl=False, err=err)
+            return f("")
+        except (KeyboardInterrupt, EOFError):
+            # getpass doesn't print a newline if the user aborts input with ^C.
+            # Allegedly this behavior is inherited from getpass(3).
+            # A doc bug has been filed at https://bugs.python.org/issue24711
+            if hide_input:
+                echo(None, err=err)
+            raise Abort()
+
+    if value_proc is None:
+        value_proc = convert_type(type, default)
+
+    prompt = _build_prompt(
+        text, prompt_suffix, show_default, default, show_choices, type
+    )
+
+    while 1:
+        while 1:
+            value = prompt_func(prompt)
+            if value:
+                break
+            elif default is not None:
+                if isinstance(value_proc, Path):
+                    # validate Path default value(exists, dir_okay etc.)
+                    value = default
+                    break
+                return default
+        try:
+            result = value_proc(value)
+        except UsageError as e:
+            echo("Error: {}".format(e.message), err=err)  # noqa: B306
+            continue
+        if not confirmation_prompt:
+            return result
+        while 1:
+            value2 = prompt_func("Repeat for confirmation: ")
+            if value2:
+                break
+        if value == value2:
+            return result
+        echo("Error: the two entered values do not match", err=err)
+
+
+def confirm(
+    text, default=False, abort=False, prompt_suffix=": ", show_default=True, err=False
+):
+    """Prompts for confirmation (yes/no question).
+
+    If the user aborts the input by sending a interrupt signal this
+    function will catch it and raise a :exc:`Abort` exception.
+
+    .. versionadded:: 4.0
+       Added the `err` parameter.
+
+    :param text: the question to ask.
+    :param default: the default for the prompt.
+    :param abort: if this is set to `True` a negative answer aborts the
+                  exception by raising :exc:`Abort`.
+    :param prompt_suffix: a suffix that should be added to the prompt.
+    :param show_default: shows or hides the default value in the prompt.
+    :param err: if set to true the file defaults to ``stderr`` instead of
+                ``stdout``, the same as with echo.
+    """
+    prompt = _build_prompt(
+        text, prompt_suffix, show_default, "Y/n" if default else "y/N"
+    )
+    while 1:
+        try:
+            # Write the prompt separately so that we get nice
+            # coloring through colorama on Windows
+            echo(prompt, nl=False, err=err)
+            value = visible_prompt_func("").lower().strip()
+        except (KeyboardInterrupt, EOFError):
+            raise Abort()
+        if value in ("y", "yes"):
+            rv = True
+        elif value in ("n", "no"):
+            rv = False
+        elif value == "":
+            rv = default
+        else:
+            echo("Error: invalid input", err=err)
+            continue
+        break
+    if abort and not rv:
+        raise Abort()
+    return rv
+
+
+def get_terminal_size():
+    """Returns the current size of the terminal as tuple in the form
+    ``(width, height)`` in columns and rows.
+    """
+    # If shutil has get_terminal_size() (Python 3.3 and later) use that
+    if sys.version_info >= (3, 3):
+        import shutil
+
+        shutil_get_terminal_size = getattr(shutil, "get_terminal_size", None)
+        if shutil_get_terminal_size:
+            sz = shutil_get_terminal_size()
+            return sz.columns, sz.lines
+
+    # We provide a sensible default for get_winterm_size() when being invoked
+    # inside a subprocess. Without this, it would not provide a useful input.
+    if get_winterm_size is not None:
+        size = get_winterm_size()
+        if size == (0, 0):
+            return (79, 24)
+        else:
+            return size
+
+    def ioctl_gwinsz(fd):
+        try:
+            import fcntl
+            import termios
+
+            cr = struct.unpack("hh", fcntl.ioctl(fd, termios.TIOCGWINSZ, "1234"))
+        except Exception:
+            return
+        return cr
+
+    cr = ioctl_gwinsz(0) or ioctl_gwinsz(1) or ioctl_gwinsz(2)
+    if not cr:
+        try:
+            fd = os.open(os.ctermid(), os.O_RDONLY)
+            try:
+                cr = ioctl_gwinsz(fd)
+            finally:
+                os.close(fd)
+        except Exception:
+            pass
+    if not cr or not cr[0] or not cr[1]:
+        cr = (os.environ.get("LINES", 25), os.environ.get("COLUMNS", DEFAULT_COLUMNS))
+    return int(cr[1]), int(cr[0])
+
+
+def echo_via_pager(text_or_generator, color=None):
+    """This function takes a text and shows it via an environment specific
+    pager on stdout.
+
+    .. versionchanged:: 3.0
+       Added the `color` flag.
+
+    :param text_or_generator: the text to page, or alternatively, a
+                              generator emitting the text to page.
+    :param color: controls if the pager supports ANSI colors or not.  The
+                  default is autodetection.
+    """
+    color = resolve_color_default(color)
+
+    if inspect.isgeneratorfunction(text_or_generator):
+        i = text_or_generator()
+    elif isinstance(text_or_generator, string_types):
+        i = [text_or_generator]
+    else:
+        i = iter(text_or_generator)
+
+    # convert every element of i to a text type if necessary
+    text_generator = (el if isinstance(el, string_types) else text_type(el) for el in i)
+
+    from ._termui_impl import pager
+
+    return pager(itertools.chain(text_generator, "\n"), color)
+
+
+def progressbar(
+    iterable=None,
+    length=None,
+    label=None,
+    show_eta=True,
+    show_percent=None,
+    show_pos=False,
+    item_show_func=None,
+    fill_char="#",
+    empty_char="-",
+    bar_template="%(label)s  [%(bar)s]  %(info)s",
+    info_sep="  ",
+    width=36,
+    file=None,
+    color=None,
+):
+    """This function creates an iterable context manager that can be used
+    to iterate over something while showing a progress bar.  It will
+    either iterate over the `iterable` or `length` items (that are counted
+    up).  While iteration happens, this function will print a rendered
+    progress bar to the given `file` (defaults to stdout) and will attempt
+    to calculate remaining time and more.  By default, this progress bar
+    will not be rendered if the file is not a terminal.
+
+    The context manager creates the progress bar.  When the context
+    manager is entered the progress bar is already created.  With every
+    iteration over the progress bar, the iterable passed to the bar is
+    advanced and the bar is updated.  When the context manager exits,
+    a newline is printed and the progress bar is finalized on screen.
+
+    Note: The progress bar is currently designed for use cases where the
+    total progress can be expected to take at least several seconds.
+    Because of this, the ProgressBar class object won't display
+    progress that is considered too fast, and progress where the time
+    between steps is less than a second.
+
+    No printing must happen or the progress bar will be unintentionally
+    destroyed.
+
+    Example usage::
+
+        with progressbar(items) as bar:
+            for item in bar:
+                do_something_with(item)
+
+    Alternatively, if no iterable is specified, one can manually update the
+    progress bar through the `update()` method instead of directly
+    iterating over the progress bar.  The update method accepts the number
+    of steps to increment the bar with::
+
+        with progressbar(length=chunks.total_bytes) as bar:
+            for chunk in chunks:
+                process_chunk(chunk)
+                bar.update(chunks.bytes)
+
+    .. versionadded:: 2.0
+
+    .. versionadded:: 4.0
+       Added the `color` parameter.  Added a `update` method to the
+       progressbar object.
+
+    :param iterable: an iterable to iterate over.  If not provided the length
+                     is required.
+    :param length: the number of items to iterate over.  By default the
+                   progressbar will attempt to ask the iterator about its
+                   length, which might or might not work.  If an iterable is
+                   also provided this parameter can be used to override the
+                   length.  If an iterable is not provided the progress bar
+                   will iterate over a range of that length.
+    :param label: the label to show next to the progress bar.
+    :param show_eta: enables or disables the estimated time display.  This is
+                     automatically disabled if the length cannot be
+                     determined.
+    :param show_percent: enables or disables the percentage display.  The
+                         default is `True` if the iterable has a length or
+                         `False` if not.
+    :param show_pos: enables or disables the absolute position display.  The
+                     default is `False`.
+    :param item_show_func: a function called with the current item which
+                           can return a string to show the current item
+                           next to the progress bar.  Note that the current
+                           item can be `None`!
+    :param fill_char: the character to use to show the filled part of the
+                      progress bar.
+    :param empty_char: the character to use to show the non-filled part of
+                       the progress bar.
+    :param bar_template: the format string to use as template for the bar.
+                         The parameters in it are ``label`` for the label,
+                         ``bar`` for the progress bar and ``info`` for the
+                         info section.
+    :param info_sep: the separator between multiple info items (eta etc.)
+    :param width: the width of the progress bar in characters, 0 means full
+                  terminal width
+    :param file: the file to write to.  If this is not a terminal then
+                 only the label is printed.
+    :param color: controls if the terminal supports ANSI colors or not.  The
+                  default is autodetection.  This is only needed if ANSI
+                  codes are included anywhere in the progress bar output
+                  which is not the case by default.
+    """
+    from ._termui_impl import ProgressBar
+
+    color = resolve_color_default(color)
+    return ProgressBar(
+        iterable=iterable,
+        length=length,
+        show_eta=show_eta,
+        show_percent=show_percent,
+        show_pos=show_pos,
+        item_show_func=item_show_func,
+        fill_char=fill_char,
+        empty_char=empty_char,
+        bar_template=bar_template,
+        info_sep=info_sep,
+        file=file,
+        label=label,
+        width=width,
+        color=color,
+    )
+
+
+def clear():
+    """Clears the terminal screen.  This will have the effect of clearing
+    the whole visible space of the terminal and moving the cursor to the
+    top left.  This does not do anything if not connected to a terminal.
+
+    .. versionadded:: 2.0
+    """
+    if not isatty(sys.stdout):
+        return
+    # If we're on Windows and we don't have colorama available, then we
+    # clear the screen by shelling out.  Otherwise we can use an escape
+    # sequence.
+    if WIN:
+        os.system("cls")
+    else:
+        sys.stdout.write("\033[2J\033[1;1H")
+
+
+def style(
+    text,
+    fg=None,
+    bg=None,
+    bold=None,
+    dim=None,
+    underline=None,
+    blink=None,
+    reverse=None,
+    reset=True,
+):
+    """Styles a text with ANSI styles and returns the new string.  By
+    default the styling is self contained which means that at the end
+    of the string a reset code is issued.  This can be prevented by
+    passing ``reset=False``.
+
+    Examples::
+
+        click.echo(click.style('Hello World!', fg='green'))
+        click.echo(click.style('ATTENTION!', blink=True))
+        click.echo(click.style('Some things', reverse=True, fg='cyan'))
+
+    Supported color names:
+
+    * ``black`` (might be a gray)
+    * ``red``
+    * ``green``
+    * ``yellow`` (might be an orange)
+    * ``blue``
+    * ``magenta``
+    * ``cyan``
+    * ``white`` (might be light gray)
+    * ``bright_black``
+    * ``bright_red``
+    * ``bright_green``
+    * ``bright_yellow``
+    * ``bright_blue``
+    * ``bright_magenta``
+    * ``bright_cyan``
+    * ``bright_white``
+    * ``reset`` (reset the color code only)
+
+    .. versionadded:: 2.0
+
+    .. versionadded:: 7.0
+       Added support for bright colors.
+
+    :param text: the string to style with ansi codes.
+    :param fg: if provided this will become the foreground color.
+    :param bg: if provided this will become the background color.
+    :param bold: if provided this will enable or disable bold mode.
+    :param dim: if provided this will enable or disable dim mode.  This is
+                badly supported.
+    :param underline: if provided this will enable or disable underline.
+    :param blink: if provided this will enable or disable blinking.
+    :param reverse: if provided this will enable or disable inverse
+                    rendering (foreground becomes background and the
+                    other way round).
+    :param reset: by default a reset-all code is added at the end of the
+                  string which means that styles do not carry over.  This
+                  can be disabled to compose styles.
+    """
+    bits = []
+    if fg:
+        try:
+            bits.append("\033[{}m".format(_ansi_colors[fg]))
+        except KeyError:
+            raise TypeError("Unknown color '{}'".format(fg))
+    if bg:
+        try:
+            bits.append("\033[{}m".format(_ansi_colors[bg] + 10))
+        except KeyError:
+            raise TypeError("Unknown color '{}'".format(bg))
+    if bold is not None:
+        bits.append("\033[{}m".format(1 if bold else 22))
+    if dim is not None:
+        bits.append("\033[{}m".format(2 if dim else 22))
+    if underline is not None:
+        bits.append("\033[{}m".format(4 if underline else 24))
+    if blink is not None:
+        bits.append("\033[{}m".format(5 if blink else 25))
+    if reverse is not None:
+        bits.append("\033[{}m".format(7 if reverse else 27))
+    bits.append(text)
+    if reset:
+        bits.append(_ansi_reset_all)
+    return "".join(bits)
+
+
+def unstyle(text):
+    """Removes ANSI styling information from a string.  Usually it's not
+    necessary to use this function as Click's echo function will
+    automatically remove styling if necessary.
+
+    .. versionadded:: 2.0
+
+    :param text: the text to remove style information from.
+    """
+    return strip_ansi(text)
+
+
+def secho(message=None, file=None, nl=True, err=False, color=None, **styles):
+    """This function combines :func:`echo` and :func:`style` into one
+    call.  As such the following two calls are the same::
+
+        click.secho('Hello World!', fg='green')
+        click.echo(click.style('Hello World!', fg='green'))
+
+    All keyword arguments are forwarded to the underlying functions
+    depending on which one they go with.
+
+    .. versionadded:: 2.0
+    """
+    if message is not None:
+        message = style(message, **styles)
+    return echo(message, file=file, nl=nl, err=err, color=color)
+
+
+def edit(
+    text=None, editor=None, env=None, require_save=True, extension=".txt", filename=None
+):
+    r"""Edits the given text in the defined editor.  If an editor is given
+    (should be the full path to the executable but the regular operating
+    system search path is used for finding the executable) it overrides
+    the detected editor.  Optionally, some environment variables can be
+    used.  If the editor is closed without changes, `None` is returned.  In
+    case a file is edited directly the return value is always `None` and
+    `require_save` and `extension` are ignored.
+
+    If the editor cannot be opened a :exc:`UsageError` is raised.
+
+    Note for Windows: to simplify cross-platform usage, the newlines are
+    automatically converted from POSIX to Windows and vice versa.  As such,
+    the message here will have ``\n`` as newline markers.
+
+    :param text: the text to edit.
+    :param editor: optionally the editor to use.  Defaults to automatic
+                   detection.
+    :param env: environment variables to forward to the editor.
+    :param require_save: if this is true, then not saving in the editor
+                         will make the return value become `None`.
+    :param extension: the extension to tell the editor about.  This defaults
+                      to `.txt` but changing this might change syntax
+                      highlighting.
+    :param filename: if provided it will edit this file instead of the
+                     provided text contents.  It will not use a temporary
+                     file as an indirection in that case.
+    """
+    from ._termui_impl import Editor
+
+    editor = Editor(
+        editor=editor, env=env, require_save=require_save, extension=extension
+    )
+    if filename is None:
+        return editor.edit(text)
+    editor.edit_file(filename)
+
+
+def launch(url, wait=False, locate=False):
+    """This function launches the given URL (or filename) in the default
+    viewer application for this file type.  If this is an executable, it
+    might launch the executable in a new session.  The return value is
+    the exit code of the launched application.  Usually, ``0`` indicates
+    success.
+
+    Examples::
+
+        click.launch('https://click.palletsprojects.com/')
+        click.launch('/my/downloaded/file', locate=True)
+
+    .. versionadded:: 2.0
+
+    :param url: URL or filename of the thing to launch.
+    :param wait: waits for the program to stop.
+    :param locate: if this is set to `True` then instead of launching the
+                   application associated with the URL it will attempt to
+                   launch a file manager with the file located.  This
+                   might have weird effects if the URL does not point to
+                   the filesystem.
+    """
+    from ._termui_impl import open_url
+
+    return open_url(url, wait=wait, locate=locate)
+
+
+# If this is provided, getchar() calls into this instead.  This is used
+# for unittesting purposes.
+_getchar = None
+
+
+def getchar(echo=False):
+    """Fetches a single character from the terminal and returns it.  This
+    will always return a unicode character and under certain rare
+    circumstances this might return more than one character.  The
+    situations which more than one character is returned is when for
+    whatever reason multiple characters end up in the terminal buffer or
+    standard input was not actually a terminal.
+
+    Note that this will always read from the terminal, even if something
+    is piped into the standard input.
+
+    Note for Windows: in rare cases when typing non-ASCII characters, this
+    function might wait for a second character and then return both at once.
+    This is because certain Unicode characters look like special-key markers.
+
+    .. versionadded:: 2.0
+
+    :param echo: if set to `True`, the character read will also show up on
+                 the terminal.  The default is to not show it.
+    """
+    f = _getchar
+    if f is None:
+        from ._termui_impl import getchar as f
+    return f(echo)
+
+
+def raw_terminal():
+    from ._termui_impl import raw_terminal as f
+
+    return f()
+
+
+def pause(info="Press any key to continue ...", err=False):
+    """This command stops execution and waits for the user to press any
+    key to continue.  This is similar to the Windows batch "pause"
+    command.  If the program is not run through a terminal, this command
+    will instead do nothing.
+
+    .. versionadded:: 2.0
+
+    .. versionadded:: 4.0
+       Added the `err` parameter.
+
+    :param info: the info string to print before pausing.
+    :param err: if set to message goes to ``stderr`` instead of
+                ``stdout``, the same as with echo.
+    """
+    if not isatty(sys.stdin) or not isatty(sys.stdout):
+        return
+    try:
+        if info:
+            echo(info, nl=False, err=err)
+        try:
+            getchar()
+        except (KeyboardInterrupt, EOFError):
+            pass
+    finally:
+        if info:
+            echo(err=err)

tools/python/ampy/click/testing.py

@@ -0,0 +1,382 @@
+import contextlib
+import os
+import shlex
+import shutil
+import sys
+import tempfile
+
+from . import formatting
+from . import termui
+from . import utils
+from ._compat import iteritems
+from ._compat import PY2
+from ._compat import string_types
+
+
+if PY2:
+    from cStringIO import StringIO
+else:
+    import io
+    from ._compat import _find_binary_reader
+
+
+class EchoingStdin(object):
+    def __init__(self, input, output):
+        self._input = input
+        self._output = output
+
+    def __getattr__(self, x):
+        return getattr(self._input, x)
+
+    def _echo(self, rv):
+        self._output.write(rv)
+        return rv
+
+    def read(self, n=-1):
+        return self._echo(self._input.read(n))
+
+    def readline(self, n=-1):
+        return self._echo(self._input.readline(n))
+
+    def readlines(self):
+        return [self._echo(x) for x in self._input.readlines()]
+
+    def __iter__(self):
+        return iter(self._echo(x) for x in self._input)
+
+    def __repr__(self):
+        return repr(self._input)
+
+
+def make_input_stream(input, charset):
+    # Is already an input stream.
+    if hasattr(input, "read"):
+        if PY2:
+            return input
+        rv = _find_binary_reader(input)
+        if rv is not None:
+            return rv
+        raise TypeError("Could not find binary reader for input stream.")
+
+    if input is None:
+        input = b""
+    elif not isinstance(input, bytes):
+        input = input.encode(charset)
+    if PY2:
+        return StringIO(input)
+    return io.BytesIO(input)
+
+
+class Result(object):
+    """Holds the captured result of an invoked CLI script."""
+
+    def __init__(
+        self, runner, stdout_bytes, stderr_bytes, exit_code, exception, exc_info=None
+    ):
+        #: The runner that created the result
+        self.runner = runner
+        #: The standard output as bytes.
+        self.stdout_bytes = stdout_bytes
+        #: The standard error as bytes, or None if not available
+        self.stderr_bytes = stderr_bytes
+        #: The exit code as integer.
+        self.exit_code = exit_code
+        #: The exception that happened if one did.
+        self.exception = exception
+        #: The traceback
+        self.exc_info = exc_info
+
+    @property
+    def output(self):
+        """The (standard) output as unicode string."""
+        return self.stdout
+
+    @property
+    def stdout(self):
+        """The standard output as unicode string."""
+        return self.stdout_bytes.decode(self.runner.charset, "replace").replace(
+            "\r\n", "\n"
+        )
+
+    @property
+    def stderr(self):
+        """The standard error as unicode string."""
+        if self.stderr_bytes is None:
+            raise ValueError("stderr not separately captured")
+        return self.stderr_bytes.decode(self.runner.charset, "replace").replace(
+            "\r\n", "\n"
+        )
+
+    def __repr__(self):
+        return "<{} {}>".format(
+            type(self).__name__, repr(self.exception) if self.exception else "okay"
+        )
+
+
+class CliRunner(object):
+    """The CLI runner provides functionality to invoke a Click command line
+    script for unittesting purposes in a isolated environment.  This only
+    works in single-threaded systems without any concurrency as it changes the
+    global interpreter state.
+
+    :param charset: the character set for the input and output data.  This is
+                    UTF-8 by default and should not be changed currently as
+                    the reporting to Click only works in Python 2 properly.
+    :param env: a dictionary with environment variables for overriding.
+    :param echo_stdin: if this is set to `True`, then reading from stdin writes
+                       to stdout.  This is useful for showing examples in
+                       some circumstances.  Note that regular prompts
+                       will automatically echo the input.
+    :param mix_stderr: if this is set to `False`, then stdout and stderr are
+                       preserved as independent streams.  This is useful for
+                       Unix-philosophy apps that have predictable stdout and
+                       noisy stderr, such that each may be measured
+                       independently
+    """
+
+    def __init__(self, charset=None, env=None, echo_stdin=False, mix_stderr=True):
+        if charset is None:
+            charset = "utf-8"
+        self.charset = charset
+        self.env = env or {}
+        self.echo_stdin = echo_stdin
+        self.mix_stderr = mix_stderr
+
+    def get_default_prog_name(self, cli):
+        """Given a command object it will return the default program name
+        for it.  The default is the `name` attribute or ``"root"`` if not
+        set.
+        """
+        return cli.name or "root"
+
+    def make_env(self, overrides=None):
+        """Returns the environment overrides for invoking a script."""
+        rv = dict(self.env)
+        if overrides:
+            rv.update(overrides)
+        return rv
+
+    @contextlib.contextmanager
+    def isolation(self, input=None, env=None, color=False):
+        """A context manager that sets up the isolation for invoking of a
+        command line tool.  This sets up stdin with the given input data
+        and `os.environ` with the overrides from the given dictionary.
+        This also rebinds some internals in Click to be mocked (like the
+        prompt functionality).
+
+        This is automatically done in the :meth:`invoke` method.
+
+        .. versionadded:: 4.0
+           The ``color`` parameter was added.
+
+        :param input: the input stream to put into sys.stdin.
+        :param env: the environment overrides as dictionary.
+        :param color: whether the output should contain color codes. The
+                      application can still override this explicitly.
+        """
+        input = make_input_stream(input, self.charset)
+
+        old_stdin = sys.stdin
+        old_stdout = sys.stdout
+        old_stderr = sys.stderr
+        old_forced_width = formatting.FORCED_WIDTH
+        formatting.FORCED_WIDTH = 80
+
+        env = self.make_env(env)
+
+        if PY2:
+            bytes_output = StringIO()
+            if self.echo_stdin:
+                input = EchoingStdin(input, bytes_output)
+            sys.stdout = bytes_output
+            if not self.mix_stderr:
+                bytes_error = StringIO()
+                sys.stderr = bytes_error
+        else:
+            bytes_output = io.BytesIO()
+            if self.echo_stdin:
+                input = EchoingStdin(input, bytes_output)
+            input = io.TextIOWrapper(input, encoding=self.charset)
+            sys.stdout = io.TextIOWrapper(bytes_output, encoding=self.charset)
+            if not self.mix_stderr:
+                bytes_error = io.BytesIO()
+                sys.stderr = io.TextIOWrapper(bytes_error, encoding=self.charset)
+
+        if self.mix_stderr:
+            sys.stderr = sys.stdout
+
+        sys.stdin = input
+
+        def visible_input(prompt=None):
+            sys.stdout.write(prompt or "")
+            val = input.readline().rstrip("\r\n")
+            sys.stdout.write("{}\n".format(val))
+            sys.stdout.flush()
+            return val
+
+        def hidden_input(prompt=None):
+            sys.stdout.write("{}\n".format(prompt or ""))
+            sys.stdout.flush()
+            return input.readline().rstrip("\r\n")
+
+        def _getchar(echo):
+            char = sys.stdin.read(1)
+            if echo:
+                sys.stdout.write(char)
+                sys.stdout.flush()
+            return char
+
+        default_color = color
+
+        def should_strip_ansi(stream=None, color=None):
+            if color is None:
+                return not default_color
+            return not color
+
+        old_visible_prompt_func = termui.visible_prompt_func
+        old_hidden_prompt_func = termui.hidden_prompt_func
+        old__getchar_func = termui._getchar
+        old_should_strip_ansi = utils.should_strip_ansi
+        termui.visible_prompt_func = visible_input
+        termui.hidden_prompt_func = hidden_input
+        termui._getchar = _getchar
+        utils.should_strip_ansi = should_strip_ansi
+
+        old_env = {}
+        try:
+            for key, value in iteritems(env):
+                old_env[key] = os.environ.get(key)
+                if value is None:
+                    try:
+                        del os.environ[key]
+                    except Exception:
+                        pass
+                else:
+                    os.environ[key] = value
+            yield (bytes_output, not self.mix_stderr and bytes_error)
+        finally:
+            for key, value in iteritems(old_env):
+                if value is None:
+                    try:
+                        del os.environ[key]
+                    except Exception:
+                        pass
+                else:
+                    os.environ[key] = value
+            sys.stdout = old_stdout
+            sys.stderr = old_stderr
+            sys.stdin = old_stdin
+            termui.visible_prompt_func = old_visible_prompt_func
+            termui.hidden_prompt_func = old_hidden_prompt_func
+            termui._getchar = old__getchar_func
+            utils.should_strip_ansi = old_should_strip_ansi
+            formatting.FORCED_WIDTH = old_forced_width
+
+    def invoke(
+        self,
+        cli,
+        args=None,
+        input=None,
+        env=None,
+        catch_exceptions=True,
+        color=False,
+        **extra
+    ):
+        """Invokes a command in an isolated environment.  The arguments are
+        forwarded directly to the command line script, the `extra` keyword
+        arguments are passed to the :meth:`~clickpkg.Command.main` function of
+        the command.
+
+        This returns a :class:`Result` object.
+
+        .. versionadded:: 3.0
+           The ``catch_exceptions`` parameter was added.
+
+        .. versionchanged:: 3.0
+           The result object now has an `exc_info` attribute with the
+           traceback if available.
+
+        .. versionadded:: 4.0
+           The ``color`` parameter was added.
+
+        :param cli: the command to invoke
+        :param args: the arguments to invoke. It may be given as an iterable
+                     or a string. When given as string it will be interpreted
+                     as a Unix shell command. More details at
+                     :func:`shlex.split`.
+        :param input: the input data for `sys.stdin`.
+        :param env: the environment overrides.
+        :param catch_exceptions: Whether to catch any other exceptions than
+                                 ``SystemExit``.
+        :param extra: the keyword arguments to pass to :meth:`main`.
+        :param color: whether the output should contain color codes. The
+                      application can still override this explicitly.
+        """
+        exc_info = None
+        with self.isolation(input=input, env=env, color=color) as outstreams:
+            exception = None
+            exit_code = 0
+
+            if isinstance(args, string_types):
+                args = shlex.split(args)
+
+            try:
+                prog_name = extra.pop("prog_name")
+            except KeyError:
+                prog_name = self.get_default_prog_name(cli)
+
+            try:
+                cli.main(args=args or (), prog_name=prog_name, **extra)
+            except SystemExit as e:
+                exc_info = sys.exc_info()
+                exit_code = e.code
+                if exit_code is None:
+                    exit_code = 0
+
+                if exit_code != 0:
+                    exception = e
+
+                if not isinstance(exit_code, int):
+                    sys.stdout.write(str(exit_code))
+                    sys.stdout.write("\n")
+                    exit_code = 1
+
+            except Exception as e:
+                if not catch_exceptions:
+                    raise
+                exception = e
+                exit_code = 1
+                exc_info = sys.exc_info()
+            finally:
+                sys.stdout.flush()
+                stdout = outstreams[0].getvalue()
+                if self.mix_stderr:
+                    stderr = None
+                else:
+                    stderr = outstreams[1].getvalue()
+
+        return Result(
+            runner=self,
+            stdout_bytes=stdout,
+            stderr_bytes=stderr,
+            exit_code=exit_code,
+            exception=exception,
+            exc_info=exc_info,
+        )
+
+    @contextlib.contextmanager
+    def isolated_filesystem(self):
+        """A context manager that creates a temporary folder and changes
+        the current working directory to it for isolated filesystem tests.
+        """
+        cwd = os.getcwd()
+        t = tempfile.mkdtemp()
+        os.chdir(t)
+        try:
+            yield t
+        finally:
+            os.chdir(cwd)
+            try:
+                shutil.rmtree(t)
+            except (OSError, IOError):  # noqa: B014
+                pass

tools/python/ampy/click/types.py

@@ -0,0 +1,762 @@
+import os
+import stat
+from datetime import datetime
+
+from ._compat import _get_argv_encoding
+from ._compat import filename_to_ui
+from ._compat import get_filesystem_encoding
+from ._compat import get_streerror
+from ._compat import open_stream
+from ._compat import PY2
+from ._compat import text_type
+from .exceptions import BadParameter
+from .utils import LazyFile
+from .utils import safecall
+
+
+class ParamType(object):
+    """Helper for converting values through types.  The following is
+    necessary for a valid type:
+
+    *   it needs a name
+    *   it needs to pass through None unchanged
+    *   it needs to convert from a string
+    *   it needs to convert its result type through unchanged
+        (eg: needs to be idempotent)
+    *   it needs to be able to deal with param and context being `None`.
+        This can be the case when the object is used with prompt
+        inputs.
+    """
+
+    is_composite = False
+
+    #: the descriptive name of this type
+    name = None
+
+    #: if a list of this type is expected and the value is pulled from a
+    #: string environment variable, this is what splits it up.  `None`
+    #: means any whitespace.  For all parameters the general rule is that
+    #: whitespace splits them up.  The exception are paths and files which
+    #: are split by ``os.path.pathsep`` by default (":" on Unix and ";" on
+    #: Windows).
+    envvar_list_splitter = None
+
+    def __call__(self, value, param=None, ctx=None):
+        if value is not None:
+            return self.convert(value, param, ctx)
+
+    def get_metavar(self, param):
+        """Returns the metavar default for this param if it provides one."""
+
+    def get_missing_message(self, param):
+        """Optionally might return extra information about a missing
+        parameter.
+
+        .. versionadded:: 2.0
+        """
+
+    def convert(self, value, param, ctx):
+        """Converts the value.  This is not invoked for values that are
+        `None` (the missing value).
+        """
+        return value
+
+    def split_envvar_value(self, rv):
+        """Given a value from an environment variable this splits it up
+        into small chunks depending on the defined envvar list splitter.
+
+        If the splitter is set to `None`, which means that whitespace splits,
+        then leading and trailing whitespace is ignored.  Otherwise, leading
+        and trailing splitters usually lead to empty items being included.
+        """
+        return (rv or "").split(self.envvar_list_splitter)
+
+    def fail(self, message, param=None, ctx=None):
+        """Helper method to fail with an invalid value message."""
+        raise BadParameter(message, ctx=ctx, param=param)
+
+
+class CompositeParamType(ParamType):
+    is_composite = True
+
+    @property
+    def arity(self):
+        raise NotImplementedError()
+
+
+class FuncParamType(ParamType):
+    def __init__(self, func):
+        self.name = func.__name__
+        self.func = func
+
+    def convert(self, value, param, ctx):
+        try:
+            return self.func(value)
+        except ValueError:
+            try:
+                value = text_type(value)
+            except UnicodeError:
+                value = str(value).decode("utf-8", "replace")
+            self.fail(value, param, ctx)
+
+
+class UnprocessedParamType(ParamType):
+    name = "text"
+
+    def convert(self, value, param, ctx):
+        return value
+
+    def __repr__(self):
+        return "UNPROCESSED"
+
+
+class StringParamType(ParamType):
+    name = "text"
+
+    def convert(self, value, param, ctx):
+        if isinstance(value, bytes):
+            enc = _get_argv_encoding()
+            try:
+                value = value.decode(enc)
+            except UnicodeError:
+                fs_enc = get_filesystem_encoding()
+                if fs_enc != enc:
+                    try:
+                        value = value.decode(fs_enc)
+                    except UnicodeError:
+                        value = value.decode("utf-8", "replace")
+                else:
+                    value = value.decode("utf-8", "replace")
+            return value
+        return value
+
+    def __repr__(self):
+        return "STRING"
+
+
+class Choice(ParamType):
+    """The choice type allows a value to be checked against a fixed set
+    of supported values. All of these values have to be strings.
+
+    You should only pass a list or tuple of choices. Other iterables
+    (like generators) may lead to surprising results.
+
+    The resulting value will always be one of the originally passed choices
+    regardless of ``case_sensitive`` or any ``ctx.token_normalize_func``
+    being specified.
+
+    See :ref:`choice-opts` for an example.
+
+    :param case_sensitive: Set to false to make choices case
+        insensitive. Defaults to true.
+    """
+
+    name = "choice"
+
+    def __init__(self, choices, case_sensitive=True):
+        self.choices = choices
+        self.case_sensitive = case_sensitive
+
+    def get_metavar(self, param):
+        return "[{}]".format("|".join(self.choices))
+
+    def get_missing_message(self, param):
+        return "Choose from:\n\t{}.".format(",\n\t".join(self.choices))
+
+    def convert(self, value, param, ctx):
+        # Match through normalization and case sensitivity
+        # first do token_normalize_func, then lowercase
+        # preserve original `value` to produce an accurate message in
+        # `self.fail`
+        normed_value = value
+        normed_choices = {choice: choice for choice in self.choices}
+
+        if ctx is not None and ctx.token_normalize_func is not None:
+            normed_value = ctx.token_normalize_func(value)
+            normed_choices = {
+                ctx.token_normalize_func(normed_choice): original
+                for normed_choice, original in normed_choices.items()
+            }
+
+        if not self.case_sensitive:
+            if PY2:
+                lower = str.lower
+            else:
+                lower = str.casefold
+
+            normed_value = lower(normed_value)
+            normed_choices = {
+                lower(normed_choice): original
+                for normed_choice, original in normed_choices.items()
+            }
+
+        if normed_value in normed_choices:
+            return normed_choices[normed_value]
+
+        self.fail(
+            "invalid choice: {}. (choose from {})".format(
+                value, ", ".join(self.choices)
+            ),
+            param,
+            ctx,
+        )
+
+    def __repr__(self):
+        return "Choice('{}')".format(list(self.choices))
+
+
+class DateTime(ParamType):
+    """The DateTime type converts date strings into `datetime` objects.
+
+    The format strings which are checked are configurable, but default to some
+    common (non-timezone aware) ISO 8601 formats.
+
+    When specifying *DateTime* formats, you should only pass a list or a tuple.
+    Other iterables, like generators, may lead to surprising results.
+
+    The format strings are processed using ``datetime.strptime``, and this
+    consequently defines the format strings which are allowed.
+
+    Parsing is tried using each format, in order, and the first format which
+    parses successfully is used.
+
+    :param formats: A list or tuple of date format strings, in the order in
+                    which they should be tried. Defaults to
+                    ``'%Y-%m-%d'``, ``'%Y-%m-%dT%H:%M:%S'``,
+                    ``'%Y-%m-%d %H:%M:%S'``.
+    """
+
+    name = "datetime"
+
+    def __init__(self, formats=None):
+        self.formats = formats or ["%Y-%m-%d", "%Y-%m-%dT%H:%M:%S", "%Y-%m-%d %H:%M:%S"]
+
+    def get_metavar(self, param):
+        return "[{}]".format("|".join(self.formats))
+
+    def _try_to_convert_date(self, value, format):
+        try:
+            return datetime.strptime(value, format)
+        except ValueError:
+            return None
+
+    def convert(self, value, param, ctx):
+        # Exact match
+        for format in self.formats:
+            dtime = self._try_to_convert_date(value, format)
+            if dtime:
+                return dtime
+
+        self.fail(
+            "invalid datetime format: {}. (choose from {})".format(
+                value, ", ".join(self.formats)
+            )
+        )
+
+    def __repr__(self):
+        return "DateTime"
+
+
+class IntParamType(ParamType):
+    name = "integer"
+
+    def convert(self, value, param, ctx):
+        try:
+            return int(value)
+        except ValueError:
+            self.fail("{} is not a valid integer".format(value), param, ctx)
+
+    def __repr__(self):
+        return "INT"
+
+
+class IntRange(IntParamType):
+    """A parameter that works similar to :data:`click.INT` but restricts
+    the value to fit into a range.  The default behavior is to fail if the
+    value falls outside the range, but it can also be silently clamped
+    between the two edges.
+
+    See :ref:`ranges` for an example.
+    """
+
+    name = "integer range"
+
+    def __init__(self, min=None, max=None, clamp=False):
+        self.min = min
+        self.max = max
+        self.clamp = clamp
+
+    def convert(self, value, param, ctx):
+        rv = IntParamType.convert(self, value, param, ctx)
+        if self.clamp:
+            if self.min is not None and rv < self.min:
+                return self.min
+            if self.max is not None and rv > self.max:
+                return self.max
+        if (
+            self.min is not None
+            and rv < self.min
+            or self.max is not None
+            and rv > self.max
+        ):
+            if self.min is None:
+                self.fail(
+                    "{} is bigger than the maximum valid value {}.".format(
+                        rv, self.max
+                    ),
+                    param,
+                    ctx,
+                )
+            elif self.max is None:
+                self.fail(
+                    "{} is smaller than the minimum valid value {}.".format(
+                        rv, self.min
+                    ),
+                    param,
+                    ctx,
+                )
+            else:
+                self.fail(
+                    "{} is not in the valid range of {} to {}.".format(
+                        rv, self.min, self.max
+                    ),
+                    param,
+                    ctx,
+                )
+        return rv
+
+    def __repr__(self):
+        return "IntRange({}, {})".format(self.min, self.max)
+
+
+class FloatParamType(ParamType):
+    name = "float"
+
+    def convert(self, value, param, ctx):
+        try:
+            return float(value)
+        except ValueError:
+            self.fail(
+                "{} is not a valid floating point value".format(value), param, ctx
+            )
+
+    def __repr__(self):
+        return "FLOAT"
+
+
+class FloatRange(FloatParamType):
+    """A parameter that works similar to :data:`click.FLOAT` but restricts
+    the value to fit into a range.  The default behavior is to fail if the
+    value falls outside the range, but it can also be silently clamped
+    between the two edges.
+
+    See :ref:`ranges` for an example.
+    """
+
+    name = "float range"
+
+    def __init__(self, min=None, max=None, clamp=False):
+        self.min = min
+        self.max = max
+        self.clamp = clamp
+
+    def convert(self, value, param, ctx):
+        rv = FloatParamType.convert(self, value, param, ctx)
+        if self.clamp:
+            if self.min is not None and rv < self.min:
+                return self.min
+            if self.max is not None and rv > self.max:
+                return self.max
+        if (
+            self.min is not None
+            and rv < self.min
+            or self.max is not None
+            and rv > self.max
+        ):
+            if self.min is None:
+                self.fail(
+                    "{} is bigger than the maximum valid value {}.".format(
+                        rv, self.max
+                    ),
+                    param,
+                    ctx,
+                )
+            elif self.max is None:
+                self.fail(
+                    "{} is smaller than the minimum valid value {}.".format(
+                        rv, self.min
+                    ),
+                    param,
+                    ctx,
+                )
+            else:
+                self.fail(
+                    "{} is not in the valid range of {} to {}.".format(
+                        rv, self.min, self.max
+                    ),
+                    param,
+                    ctx,
+                )
+        return rv
+
+    def __repr__(self):
+        return "FloatRange({}, {})".format(self.min, self.max)
+
+
+class BoolParamType(ParamType):
+    name = "boolean"
+
+    def convert(self, value, param, ctx):
+        if isinstance(value, bool):
+            return bool(value)
+        value = value.lower()
+        if value in ("true", "t", "1", "yes", "y"):
+            return True
+        elif value in ("false", "f", "0", "no", "n"):
+            return False
+        self.fail("{} is not a valid boolean".format(value), param, ctx)
+
+    def __repr__(self):
+        return "BOOL"
+
+
+class UUIDParameterType(ParamType):
+    name = "uuid"
+
+    def convert(self, value, param, ctx):
+        import uuid
+
+        try:
+            if PY2 and isinstance(value, text_type):
+                value = value.encode("ascii")
+            return uuid.UUID(value)
+        except ValueError:
+            self.fail("{} is not a valid UUID value".format(value), param, ctx)
+
+    def __repr__(self):
+        return "UUID"
+
+
+class File(ParamType):
+    """Declares a parameter to be a file for reading or writing.  The file
+    is automatically closed once the context tears down (after the command
+    finished working).
+
+    Files can be opened for reading or writing.  The special value ``-``
+    indicates stdin or stdout depending on the mode.
+
+    By default, the file is opened for reading text data, but it can also be
+    opened in binary mode or for writing.  The encoding parameter can be used
+    to force a specific encoding.
+
+    The `lazy` flag controls if the file should be opened immediately or upon
+    first IO. The default is to be non-lazy for standard input and output
+    streams as well as files opened for reading, `lazy` otherwise. When opening a
+    file lazily for reading, it is still opened temporarily for validation, but
+    will not be held open until first IO. lazy is mainly useful when opening
+    for writing to avoid creating the file until it is needed.
+
+    Starting with Click 2.0, files can also be opened atomically in which
+    case all writes go into a separate file in the same folder and upon
+    completion the file will be moved over to the original location.  This
+    is useful if a file regularly read by other users is modified.
+
+    See :ref:`file-args` for more information.
+    """
+
+    name = "filename"
+    envvar_list_splitter = os.path.pathsep
+
+    def __init__(
+        self, mode="r", encoding=None, errors="strict", lazy=None, atomic=False
+    ):
+        self.mode = mode
+        self.encoding = encoding
+        self.errors = errors
+        self.lazy = lazy
+        self.atomic = atomic
+
+    def resolve_lazy_flag(self, value):
+        if self.lazy is not None:
+            return self.lazy
+        if value == "-":
+            return False
+        elif "w" in self.mode:
+            return True
+        return False
+
+    def convert(self, value, param, ctx):
+        try:
+            if hasattr(value, "read") or hasattr(value, "write"):
+                return value
+
+            lazy = self.resolve_lazy_flag(value)
+
+            if lazy:
+                f = LazyFile(
+                    value, self.mode, self.encoding, self.errors, atomic=self.atomic
+                )
+                if ctx is not None:
+                    ctx.call_on_close(f.close_intelligently)
+                return f
+
+            f, should_close = open_stream(
+                value, self.mode, self.encoding, self.errors, atomic=self.atomic
+            )
+            # If a context is provided, we automatically close the file
+            # at the end of the context execution (or flush out).  If a
+            # context does not exist, it's the caller's responsibility to
+            # properly close the file.  This for instance happens when the
+            # type is used with prompts.
+            if ctx is not None:
+                if should_close:
+                    ctx.call_on_close(safecall(f.close))
+                else:
+                    ctx.call_on_close(safecall(f.flush))
+            return f
+        except (IOError, OSError) as e:  # noqa: B014
+            self.fail(
+                "Could not open file: {}: {}".format(
+                    filename_to_ui(value), get_streerror(e)
+                ),
+                param,
+                ctx,
+            )
+
+
+class Path(ParamType):
+    """The path type is similar to the :class:`File` type but it performs
+    different checks.  First of all, instead of returning an open file
+    handle it returns just the filename.  Secondly, it can perform various
+    basic checks about what the file or directory should be.
+
+    .. versionchanged:: 6.0
+       `allow_dash` was added.
+
+    :param exists: if set to true, the file or directory needs to exist for
+                   this value to be valid.  If this is not required and a
+                   file does indeed not exist, then all further checks are
+                   silently skipped.
+    :param file_okay: controls if a file is a possible value.
+    :param dir_okay: controls if a directory is a possible value.
+    :param writable: if true, a writable check is performed.
+    :param readable: if true, a readable check is performed.
+    :param resolve_path: if this is true, then the path is fully resolved
+                         before the value is passed onwards.  This means
+                         that it's absolute and symlinks are resolved.  It
+                         will not expand a tilde-prefix, as this is
+                         supposed to be done by the shell only.
+    :param allow_dash: If this is set to `True`, a single dash to indicate
+                       standard streams is permitted.
+    :param path_type: optionally a string type that should be used to
+                      represent the path.  The default is `None` which
+                      means the return value will be either bytes or
+                      unicode depending on what makes most sense given the
+                      input data Click deals with.
+    """
+
+    envvar_list_splitter = os.path.pathsep
+
+    def __init__(
+        self,
+        exists=False,
+        file_okay=True,
+        dir_okay=True,
+        writable=False,
+        readable=True,
+        resolve_path=False,
+        allow_dash=False,
+        path_type=None,
+    ):
+        self.exists = exists
+        self.file_okay = file_okay
+        self.dir_okay = dir_okay
+        self.writable = writable
+        self.readable = readable
+        self.resolve_path = resolve_path
+        self.allow_dash = allow_dash
+        self.type = path_type
+
+        if self.file_okay and not self.dir_okay:
+            self.name = "file"
+            self.path_type = "File"
+        elif self.dir_okay and not self.file_okay:
+            self.name = "directory"
+            self.path_type = "Directory"
+        else:
+            self.name = "path"
+            self.path_type = "Path"
+
+    def coerce_path_result(self, rv):
+        if self.type is not None and not isinstance(rv, self.type):
+            if self.type is text_type:
+                rv = rv.decode(get_filesystem_encoding())
+            else:
+                rv = rv.encode(get_filesystem_encoding())
+        return rv
+
+    def convert(self, value, param, ctx):
+        rv = value
+
+        is_dash = self.file_okay and self.allow_dash and rv in (b"-", "-")
+
+        if not is_dash:
+            if self.resolve_path:
+                rv = os.path.realpath(rv)
+
+            try:
+                st = os.stat(rv)
+            except OSError:
+                if not self.exists:
+                    return self.coerce_path_result(rv)
+                self.fail(
+                    "{} '{}' does not exist.".format(
+                        self.path_type, filename_to_ui(value)
+                    ),
+                    param,
+                    ctx,
+                )
+
+            if not self.file_okay and stat.S_ISREG(st.st_mode):
+                self.fail(
+                    "{} '{}' is a file.".format(self.path_type, filename_to_ui(value)),
+                    param,
+                    ctx,
+                )
+            if not self.dir_okay and stat.S_ISDIR(st.st_mode):
+                self.fail(
+                    "{} '{}' is a directory.".format(
+                        self.path_type, filename_to_ui(value)
+                    ),
+                    param,
+                    ctx,
+                )
+            if self.writable and not os.access(value, os.W_OK):
+                self.fail(
+                    "{} '{}' is not writable.".format(
+                        self.path_type, filename_to_ui(value)
+                    ),
+                    param,
+                    ctx,
+                )
+            if self.readable and not os.access(value, os.R_OK):
+                self.fail(
+                    "{} '{}' is not readable.".format(
+                        self.path_type, filename_to_ui(value)
+                    ),
+                    param,
+                    ctx,
+                )
+
+        return self.coerce_path_result(rv)
+
+
+class Tuple(CompositeParamType):
+    """The default behavior of Click is to apply a type on a value directly.
+    This works well in most cases, except for when `nargs` is set to a fixed
+    count and different types should be used for different items.  In this
+    case the :class:`Tuple` type can be used.  This type can only be used
+    if `nargs` is set to a fixed number.
+
+    For more information see :ref:`tuple-type`.
+
+    This can be selected by using a Python tuple literal as a type.
+
+    :param types: a list of types that should be used for the tuple items.
+    """
+
+    def __init__(self, types):
+        self.types = [convert_type(ty) for ty in types]
+
+    @property
+    def name(self):
+        return "<{}>".format(" ".join(ty.name for ty in self.types))
+
+    @property
+    def arity(self):
+        return len(self.types)
+
+    def convert(self, value, param, ctx):
+        if len(value) != len(self.types):
+            raise TypeError(
+                "It would appear that nargs is set to conflict with the"
+                " composite type arity."
+            )
+        return tuple(ty(x, param, ctx) for ty, x in zip(self.types, value))
+
+
+def convert_type(ty, default=None):
+    """Converts a callable or python type into the most appropriate
+    param type.
+    """
+    guessed_type = False
+    if ty is None and default is not None:
+        if isinstance(default, tuple):
+            ty = tuple(map(type, default))
+        else:
+            ty = type(default)
+        guessed_type = True
+
+    if isinstance(ty, tuple):
+        return Tuple(ty)
+    if isinstance(ty, ParamType):
+        return ty
+    if ty is text_type or ty is str or ty is None:
+        return STRING
+    if ty is int:
+        return INT
+    # Booleans are only okay if not guessed.  This is done because for
+    # flags the default value is actually a bit of a lie in that it
+    # indicates which of the flags is the one we want.  See get_default()
+    # for more information.
+    if ty is bool and not guessed_type:
+        return BOOL
+    if ty is float:
+        return FLOAT
+    if guessed_type:
+        return STRING
+
+    # Catch a common mistake
+    if __debug__:
+        try:
+            if issubclass(ty, ParamType):
+                raise AssertionError(
+                    "Attempted to use an uninstantiated parameter type ({}).".format(ty)
+                )
+        except TypeError:
+            pass
+    return FuncParamType(ty)
+
+
+#: A dummy parameter type that just does nothing.  From a user's
+#: perspective this appears to just be the same as `STRING` but internally
+#: no string conversion takes place.  This is necessary to achieve the
+#: same bytes/unicode behavior on Python 2/3 in situations where you want
+#: to not convert argument types.  This is usually useful when working
+#: with file paths as they can appear in bytes and unicode.
+#:
+#: For path related uses the :class:`Path` type is a better choice but
+#: there are situations where an unprocessed type is useful which is why
+#: it is is provided.
+#:
+#: .. versionadded:: 4.0
+UNPROCESSED = UnprocessedParamType()
+
+#: A unicode string parameter type which is the implicit default.  This
+#: can also be selected by using ``str`` as type.
+STRING = StringParamType()
+
+#: An integer parameter.  This can also be selected by using ``int`` as
+#: type.
+INT = IntParamType()
+
+#: A floating point value parameter.  This can also be selected by using
+#: ``float`` as type.
+FLOAT = FloatParamType()
+
+#: A boolean parameter.  This is the default for boolean flags.  This can
+#: also be selected by using ``bool`` as a type.
+BOOL = BoolParamType()
+
+#: A UUID parameter.
+UUID = UUIDParameterType()

tools/python/ampy/click/utils.py

@@ -0,0 +1,455 @@
+import os
+import sys
+
+from ._compat import _default_text_stderr
+from ._compat import _default_text_stdout
+from ._compat import auto_wrap_for_ansi
+from ._compat import binary_streams
+from ._compat import filename_to_ui
+from ._compat import get_filesystem_encoding
+from ._compat import get_streerror
+from ._compat import is_bytes
+from ._compat import open_stream
+from ._compat import PY2
+from ._compat import should_strip_ansi
+from ._compat import string_types
+from ._compat import strip_ansi
+from ._compat import text_streams
+from ._compat import text_type
+from ._compat import WIN
+from .globals import resolve_color_default
+
+if not PY2:
+    from ._compat import _find_binary_writer
+elif WIN:
+    from ._winconsole import _get_windows_argv
+    from ._winconsole import _hash_py_argv
+    from ._winconsole import _initial_argv_hash
+
+echo_native_types = string_types + (bytes, bytearray)
+
+
+def _posixify(name):
+    return "-".join(name.split()).lower()
+
+
+def safecall(func):
+    """Wraps a function so that it swallows exceptions."""
+
+    def wrapper(*args, **kwargs):
+        try:
+            return func(*args, **kwargs)
+        except Exception:
+            pass
+
+    return wrapper
+
+
+def make_str(value):
+    """Converts a value into a valid string."""
+    if isinstance(value, bytes):
+        try:
+            return value.decode(get_filesystem_encoding())
+        except UnicodeError:
+            return value.decode("utf-8", "replace")
+    return text_type(value)
+
+
+def make_default_short_help(help, max_length=45):
+    """Return a condensed version of help string."""
+    words = help.split()
+    total_length = 0
+    result = []
+    done = False
+
+    for word in words:
+        if word[-1:] == ".":
+            done = True
+        new_length = 1 + len(word) if result else len(word)
+        if total_length + new_length > max_length:
+            result.append("...")
+            done = True
+        else:
+            if result:
+                result.append(" ")
+            result.append(word)
+        if done:
+            break
+        total_length += new_length
+
+    return "".join(result)
+
+
+class LazyFile(object):
+    """A lazy file works like a regular file but it does not fully open
+    the file but it does perform some basic checks early to see if the
+    filename parameter does make sense.  This is useful for safely opening
+    files for writing.
+    """
+
+    def __init__(
+        self, filename, mode="r", encoding=None, errors="strict", atomic=False
+    ):
+        self.name = filename
+        self.mode = mode
+        self.encoding = encoding
+        self.errors = errors
+        self.atomic = atomic
+
+        if filename == "-":
+            self._f, self.should_close = open_stream(filename, mode, encoding, errors)
+        else:
+            if "r" in mode:
+                # Open and close the file in case we're opening it for
+                # reading so that we can catch at least some errors in
+                # some cases early.
+                open(filename, mode).close()
+            self._f = None
+            self.should_close = True
+
+    def __getattr__(self, name):
+        return getattr(self.open(), name)
+
+    def __repr__(self):
+        if self._f is not None:
+            return repr(self._f)
+        return "<unopened file '{}' {}>".format(self.name, self.mode)
+
+    def open(self):
+        """Opens the file if it's not yet open.  This call might fail with
+        a :exc:`FileError`.  Not handling this error will produce an error
+        that Click shows.
+        """
+        if self._f is not None:
+            return self._f
+        try:
+            rv, self.should_close = open_stream(
+                self.name, self.mode, self.encoding, self.errors, atomic=self.atomic
+            )
+        except (IOError, OSError) as e:  # noqa: E402
+            from .exceptions import FileError
+
+            raise FileError(self.name, hint=get_streerror(e))
+        self._f = rv
+        return rv
+
+    def close(self):
+        """Closes the underlying file, no matter what."""
+        if self._f is not None:
+            self._f.close()
+
+    def close_intelligently(self):
+        """This function only closes the file if it was opened by the lazy
+        file wrapper.  For instance this will never close stdin.
+        """
+        if self.should_close:
+            self.close()
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_value, tb):
+        self.close_intelligently()
+
+    def __iter__(self):
+        self.open()
+        return iter(self._f)
+
+
+class KeepOpenFile(object):
+    def __init__(self, file):
+        self._file = file
+
+    def __getattr__(self, name):
+        return getattr(self._file, name)
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_value, tb):
+        pass
+
+    def __repr__(self):
+        return repr(self._file)
+
+    def __iter__(self):
+        return iter(self._file)
+
+
+def echo(message=None, file=None, nl=True, err=False, color=None):
+    """Prints a message plus a newline to the given file or stdout.  On
+    first sight, this looks like the print function, but it has improved
+    support for handling Unicode and binary data that does not fail no
+    matter how badly configured the system is.
+
+    Primarily it means that you can print binary data as well as Unicode
+    data on both 2.x and 3.x to the given file in the most appropriate way
+    possible.  This is a very carefree function in that it will try its
+    best to not fail.  As of Click 6.0 this includes support for unicode
+    output on the Windows console.
+
+    In addition to that, if `colorama`_ is installed, the echo function will
+    also support clever handling of ANSI codes.  Essentially it will then
+    do the following:
+
+    -   add transparent handling of ANSI color codes on Windows.
+    -   hide ANSI codes automatically if the destination file is not a
+        terminal.
+
+    .. _colorama: https://pypi.org/project/colorama/
+
+    .. versionchanged:: 6.0
+       As of Click 6.0 the echo function will properly support unicode
+       output on the windows console.  Not that click does not modify
+       the interpreter in any way which means that `sys.stdout` or the
+       print statement or function will still not provide unicode support.
+
+    .. versionchanged:: 2.0
+       Starting with version 2.0 of Click, the echo function will work
+       with colorama if it's installed.
+
+    .. versionadded:: 3.0
+       The `err` parameter was added.
+
+    .. versionchanged:: 4.0
+       Added the `color` flag.
+
+    :param message: the message to print
+    :param file: the file to write to (defaults to ``stdout``)
+    :param err: if set to true the file defaults to ``stderr`` instead of
+                ``stdout``.  This is faster and easier than calling
+                :func:`get_text_stderr` yourself.
+    :param nl: if set to `True` (the default) a newline is printed afterwards.
+    :param color: controls if the terminal supports ANSI colors or not.  The
+                  default is autodetection.
+    """
+    if file is None:
+        if err:
+            file = _default_text_stderr()
+        else:
+            file = _default_text_stdout()
+
+    # Convert non bytes/text into the native string type.
+    if message is not None and not isinstance(message, echo_native_types):
+        message = text_type(message)
+
+    if nl:
+        message = message or u""
+        if isinstance(message, text_type):
+            message += u"\n"
+        else:
+            message += b"\n"
+
+    # If there is a message, and we're in Python 3, and the value looks
+    # like bytes, we manually need to find the binary stream and write the
+    # message in there.  This is done separately so that most stream
+    # types will work as you would expect.  Eg: you can write to StringIO
+    # for other cases.
+    if message and not PY2 and is_bytes(message):
+        binary_file = _find_binary_writer(file)
+        if binary_file is not None:
+            file.flush()
+            binary_file.write(message)
+            binary_file.flush()
+            return
+
+    # ANSI-style support.  If there is no message or we are dealing with
+    # bytes nothing is happening.  If we are connected to a file we want
+    # to strip colors.  If we are on windows we either wrap the stream
+    # to strip the color or we use the colorama support to translate the
+    # ansi codes to API calls.
+    if message and not is_bytes(message):
+        color = resolve_color_default(color)
+        if should_strip_ansi(file, color):
+            message = strip_ansi(message)
+        elif WIN:
+            if auto_wrap_for_ansi is not None:
+                file = auto_wrap_for_ansi(file)
+            elif not color:
+                message = strip_ansi(message)
+
+    if message:
+        file.write(message)
+    file.flush()
+
+
+def get_binary_stream(name):
+    """Returns a system stream for byte processing.  This essentially
+    returns the stream from the sys module with the given name but it
+    solves some compatibility issues between different Python versions.
+    Primarily this function is necessary for getting binary streams on
+    Python 3.
+
+    :param name: the name of the stream to open.  Valid names are ``'stdin'``,
+                 ``'stdout'`` and ``'stderr'``
+    """
+    opener = binary_streams.get(name)
+    if opener is None:
+        raise TypeError("Unknown standard stream '{}'".format(name))
+    return opener()
+
+
+def get_text_stream(name, encoding=None, errors="strict"):
+    """Returns a system stream for text processing.  This usually returns
+    a wrapped stream around a binary stream returned from
+    :func:`get_binary_stream` but it also can take shortcuts on Python 3
+    for already correctly configured streams.
+
+    :param name: the name of the stream to open.  Valid names are ``'stdin'``,
+                 ``'stdout'`` and ``'stderr'``
+    :param encoding: overrides the detected default encoding.
+    :param errors: overrides the default error mode.
+    """
+    opener = text_streams.get(name)
+    if opener is None:
+        raise TypeError("Unknown standard stream '{}'".format(name))
+    return opener(encoding, errors)
+
+
+def open_file(
+    filename, mode="r", encoding=None, errors="strict", lazy=False, atomic=False
+):
+    """This is similar to how the :class:`File` works but for manual
+    usage.  Files are opened non lazy by default.  This can open regular
+    files as well as stdin/stdout if ``'-'`` is passed.
+
+    If stdin/stdout is returned the stream is wrapped so that the context
+    manager will not close the stream accidentally.  This makes it possible
+    to always use the function like this without having to worry to
+    accidentally close a standard stream::
+
+        with open_file(filename) as f:
+            ...
+
+    .. versionadded:: 3.0
+
+    :param filename: the name of the file to open (or ``'-'`` for stdin/stdout).
+    :param mode: the mode in which to open the file.
+    :param encoding: the encoding to use.
+    :param errors: the error handling for this file.
+    :param lazy: can be flipped to true to open the file lazily.
+    :param atomic: in atomic mode writes go into a temporary file and it's
+                   moved on close.
+    """
+    if lazy:
+        return LazyFile(filename, mode, encoding, errors, atomic=atomic)
+    f, should_close = open_stream(filename, mode, encoding, errors, atomic=atomic)
+    if not should_close:
+        f = KeepOpenFile(f)
+    return f
+
+
+def get_os_args():
+    """This returns the argument part of sys.argv in the most appropriate
+    form for processing.  What this means is that this return value is in
+    a format that works for Click to process but does not necessarily
+    correspond well to what's actually standard for the interpreter.
+
+    On most environments the return value is ``sys.argv[:1]`` unchanged.
+    However if you are on Windows and running Python 2 the return value
+    will actually be a list of unicode strings instead because the
+    default behavior on that platform otherwise will not be able to
+    carry all possible values that sys.argv can have.
+
+    .. versionadded:: 6.0
+    """
+    # We can only extract the unicode argv if sys.argv has not been
+    # changed since the startup of the application.
+    if PY2 and WIN and _initial_argv_hash == _hash_py_argv():
+        return _get_windows_argv()
+    return sys.argv[1:]
+
+
+def format_filename(filename, shorten=False):
+    """Formats a filename for user display.  The main purpose of this
+    function is to ensure that the filename can be displayed at all.  This
+    will decode the filename to unicode if necessary in a way that it will
+    not fail.  Optionally, it can shorten the filename to not include the
+    full path to the filename.
+
+    :param filename: formats a filename for UI display.  This will also convert
+                     the filename into unicode without failing.
+    :param shorten: this optionally shortens the filename to strip of the
+                    path that leads up to it.
+    """
+    if shorten:
+        filename = os.path.basename(filename)
+    return filename_to_ui(filename)
+
+
+def get_app_dir(app_name, roaming=True, force_posix=False):
+    r"""Returns the config folder for the application.  The default behavior
+    is to return whatever is most appropriate for the operating system.
+
+    To give you an idea, for an app called ``"Foo Bar"``, something like
+    the following folders could be returned:
+
+    Mac OS X:
+      ``~/Library/Application Support/Foo Bar``
+    Mac OS X (POSIX):
+      ``~/.foo-bar``
+    Unix:
+      ``~/.config/foo-bar``
+    Unix (POSIX):
+      ``~/.foo-bar``
+    Win XP (roaming):
+      ``C:\Documents and Settings\<user>\Local Settings\Application Data\Foo Bar``
+    Win XP (not roaming):
+      ``C:\Documents and Settings\<user>\Application Data\Foo Bar``
+    Win 7 (roaming):
+      ``C:\Users\<user>\AppData\Roaming\Foo Bar``
+    Win 7 (not roaming):
+      ``C:\Users\<user>\AppData\Local\Foo Bar``
+
+    .. versionadded:: 2.0
+
+    :param app_name: the application name.  This should be properly capitalized
+                     and can contain whitespace.
+    :param roaming: controls if the folder should be roaming or not on Windows.
+                    Has no affect otherwise.
+    :param force_posix: if this is set to `True` then on any POSIX system the
+                        folder will be stored in the home folder with a leading
+                        dot instead of the XDG config home or darwin's
+                        application support folder.
+    """
+    if WIN:
+        key = "APPDATA" if roaming else "LOCALAPPDATA"
+        folder = os.environ.get(key)
+        if folder is None:
+            folder = os.path.expanduser("~")
+        return os.path.join(folder, app_name)
+    if force_posix:
+        return os.path.join(os.path.expanduser("~/.{}".format(_posixify(app_name))))
+    if sys.platform == "darwin":
+        return os.path.join(
+            os.path.expanduser("~/Library/Application Support"), app_name
+        )
+    return os.path.join(
+        os.environ.get("XDG_CONFIG_HOME", os.path.expanduser("~/.config")),
+        _posixify(app_name),
+    )
+
+
+class PacifyFlushWrapper(object):
+    """This wrapper is used to catch and suppress BrokenPipeErrors resulting
+    from ``.flush()`` being called on broken pipe during the shutdown/final-GC
+    of the Python interpreter. Notably ``.flush()`` is always called on
+    ``sys.stdout`` and ``sys.stderr``. So as to have minimal impact on any
+    other cleanup code, and the case where the underlying file is not a broken
+    pipe, all calls and attributes are proxied.
+    """
+
+    def __init__(self, wrapped):
+        self.wrapped = wrapped
+
+    def flush(self):
+        try:
+            self.wrapped.flush()
+        except IOError as e:
+            import errno
+
+            if e.errno != errno.EPIPE:
+                raise
+
+    def __getattr__(self, attr):
+        return getattr(self.wrapped, attr)

tools/python/ampy/dotenv/__init__.py

@@ -0,0 +1,46 @@
+from .compat import IS_TYPE_CHECKING
+from .main import load_dotenv, get_key, set_key, unset_key, find_dotenv, dotenv_values
+
+if IS_TYPE_CHECKING:
+    from typing import Any, Optional
+
+
+def load_ipython_extension(ipython):
+    # type: (Any) -> None
+    from .ipython import load_ipython_extension
+    load_ipython_extension(ipython)
+
+
+def get_cli_string(path=None, action=None, key=None, value=None, quote=None):
+    # type: (Optional[str], Optional[str], Optional[str], Optional[str], Optional[str]) -> str
+    """Returns a string suitable for running as a shell script.
+
+    Useful for converting a arguments passed to a fabric task
+    to be passed to a `local` or `run` command.
+    """
+    command = ['dotenv']
+    if quote:
+        command.append('-q %s' % quote)
+    if path:
+        command.append('-f %s' % path)
+    if action:
+        command.append(action)
+        if key:
+            command.append(key)
+            if value:
+                if ' ' in value:
+                    command.append('"%s"' % value)
+                else:
+                    command.append(value)
+
+    return ' '.join(command).strip()
+
+
+__all__ = ['get_cli_string',
+           'load_dotenv',
+           'dotenv_values',
+           'get_key',
+           'set_key',
+           'unset_key',
+           'find_dotenv',
+           'load_ipython_extension']

tools/python/ampy/dotenv/cli.py

@@ -0,0 +1,174 @@
+import os
+import sys
+from subprocess import Popen
+
+try:
+    import click
+except ImportError:
+    sys.stderr.write('It seems python-dotenv is not installed with cli option. \n'
+                     'Run pip install "python-dotenv[cli]" to fix this.')
+    sys.exit(1)
+
+from .compat import IS_TYPE_CHECKING, to_env
+from .main import dotenv_values, get_key, set_key, unset_key
+from .version import __version__
+
+if IS_TYPE_CHECKING:
+    from typing import Any, List, Dict
+
+
+@click.group()
+@click.option('-f', '--file', default=os.path.join(os.getcwd(), '.env'),
+              type=click.Path(file_okay=True),
+              help="Location of the .env file, defaults to .env file in current working directory.")
+@click.option('-q', '--quote', default='always',
+              type=click.Choice(['always', 'never', 'auto']),
+              help="Whether to quote or not the variable values. Default mode is always. This does not affect parsing.")
+@click.option('-e', '--export', default=False,
+              type=click.BOOL,
+              help="Whether to write the dot file as an executable bash script.")
+@click.version_option(version=__version__)
+@click.pass_context
+def cli(ctx, file, quote, export):
+    # type: (click.Context, Any, Any, Any) -> None
+    '''This script is used to set, get or unset values from a .env file.'''
+    ctx.obj = {}
+    ctx.obj['QUOTE'] = quote
+    ctx.obj['EXPORT'] = export
+    ctx.obj['FILE'] = file
+
+
+@cli.command()
+@click.pass_context
+def list(ctx):
+    # type: (click.Context) -> None
+    '''Display all the stored key/value.'''
+    file = ctx.obj['FILE']
+    if not os.path.isfile(file):
+        raise click.BadParameter(
+            'Path "%s" does not exist.' % (file),
+            ctx=ctx
+        )
+    dotenv_as_dict = dotenv_values(file)
+    for k, v in dotenv_as_dict.items():
+        click.echo('%s=%s' % (k, v))
+
+
+@cli.command()
+@click.pass_context
+@click.argument('key', required=True)
+@click.argument('value', required=True)
+def set(ctx, key, value):
+    # type: (click.Context, Any, Any) -> None
+    '''Store the given key/value.'''
+    file = ctx.obj['FILE']
+    quote = ctx.obj['QUOTE']
+    export = ctx.obj['EXPORT']
+    success, key, value = set_key(file, key, value, quote, export)
+    if success:
+        click.echo('%s=%s' % (key, value))
+    else:
+        exit(1)
+
+
+@cli.command()
+@click.pass_context
+@click.argument('key', required=True)
+def get(ctx, key):
+    # type: (click.Context, Any) -> None
+    '''Retrieve the value for the given key.'''
+    file = ctx.obj['FILE']
+    if not os.path.isfile(file):
+        raise click.BadParameter(
+            'Path "%s" does not exist.' % (file),
+            ctx=ctx
+        )
+    stored_value = get_key(file, key)
+    if stored_value:
+        click.echo(stored_value)
+    else:
+        exit(1)
+
+
+@cli.command()
+@click.pass_context
+@click.argument('key', required=True)
+def unset(ctx, key):
+    # type: (click.Context, Any) -> None
+    '''Removes the given key.'''
+    file = ctx.obj['FILE']
+    quote = ctx.obj['QUOTE']
+    success, key = unset_key(file, key, quote)
+    if success:
+        click.echo("Successfully removed %s" % key)
+    else:
+        exit(1)
+
+
+@cli.command(context_settings={'ignore_unknown_options': True})
+@click.pass_context
+@click.option(
+    "--override/--no-override",
+    default=True,
+    help="Override variables from the environment file with those from the .env file.",
+)
+@click.argument('commandline', nargs=-1, type=click.UNPROCESSED)
+def run(ctx, override, commandline):
+    # type: (click.Context, bool, List[str]) -> None
+    """Run command with environment variables present."""
+    file = ctx.obj['FILE']
+    if not os.path.isfile(file):
+        raise click.BadParameter(
+            'Invalid value for \'-f\' "%s" does not exist.' % (file),
+            ctx=ctx
+        )
+    dotenv_as_dict = {
+        to_env(k): to_env(v)
+        for (k, v) in dotenv_values(file).items()
+        if v is not None and (override or to_env(k) not in os.environ)
+    }
+
+    if not commandline:
+        click.echo('No command given.')
+        exit(1)
+    ret = run_command(commandline, dotenv_as_dict)
+    exit(ret)
+
+
+def run_command(command, env):
+    # type: (List[str], Dict[str, str]) -> int
+    """Run command in sub process.
+
+    Runs the command in a sub process with the variables from `env`
+    added in the current environment variables.
+
+    Parameters
+    ----------
+    command: List[str]
+        The command and it's parameters
+    env: Dict
+        The additional environment variables
+
+    Returns
+    -------
+    int
+        The return code of the command
+
+    """
+    # copy the current environment variables and add the vales from
+    # `env`
+    cmd_env = os.environ.copy()
+    cmd_env.update(env)
+
+    p = Popen(command,
+              universal_newlines=True,
+              bufsize=0,
+              shell=False,
+              env=cmd_env)
+    _, _ = p.communicate()
+
+    return p.returncode
+
+
+if __name__ == "__main__":
+    cli()

tools/python/ampy/dotenv/compat.py

@@ -0,0 +1,49 @@
+import sys
+
+PY2 = sys.version_info[0] == 2  # type: bool
+
+if PY2:
+    from StringIO import StringIO  # noqa
+else:
+    from io import StringIO  # noqa
+
+
+def is_type_checking():
+    # type: () -> bool
+    try:
+        from typing import TYPE_CHECKING
+    except ImportError:
+        return False
+    return TYPE_CHECKING
+
+
+IS_TYPE_CHECKING = is_type_checking()
+
+
+if IS_TYPE_CHECKING:
+    from typing import Text
+
+
+def to_env(text):
+    # type: (Text) -> str
+    """
+    Encode a string the same way whether it comes from the environment or a `.env` file.
+    """
+    if PY2:
+        return text.encode(sys.getfilesystemencoding() or "utf-8")
+    else:
+        return text
+
+
+def to_text(string):
+    # type: (str) -> Text
+    """
+    Make a string Unicode if it isn't already.
+
+    This is useful for defining raw unicode strings because `ur"foo"` isn't valid in
+    Python 3.
+    """
+    if PY2:
+        return string.decode("utf-8")
+    else:
+        return string

tools/python/ampy/dotenv/ipython.py

@@ -0,0 +1,41 @@
+from __future__ import print_function
+
+from IPython.core.magic import Magics, line_magic, magics_class  # type: ignore
+from IPython.core.magic_arguments import (argument, magic_arguments,  # type: ignore
+                                          parse_argstring)  # type: ignore
+
+from .main import find_dotenv, load_dotenv
+
+
+@magics_class
+class IPythonDotEnv(Magics):
+
+    @magic_arguments()
+    @argument(
+        '-o', '--override', action='store_true',
+        help="Indicate to override existing variables"
+    )
+    @argument(
+        '-v', '--verbose', action='store_true',
+        help="Indicate function calls to be verbose"
+    )
+    @argument('dotenv_path', nargs='?', type=str, default='.env',
+              help='Search in increasingly higher folders for the `dotenv_path`')
+    @line_magic
+    def dotenv(self, line):
+        args = parse_argstring(self.dotenv, line)
+        # Locate the .env file
+        dotenv_path = args.dotenv_path
+        try:
+            dotenv_path = find_dotenv(dotenv_path, True, True)
+        except IOError:
+            print("cannot find .env file")
+            return
+
+        # Load the .env file
+        load_dotenv(dotenv_path, verbose=args.verbose, override=args.override)
+
+
+def load_ipython_extension(ipython):
+    """Register the %dotenv magic."""
+    ipython.register_magics(IPythonDotEnv)

tools/python/ampy/dotenv/main.py

@@ -0,0 +1,355 @@
+# -*- coding: utf-8 -*-
+from __future__ import absolute_import, print_function, unicode_literals
+
+import io
+import logging
+import os
+import shutil
+import sys
+import tempfile
+from collections import OrderedDict
+from contextlib import contextmanager
+
+from .compat import IS_TYPE_CHECKING, PY2, StringIO, to_env
+from .parser import Binding, parse_stream
+from .variables import parse_variables
+
+logger = logging.getLogger(__name__)
+
+if IS_TYPE_CHECKING:
+    from typing import (IO, Dict, Iterable, Iterator, Mapping, Optional, Text,
+                        Tuple, Union)
+    if sys.version_info >= (3, 6):
+        _PathLike = os.PathLike
+    else:
+        _PathLike = Text
+
+    if sys.version_info >= (3, 0):
+        _StringIO = StringIO
+    else:
+        _StringIO = StringIO[Text]
+
+
+def with_warn_for_invalid_lines(mappings):
+    # type: (Iterator[Binding]) -> Iterator[Binding]
+    for mapping in mappings:
+        if mapping.error:
+            logger.warning(
+                "Python-dotenv could not parse statement starting at line %s",
+                mapping.original.line,
+            )
+        yield mapping
+
+
+class DotEnv():
+
+    def __init__(self, dotenv_path, verbose=False, encoding=None, interpolate=True, override=True):
+        # type: (Union[Text, _PathLike, _StringIO], bool, Union[None, Text], bool, bool) -> None
+        self.dotenv_path = dotenv_path  # type: Union[Text,_PathLike, _StringIO]
+        self._dict = None  # type: Optional[Dict[Text, Optional[Text]]]
+        self.verbose = verbose  # type: bool
+        self.encoding = encoding  # type: Union[None, Text]
+        self.interpolate = interpolate  # type: bool
+        self.override = override  # type: bool
+
+    @contextmanager
+    def _get_stream(self):
+        # type: () -> Iterator[IO[Text]]
+        if isinstance(self.dotenv_path, StringIO):
+            yield self.dotenv_path
+        elif os.path.isfile(self.dotenv_path):
+            with io.open(self.dotenv_path, encoding=self.encoding) as stream:
+                yield stream
+        else:
+            if self.verbose:
+                logger.info("Python-dotenv could not find configuration file %s.", self.dotenv_path or '.env')
+            yield StringIO('')
+
+    def dict(self):
+        # type: () -> Dict[Text, Optional[Text]]
+        """Return dotenv as dict"""
+        if self._dict:
+            return self._dict
+
+        raw_values = self.parse()
+
+        if self.interpolate:
+            self._dict = OrderedDict(resolve_variables(raw_values, override=self.override))
+        else:
+            self._dict = OrderedDict(raw_values)
+
+        return self._dict
+
+    def parse(self):
+        # type: () -> Iterator[Tuple[Text, Optional[Text]]]
+        with self._get_stream() as stream:
+            for mapping in with_warn_for_invalid_lines(parse_stream(stream)):
+                if mapping.key is not None:
+                    yield mapping.key, mapping.value
+
+    def set_as_environment_variables(self):
+        # type: () -> bool
+        """
+        Load the current dotenv as system environment variable.
+        """
+        for k, v in self.dict().items():
+            if k in os.environ and not self.override:
+                continue
+            if v is not None:
+                os.environ[to_env(k)] = to_env(v)
+
+        return True
+
+    def get(self, key):
+        # type: (Text) -> Optional[Text]
+        """
+        """
+        data = self.dict()
+
+        if key in data:
+            return data[key]
+
+        if self.verbose:
+            logger.warning("Key %s not found in %s.", key, self.dotenv_path)
+
+        return None
+
+
+def get_key(dotenv_path, key_to_get):
+    # type: (Union[Text, _PathLike], Text) -> Optional[Text]
+    """
+    Gets the value of a given key from the given .env
+
+    If the .env path given doesn't exist, fails
+    """
+    return DotEnv(dotenv_path, verbose=True).get(key_to_get)
+
+
+@contextmanager
+def rewrite(path):
+    # type: (_PathLike) -> Iterator[Tuple[IO[Text], IO[Text]]]
+    try:
+        if not os.path.isfile(path):
+            with io.open(path, "w+") as source:
+                source.write("")
+        with tempfile.NamedTemporaryFile(mode="w+", delete=False) as dest:
+            with io.open(path) as source:
+                yield (source, dest)  # type: ignore
+    except BaseException:
+        if os.path.isfile(dest.name):
+            os.unlink(dest.name)
+        raise
+    else:
+        shutil.move(dest.name, path)
+
+
+def set_key(dotenv_path, key_to_set, value_to_set, quote_mode="always", export=False):
+    # type: (_PathLike, Text, Text, Text, bool) -> Tuple[Optional[bool], Text, Text]
+    """
+    Adds or Updates a key/value to the given .env
+
+    If the .env path given doesn't exist, fails instead of risking creating
+    an orphan .env somewhere in the filesystem
+    """
+    value_to_set = value_to_set.strip("'").strip('"')
+
+    if " " in value_to_set:
+        quote_mode = "always"
+
+    if quote_mode == "always":
+        value_out = '"{}"'.format(value_to_set.replace('"', '\\"'))
+    else:
+        value_out = value_to_set
+    if export:
+        line_out = 'export {}={}\n'.format(key_to_set, value_out)
+    else:
+        line_out = "{}={}\n".format(key_to_set, value_out)
+
+    with rewrite(dotenv_path) as (source, dest):
+        replaced = False
+        for mapping in with_warn_for_invalid_lines(parse_stream(source)):
+            if mapping.key == key_to_set:
+                dest.write(line_out)
+                replaced = True
+            else:
+                dest.write(mapping.original.string)
+        if not replaced:
+            dest.write(line_out)
+
+    return True, key_to_set, value_to_set
+
+
+def unset_key(dotenv_path, key_to_unset, quote_mode="always"):
+    # type: (_PathLike, Text, Text) -> Tuple[Optional[bool], Text]
+    """
+    Removes a given key from the given .env
+
+    If the .env path given doesn't exist, fails
+    If the given key doesn't exist in the .env, fails
+    """
+    if not os.path.exists(dotenv_path):
+        logger.warning("Can't delete from %s - it doesn't exist.", dotenv_path)
+        return None, key_to_unset
+
+    removed = False
+    with rewrite(dotenv_path) as (source, dest):
+        for mapping in with_warn_for_invalid_lines(parse_stream(source)):
+            if mapping.key == key_to_unset:
+                removed = True
+            else:
+                dest.write(mapping.original.string)
+
+    if not removed:
+        logger.warning("Key %s not removed from %s - key doesn't exist.", key_to_unset, dotenv_path)
+        return None, key_to_unset
+
+    return removed, key_to_unset
+
+
+def resolve_variables(values, override):
+    # type: (Iterable[Tuple[Text, Optional[Text]]], bool) -> Mapping[Text, Optional[Text]]
+
+    new_values = {}  # type: Dict[Text, Optional[Text]]
+
+    for (name, value) in values:
+        if value is None:
+            result = None
+        else:
+            atoms = parse_variables(value)
+            env = {}  # type: Dict[Text, Optional[Text]]
+            if override:
+                env.update(os.environ)  # type: ignore
+                env.update(new_values)
+            else:
+                env.update(new_values)
+                env.update(os.environ)  # type: ignore
+            result = "".join(atom.resolve(env) for atom in atoms)
+
+        new_values[name] = result
+
+    return new_values
+
+
+def _walk_to_root(path):
+    # type: (Text) -> Iterator[Text]
+    """
+    Yield directories starting from the given directory up to the root
+    """
+    if not os.path.exists(path):
+        raise IOError('Starting path not found')
+
+    if os.path.isfile(path):
+        path = os.path.dirname(path)
+
+    last_dir = None
+    current_dir = os.path.abspath(path)
+    while last_dir != current_dir:
+        yield current_dir
+        parent_dir = os.path.abspath(os.path.join(current_dir, os.path.pardir))
+        last_dir, current_dir = current_dir, parent_dir
+
+
+def find_dotenv(filename='.env', raise_error_if_not_found=False, usecwd=False):
+    # type: (Text, bool, bool) -> Text
+    """
+    Search in increasingly higher folders for the given file
+
+    Returns path to the file if found, or an empty string otherwise
+    """
+
+    def _is_interactive():
+        """ Decide whether this is running in a REPL or IPython notebook """
+        main = __import__('__main__', None, None, fromlist=['__file__'])
+        return not hasattr(main, '__file__')
+
+    if usecwd or _is_interactive() or getattr(sys, 'frozen', False):
+        # Should work without __file__, e.g. in REPL or IPython notebook.
+        path = os.getcwd()
+    else:
+        # will work for .py files
+        frame = sys._getframe()
+        # find first frame that is outside of this file
+        if PY2 and not __file__.endswith('.py'):
+            # in Python2 __file__ extension could be .pyc or .pyo (this doesn't account
+            # for edge case of Python compiled for non-standard extension)
+            current_file = __file__.rsplit('.', 1)[0] + '.py'
+        else:
+            current_file = __file__
+
+        while frame.f_code.co_filename == current_file:
+            assert frame.f_back is not None
+            frame = frame.f_back
+        frame_filename = frame.f_code.co_filename
+        path = os.path.dirname(os.path.abspath(frame_filename))
+
+    for dirname in _walk_to_root(path):
+        check_path = os.path.join(dirname, filename)
+        if os.path.isfile(check_path):
+            return check_path
+
+    if raise_error_if_not_found:
+        raise IOError('File not found')
+
+    return ''
+
+
+def load_dotenv(
+    dotenv_path=None,
+    stream=None,
+    verbose=False,
+    override=False,
+    interpolate=True,
+    encoding="utf-8",
+):
+    # type: (Union[Text, _PathLike, None], Optional[_StringIO], bool, bool, bool, Optional[Text]) -> bool  # noqa
+    """Parse a .env file and then load all the variables found as environment variables.
+
+    - *dotenv_path*: absolute or relative path to .env file.
+    - *stream*: `StringIO` object with .env content, used if `dotenv_path` is `None`.
+    - *verbose*: whether to output a warning the .env file is missing. Defaults to
+      `False`.
+    - *override*: whether to override the system environment variables with the variables
+      in `.env` file.  Defaults to `False`.
+    - *encoding*: encoding to be used to read the file.
+
+    If both `dotenv_path` and `stream`, `find_dotenv()` is used to find the .env file.
+    """
+    f = dotenv_path or stream or find_dotenv()
+    dotenv = DotEnv(
+        f,
+        verbose=verbose,
+        interpolate=interpolate,
+        override=override,
+        encoding=encoding,
+    )
+    return dotenv.set_as_environment_variables()
+
+
+def dotenv_values(
+    dotenv_path=None,
+    stream=None,
+    verbose=False,
+    interpolate=True,
+    encoding="utf-8",
+):
+    # type: (Union[Text, _PathLike, None], Optional[_StringIO], bool, bool, Optional[Text]) -> Dict[Text, Optional[Text]]  # noqa: E501
+    """
+    Parse a .env file and return its content as a dict.
+
+    - *dotenv_path*: absolute or relative path to .env file.
+    - *stream*: `StringIO` object with .env content, used if `dotenv_path` is `None`.
+    - *verbose*: whether to output a warning the .env file is missing. Defaults to
+      `False`.
+      in `.env` file.  Defaults to `False`.
+    - *encoding*: encoding to be used to read the file.
+
+    If both `dotenv_path` and `stream`, `find_dotenv()` is used to find the .env file.
+    """
+    f = dotenv_path or stream or find_dotenv()
+    return DotEnv(
+        f,
+        verbose=verbose,
+        interpolate=interpolate,
+        override=True,
+        encoding=encoding,
+    ).dict()

tools/python/ampy/dotenv/parser.py

@@ -0,0 +1,231 @@
+import codecs
+import re
+
+from .compat import IS_TYPE_CHECKING, to_text
+
+if IS_TYPE_CHECKING:
+    from typing import (  # noqa:F401
+        IO, Iterator, Match, NamedTuple, Optional, Pattern, Sequence, Text,
+        Tuple
+    )
+
+
+def make_regex(string, extra_flags=0):
+    # type: (str, int) -> Pattern[Text]
+    return re.compile(to_text(string), re.UNICODE | extra_flags)
+
+
+_newline = make_regex(r"(\r\n|\n|\r)")
+_multiline_whitespace = make_regex(r"\s*", extra_flags=re.MULTILINE)
+_whitespace = make_regex(r"[^\S\r\n]*")
+_export = make_regex(r"(?:export[^\S\r\n]+)?")
+_single_quoted_key = make_regex(r"'([^']+)'")
+_unquoted_key = make_regex(r"([^=\#\s]+)")
+_equal_sign = make_regex(r"(=[^\S\r\n]*)")
+_single_quoted_value = make_regex(r"'((?:\\'|[^'])*)'")
+_double_quoted_value = make_regex(r'"((?:\\"|[^"])*)"')
+_unquoted_value = make_regex(r"([^\r\n]*)")
+_comment = make_regex(r"(?:[^\S\r\n]*#[^\r\n]*)?")
+_end_of_line = make_regex(r"[^\S\r\n]*(?:\r\n|\n|\r|$)")
+_rest_of_line = make_regex(r"[^\r\n]*(?:\r|\n|\r\n)?")
+_double_quote_escapes = make_regex(r"\\[\\'\"abfnrtv]")
+_single_quote_escapes = make_regex(r"\\[\\']")
+
+
+try:
+    # this is necessary because we only import these from typing
+    # when we are type checking, and the linter is upset if we
+    # re-import
+    import typing
+
+    Original = typing.NamedTuple(
+        "Original",
+        [
+            ("string", typing.Text),
+            ("line", int),
+        ],
+    )
+
+    Binding = typing.NamedTuple(
+        "Binding",
+        [
+            ("key", typing.Optional[typing.Text]),
+            ("value", typing.Optional[typing.Text]),
+            ("original", Original),
+            ("error", bool),
+        ],
+    )
+except (ImportError, AttributeError):
+    from collections import namedtuple
+    Original = namedtuple(  # type: ignore
+        "Original",
+        [
+            "string",
+            "line",
+        ],
+    )
+    Binding = namedtuple(  # type: ignore
+        "Binding",
+        [
+            "key",
+            "value",
+            "original",
+            "error",
+        ],
+    )
+
+
+class Position:
+    def __init__(self, chars, line):
+        # type: (int, int) -> None
+        self.chars = chars
+        self.line = line
+
+    @classmethod
+    def start(cls):
+        # type: () -> Position
+        return cls(chars=0, line=1)
+
+    def set(self, other):
+        # type: (Position) -> None
+        self.chars = other.chars
+        self.line = other.line
+
+    def advance(self, string):
+        # type: (Text) -> None
+        self.chars += len(string)
+        self.line += len(re.findall(_newline, string))
+
+
+class Error(Exception):
+    pass
+
+
+class Reader:
+    def __init__(self, stream):
+        # type: (IO[Text]) -> None
+        self.string = stream.read()
+        self.position = Position.start()
+        self.mark = Position.start()
+
+    def has_next(self):
+        # type: () -> bool
+        return self.position.chars < len(self.string)
+
+    def set_mark(self):
+        # type: () -> None
+        self.mark.set(self.position)
+
+    def get_marked(self):
+        # type: () -> Original
+        return Original(
+            string=self.string[self.mark.chars:self.position.chars],
+            line=self.mark.line,
+        )
+
+    def peek(self, count):
+        # type: (int) -> Text
+        return self.string[self.position.chars:self.position.chars + count]
+
+    def read(self, count):
+        # type: (int) -> Text
+        result = self.string[self.position.chars:self.position.chars + count]
+        if len(result) < count:
+            raise Error("read: End of string")
+        self.position.advance(result)
+        return result
+
+    def read_regex(self, regex):
+        # type: (Pattern[Text]) -> Sequence[Text]
+        match = regex.match(self.string, self.position.chars)
+        if match is None:
+            raise Error("read_regex: Pattern not found")
+        self.position.advance(self.string[match.start():match.end()])
+        return match.groups()
+
+
+def decode_escapes(regex, string):
+    # type: (Pattern[Text], Text) -> Text
+    def decode_match(match):
+        # type: (Match[Text]) -> Text
+        return codecs.decode(match.group(0), 'unicode-escape')  # type: ignore
+
+    return regex.sub(decode_match, string)
+
+
+def parse_key(reader):
+    # type: (Reader) -> Optional[Text]
+    char = reader.peek(1)
+    if char == "#":
+        return None
+    elif char == "'":
+        (key,) = reader.read_regex(_single_quoted_key)
+    else:
+        (key,) = reader.read_regex(_unquoted_key)
+    return key
+
+
+def parse_unquoted_value(reader):
+    # type: (Reader) -> Text
+    (part,) = reader.read_regex(_unquoted_value)
+    return re.sub(r"\s+#.*", "", part).rstrip()
+
+
+def parse_value(reader):
+    # type: (Reader) -> Text
+    char = reader.peek(1)
+    if char == u"'":
+        (value,) = reader.read_regex(_single_quoted_value)
+        return decode_escapes(_single_quote_escapes, value)
+    elif char == u'"':
+        (value,) = reader.read_regex(_double_quoted_value)
+        return decode_escapes(_double_quote_escapes, value)
+    elif char in (u"", u"\n", u"\r"):
+        return u""
+    else:
+        return parse_unquoted_value(reader)
+
+
+def parse_binding(reader):
+    # type: (Reader) -> Binding
+    reader.set_mark()
+    try:
+        reader.read_regex(_multiline_whitespace)
+        if not reader.has_next():
+            return Binding(
+                key=None,
+                value=None,
+                original=reader.get_marked(),
+                error=False,
+            )
+        reader.read_regex(_export)
+        key = parse_key(reader)
+        reader.read_regex(_whitespace)
+        if reader.peek(1) == "=":
+            reader.read_regex(_equal_sign)
+            value = parse_value(reader)  # type: Optional[Text]
+        else:
+            value = None
+        reader.read_regex(_comment)
+        reader.read_regex(_end_of_line)
+        return Binding(
+            key=key,
+            value=value,
+            original=reader.get_marked(),
+            error=False,
+        )
+    except Error:
+        reader.read_regex(_rest_of_line)
+        return Binding(
+            key=None,
+            value=None,
+            original=reader.get_marked(),
+            error=True,
+        )
+
+
+def parse_stream(stream):
+    # type: (IO[Text]) -> Iterator[Binding]
+    reader = Reader(stream)
+    while reader.has_next():
+        yield parse_binding(reader)

tools/python/ampy/dotenv/py.typed

@@ -0,0 +1 @@
+# Marker file for PEP 561

tools/python/ampy/dotenv/variables.py

@@ -0,0 +1,106 @@
+import re
+from abc import ABCMeta
+
+from .compat import IS_TYPE_CHECKING
+
+if IS_TYPE_CHECKING:
+    from typing import Iterator, Mapping, Optional, Pattern, Text
+
+
+_posix_variable = re.compile(
+    r"""
+    \$\{
+        (?P<name>[^\}:]*)
+        (?::-
+            (?P<default>[^\}]*)
+        )?
+    \}
+    """,
+    re.VERBOSE,
+)  # type: Pattern[Text]
+
+
+class Atom():
+    __metaclass__ = ABCMeta
+
+    def __ne__(self, other):
+        # type: (object) -> bool
+        result = self.__eq__(other)
+        if result is NotImplemented:
+            return NotImplemented
+        return not result
+
+    def resolve(self, env):
+        # type: (Mapping[Text, Optional[Text]]) -> Text
+        raise NotImplementedError
+
+
+class Literal(Atom):
+    def __init__(self, value):
+        # type: (Text) -> None
+        self.value = value
+
+    def __repr__(self):
+        # type: () -> str
+        return "Literal(value={})".format(self.value)
+
+    def __eq__(self, other):
+        # type: (object) -> bool
+        if not isinstance(other, self.__class__):
+            return NotImplemented
+        return self.value == other.value
+
+    def __hash__(self):
+        # type: () -> int
+        return hash((self.__class__, self.value))
+
+    def resolve(self, env):
+        # type: (Mapping[Text, Optional[Text]]) -> Text
+        return self.value
+
+
+class Variable(Atom):
+    def __init__(self, name, default):
+        # type: (Text, Optional[Text]) -> None
+        self.name = name
+        self.default = default
+
+    def __repr__(self):
+        # type: () -> str
+        return "Variable(name={}, default={})".format(self.name, self.default)
+
+    def __eq__(self, other):
+        # type: (object) -> bool
+        if not isinstance(other, self.__class__):
+            return NotImplemented
+        return (self.name, self.default) == (other.name, other.default)
+
+    def __hash__(self):
+        # type: () -> int
+        return hash((self.__class__, self.name, self.default))
+
+    def resolve(self, env):
+        # type: (Mapping[Text, Optional[Text]]) -> Text
+        default = self.default if self.default is not None else ""
+        result = env.get(self.name, default)
+        return result if result is not None else ""
+
+
+def parse_variables(value):
+    # type: (Text) -> Iterator[Atom]
+    cursor = 0
+
+    for match in _posix_variable.finditer(value):
+        (start, end) = match.span()
+        name = match.groupdict()["name"]
+        default = match.groupdict()["default"]
+
+        if start > cursor:
+            yield Literal(value=value[cursor:start])
+
+        yield Variable(name=name, default=default)
+        cursor = end
+
+    length = len(value)
+    if cursor < length:
+        yield Literal(value=value[cursor:length])

tools/python/ampy/dotenv/version.py

@@ -0,0 +1 @@
+__version__ = "0.17.1"

tools/python/ampy/files.py

@@ -0,0 +1,527 @@
+# Adafruit MicroPython Tool - File Operations
+# Author: Tony DiCola
+# Copyright (c) 2016 Adafruit Industries
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+import ast
+import textwrap
+import binascii
+import sys
+
+from pyboard import PyboardError
+
+
+BUFFER_SIZE = 32  # Amount of data to read or write to the serial port at a time.
+# This is kept small because small chips and USB to serial
+# bridges usually have very small buffers.
+
+
+class DirectoryExistsError(Exception):
+    pass
+
+
+class Files(object):
+    """Class to interact with a MicroPython board files over a serial connection.
+    Provides functions for listing, uploading, and downloading files from the
+    board's filesystem.
+    """
+
+    def __init__(self, pyboard):
+        """Initialize the MicroPython board files class using the provided pyboard
+        instance.  In most cases you should create a Pyboard instance (from
+        pyboard.py) which connects to a board over a serial connection and pass
+        it in, but you can pass in other objects for testing, etc.
+        """
+        self._pyboard = pyboard
+
+    def get(self, filename):
+        """Retrieve the contents of the specified file and return its contents
+        as a byte string.
+        """
+        # Open the file and read it a few bytes at a time and print out the
+        # raw bytes.  Be careful not to overload the UART buffer so only write
+        # a few bytes at a time, and don't use print since it adds newlines and
+        # expects string data.
+        command = """
+            import sys
+            import ubinascii
+            with open('{0}', 'rb') as infile:
+                while True:
+                    result = infile.read({1})
+                    if result == b'':
+                        break
+                    len = sys.stdout.write(ubinascii.hexlify(result))
+        """.format(
+            filename, BUFFER_SIZE
+        )
+        self._pyboard.enter_raw_repl()
+        try:
+            out = self._pyboard.exec_(textwrap.dedent(command))
+        except PyboardError as ex:
+            # Check if this is an OSError #2, i.e. file doesn't exist and
+            # rethrow it as something more descriptive.
+            try:
+                if ex.args[2].decode("utf-8").find("OSError: [Errno 2] ENOENT") != -1:
+                    raise RuntimeError("No such file: {0}".format(filename))
+                else:
+                    raise ex
+            except UnicodeDecodeError:
+                raise ex
+        self._pyboard.exit_raw_repl()
+        return binascii.unhexlify(out)
+
+    def ls(self, directory="/", long_format=True, recursive=False, exit_repl=True):
+        """List the contents of the specified directory (or root if none is
+        specified).  Returns a list of strings with the names of files in the
+        specified directory.  If long_format is True then a list of 2-tuples
+        with the name and size (in bytes) of the item is returned.  Note that
+        it appears the size of directories is not supported by MicroPython and
+        will always return 0 (i.e. no recursive size computation).
+        """
+
+        # Disabling for now, see https://github.com/adafruit/ampy/issues/55.
+        # # Make sure directory ends in a slash.
+        # if not directory.endswith("/"):
+        #     directory += "/"
+
+        # Make sure directory starts with slash, for consistency.
+        if not directory.startswith("/"):
+            directory = "/" + directory
+
+        command = """\
+                try:
+                    import os
+                except ImportError:
+                    import uos as os\n"""
+
+        if recursive:
+            command += """\
+                def listdir(directory):
+                    result = set()
+
+                    def _listdir(dir_or_file):
+                        try:
+                            # if its a directory, then it should provide some children.
+                            children = os.listdir(dir_or_file)
+                        except OSError:
+                            # probably a file. run stat() to confirm.
+                            os.stat(dir_or_file)
+                            result.add(dir_or_file) 
+                        else:
+                            # probably a directory, add to result if empty.
+                            if children:
+                                # queue the children to be dealt with in next iteration.
+                                for child in children:
+                                    # create the full path.
+                                    if dir_or_file == '/':
+                                        next = dir_or_file + child
+                                    else:
+                                        next = dir_or_file + '/' + child
+                                    
+                                    _listdir(next)
+                            else:
+                                result.add(dir_or_file)
+
+                    _listdir(directory)
+                    return sorted(result)\n"""
+        else:
+            command += """\
+                def check_path(path):
+                    try:
+                        stat = os.stat(path)
+                        # The first element of stat contains the file type and permission information
+                        # The mode index of the tuple returned by os.stat() is 0
+                        mode = stat[0]
+                        # To determine whether it is a directory, check the directory bit in stat mode
+                        if mode & 0o170000 == 0o040000:
+                            if len(os.listdir(path)):
+                                return 'dir'
+                            else:
+                                return 'empty dir'
+                        # To determine whether it is a file, check the file position in stat mode
+                        elif mode & 0o170000 == 0o100000:
+                            return 'file'
+                        else:
+                            return 'special file'
+                    except OSError:
+                        return 'none'
+
+                def listdir(directory):
+                    output = []
+                    if directory == '/':
+                        dirs = sorted([directory + f for f in os.listdir(directory)])
+                    else:
+                        dirs = sorted([directory + '/' + f for f in os.listdir(directory)])
+
+                    for dir in dirs:
+                        output.append([dir, check_path(dir)])
+                    return output\n"""
+
+        # Execute os.listdir() command on the board.
+        if long_format:
+            command += """
+                r = []
+                for f in listdir('{0}'):
+                    size = os.stat(f)[6]
+                    r.append('{{0}} - {{1}} bytes'.format(f, size))
+                print(r)
+            """.format(
+                directory
+            )
+        else:
+            command += """
+                print(listdir('{0}'))
+            """.format(
+                directory
+            )
+        self._pyboard.enter_raw_repl()
+        try:
+            out = self._pyboard.exec_(textwrap.dedent(command))
+        except PyboardError as ex:
+            # Check if this is an OSError #2, i.e. directory doesn't exist and
+            # rethrow it as something more descriptive.
+            if ex.args[2].decode("utf-8").find("OSError: [Errno 2] ENOENT") != -1:
+                raise RuntimeError("No such directory: {0}".format(directory))
+            else:
+                raise ex
+        if exit_repl:
+            self._pyboard.exit_raw_repl()
+        # Parse the result list and return it.
+        return ast.literal_eval(out.decode("utf-8"))
+
+    def getFilesInfo(self, directory="/", recursive=False):
+        """List the contents of the specified directory (or root if none is
+        specified).  Returns a list of strings with the names of files in the
+        specified directory.  If long_format is True then a list of 2-tuples
+        with the name and size (in bytes) of the item is returned.  Note that
+        it appears the size of directories is not supported by MicroPython and
+        will always return 0 (i.e. no recursive size computation).
+        """
+
+        # Disabling for now, see https://github.com/adafruit/ampy/issues/55.
+        # # Make sure directory ends in a slash.
+        # if not directory.endswith("/"):
+        #     directory += "/"
+
+        # Make sure directory starts with slash, for consistency.
+        # if not directory.startswith("/"):
+        #     directory = "/" + directory
+
+        command = """\
+                try:
+                    import os
+                except ImportError:
+                    import uos as os\n"""
+
+        if recursive:
+            command += """\
+                def listdir(directory):
+                    result = set()
+
+                    def _listdir(dir_or_file):
+                        try:
+                            # if its a directory, then it should provide some children.
+                            children = os.listdir(dir_or_file)
+                        except OSError:
+                            # probably a file. run stat() to confirm.
+                            os.stat(dir_or_file)
+                            result.add(dir_or_file) 
+                        else:
+                            # probably a directory, add to result if empty.
+                            if children:
+                                # queue the children to be dealt with in next iteration.
+                                for child in children:
+                                    # create the full path.
+                                    if dir_or_file == '/':
+                                        next = dir_or_file + child
+                                    else:
+                                        next = dir_or_file + '/' + child
+                                    
+                                    _listdir(next)
+                            else:
+                                result.add(dir_or_file)
+
+                    _listdir(directory)
+                    return sorted(result)\n"""
+        else:
+            command += """\
+                def listdir(directory):
+                    try:
+                        if directory == '/':
+                            return sorted([directory + f for f in os.listdir(directory)])
+                        else:
+                            return sorted([directory + '/' + f for f in os.listdir(directory)])
+                    except:
+                        return sorted([f for f in os.listdir()])\n"""
+
+        # Execute os.listdir() command on the board.
+        # 当command执行出错时执行command1
+        command2 = command
+        command2 += """
+                r = []
+                for f in listdir('{0}'):
+                    try:
+                        size = os.stat(f)[6]
+                    except:
+                        size = os.size(f)
+                    r.append([f, size])
+                print(r)
+            """.format(
+            (directory if directory else "/")
+        )
+        command += """
+                r = []
+                for f in listdir('{0}'):
+                    try:
+                        size = os.stat(f)[6]
+                    except:
+                        size = os.size(f)
+                    r.append([f, size])
+                print(r)
+            """.format(
+            directory
+        )
+        try:
+            out = self._pyboard.exec_(textwrap.dedent(command))
+        except PyboardError as ex:
+            out = self._pyboard.exec_(textwrap.dedent(command2))
+            # Check if this is an OSError #2, i.e. directory doesn't exist and
+            # rethrow it as something more descriptive.
+        except PyboardError as ex:
+            if ex.args[2].decode("utf-8").find("OSError: [Errno 2] ENOENT") != -1:
+                raise RuntimeError("No such directory: {0}".format(directory))
+            else:
+                raise ex
+        # Parse the result list and return it.
+        try:
+            return ast.literal_eval(out.decode("utf-8"))
+        except:
+            return ''
+
+    def mkdir(self, directory, exists_okay=False):
+        """Create the specified directory.  Note this cannot create a recursive
+        hierarchy of directories, instead each one should be created separately.
+        """
+        # Execute os.mkdir command on the board.
+        command = """
+            try:
+                import os
+            except ImportError:
+                import uos as os
+            os.mkdir('{0}')
+        """.format(
+            directory
+        )
+        self._pyboard.enter_raw_repl()
+        try:
+            out = self._pyboard.exec_(textwrap.dedent(command))
+        except PyboardError as ex:
+            # Check if this is an OSError #17, i.e. directory already exists.
+            if ex.args[2].decode("utf-8").find("OSError: [Errno 17] EEXIST") != -1:
+                if not exists_okay:
+                    raise DirectoryExistsError(
+                        "Directory already exists: {0}".format(directory)
+                    )
+            else:
+                raise ex
+        self._pyboard.exit_raw_repl()
+
+    def mkfile(self, file, exists_okay=False):
+        command = """
+            try:
+                import os
+            except ImportError:
+                import uos as os
+
+            try:
+                os.stat('{0}')
+            except OSError:
+                f = open('{0}', 'w')
+                f.close()
+        """.format(
+            file
+        )
+        self._pyboard.enter_raw_repl()
+        try:
+            out = self._pyboard.exec_(textwrap.dedent(command))
+        except PyboardError as ex:
+                raise ex
+        self._pyboard.exit_raw_repl()
+
+    def put(self, filename, data, enter_repl=True, exit_repl=True):
+        """Create or update the specified file with the provided data.
+        """
+        # Open the file for writing on the board and write chunks of data.
+        if enter_repl:
+            self._pyboard.enter_raw_repl()
+        self._pyboard.exec_("f = open('{0}', 'wb')".format(filename))
+        sys.stdout.write("Write " + filename)
+        size = len(data)
+        # Loop through and write a buffer size chunk of data at a time.
+        for i in range(0, size, BUFFER_SIZE):
+            chunk_size = min(BUFFER_SIZE, size - i)
+            chunk = repr(data[i : i + chunk_size])
+            # Make sure to send explicit byte strings (handles python 2 compatibility).
+            if not chunk.startswith("b"):
+                chunk = "b" + chunk
+            self._pyboard.exec_("f.write({0})".format(chunk))
+        self._pyboard.exec_("f.close()")
+        sys.stdout.write(" Done!\n")
+        if exit_repl:
+            self._pyboard.exit_raw_repl()
+
+    def putDir(self, fileNameList, dataList, enter_repl=True, exit_repl=True):
+        """Create or update the specified file with the provided data.
+        """
+        # Open the file for writing on the board and write chunks of data.
+        if enter_repl:
+            self._pyboard.enter_raw_repl() 
+        for i in range(0, len(fileNameList), 1):
+            self._pyboard.exec_("f = open('{0}', 'wb')".format(fileNameList[i]))
+            sys.stdout.write("Writing " + fileNameList[i])
+            sys.stdout.flush()
+            #print("write " + fileNameList[i])
+            data = dataList[i]
+            size = len(data)
+            # Loop through and write a buffer size chunk of data at a time.
+            for i in range(0, size, BUFFER_SIZE):
+                chunk_size = min(BUFFER_SIZE, size - i)
+                chunk = repr(data[i : i + chunk_size])
+                # Make sure to send explicit byte strings (handles python 2 compatibility).
+                if not chunk.startswith("b"):
+                    chunk = "b" + chunk
+                self._pyboard.exec_("f.write({0})".format(chunk))
+            self._pyboard.exec_("f.close()")
+            sys.stdout.write(" Done!\n")
+            sys.stdout.flush()
+        if exit_repl:
+            self._pyboard.exit_raw_repl()
+
+    def rm(self, filename):
+        """Remove the specified file or directory."""
+        command = """
+            try:
+                import os
+            except ImportError:
+                import uos as os
+            os.remove('{0}')
+        """.format(
+            filename
+        )
+        self._pyboard.enter_raw_repl()
+        try:
+            out = self._pyboard.exec_(textwrap.dedent(command))
+        except PyboardError as ex:
+            message = ex.args[2].decode("utf-8")
+            # Check if this is an OSError #2, i.e. file/directory doesn't exist
+            # and rethrow it as something more descriptive.
+            if message.find("OSError: [Errno 2] ENOENT") != -1:
+                raise RuntimeError("No such file/directory: {0}".format(filename))
+            # Check for OSError #13, the directory isn't empty.
+            if message.find("OSError: [Errno 13] EACCES") != -1:
+                raise RuntimeError("Directory is not empty: {0}".format(filename))
+            else:
+                raise ex
+        self._pyboard.exit_raw_repl()
+
+    def rmdir(self, directory, missing_okay=False):
+        """Forcefully remove the specified directory and all its children."""
+        # Build a script to walk an entire directory structure and delete every
+        # file and subfolder.  This is tricky because MicroPython has no os.walk
+        # or similar function to walk folders, so this code does it manually
+        # with recursion and changing directories.  For each directory it lists
+        # the files and deletes everything it can, i.e. all the files.  Then
+        # it lists the files again and assumes they are directories (since they
+        # couldn't be deleted in the first pass) and recursively clears those
+        # subdirectories.  Finally when finished clearing all the children the
+        # parent directory is deleted.
+        command = """
+            try:
+                import os
+            except ImportError:
+                import uos as os
+            def rmdir(directory):
+                os.chdir(directory)
+                for f in os.listdir():
+                    try:
+                        os.remove(f)
+                    except OSError:
+                        pass
+                for f in os.listdir():
+                    rmdir(f)
+                os.chdir('..')
+                os.rmdir(directory)
+            rmdir('{0}')
+        """.format(
+            directory
+        )
+        self._pyboard.enter_raw_repl()
+        try:
+            out = self._pyboard.exec_(textwrap.dedent(command))
+        except PyboardError as ex:
+            message = ex.args[2].decode("utf-8")
+            # Check if this is an OSError #2, i.e. directory doesn't exist
+            # and rethrow it as something more descriptive.
+            if message.find("OSError: [Errno 2] ENOENT") != -1:
+                if not missing_okay:
+                    raise RuntimeError("No such directory: {0}".format(directory))
+            else:
+                raise ex
+        self._pyboard.exit_raw_repl()
+
+    def rename(self, oldname, newname):
+        command = """
+            try:
+                import os
+            except ImportError:
+                import uos as os
+
+            os.rename('{0}', '{1}')
+        """.format(
+            oldname, newname
+        )
+        self._pyboard.enter_raw_repl()
+        try:
+            out = self._pyboard.exec_(textwrap.dedent(command))
+        except PyboardError as ex:
+            message = ex.args[2].decode("utf-8")
+            raise ex
+        self._pyboard.exit_raw_repl()
+
+    def run(self, filename, wait_output=True, stream_output=True):
+        """Run the provided script and return its output.  If wait_output is True
+        (default) then wait for the script to finish and then return its output,
+        otherwise just run the script and don't wait for any output.
+        If stream_output is True(default) then return None and print outputs to
+        stdout without buffering.
+        """
+        self._pyboard.enter_raw_repl()
+        out = None
+        if stream_output:
+            self._pyboard.execfile(filename, stream_output=True)
+        elif wait_output:
+            # Run the file and wait for output to return.
+            out = self._pyboard.execfile(filename)
+        else:
+            # Read the file and run it using lower level pyboard functions that
+            # won't wait for it to finish or return output.
+            with open(filename, "rb") as infile:
+                self._pyboard.exec_raw_no_follow(infile.read())
+        self._pyboard.exit_raw_repl()
+        return out

tools/python/ampy/pyboard.py

@@ -0,0 +1,448 @@
+#!/usr/bin/env python
+
+"""
+pyboard interface
+
+This module provides the Pyboard class, used to communicate with and
+control the pyboard over a serial USB connection.
+
+Example usage:
+
+    import pyboard
+    pyb = pyboard.Pyboard('/dev/ttyACM0')
+
+Or:
+
+    pyb = pyboard.Pyboard('192.168.1.1')
+
+Then:
+
+    pyb.enter_raw_repl()
+    pyb.exec('pyb.LED(1).on()')
+    pyb.exit_raw_repl()
+
+Note: if using Python2 then pyb.exec must be written as pyb.exec_.
+To run a script from the local machine on the board and print out the results:
+
+    import pyboard
+    pyboard.execfile('test.py', device='/dev/ttyACM0')
+
+This script can also be run directly.  To execute a local script, use:
+
+    ./pyboard.py test.py
+
+Or:
+
+    python pyboard.py test.py
+
+"""
+
+import sys
+import time
+import json
+
+_rawdelay = None
+
+try:
+    stdout = sys.stdout.buffer
+except AttributeError:
+    # Python2 doesn't have buffer attr
+    stdout = sys.stdout
+
+def stdout_write_bytes(b):
+    b = b.replace(b"\x04", b"")
+    stdout.write(b)
+    stdout.flush()
+
+class PyboardError(BaseException):
+    pass
+
+class TelnetToSerial:
+    def __init__(self, ip, user, password, read_timeout=None):
+        import telnetlib
+        self.tn = telnetlib.Telnet(ip, timeout=15)
+        self.read_timeout = read_timeout
+        if b'Login as:' in self.tn.read_until(b'Login as:', timeout=read_timeout):
+            self.tn.write(bytes(user, 'ascii') + b"\r\n")
+
+            if b'Password:' in self.tn.read_until(b'Password:', timeout=read_timeout):
+                # needed because of internal implementation details of the telnet server
+                time.sleep(0.2)
+                self.tn.write(bytes(password, 'ascii') + b"\r\n")
+
+                if b'for more information.' in self.tn.read_until(b'Type "help()" for more information.', timeout=read_timeout):
+                    # login succesful
+                    from collections import deque
+                    self.fifo = deque()
+                    return
+
+        raise PyboardError('Failed to establish a telnet connection with the board')
+
+    def __del__(self):
+        self.close()
+
+    def close(self):
+        try:
+            self.tn.close()
+        except:
+            # the telnet object might not exist yet, so ignore this one
+            pass
+
+    def read(self, size=1):
+        while len(self.fifo) < size:
+            timeout_count = 0
+            data = self.tn.read_eager()
+            if len(data):
+                self.fifo.extend(data)
+                timeout_count = 0
+            else:
+                time.sleep(0.25)
+                if self.read_timeout is not None and timeout_count > 4 * self.read_timeout:
+                    break
+                timeout_count += 1
+
+        data = b''
+        while len(data) < size and len(self.fifo) > 0:
+            data += bytes([self.fifo.popleft()])
+        return data
+
+    def write(self, data):
+        self.tn.write(data)
+        return len(data)
+
+    def inWaiting(self):
+        n_waiting = len(self.fifo)
+        if not n_waiting:
+            data = self.tn.read_eager()
+            self.fifo.extend(data)
+            return len(data)
+        else:
+            return n_waiting
+
+class Pyboard:
+    def __init__(self, device, baudrate=115200, user='micro', password='python', wait=0, rawdelay=0, boardreset="{}", file_empty="main.py", info=True):
+        self.boardreset = json.loads(boardreset);
+        self.file_empty = file_empty;
+        self.info = info;
+        global _rawdelay
+        _rawdelay = rawdelay
+        if device and device[0].isdigit() and device[-1].isdigit() and device.count('.') == 3:
+            # device looks like an IP address
+            self.serial = TelnetToSerial(device, user, password, read_timeout=10)
+        else:
+            import serial
+            delayed = False
+            for attempt in range(wait + 1):
+                try:
+                    self.serial = serial.Serial(device, baudrate=baudrate, interCharTimeout=10, writeTimeout=5, timeout=40)
+                    break
+                except (OSError, IOError): # Py2 and Py3 have different errors
+                    if wait == 0:
+                        continue
+                    if attempt == 0:
+                        sys.stdout.write('Waiting {} seconds for pyboard '.format(wait))
+                        delayed = True
+                time.sleep(1)
+                #sys.stdout.write('.')
+                #sys.stdout.flush()
+            else:
+                if delayed:
+                    print('')
+                raise PyboardError('failed to access ' + device)
+            if delayed:
+                print('')
+
+    def close(self):
+        self.serial.close()
+
+    def read_until(self, min_num_bytes, ending, timeout=1, data_consumer=None):
+        data = self.serial.read(min_num_bytes)
+        if data_consumer:
+            data_consumer(data)
+        timeout_count = 0
+        while True:
+            if data.endswith(ending) or data.lower().endswith(ending):
+                break
+            elif self.serial.inWaiting() > 0:
+                new_data = self.serial.read(1)
+                data = data + new_data
+                if data_consumer:
+                    data_consumer(new_data)
+                timeout_count = 0
+            else:
+                timeout_count += 1
+                if timeout is not None and timeout_count >= 10 * timeout:
+                    break
+                time.sleep(0.01)
+        return data
+
+    def enter_raw_repl(self):
+        # Brief delay before sending RAW MODE char if requests
+        if _rawdelay > 0:
+            time.sleep(_rawdelay)
+        # ctrl-C twice: interrupt any running program
+        #sys.stdout.write("Try to delete ./{} ".format(self.file_empty))
+        if self.info:
+            sys.stdout.write("Try to enter REPL ")
+            sys.stdout.flush()
+        repl_ok=False
+        oldInterCharTimeout = self.serial.interCharTimeout
+        oldTimeout = self.serial.timeout
+        self.serial.interCharTimeout = 1
+        self.serial.timeout = 1
+        try:
+            for retry in range(10):
+                self.serial.write(b'\r\x02\x03')
+                time.sleep(0.1)
+                self.serial.write(b'\x02\x03')
+                time.sleep(0.1)
+                data = self.read_until(1, b'>')
+                if self.info:
+                    sys.stdout.write(".")
+                    sys.stdout.flush()
+                if data.endswith(b'>'):
+                    repl_ok=True
+                    if self.info:
+                        sys.stdout.write("\n")
+                        sys.stdout.write("Delete {} ".format(self.file_empty))
+                        sys.stdout.flush()
+                        self.serial.write(bytes("import os; os.remove('{}')\r\n".format(self.file_empty), encoding="utf8"))
+                    time.sleep(0.1)
+                    if self.info:
+                        sys.stdout.write("Done!\n")
+                        sys.stdout.flush()
+                    break
+                if retry >8:
+                    if self.info:
+                        sys.stdout.write("\n")
+                        sys.stdout.write('could not enter raw repl, Try to reset\n')
+                        sys.stdout.flush()
+                    break
+        finally:
+            self.serial.interCharTimeout = oldInterCharTimeout
+            self.serial.timeout = oldTimeout
+            if  not repl_ok:
+                if self.info:
+                    sys.stdout.write("Reset Start\n")
+                    sys.stdout.flush()
+                resetLen = len(self.boardreset)
+                if resetLen:
+                    for i in range(0, resetLen, 1):
+                        dtr = self.boardreset[i].get("dtr", -1)
+                        rts = self.boardreset[i].get("rts", -1)
+                        sleep = self.boardreset[i].get("sleep", -1)
+                        if dtr != -1 and rts != -1:
+                            self.serial.setDTR (dtr)
+                            self.serial.setRTS (rts)
+                            if self.info:
+                                sys.stdout.write("Set dtr:{}, rts:{}\n".format(dtr, rts))
+                        elif sleep != -1:
+                            time.sleep(sleep / 1000)
+                            if self.info:
+                                sys.stdout.write("Set sleep:{}s\n".format(sleep / 1000))
+                        if self.info:
+                            sys.stdout.flush()
+                else:
+                    self.serial.setDTR (False)
+                    self.serial.setRTS (False)
+                    if self.info:
+                        sys.stdout.write("Set dtr:{}, rts:{}\n".format(False, False))
+                        sys.stdout.flush()
+                    time.sleep(0.1)
+                    if self.info:
+                        sys.stdout.write("Set sleep:{}s\n".format(0.1))
+                        sys.stdout.flush()
+                    self.serial.setDTR (True)
+                    self.serial.setRTS (True)
+                    if self.info:
+                        sys.stdout.write("Set dtr:{}, rts:{}\n".format(True, True))
+                        sys.stdout.flush()
+                    time.sleep(0.1)
+                    if self.info:
+                        sys.stdout.write("Set sleep:{}s\n".format(0.1))
+                        sys.stdout.flush()
+                if self.info:
+                    sys.stdout.write("Reset Done!\n")
+                    sys.stdout.flush()
+
+        # Brief delay before sending RAW MODE char if requests
+        if _rawdelay > 0:
+            time.sleep(_rawdelay)
+        # ctrl-C twice: interrupt any running program
+        self.serial.write(b'\r\x03')
+        time.sleep(0.1)
+        self.serial.write(b'\x03')
+        time.sleep(0.1)
+
+        # flush input (without relying on serial.flushInput())
+        n = self.serial.inWaiting()
+        while n > 0:
+            self.serial.read(n)
+            n = self.serial.inWaiting()
+        #time.sleep(2)
+        #self.serial.write(b'\x03\x04')
+        for retry in range(0, 5): 
+            self.serial.write(b'\r\x01') # ctrl-A: enter raw REPL
+            data = self.read_until(1, b'raw REPL; CTRL-B to exit\r\n>')
+            if data.endswith(b'raw REPL; CTRL-B to exit\r\n>'):
+                break
+            else:
+                if retry >= 4:
+                    raise PyboardError('could not enter raw repl')
+                else:
+                    self.serial.write(b'\r\x03')
+                    time.sleep(0.1)
+                    self.serial.write(b'\x03')
+                    time.sleep(0.1)
+        self.serial.write(b'\x04') # ctrl-D: soft reset
+        data = self.read_until(1, b'soft reboot\r\n')
+        if not data.lower().endswith(b'soft reboot\r\n'):
+            self.serial.write(b'\x04') # ctrl-D: soft reset
+            raise PyboardError('could not enter raw repl')
+        # By splitting this into 2 reads, it allows boot.py to print stuff,
+        # which will show up after the soft reboot and before the raw REPL.
+        # Modification from original pyboard.py below:
+        #   Add a small delay and send Ctrl-C twice after soft reboot to ensure
+        #   any main program loop in main.py is interrupted.
+        time.sleep(0.5)
+        self.serial.write(b'\x03')
+        time.sleep(0.1)           # (slight delay before second interrupt
+        self.serial.write(b'\x03')
+        # End modification above.
+        data = self.read_until(1, b'raw REPL; CTRL-B to exit\r\n')
+        if not data.endswith(b'raw REPL; CTRL-B to exit\r\n'):
+            raise PyboardError('could not enter raw repl')
+
+    def exit_raw_repl(self):
+        self.serial.write(b'\r\x02') # ctrl-B: enter friendly REPL
+
+    def follow(self, timeout, data_consumer=None):
+        # wait for normal output
+        data = self.read_until(1, b'\x04', timeout=timeout, data_consumer=data_consumer)
+        if not data.endswith(b'\x04'):
+            raise PyboardError('timeout waiting for first EOF reception')
+        data = data[:-1]
+
+        # wait for error output
+        data_err = self.read_until(1, b'\x04', timeout=timeout)
+        if not data_err.endswith(b'\x04'):
+            raise PyboardError('timeout waiting for second EOF reception')
+        data_err = data_err[:-1]
+
+        # return normal and error output
+        return data, data_err
+
+    def exec_raw_no_follow(self, command):
+        if isinstance(command, bytes):
+            command_bytes = command
+        else:
+            command_bytes = bytes(command, encoding='utf8')
+
+        # check we have a prompt
+        data = self.read_until(1, b'>')
+        if not data.endswith(b'>'):
+            raise PyboardError('could not enter raw repl')
+
+        # write command
+        for i in range(0, len(command_bytes), 256):
+            self.serial.write(command_bytes[i:min(i + 256, len(command_bytes))])
+            time.sleep(0.01)
+        self.serial.write(b'\x04')
+
+        # check if we could exec command
+        data = self.serial.read(2)
+        if data != b'OK' and data != b'ra':
+            raise PyboardError('could not exec command')
+
+    def exec_raw(self, command, timeout=10, data_consumer=None):
+        self.exec_raw_no_follow(command);
+        return self.follow(timeout, data_consumer)
+
+    def eval(self, expression):
+        ret = self.exec_('print({})'.format(expression))
+        ret = ret.strip()
+        return ret
+
+    def exec_(self, command, stream_output=False):
+        data_consumer = None
+        if stream_output:
+            data_consumer = stdout_write_bytes
+        ret, ret_err = self.exec_raw(command, data_consumer=data_consumer)
+        if ret_err:
+            raise PyboardError('exception', ret, ret_err)
+        return ret
+
+    def execfile(self, filename, stream_output=False):
+        with open(filename, 'rb') as f:
+            pyfile = f.read()
+        return self.exec_(pyfile, stream_output=stream_output)
+
+    def get_time(self):
+        t = str(self.eval('pyb.RTC().datetime()'), encoding='utf8')[1:-1].split(', ')
+        return int(t[4]) * 3600 + int(t[5]) * 60 + int(t[6])
+
+# in Python2 exec is a keyword so one must use "exec_"
+# but for Python3 we want to provide the nicer version "exec"
+setattr(Pyboard, "exec", Pyboard.exec_)
+
+def execfile(filename, device='/dev/ttyACM0', baudrate=115200, user='micro', password='python'):
+    pyb = Pyboard(device, baudrate, user, password)
+    pyb.enter_raw_repl()
+    output = pyb.execfile(filename)
+    stdout_write_bytes(output)
+    pyb.exit_raw_repl()
+    pyb.close()
+
+def main():
+    import argparse
+    cmd_parser = argparse.ArgumentParser(description='Run scripts on the pyboard.')
+    cmd_parser.add_argument('--device', default='/dev/ttyACM0', help='the serial device or the IP address of the pyboard')
+    cmd_parser.add_argument('-b', '--baudrate', default=115200, help='the baud rate of the serial device')
+    cmd_parser.add_argument('-u', '--user', default='micro', help='the telnet login username')
+    cmd_parser.add_argument('-p', '--password', default='python', help='the telnet login password')
+    cmd_parser.add_argument('-c', '--command', help='program passed in as string')
+    cmd_parser.add_argument('-w', '--wait', default=0, type=int, help='seconds to wait for USB connected board to become available')
+    cmd_parser.add_argument('--follow', action='store_true', help='follow the output after running the scripts [default if no scripts given]')
+    cmd_parser.add_argument('files', nargs='*', help='input files')
+    args = cmd_parser.parse_args()
+
+    def execbuffer(buf):
+        try:
+            pyb = Pyboard(args.device, args.baudrate, args.user, args.password, args.wait)
+            pyb.enter_raw_repl()
+            ret, ret_err = pyb.exec_raw(buf, timeout=None, data_consumer=stdout_write_bytes)
+            pyb.exit_raw_repl()
+            pyb.close()
+        except PyboardError as er:
+            print(er)
+            sys.exit(1)
+        except KeyboardInterrupt:
+            sys.exit(1)
+        if ret_err:
+            stdout_write_bytes(ret_err)
+            sys.exit(1)
+
+    if args.command is not None:
+        execbuffer(args.command.encode('utf-8'))
+
+    for filename in args.files:
+        with open(filename, 'rb') as f:
+            pyfile = f.read()
+            execbuffer(pyfile)
+
+    if args.follow or (args.command is None and len(args.files) == 0):
+        try:
+            pyb = Pyboard(args.device, args.baudrate, args.user, args.password, args.wait)
+            ret, ret_err = pyb.follow(timeout=None, data_consumer=stdout_write_bytes)
+            pyb.close()
+        except PyboardError as er:
+            print(er)
+            sys.exit(1)
+        except KeyboardInterrupt:
+            sys.exit(1)
+        if ret_err:
+            stdout_write_bytes(ret_err)
+            sys.exit(1)
+
+if __name__ == "__main__":
+    main()

tools/python/ampy/serial/__init__.py

@@ -0,0 +1,91 @@
+#!/usr/bin/env python
+#
+# This is a wrapper module for different platform implementations
+#
+# This file is part of pySerial. https://github.com/pyserial/pyserial
+# (C) 2001-2020 Chris Liechti <cliechti@gmx.net>
+#
+# SPDX-License-Identifier:    BSD-3-Clause
+
+from __future__ import absolute_import
+
+import sys
+import importlib
+
+from serial.serialutil import *
+#~ SerialBase, SerialException, to_bytes, iterbytes
+
+__version__ = '3.5'
+
+VERSION = __version__
+
+# pylint: disable=wrong-import-position
+if sys.platform == 'cli':
+    from serial.serialcli import Serial
+else:
+    import os
+    # chose an implementation, depending on os
+    if os.name == 'nt':  # sys.platform == 'win32':
+        from serial.serialwin32 import Serial
+    elif os.name == 'posix':
+        from serial.serialposix import Serial, PosixPollSerial, VTIMESerial  # noqa
+    elif os.name == 'java':
+        from serial.serialjava import Serial
+    else:
+        raise ImportError("Sorry: no implementation for your platform ('{}') available".format(os.name))
+
+
+protocol_handler_packages = [
+    'serial.urlhandler',
+]
+
+
+def serial_for_url(url, *args, **kwargs):
+    """\
+    Get an instance of the Serial class, depending on port/url. The port is not
+    opened when the keyword parameter 'do_not_open' is true, by default it
+    is. All other parameters are directly passed to the __init__ method when
+    the port is instantiated.
+
+    The list of package names that is searched for protocol handlers is kept in
+    ``protocol_handler_packages``.
+
+    e.g. we want to support a URL ``foobar://``. A module
+    ``my_handlers.protocol_foobar`` is provided by the user. Then
+    ``protocol_handler_packages.append("my_handlers")`` would extend the search
+    path so that ``serial_for_url("foobar://"))`` would work.
+    """
+    # check and remove extra parameter to not confuse the Serial class
+    do_open = not kwargs.pop('do_not_open', False)
+    # the default is to use the native implementation
+    klass = Serial
+    try:
+        url_lowercase = url.lower()
+    except AttributeError:
+        # it's not a string, use default
+        pass
+    else:
+        # if it is an URL, try to import the handler module from the list of possible packages
+        if '://' in url_lowercase:
+            protocol = url_lowercase.split('://', 1)[0]
+            module_name = '.protocol_{}'.format(protocol)
+            for package_name in protocol_handler_packages:
+                try:
+                    importlib.import_module(package_name)
+                    handler_module = importlib.import_module(module_name, package_name)
+                except ImportError:
+                    continue
+                else:
+                    if hasattr(handler_module, 'serial_class_for_url'):
+                        url, klass = handler_module.serial_class_for_url(url)
+                    else:
+                        klass = handler_module.Serial
+                    break
+            else:
+                raise ValueError('invalid URL, protocol {!r} not known'.format(protocol))
+    # instantiate and open when desired
+    instance = klass(None, *args, **kwargs)
+    instance.port = url
+    if do_open:
+        instance.open()
+    return instance

tools/python/ampy/serial/__main__.py

@@ -0,0 +1,3 @@
+from .tools import miniterm
+
+miniterm.main()

tools/python/ampy/serial/rs485.py

@@ -0,0 +1,94 @@
+#!/usr/bin/env python
+
+# RS485 support
+#
+# This file is part of pySerial. https://github.com/pyserial/pyserial
+# (C) 2015 Chris Liechti <cliechti@gmx.net>
+#
+# SPDX-License-Identifier:    BSD-3-Clause
+
+"""\
+The settings for RS485 are stored in a dedicated object that can be applied to
+serial ports (where supported).
+NOTE: Some implementations may only support a subset of the settings.
+"""
+
+from __future__ import absolute_import
+
+import time
+import serial
+
+
+class RS485Settings(object):
+    def __init__(
+            self,
+            rts_level_for_tx=True,
+            rts_level_for_rx=False,
+            loopback=False,
+            delay_before_tx=None,
+            delay_before_rx=None):
+        self.rts_level_for_tx = rts_level_for_tx
+        self.rts_level_for_rx = rts_level_for_rx
+        self.loopback = loopback
+        self.delay_before_tx = delay_before_tx
+        self.delay_before_rx = delay_before_rx
+
+
+class RS485(serial.Serial):
+    """\
+    A subclass that replaces the write method with one that toggles RTS
+    according to the RS485 settings.
+
+    NOTE: This may work unreliably on some serial ports (control signals not
+          synchronized or delayed compared to data). Using delays may be
+          unreliable (varying times, larger than expected) as the OS may not
+          support very fine grained delays (no smaller than in the order of
+          tens of milliseconds).
+
+    NOTE: Some implementations support this natively. Better performance
+          can be expected when the native version is used.
+
+    NOTE: The loopback property is ignored by this implementation. The actual
+          behavior depends on the used hardware.
+
+    Usage:
+
+        ser = RS485(...)
+        ser.rs485_mode = RS485Settings(...)
+        ser.write(b'hello')
+    """
+
+    def __init__(self, *args, **kwargs):
+        super(RS485, self).__init__(*args, **kwargs)
+        self._alternate_rs485_settings = None
+
+    def write(self, b):
+        """Write to port, controlling RTS before and after transmitting."""
+        if self._alternate_rs485_settings is not None:
+            # apply level for TX and optional delay
+            self.setRTS(self._alternate_rs485_settings.rts_level_for_tx)
+            if self._alternate_rs485_settings.delay_before_tx is not None:
+                time.sleep(self._alternate_rs485_settings.delay_before_tx)
+            # write and wait for data to be written
+            super(RS485, self).write(b)
+            super(RS485, self).flush()
+            # optional delay and apply level for RX
+            if self._alternate_rs485_settings.delay_before_rx is not None:
+                time.sleep(self._alternate_rs485_settings.delay_before_rx)
+            self.setRTS(self._alternate_rs485_settings.rts_level_for_rx)
+        else:
+            super(RS485, self).write(b)
+
+    # redirect where the property stores the settings so that underlying Serial
+    # instance does not see them
+    @property
+    def rs485_mode(self):
+        """\
+        Enable RS485 mode and apply new settings, set to None to disable.
+        See serial.rs485.RS485Settings for more info about the value.
+        """
+        return self._alternate_rs485_settings
+
+    @rs485_mode.setter
+    def rs485_mode(self, rs485_settings):
+        self._alternate_rs485_settings = rs485_settings

tools/python/ampy/serial/serialcli.py

@@ -0,0 +1,253 @@
+#! python
+#
+# Backend for .NET/Mono (IronPython), .NET >= 2
+#
+# This file is part of pySerial. https://github.com/pyserial/pyserial
+# (C) 2008-2015 Chris Liechti <cliechti@gmx.net>
+#
+# SPDX-License-Identifier:    BSD-3-Clause
+
+from __future__ import absolute_import
+
+import System
+import System.IO.Ports
+from serial.serialutil import *
+
+# must invoke function with byte array, make a helper to convert strings
+# to byte arrays
+sab = System.Array[System.Byte]
+
+
+def as_byte_array(string):
+    return sab([ord(x) for x in string])  # XXX will require adaption when run with a 3.x compatible IronPython
+
+
+class Serial(SerialBase):
+    """Serial port implementation for .NET/Mono."""
+
+    BAUDRATES = (50, 75, 110, 134, 150, 200, 300, 600, 1200, 1800, 2400, 4800,
+                 9600, 19200, 38400, 57600, 115200)
+
+    def open(self):
+        """\
+        Open port with current settings. This may throw a SerialException
+        if the port cannot be opened.
+        """
+        if self._port is None:
+            raise SerialException("Port must be configured before it can be used.")
+        if self.is_open:
+            raise SerialException("Port is already open.")
+        try:
+            self._port_handle = System.IO.Ports.SerialPort(self.portstr)
+        except Exception as msg:
+            self._port_handle = None
+            raise SerialException("could not open port %s: %s" % (self.portstr, msg))
+
+        # if RTS and/or DTR are not set before open, they default to True
+        if self._rts_state is None:
+            self._rts_state = True
+        if self._dtr_state is None:
+            self._dtr_state = True
+
+        self._reconfigure_port()
+        self._port_handle.Open()
+        self.is_open = True
+        if not self._dsrdtr:
+            self._update_dtr_state()
+        if not self._rtscts:
+            self._update_rts_state()
+        self.reset_input_buffer()
+
+    def _reconfigure_port(self):
+        """Set communication parameters on opened port."""
+        if not self._port_handle:
+            raise SerialException("Can only operate on a valid port handle")
+
+        #~ self._port_handle.ReceivedBytesThreshold = 1
+
+        if self._timeout is None:
+            self._port_handle.ReadTimeout = System.IO.Ports.SerialPort.InfiniteTimeout
+        else:
+            self._port_handle.ReadTimeout = int(self._timeout * 1000)
+
+        # if self._timeout != 0 and self._interCharTimeout is not None:
+            # timeouts = (int(self._interCharTimeout * 1000),) + timeouts[1:]
+
+        if self._write_timeout is None:
+            self._port_handle.WriteTimeout = System.IO.Ports.SerialPort.InfiniteTimeout
+        else:
+            self._port_handle.WriteTimeout = int(self._write_timeout * 1000)
+
+        # Setup the connection info.
+        try:
+            self._port_handle.BaudRate = self._baudrate
+        except IOError as e:
+            # catch errors from illegal baudrate settings
+            raise ValueError(str(e))
+
+        if self._bytesize == FIVEBITS:
+            self._port_handle.DataBits = 5
+        elif self._bytesize == SIXBITS:
+            self._port_handle.DataBits = 6
+        elif self._bytesize == SEVENBITS:
+            self._port_handle.DataBits = 7
+        elif self._bytesize == EIGHTBITS:
+            self._port_handle.DataBits = 8
+        else:
+            raise ValueError("Unsupported number of data bits: %r" % self._bytesize)
+
+        if self._parity == PARITY_NONE:
+            self._port_handle.Parity = getattr(System.IO.Ports.Parity, 'None')  # reserved keyword in Py3k
+        elif self._parity == PARITY_EVEN:
+            self._port_handle.Parity = System.IO.Ports.Parity.Even
+        elif self._parity == PARITY_ODD:
+            self._port_handle.Parity = System.IO.Ports.Parity.Odd
+        elif self._parity == PARITY_MARK:
+            self._port_handle.Parity = System.IO.Ports.Parity.Mark
+        elif self._parity == PARITY_SPACE:
+            self._port_handle.Parity = System.IO.Ports.Parity.Space
+        else:
+            raise ValueError("Unsupported parity mode: %r" % self._parity)
+
+        if self._stopbits == STOPBITS_ONE:
+            self._port_handle.StopBits = System.IO.Ports.StopBits.One
+        elif self._stopbits == STOPBITS_ONE_POINT_FIVE:
+            self._port_handle.StopBits = System.IO.Ports.StopBits.OnePointFive
+        elif self._stopbits == STOPBITS_TWO:
+            self._port_handle.StopBits = System.IO.Ports.StopBits.Two
+        else:
+            raise ValueError("Unsupported number of stop bits: %r" % self._stopbits)
+
+        if self._rtscts and self._xonxoff:
+            self._port_handle.Handshake = System.IO.Ports.Handshake.RequestToSendXOnXOff
+        elif self._rtscts:
+            self._port_handle.Handshake = System.IO.Ports.Handshake.RequestToSend
+        elif self._xonxoff:
+            self._port_handle.Handshake = System.IO.Ports.Handshake.XOnXOff
+        else:
+            self._port_handle.Handshake = getattr(System.IO.Ports.Handshake, 'None')   # reserved keyword in Py3k
+
+    #~ def __del__(self):
+        #~ self.close()
+
+    def close(self):
+        """Close port"""
+        if self.is_open:
+            if self._port_handle:
+                try:
+                    self._port_handle.Close()
+                except System.IO.Ports.InvalidOperationException:
+                    # ignore errors. can happen for unplugged USB serial devices
+                    pass
+                self._port_handle = None
+            self.is_open = False
+
+    #  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -
+
+    @property
+    def in_waiting(self):
+        """Return the number of characters currently in the input buffer."""
+        if not self.is_open:
+            raise PortNotOpenError()
+        return self._port_handle.BytesToRead
+
+    def read(self, size=1):
+        """\
+        Read size bytes from the serial port. If a timeout is set it may
+        return less characters as requested. With no timeout it will block
+        until the requested number of bytes is read.
+        """
+        if not self.is_open:
+            raise PortNotOpenError()
+        # must use single byte reads as this is the only way to read
+        # without applying encodings
+        data = bytearray()
+        while size:
+            try:
+                data.append(self._port_handle.ReadByte())
+            except System.TimeoutException:
+                break
+            else:
+                size -= 1
+        return bytes(data)
+
+    def write(self, data):
+        """Output the given string over the serial port."""
+        if not self.is_open:
+            raise PortNotOpenError()
+        #~ if not isinstance(data, (bytes, bytearray)):
+            #~ raise TypeError('expected %s or bytearray, got %s' % (bytes, type(data)))
+        try:
+            # must call overloaded method with byte array argument
+            # as this is the only one not applying encodings
+            self._port_handle.Write(as_byte_array(data), 0, len(data))
+        except System.TimeoutException:
+            raise SerialTimeoutException('Write timeout')
+        return len(data)
+
+    def reset_input_buffer(self):
+        """Clear input buffer, discarding all that is in the buffer."""
+        if not self.is_open:
+            raise PortNotOpenError()
+        self._port_handle.DiscardInBuffer()
+
+    def reset_output_buffer(self):
+        """\
+        Clear output buffer, aborting the current output and
+        discarding all that is in the buffer.
+        """
+        if not self.is_open:
+            raise PortNotOpenError()
+        self._port_handle.DiscardOutBuffer()
+
+    def _update_break_state(self):
+        """
+        Set break: Controls TXD. When active, to transmitting is possible.
+        """
+        if not self.is_open:
+            raise PortNotOpenError()
+        self._port_handle.BreakState = bool(self._break_state)
+
+    def _update_rts_state(self):
+        """Set terminal status line: Request To Send"""
+        if not self.is_open:
+            raise PortNotOpenError()
+        self._port_handle.RtsEnable = bool(self._rts_state)
+
+    def _update_dtr_state(self):
+        """Set terminal status line: Data Terminal Ready"""
+        if not self.is_open:
+            raise PortNotOpenError()
+        self._port_handle.DtrEnable = bool(self._dtr_state)
+
+    @property
+    def cts(self):
+        """Read terminal status line: Clear To Send"""
+        if not self.is_open:
+            raise PortNotOpenError()
+        return self._port_handle.CtsHolding
+
+    @property
+    def dsr(self):
+        """Read terminal status line: Data Set Ready"""
+        if not self.is_open:
+            raise PortNotOpenError()
+        return self._port_handle.DsrHolding
+
+    @property
+    def ri(self):
+        """Read terminal status line: Ring Indicator"""
+        if not self.is_open:
+            raise PortNotOpenError()
+        #~ return self._port_handle.XXX
+        return False  # XXX an error would be better
+
+    @property
+    def cd(self):
+        """Read terminal status line: Carrier Detect"""
+        if not self.is_open:
+            raise PortNotOpenError()
+        return self._port_handle.CDHolding
+
+    # - - platform specific - - - -
+    # none

tools/python/ampy/serial/serialjava.py

@@ -0,0 +1,251 @@
+#!jython
+#
+# Backend Jython with JavaComm
+#
+# This file is part of pySerial. https://github.com/pyserial/pyserial
+# (C) 2002-2015 Chris Liechti <cliechti@gmx.net>
+#
+# SPDX-License-Identifier:    BSD-3-Clause
+
+from __future__ import absolute_import
+
+from serial.serialutil import *
+
+
+def my_import(name):
+    mod = __import__(name)
+    components = name.split('.')
+    for comp in components[1:]:
+        mod = getattr(mod, comp)
+    return mod
+
+
+def detect_java_comm(names):
+    """try given list of modules and return that imports"""
+    for name in names:
+        try:
+            mod = my_import(name)
+            mod.SerialPort
+            return mod
+        except (ImportError, AttributeError):
+            pass
+    raise ImportError("No Java Communications API implementation found")
+
+
+# Java Communications API implementations
+# http://mho.republika.pl/java/comm/
+
+comm = detect_java_comm([
+    'javax.comm',  # Sun/IBM
+    'gnu.io',      # RXTX
+])
+
+
+def device(portnumber):
+    """Turn a port number into a device name"""
+    enum = comm.CommPortIdentifier.getPortIdentifiers()
+    ports = []
+    while enum.hasMoreElements():
+        el = enum.nextElement()
+        if el.getPortType() == comm.CommPortIdentifier.PORT_SERIAL:
+            ports.append(el)
+    return ports[portnumber].getName()
+
+
+class Serial(SerialBase):
+    """\
+    Serial port class, implemented with Java Communications API and
+    thus usable with jython and the appropriate java extension.
+    """
+
+    def open(self):
+        """\
+        Open port with current settings. This may throw a SerialException
+        if the port cannot be opened.
+        """
+        if self._port is None:
+            raise SerialException("Port must be configured before it can be used.")
+        if self.is_open:
+            raise SerialException("Port is already open.")
+        if type(self._port) == type(''):      # strings are taken directly
+            portId = comm.CommPortIdentifier.getPortIdentifier(self._port)
+        else:
+            portId = comm.CommPortIdentifier.getPortIdentifier(device(self._port))     # numbers are transformed to a comport id obj
+        try:
+            self.sPort = portId.open("python serial module", 10)
+        except Exception as msg:
+            self.sPort = None
+            raise SerialException("Could not open port: %s" % msg)
+        self._reconfigurePort()
+        self._instream = self.sPort.getInputStream()
+        self._outstream = self.sPort.getOutputStream()
+        self.is_open = True
+
+    def _reconfigurePort(self):
+        """Set communication parameters on opened port."""
+        if not self.sPort:
+            raise SerialException("Can only operate on a valid port handle")
+
+        self.sPort.enableReceiveTimeout(30)
+        if self._bytesize == FIVEBITS:
+            jdatabits = comm.SerialPort.DATABITS_5
+        elif self._bytesize == SIXBITS:
+            jdatabits = comm.SerialPort.DATABITS_6
+        elif self._bytesize == SEVENBITS:
+            jdatabits = comm.SerialPort.DATABITS_7
+        elif self._bytesize == EIGHTBITS:
+            jdatabits = comm.SerialPort.DATABITS_8
+        else:
+            raise ValueError("unsupported bytesize: %r" % self._bytesize)
+
+        if self._stopbits == STOPBITS_ONE:
+            jstopbits = comm.SerialPort.STOPBITS_1
+        elif self._stopbits == STOPBITS_ONE_POINT_FIVE:
+            jstopbits = comm.SerialPort.STOPBITS_1_5
+        elif self._stopbits == STOPBITS_TWO:
+            jstopbits = comm.SerialPort.STOPBITS_2
+        else:
+            raise ValueError("unsupported number of stopbits: %r" % self._stopbits)
+
+        if self._parity == PARITY_NONE:
+            jparity = comm.SerialPort.PARITY_NONE
+        elif self._parity == PARITY_EVEN:
+            jparity = comm.SerialPort.PARITY_EVEN
+        elif self._parity == PARITY_ODD:
+            jparity = comm.SerialPort.PARITY_ODD
+        elif self._parity == PARITY_MARK:
+            jparity = comm.SerialPort.PARITY_MARK
+        elif self._parity == PARITY_SPACE:
+            jparity = comm.SerialPort.PARITY_SPACE
+        else:
+            raise ValueError("unsupported parity type: %r" % self._parity)
+
+        jflowin = jflowout = 0
+        if self._rtscts:
+            jflowin |= comm.SerialPort.FLOWCONTROL_RTSCTS_IN
+            jflowout |= comm.SerialPort.FLOWCONTROL_RTSCTS_OUT
+        if self._xonxoff:
+            jflowin |= comm.SerialPort.FLOWCONTROL_XONXOFF_IN
+            jflowout |= comm.SerialPort.FLOWCONTROL_XONXOFF_OUT
+
+        self.sPort.setSerialPortParams(self._baudrate, jdatabits, jstopbits, jparity)
+        self.sPort.setFlowControlMode(jflowin | jflowout)
+
+        if self._timeout >= 0:
+            self.sPort.enableReceiveTimeout(int(self._timeout*1000))
+        else:
+            self.sPort.disableReceiveTimeout()
+
+    def close(self):
+        """Close port"""
+        if self.is_open:
+            if self.sPort:
+                self._instream.close()
+                self._outstream.close()
+                self.sPort.close()
+                self.sPort = None
+            self.is_open = False
+
+    #  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -
+
+    @property
+    def in_waiting(self):
+        """Return the number of characters currently in the input buffer."""
+        if not self.sPort:
+            raise PortNotOpenError()
+        return self._instream.available()
+
+    def read(self, size=1):
+        """\
+        Read size bytes from the serial port. If a timeout is set it may
+        return less characters as requested. With no timeout it will block
+        until the requested number of bytes is read.
+        """
+        if not self.sPort:
+            raise PortNotOpenError()
+        read = bytearray()
+        if size > 0:
+            while len(read) < size:
+                x = self._instream.read()
+                if x == -1:
+                    if self.timeout >= 0:
+                        break
+                else:
+                    read.append(x)
+        return bytes(read)
+
+    def write(self, data):
+        """Output the given string over the serial port."""
+        if not self.sPort:
+            raise PortNotOpenError()
+        if not isinstance(data, (bytes, bytearray)):
+            raise TypeError('expected %s or bytearray, got %s' % (bytes, type(data)))
+        self._outstream.write(data)
+        return len(data)
+
+    def reset_input_buffer(self):
+        """Clear input buffer, discarding all that is in the buffer."""
+        if not self.sPort:
+            raise PortNotOpenError()
+        self._instream.skip(self._instream.available())
+
+    def reset_output_buffer(self):
+        """\
+        Clear output buffer, aborting the current output and
+        discarding all that is in the buffer.
+        """
+        if not self.sPort:
+            raise PortNotOpenError()
+        self._outstream.flush()
+
+    def send_break(self, duration=0.25):
+        """Send break condition. Timed, returns to idle state after given duration."""
+        if not self.sPort:
+            raise PortNotOpenError()
+        self.sPort.sendBreak(duration*1000.0)
+
+    def _update_break_state(self):
+        """Set break: Controls TXD. When active, to transmitting is possible."""
+        if self.fd is None:
+            raise PortNotOpenError()
+        raise SerialException("The _update_break_state function is not implemented in java.")
+
+    def _update_rts_state(self):
+        """Set terminal status line: Request To Send"""
+        if not self.sPort:
+            raise PortNotOpenError()
+        self.sPort.setRTS(self._rts_state)
+
+    def _update_dtr_state(self):
+        """Set terminal status line: Data Terminal Ready"""
+        if not self.sPort:
+            raise PortNotOpenError()
+        self.sPort.setDTR(self._dtr_state)
+
+    @property
+    def cts(self):
+        """Read terminal status line: Clear To Send"""
+        if not self.sPort:
+            raise PortNotOpenError()
+        self.sPort.isCTS()
+
+    @property
+    def dsr(self):
+        """Read terminal status line: Data Set Ready"""
+        if not self.sPort:
+            raise PortNotOpenError()
+        self.sPort.isDSR()
+
+    @property
+    def ri(self):
+        """Read terminal status line: Ring Indicator"""
+        if not self.sPort:
+            raise PortNotOpenError()
+        self.sPort.isRI()
+
+    @property
+    def cd(self):
+        """Read terminal status line: Carrier Detect"""
+        if not self.sPort:
+            raise PortNotOpenError()
+        self.sPort.isCD()

tools/python/ampy/serial/serialposix.py

@@ -0,0 +1,900 @@
+#!/usr/bin/env python
+#
+# backend for serial IO for POSIX compatible systems, like Linux, OSX
+#
+# This file is part of pySerial. https://github.com/pyserial/pyserial
+# (C) 2001-2020 Chris Liechti <cliechti@gmx.net>
+#
+# SPDX-License-Identifier:    BSD-3-Clause
+#
+# parts based on code from Grant B. Edwards  <grante@visi.com>:
+#  ftp://ftp.visi.com/users/grante/python/PosixSerial.py
+#
+# references: http://www.easysw.com/~mike/serial/serial.html
+
+# Collection of port names (was previously used by number_to_device which was
+# removed.
+# - Linux                   /dev/ttyS%d (confirmed)
+# - cygwin/win32            /dev/com%d (confirmed)
+# - openbsd (OpenBSD)       /dev/cua%02d
+# - bsd*, freebsd*          /dev/cuad%d
+# - darwin (OS X)           /dev/cuad%d
+# - netbsd                  /dev/dty%02d (NetBSD 1.6 testing by Erk)
+# - irix (IRIX)             /dev/ttyf%d (partially tested) names depending on flow control
+# - hp (HP-UX)              /dev/tty%dp0 (not tested)
+# - sunos (Solaris/SunOS)   /dev/tty%c (letters, 'a'..'z') (confirmed)
+# - aix (AIX)               /dev/tty%d
+
+
+from __future__ import absolute_import
+
+# pylint: disable=abstract-method
+import errno
+import fcntl
+import os
+import select
+import struct
+import sys
+import termios
+
+import serial
+from serial.serialutil import SerialBase, SerialException, to_bytes, \
+    PortNotOpenError, SerialTimeoutException, Timeout
+
+
+class PlatformSpecificBase(object):
+    BAUDRATE_CONSTANTS = {}
+
+    def _set_special_baudrate(self, baudrate):
+        raise NotImplementedError('non-standard baudrates are not supported on this platform')
+
+    def _set_rs485_mode(self, rs485_settings):
+        raise NotImplementedError('RS485 not supported on this platform')
+
+    def set_low_latency_mode(self, low_latency_settings):
+        raise NotImplementedError('Low latency not supported on this platform')
+
+    def _update_break_state(self):
+        """\
+        Set break: Controls TXD. When active, no transmitting is possible.
+        """
+        if self._break_state:
+            fcntl.ioctl(self.fd, TIOCSBRK)
+        else:
+            fcntl.ioctl(self.fd, TIOCCBRK)
+    
+
+# some systems support an extra flag to enable the two in POSIX unsupported
+# paritiy settings for MARK and SPACE
+CMSPAR = 0  # default, for unsupported platforms, override below
+
+# try to detect the OS so that a device can be selected...
+# this code block should supply a device() and set_special_baudrate() function
+# for the platform
+plat = sys.platform.lower()
+
+if plat[:5] == 'linux':    # Linux (confirmed)  # noqa
+    import array
+
+    # extra termios flags
+    CMSPAR = 0o10000000000  # Use "stick" (mark/space) parity
+
+    # baudrate ioctls
+    TCGETS2 = 0x802C542A
+    TCSETS2 = 0x402C542B
+    BOTHER = 0o010000
+
+    # RS485 ioctls
+    TIOCGRS485 = 0x542E
+    TIOCSRS485 = 0x542F
+    SER_RS485_ENABLED = 0b00000001
+    SER_RS485_RTS_ON_SEND = 0b00000010
+    SER_RS485_RTS_AFTER_SEND = 0b00000100
+    SER_RS485_RX_DURING_TX = 0b00010000
+
+    class PlatformSpecific(PlatformSpecificBase):
+        BAUDRATE_CONSTANTS = {
+            0:       0o000000,  # hang up
+            50:      0o000001,
+            75:      0o000002,
+            110:     0o000003,
+            134:     0o000004,
+            150:     0o000005,
+            200:     0o000006,
+            300:     0o000007,
+            600:     0o000010,
+            1200:    0o000011,
+            1800:    0o000012,
+            2400:    0o000013,
+            4800:    0o000014,
+            9600:    0o000015,
+            19200:   0o000016,
+            38400:   0o000017,
+            57600:   0o010001,
+            115200:  0o010002,
+            230400:  0o010003,
+            460800:  0o010004,
+            500000:  0o010005,
+            576000:  0o010006,
+            921600:  0o010007,
+            1000000: 0o010010,
+            1152000: 0o010011,
+            1500000: 0o010012,
+            2000000: 0o010013,
+            2500000: 0o010014,
+            3000000: 0o010015,
+            3500000: 0o010016,
+            4000000: 0o010017
+        }
+
+        def set_low_latency_mode(self, low_latency_settings):
+            buf = array.array('i', [0] * 32)
+
+            try:
+                # get serial_struct
+                fcntl.ioctl(self.fd, termios.TIOCGSERIAL, buf)
+
+                # set or unset ASYNC_LOW_LATENCY flag
+                if low_latency_settings:
+                    buf[4] |= 0x2000
+                else:
+                    buf[4] &= ~0x2000
+
+                # set serial_struct
+                fcntl.ioctl(self.fd, termios.TIOCSSERIAL, buf)
+            except IOError as e:
+                raise ValueError('Failed to update ASYNC_LOW_LATENCY flag to {}: {}'.format(low_latency_settings, e))
+
+        def _set_special_baudrate(self, baudrate):
+            # right size is 44 on x86_64, allow for some growth
+            buf = array.array('i', [0] * 64)
+            try:
+                # get serial_struct
+                fcntl.ioctl(self.fd, TCGETS2, buf)
+                # set custom speed
+                buf[2] &= ~termios.CBAUD
+                buf[2] |= BOTHER
+                buf[9] = buf[10] = baudrate
+
+                # set serial_struct
+                fcntl.ioctl(self.fd, TCSETS2, buf)
+            except IOError as e:
+                raise ValueError('Failed to set custom baud rate ({}): {}'.format(baudrate, e))
+
+        def _set_rs485_mode(self, rs485_settings):
+            buf = array.array('i', [0] * 8)  # flags, delaytx, delayrx, padding
+            try:
+                fcntl.ioctl(self.fd, TIOCGRS485, buf)
+                buf[0] |= SER_RS485_ENABLED
+                if rs485_settings is not None:
+                    if rs485_settings.loopback:
+                        buf[0] |= SER_RS485_RX_DURING_TX
+                    else:
+                        buf[0] &= ~SER_RS485_RX_DURING_TX
+                    if rs485_settings.rts_level_for_tx:
+                        buf[0] |= SER_RS485_RTS_ON_SEND
+                    else:
+                        buf[0] &= ~SER_RS485_RTS_ON_SEND
+                    if rs485_settings.rts_level_for_rx:
+                        buf[0] |= SER_RS485_RTS_AFTER_SEND
+                    else:
+                        buf[0] &= ~SER_RS485_RTS_AFTER_SEND
+                    if rs485_settings.delay_before_tx is not None:
+                        buf[1] = int(rs485_settings.delay_before_tx * 1000)
+                    if rs485_settings.delay_before_rx is not None:
+                        buf[2] = int(rs485_settings.delay_before_rx * 1000)
+                else:
+                    buf[0] = 0  # clear SER_RS485_ENABLED
+                fcntl.ioctl(self.fd, TIOCSRS485, buf)
+            except IOError as e:
+                raise ValueError('Failed to set RS485 mode: {}'.format(e))
+
+
+elif plat == 'cygwin':       # cygwin/win32 (confirmed)
+
+    class PlatformSpecific(PlatformSpecificBase):
+        BAUDRATE_CONSTANTS = {
+            128000: 0x01003,
+            256000: 0x01005,
+            500000: 0x01007,
+            576000: 0x01008,
+            921600: 0x01009,
+            1000000: 0x0100a,
+            1152000: 0x0100b,
+            1500000: 0x0100c,
+            2000000: 0x0100d,
+            2500000: 0x0100e,
+            3000000: 0x0100f
+        }
+
+
+elif plat[:6] == 'darwin':   # OS X
+    import array
+    IOSSIOSPEED = 0x80045402  # _IOW('T', 2, speed_t)
+
+    class PlatformSpecific(PlatformSpecificBase):
+        osx_version = os.uname()[2].split('.')
+        TIOCSBRK = 0x2000747B # _IO('t', 123)
+        TIOCCBRK = 0x2000747A # _IO('t', 122)
+
+        # Tiger or above can support arbitrary serial speeds
+        if int(osx_version[0]) >= 8:
+            def _set_special_baudrate(self, baudrate):
+                # use IOKit-specific call to set up high speeds
+                buf = array.array('i', [baudrate])
+                fcntl.ioctl(self.fd, IOSSIOSPEED, buf, 1)
+
+        def _update_break_state(self):
+            """\
+            Set break: Controls TXD. When active, no transmitting is possible.
+            """
+            if self._break_state:
+                fcntl.ioctl(self.fd, PlatformSpecific.TIOCSBRK)
+            else:
+                fcntl.ioctl(self.fd, PlatformSpecific.TIOCCBRK)
+
+elif plat[:3] == 'bsd' or \
+     plat[:7] == 'freebsd' or \
+     plat[:6] == 'netbsd' or \
+     plat[:7] == 'openbsd':
+
+    class ReturnBaudrate(object):
+        def __getitem__(self, key):
+            return key
+
+    class PlatformSpecific(PlatformSpecificBase):
+        # Only tested on FreeBSD:
+        # The baud rate may be passed in as
+        # a literal value.
+        BAUDRATE_CONSTANTS = ReturnBaudrate()
+
+        TIOCSBRK = 0x2000747B # _IO('t', 123)
+        TIOCCBRK = 0x2000747A # _IO('t', 122)
+
+        
+        def _update_break_state(self):
+            """\
+            Set break: Controls TXD. When active, no transmitting is possible.
+            """
+            if self._break_state:
+                fcntl.ioctl(self.fd, PlatformSpecific.TIOCSBRK)
+            else:
+                fcntl.ioctl(self.fd, PlatformSpecific.TIOCCBRK)
+
+else:
+    class PlatformSpecific(PlatformSpecificBase):
+        pass
+
+
+# load some constants for later use.
+# try to use values from termios, use defaults from linux otherwise
+TIOCMGET = getattr(termios, 'TIOCMGET', 0x5415)
+TIOCMBIS = getattr(termios, 'TIOCMBIS', 0x5416)
+TIOCMBIC = getattr(termios, 'TIOCMBIC', 0x5417)
+TIOCMSET = getattr(termios, 'TIOCMSET', 0x5418)
+
+# TIOCM_LE = getattr(termios, 'TIOCM_LE', 0x001)
+TIOCM_DTR = getattr(termios, 'TIOCM_DTR', 0x002)
+TIOCM_RTS = getattr(termios, 'TIOCM_RTS', 0x004)
+# TIOCM_ST = getattr(termios, 'TIOCM_ST', 0x008)
+# TIOCM_SR = getattr(termios, 'TIOCM_SR', 0x010)
+
+TIOCM_CTS = getattr(termios, 'TIOCM_CTS', 0x020)
+TIOCM_CAR = getattr(termios, 'TIOCM_CAR', 0x040)
+TIOCM_RNG = getattr(termios, 'TIOCM_RNG', 0x080)
+TIOCM_DSR = getattr(termios, 'TIOCM_DSR', 0x100)
+TIOCM_CD = getattr(termios, 'TIOCM_CD', TIOCM_CAR)
+TIOCM_RI = getattr(termios, 'TIOCM_RI', TIOCM_RNG)
+# TIOCM_OUT1 = getattr(termios, 'TIOCM_OUT1', 0x2000)
+# TIOCM_OUT2 = getattr(termios, 'TIOCM_OUT2', 0x4000)
+if hasattr(termios, 'TIOCINQ'):
+    TIOCINQ = termios.TIOCINQ
+else:
+    TIOCINQ = getattr(termios, 'FIONREAD', 0x541B)
+TIOCOUTQ = getattr(termios, 'TIOCOUTQ', 0x5411)
+
+TIOCM_zero_str = struct.pack('I', 0)
+TIOCM_RTS_str = struct.pack('I', TIOCM_RTS)
+TIOCM_DTR_str = struct.pack('I', TIOCM_DTR)
+
+TIOCSBRK = getattr(termios, 'TIOCSBRK', 0x5427)
+TIOCCBRK = getattr(termios, 'TIOCCBRK', 0x5428)
+
+
+class Serial(SerialBase, PlatformSpecific):
+    """\
+    Serial port class POSIX implementation. Serial port configuration is
+    done with termios and fcntl. Runs on Linux and many other Un*x like
+    systems.
+    """
+
+    def open(self):
+        """\
+        Open port with current settings. This may throw a SerialException
+        if the port cannot be opened."""
+        if self._port is None:
+            raise SerialException("Port must be configured before it can be used.")
+        if self.is_open:
+            raise SerialException("Port is already open.")
+        self.fd = None
+        # open
+        try:
+            self.fd = os.open(self.portstr, os.O_RDWR | os.O_NOCTTY | os.O_NONBLOCK)
+        except OSError as msg:
+            self.fd = None
+            raise SerialException(msg.errno, "could not open port {}: {}".format(self._port, msg))
+        #~ fcntl.fcntl(self.fd, fcntl.F_SETFL, 0)  # set blocking
+
+        self.pipe_abort_read_r, self.pipe_abort_read_w = None, None
+        self.pipe_abort_write_r, self.pipe_abort_write_w = None, None
+
+        try:
+            self._reconfigure_port(force_update=True)
+
+            try:
+                if not self._dsrdtr:
+                    self._update_dtr_state()
+                if not self._rtscts:
+                    self._update_rts_state()
+            except IOError as e:
+                # ignore Invalid argument and Inappropriate ioctl
+                if e.errno not in (errno.EINVAL, errno.ENOTTY):
+                    raise
+
+            self._reset_input_buffer()
+
+            self.pipe_abort_read_r, self.pipe_abort_read_w = os.pipe()
+            self.pipe_abort_write_r, self.pipe_abort_write_w = os.pipe()
+            fcntl.fcntl(self.pipe_abort_read_r, fcntl.F_SETFL, os.O_NONBLOCK)
+            fcntl.fcntl(self.pipe_abort_write_r, fcntl.F_SETFL, os.O_NONBLOCK)
+        except BaseException:
+            try:
+                os.close(self.fd)
+            except Exception:
+                # ignore any exception when closing the port
+                # also to keep original exception that happened when setting up
+                pass
+            self.fd = None
+
+            if self.pipe_abort_read_w is not None:
+                os.close(self.pipe_abort_read_w)
+                self.pipe_abort_read_w = None
+            if self.pipe_abort_read_r is not None:
+                os.close(self.pipe_abort_read_r)
+                self.pipe_abort_read_r = None
+            if self.pipe_abort_write_w is not None:
+                os.close(self.pipe_abort_write_w)
+                self.pipe_abort_write_w = None
+            if self.pipe_abort_write_r is not None:
+                os.close(self.pipe_abort_write_r)
+                self.pipe_abort_write_r = None
+
+            raise
+
+        self.is_open = True
+
+    def _reconfigure_port(self, force_update=False):
+        """Set communication parameters on opened port."""
+        if self.fd is None:
+            raise SerialException("Can only operate on a valid file descriptor")
+
+        # if exclusive lock is requested, create it before we modify anything else
+        if self._exclusive is not None:
+            if self._exclusive:
+                try:
+                    fcntl.flock(self.fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
+                except IOError as msg:
+                    raise SerialException(msg.errno, "Could not exclusively lock port {}: {}".format(self._port, msg))
+            else:
+                fcntl.flock(self.fd, fcntl.LOCK_UN)
+
+        custom_baud = None
+
+        vmin = vtime = 0                # timeout is done via select
+        if self._inter_byte_timeout is not None:
+            vmin = 1
+            vtime = int(self._inter_byte_timeout * 10)
+        try:
+            orig_attr = termios.tcgetattr(self.fd)
+            iflag, oflag, cflag, lflag, ispeed, ospeed, cc = orig_attr
+        except termios.error as msg:      # if a port is nonexistent but has a /dev file, it'll fail here
+            raise SerialException("Could not configure port: {}".format(msg))
+        # set up raw mode / no echo / binary
+        cflag |= (termios.CLOCAL | termios.CREAD)
+        lflag &= ~(termios.ICANON | termios.ECHO | termios.ECHOE |
+                   termios.ECHOK | termios.ECHONL |
+                   termios.ISIG | termios.IEXTEN)  # |termios.ECHOPRT
+        for flag in ('ECHOCTL', 'ECHOKE'):  # netbsd workaround for Erk
+            if hasattr(termios, flag):
+                lflag &= ~getattr(termios, flag)
+
+        oflag &= ~(termios.OPOST | termios.ONLCR | termios.OCRNL)
+        iflag &= ~(termios.INLCR | termios.IGNCR | termios.ICRNL | termios.IGNBRK)
+        if hasattr(termios, 'IUCLC'):
+            iflag &= ~termios.IUCLC
+        if hasattr(termios, 'PARMRK'):
+            iflag &= ~termios.PARMRK
+
+        # setup baud rate
+        try:
+            ispeed = ospeed = getattr(termios, 'B{}'.format(self._baudrate))
+        except AttributeError:
+            try:
+                ispeed = ospeed = self.BAUDRATE_CONSTANTS[self._baudrate]
+            except KeyError:
+                #~ raise ValueError('Invalid baud rate: %r' % self._baudrate)
+
+                # See if BOTHER is defined for this platform; if it is, use
+                # this for a speed not defined in the baudrate constants list.
+                try:
+                    ispeed = ospeed = BOTHER
+                except NameError:
+                    # may need custom baud rate, it isn't in our list.
+                    ispeed = ospeed = getattr(termios, 'B38400')
+
+                try:
+                    custom_baud = int(self._baudrate)  # store for later
+                except ValueError:
+                    raise ValueError('Invalid baud rate: {!r}'.format(self._baudrate))
+                else:
+                    if custom_baud < 0:
+                        raise ValueError('Invalid baud rate: {!r}'.format(self._baudrate))
+
+        # setup char len
+        cflag &= ~termios.CSIZE
+        if self._bytesize == 8:
+            cflag |= termios.CS8
+        elif self._bytesize == 7:
+            cflag |= termios.CS7
+        elif self._bytesize == 6:
+            cflag |= termios.CS6
+        elif self._bytesize == 5:
+            cflag |= termios.CS5
+        else:
+            raise ValueError('Invalid char len: {!r}'.format(self._bytesize))
+        # setup stop bits
+        if self._stopbits == serial.STOPBITS_ONE:
+            cflag &= ~(termios.CSTOPB)
+        elif self._stopbits == serial.STOPBITS_ONE_POINT_FIVE:
+            cflag |= (termios.CSTOPB)  # XXX same as TWO.. there is no POSIX support for 1.5
+        elif self._stopbits == serial.STOPBITS_TWO:
+            cflag |= (termios.CSTOPB)
+        else:
+            raise ValueError('Invalid stop bit specification: {!r}'.format(self._stopbits))
+        # setup parity
+        iflag &= ~(termios.INPCK | termios.ISTRIP)
+        if self._parity == serial.PARITY_NONE:
+            cflag &= ~(termios.PARENB | termios.PARODD | CMSPAR)
+        elif self._parity == serial.PARITY_EVEN:
+            cflag &= ~(termios.PARODD | CMSPAR)
+            cflag |= (termios.PARENB)
+        elif self._parity == serial.PARITY_ODD:
+            cflag &= ~CMSPAR
+            cflag |= (termios.PARENB | termios.PARODD)
+        elif self._parity == serial.PARITY_MARK and CMSPAR:
+            cflag |= (termios.PARENB | CMSPAR | termios.PARODD)
+        elif self._parity == serial.PARITY_SPACE and CMSPAR:
+            cflag |= (termios.PARENB | CMSPAR)
+            cflag &= ~(termios.PARODD)
+        else:
+            raise ValueError('Invalid parity: {!r}'.format(self._parity))
+        # setup flow control
+        # xonxoff
+        if hasattr(termios, 'IXANY'):
+            if self._xonxoff:
+                iflag |= (termios.IXON | termios.IXOFF)  # |termios.IXANY)
+            else:
+                iflag &= ~(termios.IXON | termios.IXOFF | termios.IXANY)
+        else:
+            if self._xonxoff:
+                iflag |= (termios.IXON | termios.IXOFF)
+            else:
+                iflag &= ~(termios.IXON | termios.IXOFF)
+        # rtscts
+        if hasattr(termios, 'CRTSCTS'):
+            if self._rtscts:
+                cflag |= (termios.CRTSCTS)
+            else:
+                cflag &= ~(termios.CRTSCTS)
+        elif hasattr(termios, 'CNEW_RTSCTS'):   # try it with alternate constant name
+            if self._rtscts:
+                cflag |= (termios.CNEW_RTSCTS)
+            else:
+                cflag &= ~(termios.CNEW_RTSCTS)
+        # XXX should there be a warning if setting up rtscts (and xonxoff etc) fails??
+
+        # buffer
+        # vmin "minimal number of characters to be read. 0 for non blocking"
+        if vmin < 0 or vmin > 255:
+            raise ValueError('Invalid vmin: {!r}'.format(vmin))
+        cc[termios.VMIN] = vmin
+        # vtime
+        if vtime < 0 or vtime > 255:
+            raise ValueError('Invalid vtime: {!r}'.format(vtime))
+        cc[termios.VTIME] = vtime
+        # activate settings
+        if force_update or [iflag, oflag, cflag, lflag, ispeed, ospeed, cc] != orig_attr:
+            termios.tcsetattr(
+                self.fd,
+                termios.TCSANOW,
+                [iflag, oflag, cflag, lflag, ispeed, ospeed, cc])
+
+        # apply custom baud rate, if any
+        if custom_baud is not None:
+            self._set_special_baudrate(custom_baud)
+
+        if self._rs485_mode is not None:
+            self._set_rs485_mode(self._rs485_mode)
+
+    def close(self):
+        """Close port"""
+        if self.is_open:
+            if self.fd is not None:
+                os.close(self.fd)
+                self.fd = None
+                os.close(self.pipe_abort_read_w)
+                os.close(self.pipe_abort_read_r)
+                os.close(self.pipe_abort_write_w)
+                os.close(self.pipe_abort_write_r)
+                self.pipe_abort_read_r, self.pipe_abort_read_w = None, None
+                self.pipe_abort_write_r, self.pipe_abort_write_w = None, None
+            self.is_open = False
+
+    #  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -
+
+    @property
+    def in_waiting(self):
+        """Return the number of bytes currently in the input buffer."""
+        #~ s = fcntl.ioctl(self.fd, termios.FIONREAD, TIOCM_zero_str)
+        s = fcntl.ioctl(self.fd, TIOCINQ, TIOCM_zero_str)
+        return struct.unpack('I', s)[0]
+
+    # select based implementation, proved to work on many systems
+    def read(self, size=1):
+        """\
+        Read size bytes from the serial port. If a timeout is set it may
+        return less characters as requested. With no timeout it will block
+        until the requested number of bytes is read.
+        """
+        if not self.is_open:
+            raise PortNotOpenError()
+        read = bytearray()
+        timeout = Timeout(self._timeout)
+        while len(read) < size:
+            try:
+                ready, _, _ = select.select([self.fd, self.pipe_abort_read_r], [], [], timeout.time_left())
+                if self.pipe_abort_read_r in ready:
+                    os.read(self.pipe_abort_read_r, 1000)
+                    break
+                # If select was used with a timeout, and the timeout occurs, it
+                # returns with empty lists -> thus abort read operation.
+                # For timeout == 0 (non-blocking operation) also abort when
+                # there is nothing to read.
+                if not ready:
+                    break   # timeout
+                buf = os.read(self.fd, size - len(read))
+            except OSError as e:
+                # this is for Python 3.x where select.error is a subclass of
+                # OSError ignore BlockingIOErrors and EINTR. other errors are shown
+                # https://www.python.org/dev/peps/pep-0475.
+                if e.errno not in (errno.EAGAIN, errno.EALREADY, errno.EWOULDBLOCK, errno.EINPROGRESS, errno.EINTR):
+                    raise SerialException('read failed: {}'.format(e))
+            except select.error as e:
+                # this is for Python 2.x
+                # ignore BlockingIOErrors and EINTR. all errors are shown
+                # see also http://www.python.org/dev/peps/pep-3151/#select
+                if e[0] not in (errno.EAGAIN, errno.EALREADY, errno.EWOULDBLOCK, errno.EINPROGRESS, errno.EINTR):
+                    raise SerialException('read failed: {}'.format(e))
+            else:
+                # read should always return some data as select reported it was
+                # ready to read when we get to this point.
+                if not buf:
+                    # Disconnected devices, at least on Linux, show the
+                    # behavior that they are always ready to read immediately
+                    # but reading returns nothing.
+                    raise SerialException(
+                        'device reports readiness to read but returned no data '
+                        '(device disconnected or multiple access on port?)')
+                read.extend(buf)
+
+            if timeout.expired():
+                break
+        return bytes(read)
+
+    def cancel_read(self):
+        if self.is_open:
+            os.write(self.pipe_abort_read_w, b"x")
+
+    def cancel_write(self):
+        if self.is_open:
+            os.write(self.pipe_abort_write_w, b"x")
+
+    def write(self, data):
+        """Output the given byte string over the serial port."""
+        if not self.is_open:
+            raise PortNotOpenError()
+        d = to_bytes(data)
+        tx_len = length = len(d)
+        timeout = Timeout(self._write_timeout)
+        while tx_len > 0:
+            try:
+                n = os.write(self.fd, d)
+                if timeout.is_non_blocking:
+                    # Zero timeout indicates non-blocking - simply return the
+                    # number of bytes of data actually written
+                    return n
+                elif not timeout.is_infinite:
+                    # when timeout is set, use select to wait for being ready
+                    # with the time left as timeout
+                    if timeout.expired():
+                        raise SerialTimeoutException('Write timeout')
+                    abort, ready, _ = select.select([self.pipe_abort_write_r], [self.fd], [], timeout.time_left())
+                    if abort:
+                        os.read(self.pipe_abort_write_r, 1000)
+                        break
+                    if not ready:
+                        raise SerialTimeoutException('Write timeout')
+                else:
+                    assert timeout.time_left() is None
+                    # wait for write operation
+                    abort, ready, _ = select.select([self.pipe_abort_write_r], [self.fd], [], None)
+                    if abort:
+                        os.read(self.pipe_abort_write_r, 1)
+                        break
+                    if not ready:
+                        raise SerialException('write failed (select)')
+                d = d[n:]
+                tx_len -= n
+            except SerialException:
+                raise
+            except OSError as e:
+                # this is for Python 3.x where select.error is a subclass of
+                # OSError ignore BlockingIOErrors and EINTR. other errors are shown
+                # https://www.python.org/dev/peps/pep-0475.
+                if e.errno not in (errno.EAGAIN, errno.EALREADY, errno.EWOULDBLOCK, errno.EINPROGRESS, errno.EINTR):
+                    raise SerialException('write failed: {}'.format(e))
+            except select.error as e:
+                # this is for Python 2.x
+                # ignore BlockingIOErrors and EINTR. all errors are shown
+                # see also http://www.python.org/dev/peps/pep-3151/#select
+                if e[0] not in (errno.EAGAIN, errno.EALREADY, errno.EWOULDBLOCK, errno.EINPROGRESS, errno.EINTR):
+                    raise SerialException('write failed: {}'.format(e))
+            if not timeout.is_non_blocking and timeout.expired():
+                raise SerialTimeoutException('Write timeout')
+        return length - len(d)
+
+    def flush(self):
+        """\
+        Flush of file like objects. In this case, wait until all data
+        is written.
+        """
+        if not self.is_open:
+            raise PortNotOpenError()
+        termios.tcdrain(self.fd)
+
+    def _reset_input_buffer(self):
+        """Clear input buffer, discarding all that is in the buffer."""
+        termios.tcflush(self.fd, termios.TCIFLUSH)
+
+    def reset_input_buffer(self):
+        """Clear input buffer, discarding all that is in the buffer."""
+        if not self.is_open:
+            raise PortNotOpenError()
+        self._reset_input_buffer()
+
+    def reset_output_buffer(self):
+        """\
+        Clear output buffer, aborting the current output and discarding all
+        that is in the buffer.
+        """
+        if not self.is_open:
+            raise PortNotOpenError()
+        termios.tcflush(self.fd, termios.TCOFLUSH)
+
+    def send_break(self, duration=0.25):
+        """\
+        Send break condition. Timed, returns to idle state after given
+        duration.
+        """
+        if not self.is_open:
+            raise PortNotOpenError()
+        termios.tcsendbreak(self.fd, int(duration / 0.25))
+
+    def _update_rts_state(self):
+        """Set terminal status line: Request To Send"""
+        if self._rts_state:
+            fcntl.ioctl(self.fd, TIOCMBIS, TIOCM_RTS_str)
+        else:
+            fcntl.ioctl(self.fd, TIOCMBIC, TIOCM_RTS_str)
+
+    def _update_dtr_state(self):
+        """Set terminal status line: Data Terminal Ready"""
+        if self._dtr_state:
+            fcntl.ioctl(self.fd, TIOCMBIS, TIOCM_DTR_str)
+        else:
+            fcntl.ioctl(self.fd, TIOCMBIC, TIOCM_DTR_str)
+
+    @property
+    def cts(self):
+        """Read terminal status line: Clear To Send"""
+        if not self.is_open:
+            raise PortNotOpenError()
+        s = fcntl.ioctl(self.fd, TIOCMGET, TIOCM_zero_str)
+        return struct.unpack('I', s)[0] & TIOCM_CTS != 0
+
+    @property
+    def dsr(self):
+        """Read terminal status line: Data Set Ready"""
+        if not self.is_open:
+            raise PortNotOpenError()
+        s = fcntl.ioctl(self.fd, TIOCMGET, TIOCM_zero_str)
+        return struct.unpack('I', s)[0] & TIOCM_DSR != 0
+
+    @property
+    def ri(self):
+        """Read terminal status line: Ring Indicator"""
+        if not self.is_open:
+            raise PortNotOpenError()
+        s = fcntl.ioctl(self.fd, TIOCMGET, TIOCM_zero_str)
+        return struct.unpack('I', s)[0] & TIOCM_RI != 0
+
+    @property
+    def cd(self):
+        """Read terminal status line: Carrier Detect"""
+        if not self.is_open:
+            raise PortNotOpenError()
+        s = fcntl.ioctl(self.fd, TIOCMGET, TIOCM_zero_str)
+        return struct.unpack('I', s)[0] & TIOCM_CD != 0
+
+    # - - platform specific - - - -
+
+    @property
+    def out_waiting(self):
+        """Return the number of bytes currently in the output buffer."""
+        #~ s = fcntl.ioctl(self.fd, termios.FIONREAD, TIOCM_zero_str)
+        s = fcntl.ioctl(self.fd, TIOCOUTQ, TIOCM_zero_str)
+        return struct.unpack('I', s)[0]
+
+    def fileno(self):
+        """\
+        For easier use of the serial port instance with select.
+        WARNING: this function is not portable to different platforms!
+        """
+        if not self.is_open:
+            raise PortNotOpenError()
+        return self.fd
+
+    def set_input_flow_control(self, enable=True):
+        """\
+        Manually control flow - when software flow control is enabled.
+        This will send XON (true) or XOFF (false) to the other device.
+        WARNING: this function is not portable to different platforms!
+        """
+        if not self.is_open:
+            raise PortNotOpenError()
+        if enable:
+            termios.tcflow(self.fd, termios.TCION)
+        else:
+            termios.tcflow(self.fd, termios.TCIOFF)
+
+    def set_output_flow_control(self, enable=True):
+        """\
+        Manually control flow of outgoing data - when hardware or software flow
+        control is enabled.
+        WARNING: this function is not portable to different platforms!
+        """
+        if not self.is_open:
+            raise PortNotOpenError()
+        if enable:
+            termios.tcflow(self.fd, termios.TCOON)
+        else:
+            termios.tcflow(self.fd, termios.TCOOFF)
+
+    def nonblocking(self):
+        """DEPRECATED - has no use"""
+        import warnings
+        warnings.warn("nonblocking() has no effect, already nonblocking", DeprecationWarning)
+
+
+class PosixPollSerial(Serial):
+    """\
+    Poll based read implementation. Not all systems support poll properly.
+    However this one has better handling of errors, such as a device
+    disconnecting while it's in use (e.g. USB-serial unplugged).
+    """
+
+    def read(self, size=1):
+        """\
+        Read size bytes from the serial port. If a timeout is set it may
+        return less characters as requested. With no timeout it will block
+        until the requested number of bytes is read.
+        """
+        if not self.is_open:
+            raise PortNotOpenError()
+        read = bytearray()
+        timeout = Timeout(self._timeout)
+        poll = select.poll()
+        poll.register(self.fd, select.POLLIN | select.POLLERR | select.POLLHUP | select.POLLNVAL)
+        poll.register(self.pipe_abort_read_r, select.POLLIN | select.POLLERR | select.POLLHUP | select.POLLNVAL)
+        if size > 0:
+            while len(read) < size:
+                # print "\tread(): size",size, "have", len(read)    #debug
+                # wait until device becomes ready to read (or something fails)
+                for fd, event in poll.poll(None if timeout.is_infinite else (timeout.time_left() * 1000)):
+                    if fd == self.pipe_abort_read_r:
+                        break
+                    if event & (select.POLLERR | select.POLLHUP | select.POLLNVAL):
+                        raise SerialException('device reports error (poll)')
+                    #  we don't care if it is select.POLLIN or timeout, that's
+                    #  handled below
+                if fd == self.pipe_abort_read_r:
+                    os.read(self.pipe_abort_read_r, 1000)
+                    break
+                buf = os.read(self.fd, size - len(read))
+                read.extend(buf)
+                if timeout.expired() \
+                        or (self._inter_byte_timeout is not None and self._inter_byte_timeout > 0) and not buf:
+                    break   # early abort on timeout
+        return bytes(read)
+
+
+class VTIMESerial(Serial):
+    """\
+    Implement timeout using vtime of tty device instead of using select.
+    This means that no inter character timeout can be specified and that
+    the error handling is degraded.
+
+    Overall timeout is disabled when inter-character timeout is used.
+
+    Note that this implementation does NOT support cancel_read(), it will
+    just ignore that.
+    """
+
+    def _reconfigure_port(self, force_update=True):
+        """Set communication parameters on opened port."""
+        super(VTIMESerial, self)._reconfigure_port()
+        fcntl.fcntl(self.fd, fcntl.F_SETFL, 0)  # clear O_NONBLOCK
+
+        if self._inter_byte_timeout is not None:
+            vmin = 1
+            vtime = int(self._inter_byte_timeout * 10)
+        elif self._timeout is None:
+            vmin = 1
+            vtime = 0
+        else:
+            vmin = 0
+            vtime = int(self._timeout * 10)
+        try:
+            orig_attr = termios.tcgetattr(self.fd)
+            iflag, oflag, cflag, lflag, ispeed, ospeed, cc = orig_attr
+        except termios.error as msg:      # if a port is nonexistent but has a /dev file, it'll fail here
+            raise serial.SerialException("Could not configure port: {}".format(msg))
+
+        if vtime < 0 or vtime > 255:
+            raise ValueError('Invalid vtime: {!r}'.format(vtime))
+        cc[termios.VTIME] = vtime
+        cc[termios.VMIN] = vmin
+
+        termios.tcsetattr(
+                self.fd,
+                termios.TCSANOW,
+                [iflag, oflag, cflag, lflag, ispeed, ospeed, cc])
+
+    def read(self, size=1):
+        """\
+        Read size bytes from the serial port. If a timeout is set it may
+        return less characters as requested. With no timeout it will block
+        until the requested number of bytes is read.
+        """
+        if not self.is_open:
+            raise PortNotOpenError()
+        read = bytearray()
+        while len(read) < size:
+            buf = os.read(self.fd, size - len(read))
+            if not buf:
+                break
+            read.extend(buf)
+        return bytes(read)
+
+    # hack to make hasattr return false
+    cancel_read = property()

tools/python/ampy/serial/serialutil.py

@@ -0,0 +1,697 @@
+#! python
+#
+# Base class and support functions used by various backends.
+#
+# This file is part of pySerial. https://github.com/pyserial/pyserial
+# (C) 2001-2020 Chris Liechti <cliechti@gmx.net>
+#
+# SPDX-License-Identifier:    BSD-3-Clause
+
+from __future__ import absolute_import
+
+import io
+import time
+
+# ``memoryview`` was introduced in Python 2.7 and ``bytes(some_memoryview)``
+# isn't returning the contents (very unfortunate). Therefore we need special
+# cases and test for it. Ensure that there is a ``memoryview`` object for older
+# Python versions. This is easier than making every test dependent on its
+# existence.
+try:
+    memoryview
+except (NameError, AttributeError):
+    # implementation does not matter as we do not really use it.
+    # it just must not inherit from something else we might care for.
+    class memoryview(object):   # pylint: disable=redefined-builtin,invalid-name
+        pass
+
+try:
+    unicode
+except (NameError, AttributeError):
+    unicode = str       # for Python 3, pylint: disable=redefined-builtin,invalid-name
+
+try:
+    basestring
+except (NameError, AttributeError):
+    basestring = (str,)    # for Python 3, pylint: disable=redefined-builtin,invalid-name
+
+
+# "for byte in data" fails for python3 as it returns ints instead of bytes
+def iterbytes(b):
+    """Iterate over bytes, returning bytes instead of ints (python3)"""
+    if isinstance(b, memoryview):
+        b = b.tobytes()
+    i = 0
+    while True:
+        a = b[i:i + 1]
+        i += 1
+        if a:
+            yield a
+        else:
+            break
+
+
+# all Python versions prior 3.x convert ``str([17])`` to '[17]' instead of '\x11'
+# so a simple ``bytes(sequence)`` doesn't work for all versions
+def to_bytes(seq):
+    """convert a sequence to a bytes type"""
+    if isinstance(seq, bytes):
+        return seq
+    elif isinstance(seq, bytearray):
+        return bytes(seq)
+    elif isinstance(seq, memoryview):
+        return seq.tobytes()
+    elif isinstance(seq, unicode):
+        raise TypeError('unicode strings are not supported, please encode to bytes: {!r}'.format(seq))
+    else:
+        # handle list of integers and bytes (one or more items) for Python 2 and 3
+        return bytes(bytearray(seq))
+
+
+# create control bytes
+XON = to_bytes([17])
+XOFF = to_bytes([19])
+
+CR = to_bytes([13])
+LF = to_bytes([10])
+
+
+PARITY_NONE, PARITY_EVEN, PARITY_ODD, PARITY_MARK, PARITY_SPACE = 'N', 'E', 'O', 'M', 'S'
+STOPBITS_ONE, STOPBITS_ONE_POINT_FIVE, STOPBITS_TWO = (1, 1.5, 2)
+FIVEBITS, SIXBITS, SEVENBITS, EIGHTBITS = (5, 6, 7, 8)
+
+PARITY_NAMES = {
+    PARITY_NONE: 'None',
+    PARITY_EVEN: 'Even',
+    PARITY_ODD: 'Odd',
+    PARITY_MARK: 'Mark',
+    PARITY_SPACE: 'Space',
+}
+
+
+class SerialException(IOError):
+    """Base class for serial port related exceptions."""
+
+
+class SerialTimeoutException(SerialException):
+    """Write timeouts give an exception"""
+
+
+class PortNotOpenError(SerialException):
+    """Port is not open"""
+    def __init__(self):
+        super(PortNotOpenError, self).__init__('Attempting to use a port that is not open')
+
+
+class Timeout(object):
+    """\
+    Abstraction for timeout operations. Using time.monotonic() if available
+    or time.time() in all other cases.
+
+    The class can also be initialized with 0 or None, in order to support
+    non-blocking and fully blocking I/O operations. The attributes
+    is_non_blocking and is_infinite are set accordingly.
+    """
+    if hasattr(time, 'monotonic'):
+        # Timeout implementation with time.monotonic(). This function is only
+        # supported by Python 3.3 and above. It returns a time in seconds
+        # (float) just as time.time(), but is not affected by system clock
+        # adjustments.
+        TIME = time.monotonic
+    else:
+        # Timeout implementation with time.time(). This is compatible with all
+        # Python versions but has issues if the clock is adjusted while the
+        # timeout is running.
+        TIME = time.time
+
+    def __init__(self, duration):
+        """Initialize a timeout with given duration"""
+        self.is_infinite = (duration is None)
+        self.is_non_blocking = (duration == 0)
+        self.duration = duration
+        if duration is not None:
+            self.target_time = self.TIME() + duration
+        else:
+            self.target_time = None
+
+    def expired(self):
+        """Return a boolean, telling if the timeout has expired"""
+        return self.target_time is not None and self.time_left() <= 0
+
+    def time_left(self):
+        """Return how many seconds are left until the timeout expires"""
+        if self.is_non_blocking:
+            return 0
+        elif self.is_infinite:
+            return None
+        else:
+            delta = self.target_time - self.TIME()
+            if delta > self.duration:
+                # clock jumped, recalculate
+                self.target_time = self.TIME() + self.duration
+                return self.duration
+            else:
+                return max(0, delta)
+
+    def restart(self, duration):
+        """\
+        Restart a timeout, only supported if a timeout was already set up
+        before.
+        """
+        self.duration = duration
+        self.target_time = self.TIME() + duration
+
+
+class SerialBase(io.RawIOBase):
+    """\
+    Serial port base class. Provides __init__ function and properties to
+    get/set port settings.
+    """
+
+    # default values, may be overridden in subclasses that do not support all values
+    BAUDRATES = (50, 75, 110, 134, 150, 200, 300, 600, 1200, 1800, 2400, 4800,
+                 9600, 19200, 38400, 57600, 115200, 230400, 460800, 500000,
+                 576000, 921600, 1000000, 1152000, 1500000, 2000000, 2500000,
+                 3000000, 3500000, 4000000)
+    BYTESIZES = (FIVEBITS, SIXBITS, SEVENBITS, EIGHTBITS)
+    PARITIES = (PARITY_NONE, PARITY_EVEN, PARITY_ODD, PARITY_MARK, PARITY_SPACE)
+    STOPBITS = (STOPBITS_ONE, STOPBITS_ONE_POINT_FIVE, STOPBITS_TWO)
+
+    def __init__(self,
+                 port=None,
+                 baudrate=9600,
+                 bytesize=EIGHTBITS,
+                 parity=PARITY_NONE,
+                 stopbits=STOPBITS_ONE,
+                 timeout=None,
+                 xonxoff=False,
+                 rtscts=False,
+                 write_timeout=None,
+                 dsrdtr=False,
+                 inter_byte_timeout=None,
+                 exclusive=None,
+                 **kwargs):
+        """\
+        Initialize comm port object. If a "port" is given, then the port will be
+        opened immediately. Otherwise a Serial port object in closed state
+        is returned.
+        """
+
+        self.is_open = False
+        self.portstr = None
+        self.name = None
+        # correct values are assigned below through properties
+        self._port = None
+        self._baudrate = None
+        self._bytesize = None
+        self._parity = None
+        self._stopbits = None
+        self._timeout = None
+        self._write_timeout = None
+        self._xonxoff = None
+        self._rtscts = None
+        self._dsrdtr = None
+        self._inter_byte_timeout = None
+        self._rs485_mode = None  # disabled by default
+        self._rts_state = True
+        self._dtr_state = True
+        self._break_state = False
+        self._exclusive = None
+
+        # assign values using get/set methods using the properties feature
+        self.port = port
+        self.baudrate = baudrate
+        self.bytesize = bytesize
+        self.parity = parity
+        self.stopbits = stopbits
+        self.timeout = timeout
+        self.write_timeout = write_timeout
+        self.xonxoff = xonxoff
+        self.rtscts = rtscts
+        self.dsrdtr = dsrdtr
+        self.inter_byte_timeout = inter_byte_timeout
+        self.exclusive = exclusive
+
+        # watch for backward compatible kwargs
+        if 'writeTimeout' in kwargs:
+            self.write_timeout = kwargs.pop('writeTimeout')
+        if 'interCharTimeout' in kwargs:
+            self.inter_byte_timeout = kwargs.pop('interCharTimeout')
+        if kwargs:
+            raise ValueError('unexpected keyword arguments: {!r}'.format(kwargs))
+
+        if port is not None:
+            self.open()
+
+    #  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -
+
+    # to be implemented by subclasses:
+    # def open(self):
+    # def close(self):
+
+    #  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -
+
+    @property
+    def port(self):
+        """\
+        Get the current port setting. The value that was passed on init or using
+        setPort() is passed back.
+        """
+        return self._port
+
+    @port.setter
+    def port(self, port):
+        """\
+        Change the port.
+        """
+        if port is not None and not isinstance(port, basestring):
+            raise ValueError('"port" must be None or a string, not {}'.format(type(port)))
+        was_open = self.is_open
+        if was_open:
+            self.close()
+        self.portstr = port
+        self._port = port
+        self.name = self.portstr
+        if was_open:
+            self.open()
+
+    @property
+    def baudrate(self):
+        """Get the current baud rate setting."""
+        return self._baudrate
+
+    @baudrate.setter
+    def baudrate(self, baudrate):
+        """\
+        Change baud rate. It raises a ValueError if the port is open and the
+        baud rate is not possible. If the port is closed, then the value is
+        accepted and the exception is raised when the port is opened.
+        """
+        try:
+            b = int(baudrate)
+        except TypeError:
+            raise ValueError("Not a valid baudrate: {!r}".format(baudrate))
+        else:
+            if b < 0:
+                raise ValueError("Not a valid baudrate: {!r}".format(baudrate))
+            self._baudrate = b
+            if self.is_open:
+                self._reconfigure_port()
+
+    @property
+    def bytesize(self):
+        """Get the current byte size setting."""
+        return self._bytesize
+
+    @bytesize.setter
+    def bytesize(self, bytesize):
+        """Change byte size."""
+        if bytesize not in self.BYTESIZES:
+            raise ValueError("Not a valid byte size: {!r}".format(bytesize))
+        self._bytesize = bytesize
+        if self.is_open:
+            self._reconfigure_port()
+
+    @property
+    def exclusive(self):
+        """Get the current exclusive access setting."""
+        return self._exclusive
+
+    @exclusive.setter
+    def exclusive(self, exclusive):
+        """Change the exclusive access setting."""
+        self._exclusive = exclusive
+        if self.is_open:
+            self._reconfigure_port()
+
+    @property
+    def parity(self):
+        """Get the current parity setting."""
+        return self._parity
+
+    @parity.setter
+    def parity(self, parity):
+        """Change parity setting."""
+        if parity not in self.PARITIES:
+            raise ValueError("Not a valid parity: {!r}".format(parity))
+        self._parity = parity
+        if self.is_open:
+            self._reconfigure_port()
+
+    @property
+    def stopbits(self):
+        """Get the current stop bits setting."""
+        return self._stopbits
+
+    @stopbits.setter
+    def stopbits(self, stopbits):
+        """Change stop bits size."""
+        if stopbits not in self.STOPBITS:
+            raise ValueError("Not a valid stop bit size: {!r}".format(stopbits))
+        self._stopbits = stopbits
+        if self.is_open:
+            self._reconfigure_port()
+
+    @property
+    def timeout(self):
+        """Get the current timeout setting."""
+        return self._timeout
+
+    @timeout.setter
+    def timeout(self, timeout):
+        """Change timeout setting."""
+        if timeout is not None:
+            try:
+                timeout + 1     # test if it's a number, will throw a TypeError if not...
+            except TypeError:
+                raise ValueError("Not a valid timeout: {!r}".format(timeout))
+            if timeout < 0:
+                raise ValueError("Not a valid timeout: {!r}".format(timeout))
+        self._timeout = timeout
+        if self.is_open:
+            self._reconfigure_port()
+
+    @property
+    def write_timeout(self):
+        """Get the current timeout setting."""
+        return self._write_timeout
+
+    @write_timeout.setter
+    def write_timeout(self, timeout):
+        """Change timeout setting."""
+        if timeout is not None:
+            if timeout < 0:
+                raise ValueError("Not a valid timeout: {!r}".format(timeout))
+            try:
+                timeout + 1     # test if it's a number, will throw a TypeError if not...
+            except TypeError:
+                raise ValueError("Not a valid timeout: {!r}".format(timeout))
+
+        self._write_timeout = timeout
+        if self.is_open:
+            self._reconfigure_port()
+
+    @property
+    def inter_byte_timeout(self):
+        """Get the current inter-character timeout setting."""
+        return self._inter_byte_timeout
+
+    @inter_byte_timeout.setter
+    def inter_byte_timeout(self, ic_timeout):
+        """Change inter-byte timeout setting."""
+        if ic_timeout is not None:
+            if ic_timeout < 0:
+                raise ValueError("Not a valid timeout: {!r}".format(ic_timeout))
+            try:
+                ic_timeout + 1     # test if it's a number, will throw a TypeError if not...
+            except TypeError:
+                raise ValueError("Not a valid timeout: {!r}".format(ic_timeout))
+
+        self._inter_byte_timeout = ic_timeout
+        if self.is_open:
+            self._reconfigure_port()
+
+    @property
+    def xonxoff(self):
+        """Get the current XON/XOFF setting."""
+        return self._xonxoff
+
+    @xonxoff.setter
+    def xonxoff(self, xonxoff):
+        """Change XON/XOFF setting."""
+        self._xonxoff = xonxoff
+        if self.is_open:
+            self._reconfigure_port()
+
+    @property
+    def rtscts(self):
+        """Get the current RTS/CTS flow control setting."""
+        return self._rtscts
+
+    @rtscts.setter
+    def rtscts(self, rtscts):
+        """Change RTS/CTS flow control setting."""
+        self._rtscts = rtscts
+        if self.is_open:
+            self._reconfigure_port()
+
+    @property
+    def dsrdtr(self):
+        """Get the current DSR/DTR flow control setting."""
+        return self._dsrdtr
+
+    @dsrdtr.setter
+    def dsrdtr(self, dsrdtr=None):
+        """Change DsrDtr flow control setting."""
+        if dsrdtr is None:
+            # if not set, keep backwards compatibility and follow rtscts setting
+            self._dsrdtr = self._rtscts
+        else:
+            # if defined independently, follow its value
+            self._dsrdtr = dsrdtr
+        if self.is_open:
+            self._reconfigure_port()
+
+    @property
+    def rts(self):
+        return self._rts_state
+
+    @rts.setter
+    def rts(self, value):
+        self._rts_state = value
+        if self.is_open:
+            self._update_rts_state()
+
+    @property
+    def dtr(self):
+        return self._dtr_state
+
+    @dtr.setter
+    def dtr(self, value):
+        self._dtr_state = value
+        if self.is_open:
+            self._update_dtr_state()
+
+    @property
+    def break_condition(self):
+        return self._break_state
+
+    @break_condition.setter
+    def break_condition(self, value):
+        self._break_state = value
+        if self.is_open:
+            self._update_break_state()
+
+    #  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -
+    # functions useful for RS-485 adapters
+
+    @property
+    def rs485_mode(self):
+        """\
+        Enable RS485 mode and apply new settings, set to None to disable.
+        See serial.rs485.RS485Settings for more info about the value.
+        """
+        return self._rs485_mode
+
+    @rs485_mode.setter
+    def rs485_mode(self, rs485_settings):
+        self._rs485_mode = rs485_settings
+        if self.is_open:
+            self._reconfigure_port()
+
+    #  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -
+
+    _SAVED_SETTINGS = ('baudrate', 'bytesize', 'parity', 'stopbits', 'xonxoff',
+                       'dsrdtr', 'rtscts', 'timeout', 'write_timeout',
+                       'inter_byte_timeout')
+
+    def get_settings(self):
+        """\
+        Get current port settings as a dictionary. For use with
+        apply_settings().
+        """
+        return dict([(key, getattr(self, '_' + key)) for key in self._SAVED_SETTINGS])
+
+    def apply_settings(self, d):
+        """\
+        Apply stored settings from a dictionary returned from
+        get_settings(). It's allowed to delete keys from the dictionary. These
+        values will simply left unchanged.
+        """
+        for key in self._SAVED_SETTINGS:
+            if key in d and d[key] != getattr(self, '_' + key):   # check against internal "_" value
+                setattr(self, key, d[key])          # set non "_" value to use properties write function
+
+    #  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -
+
+    def __repr__(self):
+        """String representation of the current port settings and its state."""
+        return '{name}<id=0x{id:x}, open={p.is_open}>(port={p.portstr!r}, ' \
+               'baudrate={p.baudrate!r}, bytesize={p.bytesize!r}, parity={p.parity!r}, ' \
+               'stopbits={p.stopbits!r}, timeout={p.timeout!r}, xonxoff={p.xonxoff!r}, ' \
+               'rtscts={p.rtscts!r}, dsrdtr={p.dsrdtr!r})'.format(
+                   name=self.__class__.__name__, id=id(self), p=self)
+
+    #  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -
+    # compatibility with io library
+    # pylint: disable=invalid-name,missing-docstring
+
+    def readable(self):
+        return True
+
+    def writable(self):
+        return True
+
+    def seekable(self):
+        return False
+
+    def readinto(self, b):
+        data = self.read(len(b))
+        n = len(data)
+        try:
+            b[:n] = data
+        except TypeError as err:
+            import array
+            if not isinstance(b, array.array):
+                raise err
+            b[:n] = array.array('b', data)
+        return n
+
+    #  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -
+    # context manager
+
+    def __enter__(self):
+        if self._port is not None and not self.is_open:
+            self.open()
+        return self
+
+    def __exit__(self, *args, **kwargs):
+        self.close()
+
+    #  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -
+
+    def send_break(self, duration=0.25):
+        """\
+        Send break condition. Timed, returns to idle state after given
+        duration.
+        """
+        if not self.is_open:
+            raise PortNotOpenError()
+        self.break_condition = True
+        time.sleep(duration)
+        self.break_condition = False
+
+    #  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -
+    # backwards compatibility / deprecated functions
+
+    def flushInput(self):
+        self.reset_input_buffer()
+
+    def flushOutput(self):
+        self.reset_output_buffer()
+
+    def inWaiting(self):
+        return self.in_waiting
+
+    def sendBreak(self, duration=0.25):
+        self.send_break(duration)
+
+    def setRTS(self, value=1):
+        self.rts = value
+
+    def setDTR(self, value=1):
+        self.dtr = value
+
+    def getCTS(self):
+        return self.cts
+
+    def getDSR(self):
+        return self.dsr
+
+    def getRI(self):
+        return self.ri
+
+    def getCD(self):
+        return self.cd
+
+    def setPort(self, port):
+        self.port = port
+
+    @property
+    def writeTimeout(self):
+        return self.write_timeout
+
+    @writeTimeout.setter
+    def writeTimeout(self, timeout):
+        self.write_timeout = timeout
+
+    @property
+    def interCharTimeout(self):
+        return self.inter_byte_timeout
+
+    @interCharTimeout.setter
+    def interCharTimeout(self, interCharTimeout):
+        self.inter_byte_timeout = interCharTimeout
+
+    def getSettingsDict(self):
+        return self.get_settings()
+
+    def applySettingsDict(self, d):
+        self.apply_settings(d)
+
+    def isOpen(self):
+        return self.is_open
+
+    #  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -
+    # additional functionality
+
+    def read_all(self):
+        """\
+        Read all bytes currently available in the buffer of the OS.
+        """
+        return self.read(self.in_waiting)
+
+    def read_until(self, expected=LF, size=None):
+        """\
+        Read until an expected sequence is found ('\n' by default), the size
+        is exceeded or until timeout occurs.
+        """
+        lenterm = len(expected)
+        line = bytearray()
+        timeout = Timeout(self._timeout)
+        while True:
+            c = self.read(1)
+            if c:
+                line += c
+                if line[-lenterm:] == expected:
+                    break
+                if size is not None and len(line) >= size:
+                    break
+            else:
+                break
+            if timeout.expired():
+                break
+        return bytes(line)
+
+    def iread_until(self, *args, **kwargs):
+        """\
+        Read lines, implemented as generator. It will raise StopIteration on
+        timeout (empty read).
+        """
+        while True:
+            line = self.read_until(*args, **kwargs)
+            if not line:
+                break
+            yield line
+
+
+#  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -
+if __name__ == '__main__':
+    import sys
+    s = SerialBase()
+    sys.stdout.write('port name:  {}\n'.format(s.name))
+    sys.stdout.write('baud rates: {}\n'.format(s.BAUDRATES))
+    sys.stdout.write('byte sizes: {}\n'.format(s.BYTESIZES))
+    sys.stdout.write('parities:   {}\n'.format(s.PARITIES))
+    sys.stdout.write('stop bits:  {}\n'.format(s.STOPBITS))
+    sys.stdout.write('{}\n'.format(s))

tools/python/ampy/serial/serialwin32.py

@@ -0,0 +1,477 @@
+#! python
+#
+# backend for Windows ("win32" incl. 32/64 bit support)
+#
+# (C) 2001-2020 Chris Liechti <cliechti@gmx.net>
+#
+# This file is part of pySerial. https://github.com/pyserial/pyserial
+# SPDX-License-Identifier:    BSD-3-Clause
+#
+# Initial patch to use ctypes by Giovanni Bajo <rasky@develer.com>
+
+from __future__ import absolute_import
+
+# pylint: disable=invalid-name,too-few-public-methods
+import ctypes
+import time
+from serial import win32
+
+import serial
+from serial.serialutil import SerialBase, SerialException, to_bytes, PortNotOpenError, SerialTimeoutException
+
+
+class Serial(SerialBase):
+    """Serial port implementation for Win32 based on ctypes."""
+
+    BAUDRATES = (50, 75, 110, 134, 150, 200, 300, 600, 1200, 1800, 2400, 4800,
+                 9600, 19200, 38400, 57600, 115200)
+
+    def __init__(self, *args, **kwargs):
+        self._port_handle = None
+        self._overlapped_read = None
+        self._overlapped_write = None
+        super(Serial, self).__init__(*args, **kwargs)
+
+    def open(self):
+        """\
+        Open port with current settings. This may throw a SerialException
+        if the port cannot be opened.
+        """
+        if self._port is None:
+            raise SerialException("Port must be configured before it can be used.")
+        if self.is_open:
+            raise SerialException("Port is already open.")
+        # the "\\.\COMx" format is required for devices other than COM1-COM8
+        # not all versions of windows seem to support this properly
+        # so that the first few ports are used with the DOS device name
+        port = self.name
+        try:
+            if port.upper().startswith('COM') and int(port[3:]) > 8:
+                port = '\\\\.\\' + port
+        except ValueError:
+            # for like COMnotanumber
+            pass
+        self._port_handle = win32.CreateFile(
+            port,
+            win32.GENERIC_READ | win32.GENERIC_WRITE,
+            0,  # exclusive access
+            None,  # no security
+            win32.OPEN_EXISTING,
+            win32.FILE_ATTRIBUTE_NORMAL | win32.FILE_FLAG_OVERLAPPED,
+            0)
+        if self._port_handle == win32.INVALID_HANDLE_VALUE:
+            self._port_handle = None    # 'cause __del__ is called anyway
+            raise SerialException("could not open port {!r}: {!r}".format(self.portstr, ctypes.WinError()))
+
+        try:
+            self._overlapped_read = win32.OVERLAPPED()
+            self._overlapped_read.hEvent = win32.CreateEvent(None, 1, 0, None)
+            self._overlapped_write = win32.OVERLAPPED()
+            #~ self._overlapped_write.hEvent = win32.CreateEvent(None, 1, 0, None)
+            self._overlapped_write.hEvent = win32.CreateEvent(None, 0, 0, None)
+
+            # Setup a 4k buffer
+            win32.SetupComm(self._port_handle, 4096, 4096)
+
+            # Save original timeout values:
+            self._orgTimeouts = win32.COMMTIMEOUTS()
+            win32.GetCommTimeouts(self._port_handle, ctypes.byref(self._orgTimeouts))
+
+            self._reconfigure_port()
+
+            # Clear buffers:
+            # Remove anything that was there
+            win32.PurgeComm(
+                self._port_handle,
+                win32.PURGE_TXCLEAR | win32.PURGE_TXABORT |
+                win32.PURGE_RXCLEAR | win32.PURGE_RXABORT)
+        except:
+            try:
+                self._close()
+            except:
+                # ignore any exception when closing the port
+                # also to keep original exception that happened when setting up
+                pass
+            self._port_handle = None
+            raise
+        else:
+            self.is_open = True
+
+    def _reconfigure_port(self):
+        """Set communication parameters on opened port."""
+        if not self._port_handle:
+            raise SerialException("Can only operate on a valid port handle")
+
+        # Set Windows timeout values
+        # timeouts is a tuple with the following items:
+        # (ReadIntervalTimeout,ReadTotalTimeoutMultiplier,
+        #  ReadTotalTimeoutConstant,WriteTotalTimeoutMultiplier,
+        #  WriteTotalTimeoutConstant)
+        timeouts = win32.COMMTIMEOUTS()
+        if self._timeout is None:
+            pass  # default of all zeros is OK
+        elif self._timeout == 0:
+            timeouts.ReadIntervalTimeout = win32.MAXDWORD
+        else:
+            timeouts.ReadTotalTimeoutConstant = max(int(self._timeout * 1000), 1)
+        if self._timeout != 0 and self._inter_byte_timeout is not None:
+            timeouts.ReadIntervalTimeout = max(int(self._inter_byte_timeout * 1000), 1)
+
+        if self._write_timeout is None:
+            pass
+        elif self._write_timeout == 0:
+            timeouts.WriteTotalTimeoutConstant = win32.MAXDWORD
+        else:
+            timeouts.WriteTotalTimeoutConstant = max(int(self._write_timeout * 1000), 1)
+        win32.SetCommTimeouts(self._port_handle, ctypes.byref(timeouts))
+
+        win32.SetCommMask(self._port_handle, win32.EV_ERR)
+
+        # Setup the connection info.
+        # Get state and modify it:
+        comDCB = win32.DCB()
+        win32.GetCommState(self._port_handle, ctypes.byref(comDCB))
+        comDCB.BaudRate = self._baudrate
+
+        if self._bytesize == serial.FIVEBITS:
+            comDCB.ByteSize = 5
+        elif self._bytesize == serial.SIXBITS:
+            comDCB.ByteSize = 6
+        elif self._bytesize == serial.SEVENBITS:
+            comDCB.ByteSize = 7
+        elif self._bytesize == serial.EIGHTBITS:
+            comDCB.ByteSize = 8
+        else:
+            raise ValueError("Unsupported number of data bits: {!r}".format(self._bytesize))
+
+        if self._parity == serial.PARITY_NONE:
+            comDCB.Parity = win32.NOPARITY
+            comDCB.fParity = 0  # Disable Parity Check
+        elif self._parity == serial.PARITY_EVEN:
+            comDCB.Parity = win32.EVENPARITY
+            comDCB.fParity = 1  # Enable Parity Check
+        elif self._parity == serial.PARITY_ODD:
+            comDCB.Parity = win32.ODDPARITY
+            comDCB.fParity = 1  # Enable Parity Check
+        elif self._parity == serial.PARITY_MARK:
+            comDCB.Parity = win32.MARKPARITY
+            comDCB.fParity = 1  # Enable Parity Check
+        elif self._parity == serial.PARITY_SPACE:
+            comDCB.Parity = win32.SPACEPARITY
+            comDCB.fParity = 1  # Enable Parity Check
+        else:
+            raise ValueError("Unsupported parity mode: {!r}".format(self._parity))
+
+        if self._stopbits == serial.STOPBITS_ONE:
+            comDCB.StopBits = win32.ONESTOPBIT
+        elif self._stopbits == serial.STOPBITS_ONE_POINT_FIVE:
+            comDCB.StopBits = win32.ONE5STOPBITS
+        elif self._stopbits == serial.STOPBITS_TWO:
+            comDCB.StopBits = win32.TWOSTOPBITS
+        else:
+            raise ValueError("Unsupported number of stop bits: {!r}".format(self._stopbits))
+
+        comDCB.fBinary = 1  # Enable Binary Transmission
+        # Char. w/ Parity-Err are replaced with 0xff (if fErrorChar is set to TRUE)
+        if self._rs485_mode is None:
+            if self._rtscts:
+                comDCB.fRtsControl = win32.RTS_CONTROL_HANDSHAKE
+            else:
+                comDCB.fRtsControl = win32.RTS_CONTROL_ENABLE if self._rts_state else win32.RTS_CONTROL_DISABLE
+            comDCB.fOutxCtsFlow = self._rtscts
+        else:
+            # checks for unsupported settings
+            # XXX verify if platform really does not have a setting for those
+            if not self._rs485_mode.rts_level_for_tx:
+                raise ValueError(
+                    'Unsupported value for RS485Settings.rts_level_for_tx: {!r} (only True is allowed)'.format(
+                        self._rs485_mode.rts_level_for_tx,))
+            if self._rs485_mode.rts_level_for_rx:
+                raise ValueError(
+                    'Unsupported value for RS485Settings.rts_level_for_rx: {!r} (only False is allowed)'.format(
+                        self._rs485_mode.rts_level_for_rx,))
+            if self._rs485_mode.delay_before_tx is not None:
+                raise ValueError(
+                    'Unsupported value for RS485Settings.delay_before_tx: {!r} (only None is allowed)'.format(
+                        self._rs485_mode.delay_before_tx,))
+            if self._rs485_mode.delay_before_rx is not None:
+                raise ValueError(
+                    'Unsupported value for RS485Settings.delay_before_rx: {!r} (only None is allowed)'.format(
+                        self._rs485_mode.delay_before_rx,))
+            if self._rs485_mode.loopback:
+                raise ValueError(
+                    'Unsupported value for RS485Settings.loopback: {!r} (only False is allowed)'.format(
+                        self._rs485_mode.loopback,))
+            comDCB.fRtsControl = win32.RTS_CONTROL_TOGGLE
+            comDCB.fOutxCtsFlow = 0
+
+        if self._dsrdtr:
+            comDCB.fDtrControl = win32.DTR_CONTROL_HANDSHAKE
+        else:
+            comDCB.fDtrControl = win32.DTR_CONTROL_ENABLE if self._dtr_state else win32.DTR_CONTROL_DISABLE
+        comDCB.fOutxDsrFlow = self._dsrdtr
+        comDCB.fOutX = self._xonxoff
+        comDCB.fInX = self._xonxoff
+        comDCB.fNull = 0
+        comDCB.fErrorChar = 0
+        comDCB.fAbortOnError = 0
+        comDCB.XonChar = serial.XON
+        comDCB.XoffChar = serial.XOFF
+
+        if not win32.SetCommState(self._port_handle, ctypes.byref(comDCB)):
+            raise SerialException(
+                'Cannot configure port, something went wrong. '
+                'Original message: {!r}'.format(ctypes.WinError()))
+
+    #~ def __del__(self):
+        #~ self.close()
+
+    def _close(self):
+        """internal close port helper"""
+        if self._port_handle is not None:
+            # Restore original timeout values:
+            win32.SetCommTimeouts(self._port_handle, self._orgTimeouts)
+            if self._overlapped_read is not None:
+                self.cancel_read()
+                win32.CloseHandle(self._overlapped_read.hEvent)
+                self._overlapped_read = None
+            if self._overlapped_write is not None:
+                self.cancel_write()
+                win32.CloseHandle(self._overlapped_write.hEvent)
+                self._overlapped_write = None
+            win32.CloseHandle(self._port_handle)
+            self._port_handle = None
+
+    def close(self):
+        """Close port"""
+        if self.is_open:
+            self._close()
+            self.is_open = False
+
+    #  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -
+
+    @property
+    def in_waiting(self):
+        """Return the number of bytes currently in the input buffer."""
+        flags = win32.DWORD()
+        comstat = win32.COMSTAT()
+        if not win32.ClearCommError(self._port_handle, ctypes.byref(flags), ctypes.byref(comstat)):
+            raise SerialException("ClearCommError failed ({!r})".format(ctypes.WinError()))
+        return comstat.cbInQue
+
+    def read(self, size=1):
+        """\
+        Read size bytes from the serial port. If a timeout is set it may
+        return less characters as requested. With no timeout it will block
+        until the requested number of bytes is read.
+        """
+        if not self.is_open:
+            raise PortNotOpenError()
+        if size > 0:
+            win32.ResetEvent(self._overlapped_read.hEvent)
+            flags = win32.DWORD()
+            comstat = win32.COMSTAT()
+            if not win32.ClearCommError(self._port_handle, ctypes.byref(flags), ctypes.byref(comstat)):
+                raise SerialException("ClearCommError failed ({!r})".format(ctypes.WinError()))
+            n = min(comstat.cbInQue, size) if self.timeout == 0 else size
+            if n > 0:
+                buf = ctypes.create_string_buffer(n)
+                rc = win32.DWORD()
+                read_ok = win32.ReadFile(
+                    self._port_handle,
+                    buf,
+                    n,
+                    ctypes.byref(rc),
+                    ctypes.byref(self._overlapped_read))
+                if not read_ok and win32.GetLastError() not in (win32.ERROR_SUCCESS, win32.ERROR_IO_PENDING):
+                    raise SerialException("ReadFile failed ({!r})".format(ctypes.WinError()))
+                result_ok = win32.GetOverlappedResult(
+                    self._port_handle,
+                    ctypes.byref(self._overlapped_read),
+                    ctypes.byref(rc),
+                    True)
+                if not result_ok:
+                    if win32.GetLastError() != win32.ERROR_OPERATION_ABORTED:
+                        raise SerialException("GetOverlappedResult failed ({!r})".format(ctypes.WinError()))
+                read = buf.raw[:rc.value]
+            else:
+                read = bytes()
+        else:
+            read = bytes()
+        return bytes(read)
+
+    def write(self, data):
+        """Output the given byte string over the serial port."""
+        if not self.is_open:
+            raise PortNotOpenError()
+        #~ if not isinstance(data, (bytes, bytearray)):
+            #~ raise TypeError('expected %s or bytearray, got %s' % (bytes, type(data)))
+        # convert data (needed in case of memoryview instance: Py 3.1 io lib), ctypes doesn't like memoryview
+        data = to_bytes(data)
+        if data:
+            #~ win32event.ResetEvent(self._overlapped_write.hEvent)
+            n = win32.DWORD()
+            success = win32.WriteFile(self._port_handle, data, len(data), ctypes.byref(n), self._overlapped_write)
+            if self._write_timeout != 0:  # if blocking (None) or w/ write timeout (>0)
+                if not success and win32.GetLastError() not in (win32.ERROR_SUCCESS, win32.ERROR_IO_PENDING):
+                    raise SerialException("WriteFile failed ({!r})".format(ctypes.WinError()))
+
+                # Wait for the write to complete.
+                #~ win32.WaitForSingleObject(self._overlapped_write.hEvent, win32.INFINITE)
+                win32.GetOverlappedResult(self._port_handle, self._overlapped_write, ctypes.byref(n), True)
+                if win32.GetLastError() == win32.ERROR_OPERATION_ABORTED:
+                    return n.value  # canceled IO is no error
+                if n.value != len(data):
+                    raise SerialTimeoutException('Write timeout')
+                return n.value
+            else:
+                errorcode = win32.ERROR_SUCCESS if success else win32.GetLastError()
+                if errorcode in (win32.ERROR_INVALID_USER_BUFFER, win32.ERROR_NOT_ENOUGH_MEMORY,
+                                 win32.ERROR_OPERATION_ABORTED):
+                    return 0
+                elif errorcode in (win32.ERROR_SUCCESS, win32.ERROR_IO_PENDING):
+                    # no info on true length provided by OS function in async mode
+                    return len(data)
+                else:
+                    raise SerialException("WriteFile failed ({!r})".format(ctypes.WinError()))
+        else:
+            return 0
+
+    def flush(self):
+        """\
+        Flush of file like objects. In this case, wait until all data
+        is written.
+        """
+        while self.out_waiting:
+            time.sleep(0.05)
+        # XXX could also use WaitCommEvent with mask EV_TXEMPTY, but it would
+        # require overlapped IO and it's also only possible to set a single mask
+        # on the port---
+
+    def reset_input_buffer(self):
+        """Clear input buffer, discarding all that is in the buffer."""
+        if not self.is_open:
+            raise PortNotOpenError()
+        win32.PurgeComm(self._port_handle, win32.PURGE_RXCLEAR | win32.PURGE_RXABORT)
+
+    def reset_output_buffer(self):
+        """\
+        Clear output buffer, aborting the current output and discarding all
+        that is in the buffer.
+        """
+        if not self.is_open:
+            raise PortNotOpenError()
+        win32.PurgeComm(self._port_handle, win32.PURGE_TXCLEAR | win32.PURGE_TXABORT)
+
+    def _update_break_state(self):
+        """Set break: Controls TXD. When active, to transmitting is possible."""
+        if not self.is_open:
+            raise PortNotOpenError()
+        if self._break_state:
+            win32.SetCommBreak(self._port_handle)
+        else:
+            win32.ClearCommBreak(self._port_handle)
+
+    def _update_rts_state(self):
+        """Set terminal status line: Request To Send"""
+        if self._rts_state:
+            win32.EscapeCommFunction(self._port_handle, win32.SETRTS)
+        else:
+            win32.EscapeCommFunction(self._port_handle, win32.CLRRTS)
+
+    def _update_dtr_state(self):
+        """Set terminal status line: Data Terminal Ready"""
+        if self._dtr_state:
+            win32.EscapeCommFunction(self._port_handle, win32.SETDTR)
+        else:
+            win32.EscapeCommFunction(self._port_handle, win32.CLRDTR)
+
+    def _GetCommModemStatus(self):
+        if not self.is_open:
+            raise PortNotOpenError()
+        stat = win32.DWORD()
+        win32.GetCommModemStatus(self._port_handle, ctypes.byref(stat))
+        return stat.value
+
+    @property
+    def cts(self):
+        """Read terminal status line: Clear To Send"""
+        return win32.MS_CTS_ON & self._GetCommModemStatus() != 0
+
+    @property
+    def dsr(self):
+        """Read terminal status line: Data Set Ready"""
+        return win32.MS_DSR_ON & self._GetCommModemStatus() != 0
+
+    @property
+    def ri(self):
+        """Read terminal status line: Ring Indicator"""
+        return win32.MS_RING_ON & self._GetCommModemStatus() != 0
+
+    @property
+    def cd(self):
+        """Read terminal status line: Carrier Detect"""
+        return win32.MS_RLSD_ON & self._GetCommModemStatus() != 0
+
+    # - - platform specific - - - -
+
+    def set_buffer_size(self, rx_size=4096, tx_size=None):
+        """\
+        Recommend a buffer size to the driver (device driver can ignore this
+        value). Must be called after the port is opened.
+        """
+        if tx_size is None:
+            tx_size = rx_size
+        win32.SetupComm(self._port_handle, rx_size, tx_size)
+
+    def set_output_flow_control(self, enable=True):
+        """\
+        Manually control flow - when software flow control is enabled.
+        This will do the same as if XON (true) or XOFF (false) are received
+        from the other device and control the transmission accordingly.
+        WARNING: this function is not portable to different platforms!
+        """
+        if not self.is_open:
+            raise PortNotOpenError()
+        if enable:
+            win32.EscapeCommFunction(self._port_handle, win32.SETXON)
+        else:
+            win32.EscapeCommFunction(self._port_handle, win32.SETXOFF)
+
+    @property
+    def out_waiting(self):
+        """Return how many bytes the in the outgoing buffer"""
+        flags = win32.DWORD()
+        comstat = win32.COMSTAT()
+        if not win32.ClearCommError(self._port_handle, ctypes.byref(flags), ctypes.byref(comstat)):
+            raise SerialException("ClearCommError failed ({!r})".format(ctypes.WinError()))
+        return comstat.cbOutQue
+
+    def _cancel_overlapped_io(self, overlapped):
+        """Cancel a blocking read operation, may be called from other thread"""
+        # check if read operation is pending
+        rc = win32.DWORD()
+        err = win32.GetOverlappedResult(
+            self._port_handle,
+            ctypes.byref(overlapped),
+            ctypes.byref(rc),
+            False)
+        if not err and win32.GetLastError() in (win32.ERROR_IO_PENDING, win32.ERROR_IO_INCOMPLETE):
+            # cancel, ignoring any errors (e.g. it may just have finished on its own)
+            win32.CancelIoEx(self._port_handle, overlapped)
+
+    def cancel_read(self):
+        """Cancel a blocking read operation, may be called from other thread"""
+        self._cancel_overlapped_io(self._overlapped_read)
+
+    def cancel_write(self):
+        """Cancel a blocking write operation, may be called from other thread"""
+        self._cancel_overlapped_io(self._overlapped_write)
+
+    @SerialBase.exclusive.setter
+    def exclusive(self, exclusive):
+        """Change the exclusive access setting."""
+        if exclusive is not None and not exclusive:
+            raise ValueError('win32 only supports exclusive access (not: {})'.format(exclusive))
+        else:
+            serial.SerialBase.exclusive.__set__(self, exclusive)

tools/python/ampy/serial/threaded/__init__.py

@@ -0,0 +1,297 @@
+#!/usr/bin/env python3
+#
+# Working with threading and pySerial
+#
+# This file is part of pySerial. https://github.com/pyserial/pyserial
+# (C) 2015-2016 Chris Liechti <cliechti@gmx.net>
+#
+# SPDX-License-Identifier:    BSD-3-Clause
+"""\
+Support threading with serial ports.
+"""
+from __future__ import absolute_import
+
+import serial
+import threading
+
+
+class Protocol(object):
+    """\
+    Protocol as used by the ReaderThread. This base class provides empty
+    implementations of all methods.
+    """
+
+    def connection_made(self, transport):
+        """Called when reader thread is started"""
+
+    def data_received(self, data):
+        """Called with snippets received from the serial port"""
+
+    def connection_lost(self, exc):
+        """\
+        Called when the serial port is closed or the reader loop terminated
+        otherwise.
+        """
+        if isinstance(exc, Exception):
+            raise exc
+
+
+class Packetizer(Protocol):
+    """
+    Read binary packets from serial port. Packets are expected to be terminated
+    with a TERMINATOR byte (null byte by default).
+
+    The class also keeps track of the transport.
+    """
+
+    TERMINATOR = b'\0'
+
+    def __init__(self):
+        self.buffer = bytearray()
+        self.transport = None
+
+    def connection_made(self, transport):
+        """Store transport"""
+        self.transport = transport
+
+    def connection_lost(self, exc):
+        """Forget transport"""
+        self.transport = None
+        super(Packetizer, self).connection_lost(exc)
+
+    def data_received(self, data):
+        """Buffer received data, find TERMINATOR, call handle_packet"""
+        self.buffer.extend(data)
+        while self.TERMINATOR in self.buffer:
+            packet, self.buffer = self.buffer.split(self.TERMINATOR, 1)
+            self.handle_packet(packet)
+
+    def handle_packet(self, packet):
+        """Process packets - to be overridden by subclassing"""
+        raise NotImplementedError('please implement functionality in handle_packet')
+
+
+class FramedPacket(Protocol):
+    """
+    Read binary packets. Packets are expected to have a start and stop marker.
+
+    The class also keeps track of the transport.
+    """
+
+    START = b'('
+    STOP = b')'
+
+    def __init__(self):
+        self.packet = bytearray()
+        self.in_packet = False
+        self.transport = None
+
+    def connection_made(self, transport):
+        """Store transport"""
+        self.transport = transport
+
+    def connection_lost(self, exc):
+        """Forget transport"""
+        self.transport = None
+        self.in_packet = False
+        del self.packet[:]
+        super(FramedPacket, self).connection_lost(exc)
+
+    def data_received(self, data):
+        """Find data enclosed in START/STOP, call handle_packet"""
+        for byte in serial.iterbytes(data):
+            if byte == self.START:
+                self.in_packet = True
+            elif byte == self.STOP:
+                self.in_packet = False
+                self.handle_packet(bytes(self.packet)) # make read-only copy
+                del self.packet[:]
+            elif self.in_packet:
+                self.packet.extend(byte)
+            else:
+                self.handle_out_of_packet_data(byte)
+
+    def handle_packet(self, packet):
+        """Process packets - to be overridden by subclassing"""
+        raise NotImplementedError('please implement functionality in handle_packet')
+
+    def handle_out_of_packet_data(self, data):
+        """Process data that is received outside of packets"""
+        pass
+
+
+class LineReader(Packetizer):
+    """
+    Read and write (Unicode) lines from/to serial port.
+    The encoding is applied.
+    """
+
+    TERMINATOR = b'\r\n'
+    ENCODING = 'utf-8'
+    UNICODE_HANDLING = 'replace'
+
+    def handle_packet(self, packet):
+        self.handle_line(packet.decode(self.ENCODING, self.UNICODE_HANDLING))
+
+    def handle_line(self, line):
+        """Process one line - to be overridden by subclassing"""
+        raise NotImplementedError('please implement functionality in handle_line')
+
+    def write_line(self, text):
+        """
+        Write text to the transport. ``text`` is a Unicode string and the encoding
+        is applied before sending ans also the newline is append.
+        """
+        # + is not the best choice but bytes does not support % or .format in py3 and we want a single write call
+        self.transport.write(text.encode(self.ENCODING, self.UNICODE_HANDLING) + self.TERMINATOR)
+
+
+class ReaderThread(threading.Thread):
+    """\
+    Implement a serial port read loop and dispatch to a Protocol instance (like
+    the asyncio.Protocol) but do it with threads.
+
+    Calls to close() will close the serial port but it is also possible to just
+    stop() this thread and continue the serial port instance otherwise.
+    """
+
+    def __init__(self, serial_instance, protocol_factory):
+        """\
+        Initialize thread.
+
+        Note that the serial_instance' timeout is set to one second!
+        Other settings are not changed.
+        """
+        super(ReaderThread, self).__init__()
+        self.daemon = True
+        self.serial = serial_instance
+        self.protocol_factory = protocol_factory
+        self.alive = True
+        self._lock = threading.Lock()
+        self._connection_made = threading.Event()
+        self.protocol = None
+
+    def stop(self):
+        """Stop the reader thread"""
+        self.alive = False
+        if hasattr(self.serial, 'cancel_read'):
+            self.serial.cancel_read()
+        self.join(2)
+
+    def run(self):
+        """Reader loop"""
+        if not hasattr(self.serial, 'cancel_read'):
+            self.serial.timeout = 1
+        self.protocol = self.protocol_factory()
+        try:
+            self.protocol.connection_made(self)
+        except Exception as e:
+            self.alive = False
+            self.protocol.connection_lost(e)
+            self._connection_made.set()
+            return
+        error = None
+        self._connection_made.set()
+        while self.alive and self.serial.is_open:
+            try:
+                # read all that is there or wait for one byte (blocking)
+                data = self.serial.read(self.serial.in_waiting or 1)
+            except serial.SerialException as e:
+                # probably some I/O problem such as disconnected USB serial
+                # adapters -> exit
+                error = e
+                break
+            else:
+                if data:
+                    # make a separated try-except for called user code
+                    try:
+                        self.protocol.data_received(data)
+                    except Exception as e:
+                        error = e
+                        break
+        self.alive = False
+        self.protocol.connection_lost(error)
+        self.protocol = None
+
+    def write(self, data):
+        """Thread safe writing (uses lock)"""
+        with self._lock:
+            return self.serial.write(data)
+
+    def close(self):
+        """Close the serial port and exit reader thread (uses lock)"""
+        # use the lock to let other threads finish writing
+        with self._lock:
+            # first stop reading, so that closing can be done on idle port
+            self.stop()
+            self.serial.close()
+
+    def connect(self):
+        """
+        Wait until connection is set up and return the transport and protocol
+        instances.
+        """
+        if self.alive:
+            self._connection_made.wait()
+            if not self.alive:
+                raise RuntimeError('connection_lost already called')
+            return (self, self.protocol)
+        else:
+            raise RuntimeError('already stopped')
+
+    # - -  context manager, returns protocol
+
+    def __enter__(self):
+        """\
+        Enter context handler. May raise RuntimeError in case the connection
+        could not be created.
+        """
+        self.start()
+        self._connection_made.wait()
+        if not self.alive:
+            raise RuntimeError('connection_lost already called')
+        return self.protocol
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        """Leave context: close port"""
+        self.close()
+
+
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+# test
+if __name__ == '__main__':
+    # pylint: disable=wrong-import-position
+    import sys
+    import time
+    import traceback
+
+    #~ PORT = 'spy:///dev/ttyUSB0'
+    PORT = 'loop://'
+
+    class PrintLines(LineReader):
+        def connection_made(self, transport):
+            super(PrintLines, self).connection_made(transport)
+            sys.stdout.write('port opened\n')
+            self.write_line('hello world')
+
+        def handle_line(self, data):
+            sys.stdout.write('line received: {!r}\n'.format(data))
+
+        def connection_lost(self, exc):
+            if exc:
+                traceback.print_exc(exc)
+            sys.stdout.write('port closed\n')
+
+    ser = serial.serial_for_url(PORT, baudrate=115200, timeout=1)
+    with ReaderThread(ser, PrintLines) as protocol:
+        protocol.write_line('hello')
+        time.sleep(2)
+
+    # alternative usage
+    ser = serial.serial_for_url(PORT, baudrate=115200, timeout=1)
+    t = ReaderThread(ser, PrintLines)
+    t.start()
+    transport, protocol = t.connect()
+    protocol.write_line('hello')
+    time.sleep(2)
+    t.close()

tools/python/ampy/serial/tools/hexlify_codec.py

@@ -0,0 +1,126 @@
+#! python
+#
+# This is a codec to create and decode hexdumps with spaces between characters. used by miniterm.
+#
+# This file is part of pySerial. https://github.com/pyserial/pyserial
+# (C) 2015-2016 Chris Liechti <cliechti@gmx.net>
+#
+# SPDX-License-Identifier:    BSD-3-Clause
+"""\
+Python 'hex' Codec - 2-digit hex with spaces content transfer encoding.
+
+Encode and decode may be a bit missleading at first sight...
+
+The textual representation is a hex dump: e.g. "40 41"
+The "encoded" data of this is the binary form, e.g. b"@A"
+
+Therefore decoding is binary to text and thus converting binary data to hex dump.
+
+"""
+
+from __future__ import absolute_import
+
+import codecs
+import serial
+
+
+try:
+    unicode
+except (NameError, AttributeError):
+    unicode = str       # for Python 3, pylint: disable=redefined-builtin,invalid-name
+
+
+HEXDIGITS = '0123456789ABCDEF'
+
+
+# Codec APIs
+
+def hex_encode(data, errors='strict'):
+    """'40 41 42' -> b'@ab'"""
+    return (serial.to_bytes([int(h, 16) for h in data.split()]), len(data))
+
+
+def hex_decode(data, errors='strict'):
+    """b'@ab' -> '40 41 42'"""
+    return (unicode(''.join('{:02X} '.format(ord(b)) for b in serial.iterbytes(data))), len(data))
+
+
+class Codec(codecs.Codec):
+    def encode(self, data, errors='strict'):
+        """'40 41 42' -> b'@ab'"""
+        return serial.to_bytes([int(h, 16) for h in data.split()])
+
+    def decode(self, data, errors='strict'):
+        """b'@ab' -> '40 41 42'"""
+        return unicode(''.join('{:02X} '.format(ord(b)) for b in serial.iterbytes(data)))
+
+
+class IncrementalEncoder(codecs.IncrementalEncoder):
+    """Incremental hex encoder"""
+
+    def __init__(self, errors='strict'):
+        self.errors = errors
+        self.state = 0
+
+    def reset(self):
+        self.state = 0
+
+    def getstate(self):
+        return self.state
+
+    def setstate(self, state):
+        self.state = state
+
+    def encode(self, data, final=False):
+        """\
+        Incremental encode, keep track of digits and emit a byte when a pair
+        of hex digits is found. The space is optional unless the error
+        handling is defined to be 'strict'.
+        """
+        state = self.state
+        encoded = []
+        for c in data.upper():
+            if c in HEXDIGITS:
+                z = HEXDIGITS.index(c)
+                if state:
+                    encoded.append(z + (state & 0xf0))
+                    state = 0
+                else:
+                    state = 0x100 + (z << 4)
+            elif c == ' ':      # allow spaces to separate values
+                if state and self.errors == 'strict':
+                    raise UnicodeError('odd number of hex digits')
+                state = 0
+            else:
+                if self.errors == 'strict':
+                    raise UnicodeError('non-hex digit found: {!r}'.format(c))
+        self.state = state
+        return serial.to_bytes(encoded)
+
+
+class IncrementalDecoder(codecs.IncrementalDecoder):
+    """Incremental decoder"""
+    def decode(self, data, final=False):
+        return unicode(''.join('{:02X} '.format(ord(b)) for b in serial.iterbytes(data)))
+
+
+class StreamWriter(Codec, codecs.StreamWriter):
+    """Combination of hexlify codec and StreamWriter"""
+
+
+class StreamReader(Codec, codecs.StreamReader):
+    """Combination of hexlify codec and StreamReader"""
+
+
+def getregentry():
+    """encodings module API"""
+    return codecs.CodecInfo(
+        name='hexlify',
+        encode=hex_encode,
+        decode=hex_decode,
+        incrementalencoder=IncrementalEncoder,
+        incrementaldecoder=IncrementalDecoder,
+        streamwriter=StreamWriter,
+        streamreader=StreamReader,
+        #~ _is_text_encoding=True,
+    )

tools/python/ampy/serial/tools/list_ports.py

@@ -0,0 +1,110 @@
+#!/usr/bin/env python
+#
+# Serial port enumeration. Console tool and backend selection.
+#
+# This file is part of pySerial. https://github.com/pyserial/pyserial
+# (C) 2011-2015 Chris Liechti <cliechti@gmx.net>
+#
+# SPDX-License-Identifier:    BSD-3-Clause
+
+"""\
+This module will provide a function called comports that returns an
+iterable (generator or list) that will enumerate available com ports. Note that
+on some systems non-existent ports may be listed.
+
+Additionally a grep function is supplied that can be used to search for ports
+based on their descriptions or hardware ID.
+"""
+
+from __future__ import absolute_import
+
+import sys
+import os
+import re
+
+# chose an implementation, depending on os
+#~ if sys.platform == 'cli':
+#~ else:
+if os.name == 'nt':  # sys.platform == 'win32':
+    from serial.tools.list_ports_windows import comports
+elif os.name == 'posix':
+    from serial.tools.list_ports_posix import comports
+#~ elif os.name == 'java':
+else:
+    raise ImportError("Sorry: no implementation for your platform ('{}') available".format(os.name))
+
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+
+
+def grep(regexp, include_links=False):
+    """\
+    Search for ports using a regular expression. Port name, description and
+    hardware ID are searched. The function returns an iterable that returns the
+    same tuples as comport() would do.
+    """
+    r = re.compile(regexp, re.I)
+    for info in comports(include_links):
+        port, desc, hwid = info
+        if r.search(port) or r.search(desc) or r.search(hwid):
+            yield info
+
+
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+def main():
+    import argparse
+
+    parser = argparse.ArgumentParser(description='Serial port enumeration')
+
+    parser.add_argument(
+        'regexp',
+        nargs='?',
+        help='only show ports that match this regex')
+
+    parser.add_argument(
+        '-v', '--verbose',
+        action='store_true',
+        help='show more messages')
+
+    parser.add_argument(
+        '-q', '--quiet',
+        action='store_true',
+        help='suppress all messages')
+
+    parser.add_argument(
+        '-n',
+        type=int,
+        help='only output the N-th entry')
+
+    parser.add_argument(
+        '-s', '--include-links',
+        action='store_true',
+        help='include entries that are symlinks to real devices')
+
+    args = parser.parse_args()
+
+    hits = 0
+    # get iteraror w/ or w/o filter
+    if args.regexp:
+        if not args.quiet:
+            sys.stderr.write("Filtered list with regexp: {!r}\n".format(args.regexp))
+        iterator = sorted(grep(args.regexp, include_links=args.include_links))
+    else:
+        iterator = sorted(comports(include_links=args.include_links))
+    # list them
+    for n, (port, desc, hwid) in enumerate(iterator, 1):
+        if args.n is None or args.n == n:
+            sys.stdout.write("{:20}\n".format(port))
+            if args.verbose:
+                sys.stdout.write("    desc: {}\n".format(desc))
+                sys.stdout.write("    hwid: {}\n".format(hwid))
+        hits += 1
+    if not args.quiet:
+        if hits:
+            sys.stderr.write("{} ports found\n".format(hits))
+        else:
+            sys.stderr.write("no ports found\n")
+
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+# test
+if __name__ == '__main__':
+    main()

tools/python/ampy/serial/tools/list_ports_common.py

@@ -0,0 +1,121 @@
+#!/usr/bin/env python
+#
+# This is a helper module for the various platform dependent list_port
+# implementations.
+#
+# This file is part of pySerial. https://github.com/pyserial/pyserial
+# (C) 2015 Chris Liechti <cliechti@gmx.net>
+#
+# SPDX-License-Identifier:    BSD-3-Clause
+
+from __future__ import absolute_import
+
+import re
+import glob
+import os
+import os.path
+
+
+def numsplit(text):
+    """\
+    Convert string into a list of texts and numbers in order to support a
+    natural sorting.
+    """
+    result = []
+    for group in re.split(r'(\d+)', text):
+        if group:
+            try:
+                group = int(group)
+            except ValueError:
+                pass
+            result.append(group)
+    return result
+
+
+class ListPortInfo(object):
+    """Info collection base class for serial ports"""
+
+    def __init__(self, device, skip_link_detection=False):
+        self.device = device
+        self.name = os.path.basename(device)
+        self.description = 'n/a'
+        self.hwid = 'n/a'
+        # USB specific data
+        self.vid = None
+        self.pid = None
+        self.serial_number = None
+        self.location = None
+        self.manufacturer = None
+        self.product = None
+        self.interface = None
+        # special handling for links
+        if not skip_link_detection and device is not None and os.path.islink(device):
+            self.hwid = 'LINK={}'.format(os.path.realpath(device))
+
+    def usb_description(self):
+        """return a short string to name the port based on USB info"""
+        if self.interface is not None:
+            return '{} - {}'.format(self.product, self.interface)
+        elif self.product is not None:
+            return self.product
+        else:
+            return self.name
+
+    def usb_info(self):
+        """return a string with USB related information about device"""
+        return 'USB VID:PID={:04X}:{:04X}{}{}'.format(
+            self.vid or 0,
+            self.pid or 0,
+            ' SER={}'.format(self.serial_number) if self.serial_number is not None else '',
+            ' LOCATION={}'.format(self.location) if self.location is not None else '')
+
+    def apply_usb_info(self):
+        """update description and hwid from USB data"""
+        self.description = self.usb_description()
+        self.hwid = self.usb_info()
+
+    def __eq__(self, other):
+        return isinstance(other, ListPortInfo) and self.device == other.device
+
+    def __hash__(self):
+        return hash(self.device)
+
+    def __lt__(self, other):
+        if not isinstance(other, ListPortInfo):
+            raise TypeError('unorderable types: {}() and {}()'.format(
+                type(self).__name__,
+                type(other).__name__))
+        return numsplit(self.device) < numsplit(other.device)
+
+    def __str__(self):
+        return '{} - {}'.format(self.device, self.description)
+
+    def __getitem__(self, index):
+        """Item access: backwards compatible -> (port, desc, hwid)"""
+        if index == 0:
+            return self.device
+        elif index == 1:
+            return self.description
+        elif index == 2:
+            return self.hwid
+        else:
+            raise IndexError('{} > 2'.format(index))
+
+
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+def list_links(devices):
+    """\
+    search all /dev devices and look for symlinks to known ports already
+    listed in devices.
+    """
+    links = []
+    for device in glob.glob('/dev/*'):
+        if os.path.islink(device) and os.path.realpath(device) in devices:
+            links.append(device)
+    return links
+
+
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+# test
+if __name__ == '__main__':
+    print(ListPortInfo('dummy'))

tools/python/ampy/serial/tools/list_ports_linux.py

@@ -0,0 +1,109 @@
+#!/usr/bin/env python
+#
+# This is a module that gathers a list of serial ports including details on
+# GNU/Linux systems.
+#
+# This file is part of pySerial. https://github.com/pyserial/pyserial
+# (C) 2011-2015 Chris Liechti <cliechti@gmx.net>
+#
+# SPDX-License-Identifier:    BSD-3-Clause
+
+from __future__ import absolute_import
+
+import glob
+import os
+from serial.tools import list_ports_common
+
+
+class SysFS(list_ports_common.ListPortInfo):
+    """Wrapper for easy sysfs access and device info"""
+
+    def __init__(self, device):
+        super(SysFS, self).__init__(device)
+        # special handling for links
+        if device is not None and os.path.islink(device):
+            device = os.path.realpath(device)
+            is_link = True
+        else:
+            is_link = False
+        self.usb_device_path = None
+        if os.path.exists('/sys/class/tty/{}/device'.format(self.name)):
+            self.device_path = os.path.realpath('/sys/class/tty/{}/device'.format(self.name))
+            self.subsystem = os.path.basename(os.path.realpath(os.path.join(self.device_path, 'subsystem')))
+        else:
+            self.device_path = None
+            self.subsystem = None
+        # check device type
+        if self.subsystem == 'usb-serial':
+            self.usb_interface_path = os.path.dirname(self.device_path)
+        elif self.subsystem == 'usb':
+            self.usb_interface_path = self.device_path
+        else:
+            self.usb_interface_path = None
+        # fill-in info for USB devices
+        if self.usb_interface_path is not None:
+            self.usb_device_path = os.path.dirname(self.usb_interface_path)
+
+            try:
+                num_if = int(self.read_line(self.usb_device_path, 'bNumInterfaces'))
+            except ValueError:
+                num_if = 1
+
+            self.vid = int(self.read_line(self.usb_device_path, 'idVendor'), 16)
+            self.pid = int(self.read_line(self.usb_device_path, 'idProduct'), 16)
+            self.serial_number = self.read_line(self.usb_device_path, 'serial')
+            if num_if > 1:  # multi interface devices like FT4232
+                self.location = os.path.basename(self.usb_interface_path)
+            else:
+                self.location = os.path.basename(self.usb_device_path)
+
+            self.manufacturer = self.read_line(self.usb_device_path, 'manufacturer')
+            self.product = self.read_line(self.usb_device_path, 'product')
+            self.interface = self.read_line(self.usb_interface_path, 'interface')
+
+        if self.subsystem in ('usb', 'usb-serial'):
+            self.apply_usb_info()
+        #~ elif self.subsystem in ('pnp', 'amba'):  # PCI based devices, raspi
+        elif self.subsystem == 'pnp':  # PCI based devices
+            self.description = self.name
+            self.hwid = self.read_line(self.device_path, 'id')
+        elif self.subsystem == 'amba':  # raspi
+            self.description = self.name
+            self.hwid = os.path.basename(self.device_path)
+
+        if is_link:
+            self.hwid += ' LINK={}'.format(device)
+
+    def read_line(self, *args):
+        """\
+        Helper function to read a single line from a file.
+        One or more parameters are allowed, they are joined with os.path.join.
+        Returns None on errors..
+        """
+        try:
+            with open(os.path.join(*args)) as f:
+                line = f.readline().strip()
+            return line
+        except IOError:
+            return None
+
+
+def comports(include_links=False):
+    devices = glob.glob('/dev/ttyS*')           # built-in serial ports
+    devices.extend(glob.glob('/dev/ttyUSB*'))   # usb-serial with own driver
+    devices.extend(glob.glob('/dev/ttyXRUSB*')) # xr-usb-serial port exar (DELL Edge 3001)
+    devices.extend(glob.glob('/dev/ttyACM*'))   # usb-serial with CDC-ACM profile
+    devices.extend(glob.glob('/dev/ttyAMA*'))   # ARM internal port (raspi)
+    devices.extend(glob.glob('/dev/rfcomm*'))   # BT serial devices
+    devices.extend(glob.glob('/dev/ttyAP*'))    # Advantech multi-port serial controllers
+    if include_links:
+        devices.extend(list_ports_common.list_links(devices))
+    return [info
+            for info in [SysFS(d) for d in devices]
+            if info.subsystem != "platform"]    # hide non-present internal serial ports
+
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+# test
+if __name__ == '__main__':
+    for info in sorted(comports()):
+        print("{0}: {0.subsystem}".format(info))

tools/python/ampy/serial/tools/list_ports_osx.py

@@ -0,0 +1,299 @@
+#!/usr/bin/env python
+#
+# This is a module that gathers a list of serial ports including details on OSX
+#
+# code originally from https://github.com/makerbot/pyserial/tree/master/serial/tools
+# with contributions from cibomahto, dgs3, FarMcKon, tedbrandston
+# and modifications by cliechti, hoihu, hardkrash
+#
+# This file is part of pySerial. https://github.com/pyserial/pyserial
+# (C) 2013-2020
+#
+# SPDX-License-Identifier:    BSD-3-Clause
+
+
+# List all of the callout devices in OS/X by querying IOKit.
+
+# See the following for a reference of how to do this:
+# http://developer.apple.com/library/mac/#documentation/DeviceDrivers/Conceptual/WorkingWSerial/WWSerial_SerialDevs/SerialDevices.html#//apple_ref/doc/uid/TP30000384-CIHGEAFD
+
+# More help from darwin_hid.py
+
+# Also see the 'IORegistryExplorer' for an idea of what we are actually searching
+
+from __future__ import absolute_import
+
+import ctypes
+
+from serial.tools import list_ports_common
+
+iokit = ctypes.cdll.LoadLibrary('/System/Library/Frameworks/IOKit.framework/IOKit')
+cf = ctypes.cdll.LoadLibrary('/System/Library/Frameworks/CoreFoundation.framework/CoreFoundation')
+
+# kIOMasterPortDefault is no longer exported in BigSur but no biggie, using NULL works just the same
+kIOMasterPortDefault = 0 # WAS: ctypes.c_void_p.in_dll(iokit, "kIOMasterPortDefault")
+kCFAllocatorDefault = ctypes.c_void_p.in_dll(cf, "kCFAllocatorDefault")
+
+kCFStringEncodingMacRoman = 0
+kCFStringEncodingUTF8 = 0x08000100
+
+# defined in `IOKit/usb/USBSpec.h`
+kUSBVendorString = 'USB Vendor Name'
+kUSBSerialNumberString = 'USB Serial Number'
+
+# `io_name_t` defined as `typedef char io_name_t[128];`
+# in `device/device_types.h`
+io_name_size = 128
+
+# defined in `mach/kern_return.h`
+KERN_SUCCESS = 0
+# kern_return_t defined as `typedef int kern_return_t;` in `mach/i386/kern_return.h`
+kern_return_t = ctypes.c_int
+
+iokit.IOServiceMatching.restype = ctypes.c_void_p
+
+iokit.IOServiceGetMatchingServices.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p]
+iokit.IOServiceGetMatchingServices.restype = kern_return_t
+
+iokit.IORegistryEntryGetParentEntry.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p]
+iokit.IOServiceGetMatchingServices.restype = kern_return_t
+
+iokit.IORegistryEntryCreateCFProperty.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p, ctypes.c_uint32]
+iokit.IORegistryEntryCreateCFProperty.restype = ctypes.c_void_p
+
+iokit.IORegistryEntryGetPath.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p]
+iokit.IORegistryEntryGetPath.restype = kern_return_t
+
+iokit.IORegistryEntryGetName.argtypes = [ctypes.c_void_p, ctypes.c_void_p]
+iokit.IORegistryEntryGetName.restype = kern_return_t
+
+iokit.IOObjectGetClass.argtypes = [ctypes.c_void_p, ctypes.c_void_p]
+iokit.IOObjectGetClass.restype = kern_return_t
+
+iokit.IOObjectRelease.argtypes = [ctypes.c_void_p]
+
+
+cf.CFStringCreateWithCString.argtypes = [ctypes.c_void_p, ctypes.c_char_p, ctypes.c_int32]
+cf.CFStringCreateWithCString.restype = ctypes.c_void_p
+
+cf.CFStringGetCStringPtr.argtypes = [ctypes.c_void_p, ctypes.c_uint32]
+cf.CFStringGetCStringPtr.restype = ctypes.c_char_p
+
+cf.CFStringGetCString.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_long, ctypes.c_uint32]
+cf.CFStringGetCString.restype = ctypes.c_bool
+
+cf.CFNumberGetValue.argtypes = [ctypes.c_void_p, ctypes.c_uint32, ctypes.c_void_p]
+cf.CFNumberGetValue.restype = ctypes.c_void_p
+
+# void CFRelease ( CFTypeRef cf );
+cf.CFRelease.argtypes = [ctypes.c_void_p]
+cf.CFRelease.restype = None
+
+# CFNumber type defines
+kCFNumberSInt8Type = 1
+kCFNumberSInt16Type = 2
+kCFNumberSInt32Type = 3
+kCFNumberSInt64Type = 4
+
+
+def get_string_property(device_type, property):
+    """
+    Search the given device for the specified string property
+
+    @param device_type Type of Device
+    @param property String to search for
+    @return Python string containing the value, or None if not found.
+    """
+    key = cf.CFStringCreateWithCString(
+            kCFAllocatorDefault,
+            property.encode("utf-8"),
+            kCFStringEncodingUTF8)
+
+    CFContainer = iokit.IORegistryEntryCreateCFProperty(
+            device_type,
+            key,
+            kCFAllocatorDefault,
+            0)
+    output = None
+
+    if CFContainer:
+        output = cf.CFStringGetCStringPtr(CFContainer, 0)
+        if output is not None:
+            output = output.decode('utf-8')
+        else:
+            buffer = ctypes.create_string_buffer(io_name_size);
+            success = cf.CFStringGetCString(CFContainer, ctypes.byref(buffer), io_name_size, kCFStringEncodingUTF8)
+            if success:
+                output = buffer.value.decode('utf-8')
+        cf.CFRelease(CFContainer)
+    return output
+
+
+def get_int_property(device_type, property, cf_number_type):
+    """
+    Search the given device for the specified string property
+
+    @param device_type Device to search
+    @param property String to search for
+    @param cf_number_type CFType number
+
+    @return Python string containing the value, or None if not found.
+    """
+    key = cf.CFStringCreateWithCString(
+            kCFAllocatorDefault,
+            property.encode("utf-8"),
+            kCFStringEncodingUTF8)
+
+    CFContainer = iokit.IORegistryEntryCreateCFProperty(
+            device_type,
+            key,
+            kCFAllocatorDefault,
+            0)
+
+    if CFContainer:
+        if (cf_number_type == kCFNumberSInt32Type):
+            number = ctypes.c_uint32()
+        elif (cf_number_type == kCFNumberSInt16Type):
+            number = ctypes.c_uint16()
+        cf.CFNumberGetValue(CFContainer, cf_number_type, ctypes.byref(number))
+        cf.CFRelease(CFContainer)
+        return number.value
+    return None
+
+def IORegistryEntryGetName(device):
+    devicename = ctypes.create_string_buffer(io_name_size);
+    res = iokit.IORegistryEntryGetName(device, ctypes.byref(devicename))
+    if res != KERN_SUCCESS:
+        return None
+    # this works in python2 but may not be valid. Also I don't know if
+    # this encoding is guaranteed. It may be dependent on system locale.
+    return devicename.value.decode('utf-8')
+
+def IOObjectGetClass(device):
+    classname = ctypes.create_string_buffer(io_name_size)
+    iokit.IOObjectGetClass(device, ctypes.byref(classname))
+    return classname.value
+
+def GetParentDeviceByType(device, parent_type):
+    """ Find the first parent of a device that implements the parent_type
+        @param IOService Service to inspect
+        @return Pointer to the parent type, or None if it was not found.
+    """
+    # First, try to walk up the IOService tree to find a parent of this device that is a IOUSBDevice.
+    parent_type = parent_type.encode('utf-8')
+    while IOObjectGetClass(device) != parent_type:
+        parent = ctypes.c_void_p()
+        response = iokit.IORegistryEntryGetParentEntry(
+                device,
+                "IOService".encode("utf-8"),
+                ctypes.byref(parent))
+        # If we weren't able to find a parent for the device, we're done.
+        if response != KERN_SUCCESS:
+            return None
+        device = parent
+    return device
+
+
+def GetIOServicesByType(service_type):
+    """
+    returns iterator over specified service_type
+    """
+    serial_port_iterator = ctypes.c_void_p()
+
+    iokit.IOServiceGetMatchingServices(
+            kIOMasterPortDefault,
+            iokit.IOServiceMatching(service_type.encode('utf-8')),
+            ctypes.byref(serial_port_iterator))
+
+    services = []
+    while iokit.IOIteratorIsValid(serial_port_iterator):
+        service = iokit.IOIteratorNext(serial_port_iterator)
+        if not service:
+            break
+        services.append(service)
+    iokit.IOObjectRelease(serial_port_iterator)
+    return services
+
+
+def location_to_string(locationID):
+    """
+    helper to calculate port and bus number from locationID
+    """
+    loc = ['{}-'.format(locationID >> 24)]
+    while locationID & 0xf00000:
+        if len(loc) > 1:
+            loc.append('.')
+        loc.append('{}'.format((locationID >> 20) & 0xf))
+        locationID <<= 4
+    return ''.join(loc)
+
+
+class SuitableSerialInterface(object):
+    pass
+
+
+def scan_interfaces():
+    """
+    helper function to scan USB interfaces
+    returns a list of SuitableSerialInterface objects with name and id attributes
+    """
+    interfaces = []
+    for service in GetIOServicesByType('IOSerialBSDClient'):
+        device = get_string_property(service, "IOCalloutDevice")
+        if device:
+            usb_device = GetParentDeviceByType(service, "IOUSBInterface")
+            if usb_device:
+                name = get_string_property(usb_device, "USB Interface Name") or None
+                locationID = get_int_property(usb_device, "locationID", kCFNumberSInt32Type) or ''
+                i = SuitableSerialInterface()
+                i.id = locationID
+                i.name = name
+                interfaces.append(i)
+    return interfaces
+
+
+def search_for_locationID_in_interfaces(serial_interfaces, locationID):
+    for interface in serial_interfaces:
+        if (interface.id == locationID):
+            return interface.name
+    return None
+
+
+def comports(include_links=False):
+    # XXX include_links is currently ignored. are links in /dev even supported here?
+    # Scan for all iokit serial ports
+    services = GetIOServicesByType('IOSerialBSDClient')
+    ports = []
+    serial_interfaces = scan_interfaces()
+    for service in services:
+        # First, add the callout device file.
+        device = get_string_property(service, "IOCalloutDevice")
+        if device:
+            info = list_ports_common.ListPortInfo(device)
+            # If the serial port is implemented by IOUSBDevice
+            # NOTE IOUSBDevice was deprecated as of 10.11 and finally on Apple Silicon
+            # devices has been completely removed.  Thanks to @oskay for this patch.
+            usb_device = GetParentDeviceByType(service, "IOUSBHostDevice")
+            if not usb_device:
+                usb_device = GetParentDeviceByType(service, "IOUSBDevice")
+            if usb_device:
+                # fetch some useful informations from properties
+                info.vid = get_int_property(usb_device, "idVendor", kCFNumberSInt16Type)
+                info.pid = get_int_property(usb_device, "idProduct", kCFNumberSInt16Type)
+                info.serial_number = get_string_property(usb_device, kUSBSerialNumberString)
+                # We know this is a usb device, so the
+                # IORegistryEntryName should always be aliased to the
+                # usb product name string descriptor.
+                info.product = IORegistryEntryGetName(usb_device) or 'n/a'
+                info.manufacturer = get_string_property(usb_device, kUSBVendorString)
+                locationID = get_int_property(usb_device, "locationID", kCFNumberSInt32Type)
+                info.location = location_to_string(locationID)
+                info.interface = search_for_locationID_in_interfaces(serial_interfaces, locationID)
+                info.apply_usb_info()
+            ports.append(info)
+    return ports
+
+# test
+if __name__ == '__main__':
+    for port, desc, hwid in sorted(comports()):
+        print("{}: {} [{}]".format(port, desc, hwid))

tools/python/ampy/serial/tools/list_ports_posix.py

@@ -0,0 +1,119 @@
+#!/usr/bin/env python
+#
+# This is a module that gathers a list of serial ports on POSIXy systems.
+# For some specific implementations, see also list_ports_linux, list_ports_osx
+#
+# This file is part of pySerial. https://github.com/pyserial/pyserial
+# (C) 2011-2015 Chris Liechti <cliechti@gmx.net>
+#
+# SPDX-License-Identifier:    BSD-3-Clause
+
+"""\
+The ``comports`` function is expected to return an iterable that yields tuples
+of 3 strings: port name, human readable description and a hardware ID.
+
+As currently no method is known to get the second two strings easily, they are
+currently just identical to the port name.
+"""
+
+from __future__ import absolute_import
+
+import glob
+import sys
+import os
+from serial.tools import list_ports_common
+
+# try to detect the OS so that a device can be selected...
+plat = sys.platform.lower()
+
+if plat[:5] == 'linux':    # Linux (confirmed)  # noqa
+    from serial.tools.list_ports_linux import comports
+
+elif plat[:6] == 'darwin':   # OS X (confirmed)
+    from serial.tools.list_ports_osx import comports
+
+elif plat == 'cygwin':       # cygwin/win32
+    # cygwin accepts /dev/com* in many contexts
+    # (such as 'open' call, explicit 'ls'), but 'glob.glob'
+    # and bare 'ls' do not; so use /dev/ttyS* instead
+    def comports(include_links=False):
+        devices = glob.glob('/dev/ttyS*')
+        if include_links:
+            devices.extend(list_ports_common.list_links(devices))
+        return [list_ports_common.ListPortInfo(d) for d in devices]
+
+elif plat[:7] == 'openbsd':    # OpenBSD
+    def comports(include_links=False):
+        devices = glob.glob('/dev/cua*')
+        if include_links:
+            devices.extend(list_ports_common.list_links(devices))
+        return [list_ports_common.ListPortInfo(d) for d in devices]
+
+elif plat[:3] == 'bsd' or plat[:7] == 'freebsd':
+    def comports(include_links=False):
+        devices = glob.glob('/dev/cua*[!.init][!.lock]')
+        if include_links:
+            devices.extend(list_ports_common.list_links(devices))
+        return [list_ports_common.ListPortInfo(d) for d in devices]
+
+elif plat[:6] == 'netbsd':   # NetBSD
+    def comports(include_links=False):
+        """scan for available ports. return a list of device names."""
+        devices = glob.glob('/dev/dty*')
+        if include_links:
+            devices.extend(list_ports_common.list_links(devices))
+        return [list_ports_common.ListPortInfo(d) for d in devices]
+
+elif plat[:4] == 'irix':     # IRIX
+    def comports(include_links=False):
+        """scan for available ports. return a list of device names."""
+        devices = glob.glob('/dev/ttyf*')
+        if include_links:
+            devices.extend(list_ports_common.list_links(devices))
+        return [list_ports_common.ListPortInfo(d) for d in devices]
+
+elif plat[:2] == 'hp':       # HP-UX (not tested)
+    def comports(include_links=False):
+        """scan for available ports. return a list of device names."""
+        devices = glob.glob('/dev/tty*p0')
+        if include_links:
+            devices.extend(list_ports_common.list_links(devices))
+        return [list_ports_common.ListPortInfo(d) for d in devices]
+
+elif plat[:5] == 'sunos':    # Solaris/SunOS
+    def comports(include_links=False):
+        """scan for available ports. return a list of device names."""
+        devices = glob.glob('/dev/tty*c')
+        if include_links:
+            devices.extend(list_ports_common.list_links(devices))
+        return [list_ports_common.ListPortInfo(d) for d in devices]
+
+elif plat[:3] == 'aix':      # AIX
+    def comports(include_links=False):
+        """scan for available ports. return a list of device names."""
+        devices = glob.glob('/dev/tty*')
+        if include_links:
+            devices.extend(list_ports_common.list_links(devices))
+        return [list_ports_common.ListPortInfo(d) for d in devices]
+
+else:
+    # platform detection has failed...
+    import serial
+    sys.stderr.write("""\
+don't know how to enumerate ttys on this system.
+! I you know how the serial ports are named send this information to
+! the author of this module:
+
+sys.platform = {!r}
+os.name = {!r}
+pySerial version = {}
+
+also add the naming scheme of the serial ports and with a bit luck you can get
+this module running...
+""".format(sys.platform, os.name, serial.VERSION))
+    raise ImportError("Sorry: no implementation for your platform ('{}') available".format(os.name))
+
+# test
+if __name__ == '__main__':
+    for port, desc, hwid in sorted(comports()):
+        print("{}: {} [{}]".format(port, desc, hwid))

tools/python/ampy/serial/tools/list_ports_windows.py

@@ -0,0 +1,427 @@
+#! python
+#
+# Enumerate serial ports on Windows including a human readable description
+# and hardware information.
+#
+# This file is part of pySerial. https://github.com/pyserial/pyserial
+# (C) 2001-2016 Chris Liechti <cliechti@gmx.net>
+#
+# SPDX-License-Identifier:    BSD-3-Clause
+
+from __future__ import absolute_import
+
+# pylint: disable=invalid-name,too-few-public-methods
+import re
+import ctypes
+from ctypes.wintypes import BOOL
+from ctypes.wintypes import HWND
+from ctypes.wintypes import DWORD
+from ctypes.wintypes import WORD
+from ctypes.wintypes import LONG
+from ctypes.wintypes import ULONG
+from ctypes.wintypes import HKEY
+from ctypes.wintypes import BYTE
+import serial
+from serial.win32 import ULONG_PTR
+from serial.tools import list_ports_common
+
+
+def ValidHandle(value, func, arguments):
+    if value == 0:
+        raise ctypes.WinError()
+    return value
+
+
+NULL = 0
+HDEVINFO = ctypes.c_void_p
+LPCTSTR = ctypes.c_wchar_p
+PCTSTR = ctypes.c_wchar_p
+PTSTR = ctypes.c_wchar_p
+LPDWORD = PDWORD = ctypes.POINTER(DWORD)
+#~ LPBYTE = PBYTE = ctypes.POINTER(BYTE)
+LPBYTE = PBYTE = ctypes.c_void_p        # XXX avoids error about types
+
+ACCESS_MASK = DWORD
+REGSAM = ACCESS_MASK
+
+
+class GUID(ctypes.Structure):
+    _fields_ = [
+        ('Data1', DWORD),
+        ('Data2', WORD),
+        ('Data3', WORD),
+        ('Data4', BYTE * 8),
+    ]
+
+    def __str__(self):
+        return "{{{:08x}-{:04x}-{:04x}-{}-{}}}".format(
+            self.Data1,
+            self.Data2,
+            self.Data3,
+            ''.join(["{:02x}".format(d) for d in self.Data4[:2]]),
+            ''.join(["{:02x}".format(d) for d in self.Data4[2:]]),
+        )
+
+
+class SP_DEVINFO_DATA(ctypes.Structure):
+    _fields_ = [
+        ('cbSize', DWORD),
+        ('ClassGuid', GUID),
+        ('DevInst', DWORD),
+        ('Reserved', ULONG_PTR),
+    ]
+
+    def __str__(self):
+        return "ClassGuid:{} DevInst:{}".format(self.ClassGuid, self.DevInst)
+
+
+PSP_DEVINFO_DATA = ctypes.POINTER(SP_DEVINFO_DATA)
+
+PSP_DEVICE_INTERFACE_DETAIL_DATA = ctypes.c_void_p
+
+setupapi = ctypes.windll.LoadLibrary("setupapi")
+SetupDiDestroyDeviceInfoList = setupapi.SetupDiDestroyDeviceInfoList
+SetupDiDestroyDeviceInfoList.argtypes = [HDEVINFO]
+SetupDiDestroyDeviceInfoList.restype = BOOL
+
+SetupDiClassGuidsFromName = setupapi.SetupDiClassGuidsFromNameW
+SetupDiClassGuidsFromName.argtypes = [PCTSTR, ctypes.POINTER(GUID), DWORD, PDWORD]
+SetupDiClassGuidsFromName.restype = BOOL
+
+SetupDiEnumDeviceInfo = setupapi.SetupDiEnumDeviceInfo
+SetupDiEnumDeviceInfo.argtypes = [HDEVINFO, DWORD, PSP_DEVINFO_DATA]
+SetupDiEnumDeviceInfo.restype = BOOL
+
+SetupDiGetClassDevs = setupapi.SetupDiGetClassDevsW
+SetupDiGetClassDevs.argtypes = [ctypes.POINTER(GUID), PCTSTR, HWND, DWORD]
+SetupDiGetClassDevs.restype = HDEVINFO
+SetupDiGetClassDevs.errcheck = ValidHandle
+
+SetupDiGetDeviceRegistryProperty = setupapi.SetupDiGetDeviceRegistryPropertyW
+SetupDiGetDeviceRegistryProperty.argtypes = [HDEVINFO, PSP_DEVINFO_DATA, DWORD, PDWORD, PBYTE, DWORD, PDWORD]
+SetupDiGetDeviceRegistryProperty.restype = BOOL
+
+SetupDiGetDeviceInstanceId = setupapi.SetupDiGetDeviceInstanceIdW
+SetupDiGetDeviceInstanceId.argtypes = [HDEVINFO, PSP_DEVINFO_DATA, PTSTR, DWORD, PDWORD]
+SetupDiGetDeviceInstanceId.restype = BOOL
+
+SetupDiOpenDevRegKey = setupapi.SetupDiOpenDevRegKey
+SetupDiOpenDevRegKey.argtypes = [HDEVINFO, PSP_DEVINFO_DATA, DWORD, DWORD, DWORD, REGSAM]
+SetupDiOpenDevRegKey.restype = HKEY
+
+advapi32 = ctypes.windll.LoadLibrary("Advapi32")
+RegCloseKey = advapi32.RegCloseKey
+RegCloseKey.argtypes = [HKEY]
+RegCloseKey.restype = LONG
+
+RegQueryValueEx = advapi32.RegQueryValueExW
+RegQueryValueEx.argtypes = [HKEY, LPCTSTR, LPDWORD, LPDWORD, LPBYTE, LPDWORD]
+RegQueryValueEx.restype = LONG
+
+cfgmgr32 = ctypes.windll.LoadLibrary("Cfgmgr32")
+CM_Get_Parent = cfgmgr32.CM_Get_Parent
+CM_Get_Parent.argtypes = [PDWORD, DWORD, ULONG]
+CM_Get_Parent.restype = LONG
+
+CM_Get_Device_IDW = cfgmgr32.CM_Get_Device_IDW
+CM_Get_Device_IDW.argtypes = [DWORD, PTSTR, ULONG, ULONG]
+CM_Get_Device_IDW.restype = LONG
+
+CM_MapCrToWin32Err = cfgmgr32.CM_MapCrToWin32Err
+CM_MapCrToWin32Err.argtypes = [DWORD, DWORD]
+CM_MapCrToWin32Err.restype = DWORD
+
+
+DIGCF_PRESENT = 2
+DIGCF_DEVICEINTERFACE = 16
+INVALID_HANDLE_VALUE = 0
+ERROR_INSUFFICIENT_BUFFER = 122
+ERROR_NOT_FOUND = 1168
+SPDRP_HARDWAREID = 1
+SPDRP_FRIENDLYNAME = 12
+SPDRP_LOCATION_PATHS = 35
+SPDRP_MFG = 11
+DICS_FLAG_GLOBAL = 1
+DIREG_DEV = 0x00000001
+KEY_READ = 0x20019
+
+
+MAX_USB_DEVICE_TREE_TRAVERSAL_DEPTH = 5
+
+
+def get_parent_serial_number(child_devinst, child_vid, child_pid, depth=0, last_serial_number=None):
+    """ Get the serial number of the parent of a device.
+
+    Args:
+        child_devinst: The device instance handle to get the parent serial number of.
+        child_vid: The vendor ID of the child device.
+        child_pid: The product ID of the child device.
+        depth: The current iteration depth of the USB device tree.
+    """
+
+    # If the traversal depth is beyond the max, abandon attempting to find the serial number.
+    if depth > MAX_USB_DEVICE_TREE_TRAVERSAL_DEPTH:
+        return '' if not last_serial_number else last_serial_number
+
+    # Get the parent device instance.
+    devinst = DWORD()
+    ret = CM_Get_Parent(ctypes.byref(devinst), child_devinst, 0)
+
+    if ret:
+        win_error = CM_MapCrToWin32Err(DWORD(ret), DWORD(0))
+
+        # If there is no parent available, the child was the root device. We cannot traverse
+        # further.
+        if win_error == ERROR_NOT_FOUND:
+            return '' if not last_serial_number else last_serial_number
+
+        raise ctypes.WinError(win_error)
+
+    # Get the ID of the parent device and parse it for vendor ID, product ID, and serial number.
+    parentHardwareID = ctypes.create_unicode_buffer(250)
+
+    ret = CM_Get_Device_IDW(
+        devinst,
+        parentHardwareID,
+        ctypes.sizeof(parentHardwareID) - 1,
+        0)
+
+    if ret:
+        raise ctypes.WinError(CM_MapCrToWin32Err(DWORD(ret), DWORD(0)))
+
+    parentHardwareID_str = parentHardwareID.value
+    m = re.search(r'VID_([0-9a-f]{4})(&PID_([0-9a-f]{4}))?(&MI_(\d{2}))?(\\(.*))?',
+                  parentHardwareID_str,
+                  re.I)
+
+    # return early if we have no matches (likely malformed serial, traversed too far)
+    if not m:
+        return '' if not last_serial_number else last_serial_number
+
+    vid = None
+    pid = None
+    serial_number = None
+    if m.group(1):
+        vid = int(m.group(1), 16)
+    if m.group(3):
+        pid = int(m.group(3), 16)
+    if m.group(7):
+        serial_number = m.group(7)
+
+    # store what we found as a fallback for malformed serial values up the chain
+    found_serial_number = serial_number
+
+    # Check that the USB serial number only contains alpha-numeric characters. It may be a windows
+    # device ID (ephemeral ID).
+    if serial_number and not re.match(r'^\w+$', serial_number):
+        serial_number = None
+
+    if not vid or not pid:
+        # If pid and vid are not available at this device level, continue to the parent.
+        return get_parent_serial_number(devinst, child_vid, child_pid, depth + 1, found_serial_number)
+
+    if pid != child_pid or vid != child_vid:
+        # If the VID or PID has changed, we are no longer looking at the same physical device. The
+        # serial number is unknown.
+        return '' if not last_serial_number else last_serial_number
+
+    # In this case, the vid and pid of the parent device are identical to the child. However, if
+    # there still isn't a serial number available, continue to the next parent.
+    if not serial_number:
+        return get_parent_serial_number(devinst, child_vid, child_pid, depth + 1, found_serial_number)
+
+    # Finally, the VID and PID are identical to the child and a serial number is present, so return
+    # it.
+    return serial_number
+
+
+def iterate_comports():
+    """Return a generator that yields descriptions for serial ports"""
+    PortsGUIDs = (GUID * 8)()  # so far only seen one used, so hope 8 are enough...
+    ports_guids_size = DWORD()
+    if not SetupDiClassGuidsFromName(
+            "Ports",
+            PortsGUIDs,
+            ctypes.sizeof(PortsGUIDs),
+            ctypes.byref(ports_guids_size)):
+        raise ctypes.WinError()
+
+    ModemsGUIDs = (GUID * 8)()  # so far only seen one used, so hope 8 are enough...
+    modems_guids_size = DWORD()
+    if not SetupDiClassGuidsFromName(
+            "Modem",
+            ModemsGUIDs,
+            ctypes.sizeof(ModemsGUIDs),
+            ctypes.byref(modems_guids_size)):
+        raise ctypes.WinError()
+
+    GUIDs = PortsGUIDs[:ports_guids_size.value] + ModemsGUIDs[:modems_guids_size.value]
+
+    # repeat for all possible GUIDs
+    for index in range(len(GUIDs)):
+        bInterfaceNumber = None
+        g_hdi = SetupDiGetClassDevs(
+            ctypes.byref(GUIDs[index]),
+            None,
+            NULL,
+            DIGCF_PRESENT)  # was DIGCF_PRESENT|DIGCF_DEVICEINTERFACE which misses CDC ports
+
+        devinfo = SP_DEVINFO_DATA()
+        devinfo.cbSize = ctypes.sizeof(devinfo)
+        index = 0
+        while SetupDiEnumDeviceInfo(g_hdi, index, ctypes.byref(devinfo)):
+            index += 1
+
+            # get the real com port name
+            hkey = SetupDiOpenDevRegKey(
+                g_hdi,
+                ctypes.byref(devinfo),
+                DICS_FLAG_GLOBAL,
+                0,
+                DIREG_DEV,  # DIREG_DRV for SW info
+                KEY_READ)
+            port_name_buffer = ctypes.create_unicode_buffer(250)
+            port_name_length = ULONG(ctypes.sizeof(port_name_buffer))
+            RegQueryValueEx(
+                hkey,
+                "PortName",
+                None,
+                None,
+                ctypes.byref(port_name_buffer),
+                ctypes.byref(port_name_length))
+            RegCloseKey(hkey)
+
+            # unfortunately does this method also include parallel ports.
+            # we could check for names starting with COM or just exclude LPT
+            # and hope that other "unknown" names are serial ports...
+            if port_name_buffer.value.startswith('LPT'):
+                continue
+
+            # hardware ID
+            szHardwareID = ctypes.create_unicode_buffer(250)
+            # try to get ID that includes serial number
+            if not SetupDiGetDeviceInstanceId(
+                    g_hdi,
+                    ctypes.byref(devinfo),
+                    #~ ctypes.byref(szHardwareID),
+                    szHardwareID,
+                    ctypes.sizeof(szHardwareID) - 1,
+                    None):
+                # fall back to more generic hardware ID if that would fail
+                if not SetupDiGetDeviceRegistryProperty(
+                        g_hdi,
+                        ctypes.byref(devinfo),
+                        SPDRP_HARDWAREID,
+                        None,
+                        ctypes.byref(szHardwareID),
+                        ctypes.sizeof(szHardwareID) - 1,
+                        None):
+                    # Ignore ERROR_INSUFFICIENT_BUFFER
+                    if ctypes.GetLastError() != ERROR_INSUFFICIENT_BUFFER:
+                        raise ctypes.WinError()
+            # stringify
+            szHardwareID_str = szHardwareID.value
+
+            info = list_ports_common.ListPortInfo(port_name_buffer.value, skip_link_detection=True)
+
+            # in case of USB, make a more readable string, similar to that form
+            # that we also generate on other platforms
+            if szHardwareID_str.startswith('USB'):
+                m = re.search(r'VID_([0-9a-f]{4})(&PID_([0-9a-f]{4}))?(&MI_(\d{2}))?(\\(.*))?', szHardwareID_str, re.I)
+                if m:
+                    info.vid = int(m.group(1), 16)
+                    if m.group(3):
+                        info.pid = int(m.group(3), 16)
+                    if m.group(5):
+                        bInterfaceNumber = int(m.group(5))
+
+                    # Check that the USB serial number only contains alpha-numeric characters. It
+                    # may be a windows device ID (ephemeral ID) for composite devices.
+                    if m.group(7) and re.match(r'^\w+$', m.group(7)):
+                        info.serial_number = m.group(7)
+                    else:
+                        info.serial_number = get_parent_serial_number(devinfo.DevInst, info.vid, info.pid)
+
+                # calculate a location string
+                loc_path_str = ctypes.create_unicode_buffer(250)
+                if SetupDiGetDeviceRegistryProperty(
+                        g_hdi,
+                        ctypes.byref(devinfo),
+                        SPDRP_LOCATION_PATHS,
+                        None,
+                        ctypes.byref(loc_path_str),
+                        ctypes.sizeof(loc_path_str) - 1,
+                        None):
+                    m = re.finditer(r'USBROOT\((\w+)\)|#USB\((\w+)\)', loc_path_str.value)
+                    location = []
+                    for g in m:
+                        if g.group(1):
+                            location.append('{:d}'.format(int(g.group(1)) + 1))
+                        else:
+                            if len(location) > 1:
+                                location.append('.')
+                            else:
+                                location.append('-')
+                            location.append(g.group(2))
+                    if bInterfaceNumber is not None:
+                        location.append(':{}.{}'.format(
+                            'x',  # XXX how to determine correct bConfigurationValue?
+                            bInterfaceNumber))
+                    if location:
+                        info.location = ''.join(location)
+                info.hwid = info.usb_info()
+            elif szHardwareID_str.startswith('FTDIBUS'):
+                m = re.search(r'VID_([0-9a-f]{4})\+PID_([0-9a-f]{4})(\+(\w+))?', szHardwareID_str, re.I)
+                if m:
+                    info.vid = int(m.group(1), 16)
+                    info.pid = int(m.group(2), 16)
+                    if m.group(4):
+                        info.serial_number = m.group(4)
+                # USB location is hidden by FDTI driver :(
+                info.hwid = info.usb_info()
+            else:
+                info.hwid = szHardwareID_str
+
+            # friendly name
+            szFriendlyName = ctypes.create_unicode_buffer(250)
+            if SetupDiGetDeviceRegistryProperty(
+                    g_hdi,
+                    ctypes.byref(devinfo),
+                    SPDRP_FRIENDLYNAME,
+                    #~ SPDRP_DEVICEDESC,
+                    None,
+                    ctypes.byref(szFriendlyName),
+                    ctypes.sizeof(szFriendlyName) - 1,
+                    None):
+                info.description = szFriendlyName.value
+            #~ else:
+                # Ignore ERROR_INSUFFICIENT_BUFFER
+                #~ if ctypes.GetLastError() != ERROR_INSUFFICIENT_BUFFER:
+                    #~ raise IOError("failed to get details for %s (%s)" % (devinfo, szHardwareID.value))
+                # ignore errors and still include the port in the list, friendly name will be same as port name
+
+            # manufacturer
+            szManufacturer = ctypes.create_unicode_buffer(250)
+            if SetupDiGetDeviceRegistryProperty(
+                    g_hdi,
+                    ctypes.byref(devinfo),
+                    SPDRP_MFG,
+                    #~ SPDRP_DEVICEDESC,
+                    None,
+                    ctypes.byref(szManufacturer),
+                    ctypes.sizeof(szManufacturer) - 1,
+                    None):
+                info.manufacturer = szManufacturer.value
+            yield info
+        SetupDiDestroyDeviceInfoList(g_hdi)
+
+
+def comports(include_links=False):
+    """Return a list of info objects about serial ports"""
+    return list(iterate_comports())
+
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+# test
+if __name__ == '__main__':
+    for port, desc, hwid in sorted(comports()):
+        print("{}: {} [{}]".format(port, desc, hwid))

tools/python/ampy/serial/urlhandler/protocol_alt.py

@@ -0,0 +1,57 @@
+#! python
+#
+# This module implements a special URL handler that allows selecting an
+# alternate implementation provided by some backends.
+#
+# This file is part of pySerial. https://github.com/pyserial/pyserial
+# (C) 2015 Chris Liechti <cliechti@gmx.net>
+#
+# SPDX-License-Identifier:    BSD-3-Clause
+#
+# URL format:    alt://port[?option[=value][&option[=value]]]
+# options:
+# - class=X used class named X instead of Serial
+#
+# example:
+#   use poll based implementation on Posix (Linux):
+#   python -m serial.tools.miniterm alt:///dev/ttyUSB0?class=PosixPollSerial
+
+from __future__ import absolute_import
+
+try:
+    import urlparse
+except ImportError:
+    import urllib.parse as urlparse
+
+import serial
+
+
+def serial_class_for_url(url):
+    """extract host and port from an URL string"""
+    parts = urlparse.urlsplit(url)
+    if parts.scheme != 'alt':
+        raise serial.SerialException(
+            'expected a string in the form "alt://port[?option[=value][&option[=value]]]": '
+            'not starting with alt:// ({!r})'.format(parts.scheme))
+    class_name = 'Serial'
+    try:
+        for option, values in urlparse.parse_qs(parts.query, True).items():
+            if option == 'class':
+                class_name = values[0]
+            else:
+                raise ValueError('unknown option: {!r}'.format(option))
+    except ValueError as e:
+        raise serial.SerialException(
+            'expected a string in the form '
+            '"alt://port[?option[=value][&option[=value]]]": {!r}'.format(e))
+    if not hasattr(serial, class_name):
+        raise ValueError('unknown class: {!r}'.format(class_name))
+    cls = getattr(serial, class_name)
+    if not issubclass(cls, serial.Serial):
+        raise ValueError('class {!r} is not an instance of Serial'.format(class_name))
+    return (''.join([parts.netloc, parts.path]), cls)
+
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+if __name__ == '__main__':
+    s = serial.serial_for_url('alt:///dev/ttyS0?class=PosixPollSerial')
+    print(s)

tools/python/ampy/serial/urlhandler/protocol_cp2110.py

@@ -0,0 +1,258 @@
+#! python
+#
+# Backend for Silicon Labs CP2110/4 HID-to-UART devices.
+#
+# This file is part of pySerial. https://github.com/pyserial/pyserial
+# (C) 2001-2015 Chris Liechti <cliechti@gmx.net>
+# (C) 2019 Google LLC
+#
+# SPDX-License-Identifier:    BSD-3-Clause
+
+# This backend implements support for HID-to-UART devices manufactured
+# by Silicon Labs and marketed as CP2110 and CP2114. The
+# implementation is (mostly) OS-independent and in userland. It relies
+# on cython-hidapi (https://github.com/trezor/cython-hidapi).
+
+# The HID-to-UART protocol implemented by CP2110/4 is described in the
+# AN434 document from Silicon Labs:
+# https://www.silabs.com/documents/public/application-notes/AN434-CP2110-4-Interface-Specification.pdf
+
+# TODO items:
+
+# - rtscts support is configured for hardware flow control, but the
+#   signaling is missing (AN434 suggests this is done through GPIO).
+# - Cancelling reads and writes is not supported.
+# - Baudrate validation is not implemented, as it depends on model and configuration.
+
+import struct
+import threading
+
+try:
+    import urlparse
+except ImportError:
+    import urllib.parse as urlparse
+
+try:
+    import Queue
+except ImportError:
+    import queue as Queue
+
+import hid  # hidapi
+
+import serial
+from serial.serialutil import SerialBase, SerialException, PortNotOpenError, to_bytes, Timeout
+
+
+# Report IDs and related constant
+_REPORT_GETSET_UART_ENABLE = 0x41
+_DISABLE_UART = 0x00
+_ENABLE_UART = 0x01
+
+_REPORT_SET_PURGE_FIFOS = 0x43
+_PURGE_TX_FIFO = 0x01
+_PURGE_RX_FIFO = 0x02
+
+_REPORT_GETSET_UART_CONFIG = 0x50
+
+_REPORT_SET_TRANSMIT_LINE_BREAK = 0x51
+_REPORT_SET_STOP_LINE_BREAK = 0x52
+
+
+class Serial(SerialBase):
+    # This is not quite correct. AN343 specifies that the minimum
+    # baudrate is different between CP2110 and CP2114, and it's halved
+    # when using non-8-bit symbols.
+    BAUDRATES = (300, 375, 600, 1200, 1800, 2400, 4800, 9600, 19200,
+                 38400, 57600, 115200, 230400, 460800, 500000, 576000,
+                 921600, 1000000)
+
+    def __init__(self, *args, **kwargs):
+        self._hid_handle = None
+        self._read_buffer = None
+        self._thread = None
+        super(Serial, self).__init__(*args, **kwargs)
+
+    def open(self):
+        if self._port is None:
+            raise SerialException("Port must be configured before it can be used.")
+        if self.is_open:
+            raise SerialException("Port is already open.")
+
+        self._read_buffer = Queue.Queue()
+
+        self._hid_handle = hid.device()
+        try:
+            portpath = self.from_url(self.portstr)
+            self._hid_handle.open_path(portpath)
+        except OSError as msg:
+            raise SerialException(msg.errno, "could not open port {}: {}".format(self._port, msg))
+
+        try:
+            self._reconfigure_port()
+        except:
+            try:
+                self._hid_handle.close()
+            except:
+                pass
+            self._hid_handle = None
+            raise
+        else:
+            self.is_open = True
+            self._thread = threading.Thread(target=self._hid_read_loop)
+            self._thread.setDaemon(True)
+            self._thread.setName('pySerial CP2110 reader thread for {}'.format(self._port))
+            self._thread.start()
+
+    def from_url(self, url):
+        parts = urlparse.urlsplit(url)
+        if parts.scheme != "cp2110":
+            raise SerialException(
+                'expected a string in the forms '
+                '"cp2110:///dev/hidraw9" or "cp2110://0001:0023:00": '
+                'not starting with cp2110:// {{!r}}'.format(parts.scheme))
+        if parts.netloc:  # cp2100://BUS:DEVICE:ENDPOINT, for libusb
+            return parts.netloc.encode('utf-8')
+        return parts.path.encode('utf-8')
+
+    def close(self):
+        self.is_open = False
+        if self._thread:
+            self._thread.join(1)  # read timeout is 0.1
+            self._thread = None
+        self._hid_handle.close()
+        self._hid_handle = None
+
+    def _reconfigure_port(self):
+        parity_value = None
+        if self._parity == serial.PARITY_NONE:
+            parity_value = 0x00
+        elif self._parity == serial.PARITY_ODD:
+            parity_value = 0x01
+        elif self._parity == serial.PARITY_EVEN:
+            parity_value = 0x02
+        elif self._parity == serial.PARITY_MARK:
+            parity_value = 0x03
+        elif self._parity == serial.PARITY_SPACE:
+            parity_value = 0x04
+        else:
+            raise ValueError('Invalid parity: {!r}'.format(self._parity))
+
+        if self.rtscts:
+            flow_control_value = 0x01
+        else:
+            flow_control_value = 0x00
+
+        data_bits_value = None
+        if self._bytesize == 5:
+            data_bits_value = 0x00
+        elif self._bytesize == 6:
+            data_bits_value = 0x01
+        elif self._bytesize == 7:
+            data_bits_value = 0x02
+        elif self._bytesize == 8:
+            data_bits_value = 0x03
+        else:
+            raise ValueError('Invalid char len: {!r}'.format(self._bytesize))
+
+        stop_bits_value = None
+        if self._stopbits == serial.STOPBITS_ONE:
+            stop_bits_value = 0x00
+        elif self._stopbits == serial.STOPBITS_ONE_POINT_FIVE:
+            stop_bits_value = 0x01
+        elif self._stopbits == serial.STOPBITS_TWO:
+            stop_bits_value = 0x01
+        else:
+            raise ValueError('Invalid stop bit specification: {!r}'.format(self._stopbits))
+
+        configuration_report = struct.pack(
+            '>BLBBBB',
+            _REPORT_GETSET_UART_CONFIG,
+            self._baudrate,
+            parity_value,
+            flow_control_value,
+            data_bits_value,
+            stop_bits_value)
+
+        self._hid_handle.send_feature_report(configuration_report)
+
+        self._hid_handle.send_feature_report(
+            bytes((_REPORT_GETSET_UART_ENABLE, _ENABLE_UART)))
+        self._update_break_state()
+
+    @property
+    def in_waiting(self):
+        return self._read_buffer.qsize()
+
+    def reset_input_buffer(self):
+        if not self.is_open:
+            raise PortNotOpenError()
+        self._hid_handle.send_feature_report(
+            bytes((_REPORT_SET_PURGE_FIFOS, _PURGE_RX_FIFO)))
+        # empty read buffer
+        while self._read_buffer.qsize():
+            self._read_buffer.get(False)
+
+    def reset_output_buffer(self):
+        if not self.is_open:
+            raise PortNotOpenError()
+        self._hid_handle.send_feature_report(
+            bytes((_REPORT_SET_PURGE_FIFOS, _PURGE_TX_FIFO)))
+
+    def _update_break_state(self):
+        if not self._hid_handle:
+            raise PortNotOpenError()
+
+        if self._break_state:
+            self._hid_handle.send_feature_report(
+                bytes((_REPORT_SET_TRANSMIT_LINE_BREAK, 0)))
+        else:
+            # Note that while AN434 states "There are no data bytes in
+            # the payload other than the Report ID", either hidapi or
+            # Linux does not seem to send the report otherwise.
+            self._hid_handle.send_feature_report(
+                bytes((_REPORT_SET_STOP_LINE_BREAK, 0)))
+
+    def read(self, size=1):
+        if not self.is_open:
+            raise PortNotOpenError()
+
+        data = bytearray()
+        try:
+            timeout = Timeout(self._timeout)
+            while len(data) < size:
+                if self._thread is None:
+                    raise SerialException('connection failed (reader thread died)')
+                buf = self._read_buffer.get(True, timeout.time_left())
+                if buf is None:
+                    return bytes(data)
+                data += buf
+                if timeout.expired():
+                    break
+        except Queue.Empty:  # -> timeout
+            pass
+        return bytes(data)
+
+    def write(self, data):
+        if not self.is_open:
+            raise PortNotOpenError()
+        data = to_bytes(data)
+        tx_len = len(data)
+        while tx_len > 0:
+            to_be_sent = min(tx_len, 0x3F)
+            report = to_bytes([to_be_sent]) + data[:to_be_sent]
+            self._hid_handle.write(report)
+
+            data = data[to_be_sent:]
+            tx_len = len(data)
+
+    def _hid_read_loop(self):
+        try:
+            while self.is_open:
+                data = self._hid_handle.read(64, timeout_ms=100)
+                if not data:
+                    continue
+                data_len = data.pop(0)
+                assert data_len == len(data)
+                self._read_buffer.put(bytearray(data))
+        finally:
+            self._thread = None

tools/python/ampy/serial/urlhandler/protocol_hwgrep.py

@@ -0,0 +1,91 @@
+#! python
+#
+# This module implements a special URL handler that uses the port listing to
+# find ports by searching the string descriptions.
+#
+# This file is part of pySerial. https://github.com/pyserial/pyserial
+# (C) 2011-2015 Chris Liechti <cliechti@gmx.net>
+#
+# SPDX-License-Identifier:    BSD-3-Clause
+#
+# URL format:    hwgrep://<regexp>&<option>
+#
+# where <regexp> is a Python regexp according to the re module
+#
+# violating the normal definition for URLs, the charachter `&` is used to
+# separate parameters from the arguments (instead of `?`, but the question mark
+# is heavily used in regexp'es)
+#
+# options:
+# n=<N>     pick the N'th entry instead of the first one (numbering starts at 1)
+# skip_busy tries to open port to check if it is busy, fails on posix as ports are not locked!
+
+from __future__ import absolute_import
+
+import serial
+import serial.tools.list_ports
+
+try:
+    basestring
+except NameError:
+    basestring = str    # python 3  pylint: disable=redefined-builtin
+
+
+class Serial(serial.Serial):
+    """Just inherit the native Serial port implementation and patch the port property."""
+    # pylint: disable=no-member
+
+    @serial.Serial.port.setter
+    def port(self, value):
+        """translate port name before storing it"""
+        if isinstance(value, basestring) and value.startswith('hwgrep://'):
+            serial.Serial.port.__set__(self, self.from_url(value))
+        else:
+            serial.Serial.port.__set__(self, value)
+
+    def from_url(self, url):
+        """extract host and port from an URL string"""
+        if url.lower().startswith("hwgrep://"):
+            url = url[9:]
+        n = 0
+        test_open = False
+        args = url.split('&')
+        regexp = args.pop(0)
+        for arg in args:
+            if '=' in arg:
+                option, value = arg.split('=', 1)
+            else:
+                option = arg
+                value = None
+            if option == 'n':
+                # pick n'th element
+                n = int(value) - 1
+                if n < 1:
+                    raise ValueError('option "n" expects a positive integer larger than 1: {!r}'.format(value))
+            elif option == 'skip_busy':
+                # open to test if port is available. not the nicest way..
+                test_open = True
+            else:
+                raise ValueError('unknown option: {!r}'.format(option))
+        # use a for loop to get the 1st element from the generator
+        for port, desc, hwid in sorted(serial.tools.list_ports.grep(regexp)):
+            if test_open:
+                try:
+                    s = serial.Serial(port)
+                except serial.SerialException:
+                    # it has some error, skip this one
+                    continue
+                else:
+                    s.close()
+            if n:
+                n -= 1
+                continue
+            return port
+        else:
+            raise serial.SerialException('no ports found matching regexp {!r}'.format(url))
+
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+if __name__ == '__main__':
+    s = Serial(None)
+    s.port = 'hwgrep://ttyS0'
+    print(s)

tools/python/ampy/serial/urlhandler/protocol_loop.py

@@ -0,0 +1,308 @@
+#! python
+#
+# This module implements a loop back connection receiving itself what it sent.
+#
+# The purpose of this module is.. well... You can run the unit tests with it.
+# and it was so easy to implement ;-)
+#
+# This file is part of pySerial. https://github.com/pyserial/pyserial
+# (C) 2001-2020 Chris Liechti <cliechti@gmx.net>
+#
+# SPDX-License-Identifier:    BSD-3-Clause
+#
+# URL format:    loop://[option[/option...]]
+# options:
+# - "debug" print diagnostic messages
+from __future__ import absolute_import
+
+import logging
+import numbers
+import time
+try:
+    import urlparse
+except ImportError:
+    import urllib.parse as urlparse
+try:
+    import queue
+except ImportError:
+    import Queue as queue
+
+from serial.serialutil import SerialBase, SerialException, to_bytes, iterbytes, SerialTimeoutException, PortNotOpenError
+
+# map log level names to constants. used in from_url()
+LOGGER_LEVELS = {
+    'debug': logging.DEBUG,
+    'info': logging.INFO,
+    'warning': logging.WARNING,
+    'error': logging.ERROR,
+}
+
+
+class Serial(SerialBase):
+    """Serial port implementation that simulates a loop back connection in plain software."""
+
+    BAUDRATES = (50, 75, 110, 134, 150, 200, 300, 600, 1200, 1800, 2400, 4800,
+                 9600, 19200, 38400, 57600, 115200)
+
+    def __init__(self, *args, **kwargs):
+        self.buffer_size = 4096
+        self.queue = None
+        self.logger = None
+        self._cancel_write = False
+        super(Serial, self).__init__(*args, **kwargs)
+
+    def open(self):
+        """\
+        Open port with current settings. This may throw a SerialException
+        if the port cannot be opened.
+        """
+        if self.is_open:
+            raise SerialException("Port is already open.")
+        self.logger = None
+        self.queue = queue.Queue(self.buffer_size)
+
+        if self._port is None:
+            raise SerialException("Port must be configured before it can be used.")
+        # not that there is anything to open, but the function applies the
+        # options found in the URL
+        self.from_url(self.port)
+
+        # not that there anything to configure...
+        self._reconfigure_port()
+        # all things set up get, now a clean start
+        self.is_open = True
+        if not self._dsrdtr:
+            self._update_dtr_state()
+        if not self._rtscts:
+            self._update_rts_state()
+        self.reset_input_buffer()
+        self.reset_output_buffer()
+
+    def close(self):
+        if self.is_open:
+            self.is_open = False
+            try:
+                self.queue.put_nowait(None)
+            except queue.Full:
+                pass
+        super(Serial, self).close()
+
+    def _reconfigure_port(self):
+        """\
+        Set communication parameters on opened port. For the loop://
+        protocol all settings are ignored!
+        """
+        # not that's it of any real use, but it helps in the unit tests
+        if not isinstance(self._baudrate, numbers.Integral) or not 0 < self._baudrate < 2 ** 32:
+            raise ValueError("invalid baudrate: {!r}".format(self._baudrate))
+        if self.logger:
+            self.logger.info('_reconfigure_port()')
+
+    def from_url(self, url):
+        """extract host and port from an URL string"""
+        parts = urlparse.urlsplit(url)
+        if parts.scheme != "loop":
+            raise SerialException(
+                'expected a string in the form '
+                '"loop://[?logging={debug|info|warning|error}]": not starting '
+                'with loop:// ({!r})'.format(parts.scheme))
+        try:
+            # process options now, directly altering self
+            for option, values in urlparse.parse_qs(parts.query, True).items():
+                if option == 'logging':
+                    logging.basicConfig()   # XXX is that good to call it here?
+                    self.logger = logging.getLogger('pySerial.loop')
+                    self.logger.setLevel(LOGGER_LEVELS[values[0]])
+                    self.logger.debug('enabled logging')
+                else:
+                    raise ValueError('unknown option: {!r}'.format(option))
+        except ValueError as e:
+            raise SerialException(
+                'expected a string in the form '
+                '"loop://[?logging={debug|info|warning|error}]": {}'.format(e))
+
+    #  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -
+
+    @property
+    def in_waiting(self):
+        """Return the number of bytes currently in the input buffer."""
+        if not self.is_open:
+            raise PortNotOpenError()
+        if self.logger:
+            # attention the logged value can differ from return value in
+            # threaded environments...
+            self.logger.debug('in_waiting -> {:d}'.format(self.queue.qsize()))
+        return self.queue.qsize()
+
+    def read(self, size=1):
+        """\
+        Read size bytes from the serial port. If a timeout is set it may
+        return less characters as requested. With no timeout it will block
+        until the requested number of bytes is read.
+        """
+        if not self.is_open:
+            raise PortNotOpenError()
+        if self._timeout is not None and self._timeout != 0:
+            timeout = time.time() + self._timeout
+        else:
+            timeout = None
+        data = bytearray()
+        while size > 0 and self.is_open:
+            try:
+                b = self.queue.get(timeout=self._timeout)  # XXX inter char timeout
+            except queue.Empty:
+                if self._timeout == 0:
+                    break
+            else:
+                if b is not None:
+                    data += b
+                    size -= 1
+                else:
+                    break
+            # check for timeout now, after data has been read.
+            # useful for timeout = 0 (non blocking) read
+            if timeout and time.time() > timeout:
+                if self.logger:
+                    self.logger.info('read timeout')
+                break
+        return bytes(data)
+
+    def cancel_read(self):
+        self.queue.put_nowait(None)
+
+    def cancel_write(self):
+        self._cancel_write = True
+
+    def write(self, data):
+        """\
+        Output the given byte string over the serial port. Can block if the
+        connection is blocked. May raise SerialException if the connection is
+        closed.
+        """
+        self._cancel_write = False
+        if not self.is_open:
+            raise PortNotOpenError()
+        data = to_bytes(data)
+        # calculate aprox time that would be used to send the data
+        time_used_to_send = 10.0 * len(data) / self._baudrate
+        # when a write timeout is configured check if we would be successful
+        # (not sending anything, not even the part that would have time)
+        if self._write_timeout is not None and time_used_to_send > self._write_timeout:
+            # must wait so that unit test succeeds
+            time_left = self._write_timeout
+            while time_left > 0 and not self._cancel_write:
+                time.sleep(min(time_left, 0.5))
+                time_left -= 0.5
+            if self._cancel_write:
+                return 0  # XXX
+            raise SerialTimeoutException('Write timeout')
+        for byte in iterbytes(data):
+            self.queue.put(byte, timeout=self._write_timeout)
+        return len(data)
+
+    def reset_input_buffer(self):
+        """Clear input buffer, discarding all that is in the buffer."""
+        if not self.is_open:
+            raise PortNotOpenError()
+        if self.logger:
+            self.logger.info('reset_input_buffer()')
+        try:
+            while self.queue.qsize():
+                self.queue.get_nowait()
+        except queue.Empty:
+            pass
+
+    def reset_output_buffer(self):
+        """\
+        Clear output buffer, aborting the current output and
+        discarding all that is in the buffer.
+        """
+        if not self.is_open:
+            raise PortNotOpenError()
+        if self.logger:
+            self.logger.info('reset_output_buffer()')
+        try:
+            while self.queue.qsize():
+                self.queue.get_nowait()
+        except queue.Empty:
+            pass
+
+    @property
+    def out_waiting(self):
+        """Return how many bytes the in the outgoing buffer"""
+        if not self.is_open:
+            raise PortNotOpenError()
+        if self.logger:
+            # attention the logged value can differ from return value in
+            # threaded environments...
+            self.logger.debug('out_waiting -> {:d}'.format(self.queue.qsize()))
+        return self.queue.qsize()
+
+    def _update_break_state(self):
+        """\
+        Set break: Controls TXD. When active, to transmitting is
+        possible.
+        """
+        if self.logger:
+            self.logger.info('_update_break_state({!r})'.format(self._break_state))
+
+    def _update_rts_state(self):
+        """Set terminal status line: Request To Send"""
+        if self.logger:
+            self.logger.info('_update_rts_state({!r}) -> state of CTS'.format(self._rts_state))
+
+    def _update_dtr_state(self):
+        """Set terminal status line: Data Terminal Ready"""
+        if self.logger:
+            self.logger.info('_update_dtr_state({!r}) -> state of DSR'.format(self._dtr_state))
+
+    @property
+    def cts(self):
+        """Read terminal status line: Clear To Send"""
+        if not self.is_open:
+            raise PortNotOpenError()
+        if self.logger:
+            self.logger.info('CTS -> state of RTS ({!r})'.format(self._rts_state))
+        return self._rts_state
+
+    @property
+    def dsr(self):
+        """Read terminal status line: Data Set Ready"""
+        if self.logger:
+            self.logger.info('DSR -> state of DTR ({!r})'.format(self._dtr_state))
+        return self._dtr_state
+
+    @property
+    def ri(self):
+        """Read terminal status line: Ring Indicator"""
+        if not self.is_open:
+            raise PortNotOpenError()
+        if self.logger:
+            self.logger.info('returning dummy for RI')
+        return False
+
+    @property
+    def cd(self):
+        """Read terminal status line: Carrier Detect"""
+        if not self.is_open:
+            raise PortNotOpenError()
+        if self.logger:
+            self.logger.info('returning dummy for CD')
+        return True
+
+    # - - - platform specific - - -
+    # None so far
+
+
+# simple client test
+if __name__ == '__main__':
+    import sys
+    s = Serial('loop://')
+    sys.stdout.write('{}\n'.format(s))
+
+    sys.stdout.write("write...\n")
+    s.write("hello\n")
+    s.flush()
+    sys.stdout.write("read: {!r}\n".format(s.read(5)))
+
+    s.close()

tools/python/ampy/serial/urlhandler/protocol_rfc2217.py

@@ -0,0 +1,12 @@
+#! python
+#
+# This is a thin wrapper to load the rfc2217 implementation.
+#
+# This file is part of pySerial. https://github.com/pyserial/pyserial
+# (C) 2011 Chris Liechti <cliechti@gmx.net>
+#
+# SPDX-License-Identifier:    BSD-3-Clause
+
+from __future__ import absolute_import
+
+from serial.rfc2217 import Serial  # noqa

tools/python/ampy/serial/urlhandler/protocol_socket.py

@@ -0,0 +1,359 @@
+#! python
+#
+# This module implements a simple socket based client.
+# It does not support changing any port parameters and will silently ignore any
+# requests to do so.
+#
+# The purpose of this module is that applications using pySerial can connect to
+# TCP/IP to serial port converters that do not support RFC 2217.
+#
+# This file is part of pySerial. https://github.com/pyserial/pyserial
+# (C) 2001-2015 Chris Liechti <cliechti@gmx.net>
+#
+# SPDX-License-Identifier:    BSD-3-Clause
+#
+# URL format:    socket://<host>:<port>[/option[/option...]]
+# options:
+# - "debug" print diagnostic messages
+
+from __future__ import absolute_import
+
+import errno
+import logging
+import select
+import socket
+import time
+try:
+    import urlparse
+except ImportError:
+    import urllib.parse as urlparse
+
+from serial.serialutil import SerialBase, SerialException, to_bytes, \
+    PortNotOpenError, SerialTimeoutException, Timeout
+
+# map log level names to constants. used in from_url()
+LOGGER_LEVELS = {
+    'debug': logging.DEBUG,
+    'info': logging.INFO,
+    'warning': logging.WARNING,
+    'error': logging.ERROR,
+}
+
+POLL_TIMEOUT = 5
+
+
+class Serial(SerialBase):
+    """Serial port implementation for plain sockets."""
+
+    BAUDRATES = (50, 75, 110, 134, 150, 200, 300, 600, 1200, 1800, 2400, 4800,
+                 9600, 19200, 38400, 57600, 115200)
+
+    def open(self):
+        """\
+        Open port with current settings. This may throw a SerialException
+        if the port cannot be opened.
+        """
+        self.logger = None
+        if self._port is None:
+            raise SerialException("Port must be configured before it can be used.")
+        if self.is_open:
+            raise SerialException("Port is already open.")
+        try:
+            # timeout is used for write timeout support :/ and to get an initial connection timeout
+            self._socket = socket.create_connection(self.from_url(self.portstr), timeout=POLL_TIMEOUT)
+        except Exception as msg:
+            self._socket = None
+            raise SerialException("Could not open port {}: {}".format(self.portstr, msg))
+        # after connecting, switch to non-blocking, we're using select
+        self._socket.setblocking(False)
+
+        # not that there is anything to configure...
+        self._reconfigure_port()
+        # all things set up get, now a clean start
+        self.is_open = True
+        if not self._dsrdtr:
+            self._update_dtr_state()
+        if not self._rtscts:
+            self._update_rts_state()
+        self.reset_input_buffer()
+        self.reset_output_buffer()
+
+    def _reconfigure_port(self):
+        """\
+        Set communication parameters on opened port. For the socket://
+        protocol all settings are ignored!
+        """
+        if self._socket is None:
+            raise SerialException("Can only operate on open ports")
+        if self.logger:
+            self.logger.info('ignored port configuration change')
+
+    def close(self):
+        """Close port"""
+        if self.is_open:
+            if self._socket:
+                try:
+                    self._socket.shutdown(socket.SHUT_RDWR)
+                    self._socket.close()
+                except:
+                    # ignore errors.
+                    pass
+                self._socket = None
+            self.is_open = False
+            # in case of quick reconnects, give the server some time
+            time.sleep(0.3)
+
+    def from_url(self, url):
+        """extract host and port from an URL string"""
+        parts = urlparse.urlsplit(url)
+        if parts.scheme != "socket":
+            raise SerialException(
+                'expected a string in the form '
+                '"socket://<host>:<port>[?logging={debug|info|warning|error}]": '
+                'not starting with socket:// ({!r})'.format(parts.scheme))
+        try:
+            # process options now, directly altering self
+            for option, values in urlparse.parse_qs(parts.query, True).items():
+                if option == 'logging':
+                    logging.basicConfig()   # XXX is that good to call it here?
+                    self.logger = logging.getLogger('pySerial.socket')
+                    self.logger.setLevel(LOGGER_LEVELS[values[0]])
+                    self.logger.debug('enabled logging')
+                else:
+                    raise ValueError('unknown option: {!r}'.format(option))
+            if not 0 <= parts.port < 65536:
+                raise ValueError("port not in range 0...65535")
+        except ValueError as e:
+            raise SerialException(
+                'expected a string in the form '
+                '"socket://<host>:<port>[?logging={debug|info|warning|error}]": {}'.format(e))
+
+        return (parts.hostname, parts.port)
+
+    #  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -
+
+    @property
+    def in_waiting(self):
+        """Return the number of bytes currently in the input buffer."""
+        if not self.is_open:
+            raise PortNotOpenError()
+        # Poll the socket to see if it is ready for reading.
+        # If ready, at least one byte will be to read.
+        lr, lw, lx = select.select([self._socket], [], [], 0)
+        return len(lr)
+
+    # select based implementation, similar to posix, but only using socket API
+    # to be portable, additionally handle socket timeout which is used to
+    # emulate write timeouts
+    def read(self, size=1):
+        """\
+        Read size bytes from the serial port. If a timeout is set it may
+        return less characters as requested. With no timeout it will block
+        until the requested number of bytes is read.
+        """
+        if not self.is_open:
+            raise PortNotOpenError()
+        read = bytearray()
+        timeout = Timeout(self._timeout)
+        while len(read) < size:
+            try:
+                ready, _, _ = select.select([self._socket], [], [], timeout.time_left())
+                # If select was used with a timeout, and the timeout occurs, it
+                # returns with empty lists -> thus abort read operation.
+                # For timeout == 0 (non-blocking operation) also abort when
+                # there is nothing to read.
+                if not ready:
+                    break   # timeout
+                buf = self._socket.recv(size - len(read))
+                # read should always return some data as select reported it was
+                # ready to read when we get to this point, unless it is EOF
+                if not buf:
+                    raise SerialException('socket disconnected')
+                read.extend(buf)
+            except OSError as e:
+                # this is for Python 3.x where select.error is a subclass of
+                # OSError ignore BlockingIOErrors and EINTR. other errors are shown
+                # https://www.python.org/dev/peps/pep-0475.
+                if e.errno not in (errno.EAGAIN, errno.EALREADY, errno.EWOULDBLOCK, errno.EINPROGRESS, errno.EINTR):
+                    raise SerialException('read failed: {}'.format(e))
+            except (select.error, socket.error) as e:
+                # this is for Python 2.x
+                # ignore BlockingIOErrors and EINTR. all errors are shown
+                # see also http://www.python.org/dev/peps/pep-3151/#select
+                if e[0] not in (errno.EAGAIN, errno.EALREADY, errno.EWOULDBLOCK, errno.EINPROGRESS, errno.EINTR):
+                    raise SerialException('read failed: {}'.format(e))
+            if timeout.expired():
+                break
+        return bytes(read)
+
+    def write(self, data):
+        """\
+        Output the given byte string over the serial port. Can block if the
+        connection is blocked. May raise SerialException if the connection is
+        closed.
+        """
+        if not self.is_open:
+            raise PortNotOpenError()
+
+        d = to_bytes(data)
+        tx_len = length = len(d)
+        timeout = Timeout(self._write_timeout)
+        while tx_len > 0:
+            try:
+                n = self._socket.send(d)
+                if timeout.is_non_blocking:
+                    # Zero timeout indicates non-blocking - simply return the
+                    # number of bytes of data actually written
+                    return n
+                elif not timeout.is_infinite:
+                    # when timeout is set, use select to wait for being ready
+                    # with the time left as timeout
+                    if timeout.expired():
+                        raise SerialTimeoutException('Write timeout')
+                    _, ready, _ = select.select([], [self._socket], [], timeout.time_left())
+                    if not ready:
+                        raise SerialTimeoutException('Write timeout')
+                else:
+                    assert timeout.time_left() is None
+                    # wait for write operation
+                    _, ready, _ = select.select([], [self._socket], [], None)
+                    if not ready:
+                        raise SerialException('write failed (select)')
+                d = d[n:]
+                tx_len -= n
+            except SerialException:
+                raise
+            except OSError as e:
+                # this is for Python 3.x where select.error is a subclass of
+                # OSError ignore BlockingIOErrors and EINTR. other errors are shown
+                # https://www.python.org/dev/peps/pep-0475.
+                if e.errno not in (errno.EAGAIN, errno.EALREADY, errno.EWOULDBLOCK, errno.EINPROGRESS, errno.EINTR):
+                    raise SerialException('write failed: {}'.format(e))
+            except select.error as e:
+                # this is for Python 2.x
+                # ignore BlockingIOErrors and EINTR. all errors are shown
+                # see also http://www.python.org/dev/peps/pep-3151/#select
+                if e[0] not in (errno.EAGAIN, errno.EALREADY, errno.EWOULDBLOCK, errno.EINPROGRESS, errno.EINTR):
+                    raise SerialException('write failed: {}'.format(e))
+            if not timeout.is_non_blocking and timeout.expired():
+                raise SerialTimeoutException('Write timeout')
+        return length - len(d)
+
+    def reset_input_buffer(self):
+        """Clear input buffer, discarding all that is in the buffer."""
+        if not self.is_open:
+            raise PortNotOpenError()
+
+        # just use recv to remove input, while there is some
+        ready = True
+        while ready:
+            ready, _, _ = select.select([self._socket], [], [], 0)
+            try:
+                if ready:
+                    ready = self._socket.recv(4096)
+            except OSError as e:
+                # this is for Python 3.x where select.error is a subclass of
+                # OSError ignore BlockingIOErrors and EINTR. other errors are shown
+                # https://www.python.org/dev/peps/pep-0475.
+                if e.errno not in (errno.EAGAIN, errno.EALREADY, errno.EWOULDBLOCK, errno.EINPROGRESS, errno.EINTR):
+                    raise SerialException('read failed: {}'.format(e))
+            except (select.error, socket.error) as e:
+                # this is for Python 2.x
+                # ignore BlockingIOErrors and EINTR. all errors are shown
+                # see also http://www.python.org/dev/peps/pep-3151/#select
+                if e[0] not in (errno.EAGAIN, errno.EALREADY, errno.EWOULDBLOCK, errno.EINPROGRESS, errno.EINTR):
+                    raise SerialException('read failed: {}'.format(e))
+
+    def reset_output_buffer(self):
+        """\
+        Clear output buffer, aborting the current output and
+        discarding all that is in the buffer.
+        """
+        if not self.is_open:
+            raise PortNotOpenError()
+        if self.logger:
+            self.logger.info('ignored reset_output_buffer')
+
+    def send_break(self, duration=0.25):
+        """\
+        Send break condition. Timed, returns to idle state after given
+        duration.
+        """
+        if not self.is_open:
+            raise PortNotOpenError()
+        if self.logger:
+            self.logger.info('ignored send_break({!r})'.format(duration))
+
+    def _update_break_state(self):
+        """Set break: Controls TXD. When active, to transmitting is
+        possible."""
+        if self.logger:
+            self.logger.info('ignored _update_break_state({!r})'.format(self._break_state))
+
+    def _update_rts_state(self):
+        """Set terminal status line: Request To Send"""
+        if self.logger:
+            self.logger.info('ignored _update_rts_state({!r})'.format(self._rts_state))
+
+    def _update_dtr_state(self):
+        """Set terminal status line: Data Terminal Ready"""
+        if self.logger:
+            self.logger.info('ignored _update_dtr_state({!r})'.format(self._dtr_state))
+
+    @property
+    def cts(self):
+        """Read terminal status line: Clear To Send"""
+        if not self.is_open:
+            raise PortNotOpenError()
+        if self.logger:
+            self.logger.info('returning dummy for cts')
+        return True
+
+    @property
+    def dsr(self):
+        """Read terminal status line: Data Set Ready"""
+        if not self.is_open:
+            raise PortNotOpenError()
+        if self.logger:
+            self.logger.info('returning dummy for dsr')
+        return True
+
+    @property
+    def ri(self):
+        """Read terminal status line: Ring Indicator"""
+        if not self.is_open:
+            raise PortNotOpenError()
+        if self.logger:
+            self.logger.info('returning dummy for ri')
+        return False
+
+    @property
+    def cd(self):
+        """Read terminal status line: Carrier Detect"""
+        if not self.is_open:
+            raise PortNotOpenError()
+        if self.logger:
+            self.logger.info('returning dummy for cd)')
+        return True
+
+    # - - - platform specific - - -
+
+    # works on Linux and probably all the other POSIX systems
+    def fileno(self):
+        """Get the file handle of the underlying socket for use with select"""
+        return self._socket.fileno()
+
+
+#
+# simple client test
+if __name__ == '__main__':
+    import sys
+    s = Serial('socket://localhost:7000')
+    sys.stdout.write('{}\n'.format(s))
+
+    sys.stdout.write("write...\n")
+    s.write(b"hello\n")
+    s.flush()
+    sys.stdout.write("read: {}\n".format(s.read(5)))
+
+    s.close()

tools/python/ampy/serial/urlhandler/protocol_spy.py

@@ -0,0 +1,290 @@
+#! python
+#
+# This module implements a special URL handler that wraps an other port,
+# print the traffic for debugging purposes. With this, it is possible
+# to debug the serial port traffic on every application that uses
+# serial_for_url.
+#
+# This file is part of pySerial. https://github.com/pyserial/pyserial
+# (C) 2015 Chris Liechti <cliechti@gmx.net>
+#
+# SPDX-License-Identifier:    BSD-3-Clause
+#
+# URL format:    spy://port[?option[=value][&option[=value]]]
+# options:
+# - dev=X   a file or device to write to
+# - color   use escape code to colorize output
+# - raw     forward raw bytes instead of hexdump
+#
+# example:
+#   redirect output to an other terminal window on Posix (Linux):
+#   python -m serial.tools.miniterm spy:///dev/ttyUSB0?dev=/dev/pts/14\&color
+
+from __future__ import absolute_import
+
+import sys
+import time
+
+import serial
+from serial.serialutil import  to_bytes
+
+try:
+    import urlparse
+except ImportError:
+    import urllib.parse as urlparse
+
+
+def sixteen(data):
+    """\
+    yield tuples of hex and ASCII display in multiples of 16. Includes a
+    space after 8 bytes and (None, None) after 16 bytes and at the end.
+    """
+    n = 0
+    for b in serial.iterbytes(data):
+        yield ('{:02X} '.format(ord(b)), b.decode('ascii') if b' ' <= b < b'\x7f' else '.')
+        n += 1
+        if n == 8:
+            yield (' ', '')
+        elif n >= 16:
+            yield (None, None)
+            n = 0
+    if n > 0:
+        while n < 16:
+            n += 1
+            if n == 8:
+                yield (' ', '')
+            yield ('   ', ' ')
+        yield (None, None)
+
+
+def hexdump(data):
+    """yield lines with hexdump of data"""
+    values = []
+    ascii = []
+    offset = 0
+    for h, a in sixteen(data):
+        if h is None:
+            yield (offset, ' '.join([''.join(values), ''.join(ascii)]))
+            del values[:]
+            del ascii[:]
+            offset += 0x10
+        else:
+            values.append(h)
+            ascii.append(a)
+
+
+class FormatRaw(object):
+    """Forward only RX and TX data to output."""
+
+    def __init__(self, output, color):
+        self.output = output
+        self.color = color
+        self.rx_color = '\x1b[32m'
+        self.tx_color = '\x1b[31m'
+
+    def rx(self, data):
+        """show received data"""
+        if self.color:
+            self.output.write(self.rx_color)
+        self.output.write(data)
+        self.output.flush()
+
+    def tx(self, data):
+        """show transmitted data"""
+        if self.color:
+            self.output.write(self.tx_color)
+        self.output.write(data)
+        self.output.flush()
+
+    def control(self, name, value):
+        """(do not) show control calls"""
+        pass
+
+
+class FormatHexdump(object):
+    """\
+    Create a hex dump of RX ad TX data, show when control lines are read or
+    written.
+
+    output example::
+
+        000000.000 Q-RX flushInput
+        000002.469 RTS  inactive
+        000002.773 RTS  active
+        000003.001 TX   48 45 4C 4C 4F                                    HELLO
+        000003.102 RX   48 45 4C 4C 4F                                    HELLO
+
+    """
+
+    def __init__(self, output, color):
+        self.start_time = time.time()
+        self.output = output
+        self.color = color
+        self.rx_color = '\x1b[32m'
+        self.tx_color = '\x1b[31m'
+        self.control_color = '\x1b[37m'
+
+    def write_line(self, timestamp, label, value, value2=''):
+        self.output.write('{:010.3f} {:4} {}{}\n'.format(timestamp, label, value, value2))
+        self.output.flush()
+
+    def rx(self, data):
+        """show received data as hex dump"""
+        if self.color:
+            self.output.write(self.rx_color)
+        if data:
+            for offset, row in hexdump(data):
+                self.write_line(time.time() - self.start_time, 'RX', '{:04X}  '.format(offset), row)
+        else:
+            self.write_line(time.time() - self.start_time, 'RX', '<empty>')
+
+    def tx(self, data):
+        """show transmitted data as hex dump"""
+        if self.color:
+            self.output.write(self.tx_color)
+        for offset, row in hexdump(data):
+            self.write_line(time.time() - self.start_time, 'TX', '{:04X}  '.format(offset), row)
+
+    def control(self, name, value):
+        """show control calls"""
+        if self.color:
+            self.output.write(self.control_color)
+        self.write_line(time.time() - self.start_time, name, value)
+
+
+class Serial(serial.Serial):
+    """\
+    Inherit the native Serial port implementation and wrap all the methods and
+    attributes.
+    """
+    # pylint: disable=no-member
+
+    def __init__(self, *args, **kwargs):
+        super(Serial, self).__init__(*args, **kwargs)
+        self.formatter = None
+        self.show_all = False
+
+    @serial.Serial.port.setter
+    def port(self, value):
+        if value is not None:
+            serial.Serial.port.__set__(self, self.from_url(value))
+
+    def from_url(self, url):
+        """extract host and port from an URL string"""
+        parts = urlparse.urlsplit(url)
+        if parts.scheme != 'spy':
+            raise serial.SerialException(
+                'expected a string in the form '
+                '"spy://port[?option[=value][&option[=value]]]": '
+                'not starting with spy:// ({!r})'.format(parts.scheme))
+        # process options now, directly altering self
+        formatter = FormatHexdump
+        color = False
+        output = sys.stderr
+        try:
+            for option, values in urlparse.parse_qs(parts.query, True).items():
+                if option == 'file':
+                    output = open(values[0], 'w')
+                elif option == 'color':
+                    color = True
+                elif option == 'raw':
+                    formatter = FormatRaw
+                elif option == 'all':
+                    self.show_all = True
+                else:
+                    raise ValueError('unknown option: {!r}'.format(option))
+        except ValueError as e:
+            raise serial.SerialException(
+                'expected a string in the form '
+                '"spy://port[?option[=value][&option[=value]]]": {}'.format(e))
+        self.formatter = formatter(output, color)
+        return ''.join([parts.netloc, parts.path])
+
+    def write(self, tx):
+        tx = to_bytes(tx)
+        self.formatter.tx(tx)
+        return super(Serial, self).write(tx)
+
+    def read(self, size=1):
+        rx = super(Serial, self).read(size)
+        if rx or self.show_all:
+            self.formatter.rx(rx)
+        return rx
+
+    if hasattr(serial.Serial, 'cancel_read'):
+        def cancel_read(self):
+            self.formatter.control('Q-RX', 'cancel_read')
+            super(Serial, self).cancel_read()
+
+    if hasattr(serial.Serial, 'cancel_write'):
+        def cancel_write(self):
+            self.formatter.control('Q-TX', 'cancel_write')
+            super(Serial, self).cancel_write()
+
+    @property
+    def in_waiting(self):
+        n = super(Serial, self).in_waiting
+        if self.show_all:
+            self.formatter.control('Q-RX', 'in_waiting -> {}'.format(n))
+        return n
+
+    def flush(self):
+        self.formatter.control('Q-TX', 'flush')
+        super(Serial, self).flush()
+
+    def reset_input_buffer(self):
+        self.formatter.control('Q-RX', 'reset_input_buffer')
+        super(Serial, self).reset_input_buffer()
+
+    def reset_output_buffer(self):
+        self.formatter.control('Q-TX', 'reset_output_buffer')
+        super(Serial, self).reset_output_buffer()
+
+    def send_break(self, duration=0.25):
+        self.formatter.control('BRK', 'send_break {}s'.format(duration))
+        super(Serial, self).send_break(duration)
+
+    @serial.Serial.break_condition.setter
+    def break_condition(self, level):
+        self.formatter.control('BRK', 'active' if level else 'inactive')
+        serial.Serial.break_condition.__set__(self, level)
+
+    @serial.Serial.rts.setter
+    def rts(self, level):
+        self.formatter.control('RTS', 'active' if level else 'inactive')
+        serial.Serial.rts.__set__(self, level)
+
+    @serial.Serial.dtr.setter
+    def dtr(self, level):
+        self.formatter.control('DTR', 'active' if level else 'inactive')
+        serial.Serial.dtr.__set__(self, level)
+
+    @serial.Serial.cts.getter
+    def cts(self):
+        level = super(Serial, self).cts
+        self.formatter.control('CTS', 'active' if level else 'inactive')
+        return level
+
+    @serial.Serial.dsr.getter
+    def dsr(self):
+        level = super(Serial, self).dsr
+        self.formatter.control('DSR', 'active' if level else 'inactive')
+        return level
+
+    @serial.Serial.ri.getter
+    def ri(self):
+        level = super(Serial, self).ri
+        self.formatter.control('RI', 'active' if level else 'inactive')
+        return level
+
+    @serial.Serial.cd.getter
+    def cd(self):
+        level = super(Serial, self).cd
+        self.formatter.control('CD', 'active' if level else 'inactive')
+        return level
+
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+if __name__ == '__main__':
+    ser = Serial(None)
+    ser.port = 'spy:///dev/ttyS0'
+    print(ser)

tools/python/ampy/serial/win32.py

@@ -0,0 +1,366 @@
+#! python
+#
+# Constants and types for use with Windows API, used by serialwin32.py
+#
+# This file is part of pySerial. https://github.com/pyserial/pyserial
+# (C) 2001-2015 Chris Liechti <cliechti@gmx.net>
+#
+# SPDX-License-Identifier:    BSD-3-Clause
+
+# pylint: disable=invalid-name,too-few-public-methods,protected-access,too-many-instance-attributes
+
+from __future__ import absolute_import
+
+from ctypes import c_ulong, c_void_p, c_int64, c_char, \
+                   WinDLL, sizeof, Structure, Union, POINTER
+from ctypes.wintypes import HANDLE
+from ctypes.wintypes import BOOL
+from ctypes.wintypes import LPCWSTR
+from ctypes.wintypes import DWORD
+from ctypes.wintypes import WORD
+from ctypes.wintypes import BYTE
+_stdcall_libraries = {}
+_stdcall_libraries['kernel32'] = WinDLL('kernel32')
+
+INVALID_HANDLE_VALUE = HANDLE(-1).value
+
+
+# some details of the windows API differ between 32 and 64 bit systems..
+def is_64bit():
+    """Returns true when running on a 64 bit system"""
+    return sizeof(c_ulong) != sizeof(c_void_p)
+
+# ULONG_PTR is a an ordinary number, not a pointer and contrary to the name it
+# is either 32 or 64 bits, depending on the type of windows...
+# so test if this a 32 bit windows...
+if is_64bit():
+    ULONG_PTR = c_int64
+else:
+    ULONG_PTR = c_ulong
+
+
+class _SECURITY_ATTRIBUTES(Structure):
+    pass
+LPSECURITY_ATTRIBUTES = POINTER(_SECURITY_ATTRIBUTES)
+
+
+try:
+    CreateEventW = _stdcall_libraries['kernel32'].CreateEventW
+except AttributeError:
+    # Fallback to non wide char version for old OS...
+    from ctypes.wintypes import LPCSTR
+    CreateEventA = _stdcall_libraries['kernel32'].CreateEventA
+    CreateEventA.restype = HANDLE
+    CreateEventA.argtypes = [LPSECURITY_ATTRIBUTES, BOOL, BOOL, LPCSTR]
+    CreateEvent = CreateEventA
+
+    CreateFileA = _stdcall_libraries['kernel32'].CreateFileA
+    CreateFileA.restype = HANDLE
+    CreateFileA.argtypes = [LPCSTR, DWORD, DWORD, LPSECURITY_ATTRIBUTES, DWORD, DWORD, HANDLE]
+    CreateFile = CreateFileA
+else:
+    CreateEventW.restype = HANDLE
+    CreateEventW.argtypes = [LPSECURITY_ATTRIBUTES, BOOL, BOOL, LPCWSTR]
+    CreateEvent = CreateEventW  # alias
+
+    CreateFileW = _stdcall_libraries['kernel32'].CreateFileW
+    CreateFileW.restype = HANDLE
+    CreateFileW.argtypes = [LPCWSTR, DWORD, DWORD, LPSECURITY_ATTRIBUTES, DWORD, DWORD, HANDLE]
+    CreateFile = CreateFileW  # alias
+
+
+class _OVERLAPPED(Structure):
+    pass
+
+OVERLAPPED = _OVERLAPPED
+
+
+class _COMSTAT(Structure):
+    pass
+
+COMSTAT = _COMSTAT
+
+
+class _DCB(Structure):
+    pass
+
+DCB = _DCB
+
+
+class _COMMTIMEOUTS(Structure):
+    pass
+
+COMMTIMEOUTS = _COMMTIMEOUTS
+
+GetLastError = _stdcall_libraries['kernel32'].GetLastError
+GetLastError.restype = DWORD
+GetLastError.argtypes = []
+
+LPOVERLAPPED = POINTER(_OVERLAPPED)
+LPDWORD = POINTER(DWORD)
+
+GetOverlappedResult = _stdcall_libraries['kernel32'].GetOverlappedResult
+GetOverlappedResult.restype = BOOL
+GetOverlappedResult.argtypes = [HANDLE, LPOVERLAPPED, LPDWORD, BOOL]
+
+ResetEvent = _stdcall_libraries['kernel32'].ResetEvent
+ResetEvent.restype = BOOL
+ResetEvent.argtypes = [HANDLE]
+
+LPCVOID = c_void_p
+
+WriteFile = _stdcall_libraries['kernel32'].WriteFile
+WriteFile.restype = BOOL
+WriteFile.argtypes = [HANDLE, LPCVOID, DWORD, LPDWORD, LPOVERLAPPED]
+
+LPVOID = c_void_p
+
+ReadFile = _stdcall_libraries['kernel32'].ReadFile
+ReadFile.restype = BOOL
+ReadFile.argtypes = [HANDLE, LPVOID, DWORD, LPDWORD, LPOVERLAPPED]
+
+CloseHandle = _stdcall_libraries['kernel32'].CloseHandle
+CloseHandle.restype = BOOL
+CloseHandle.argtypes = [HANDLE]
+
+ClearCommBreak = _stdcall_libraries['kernel32'].ClearCommBreak
+ClearCommBreak.restype = BOOL
+ClearCommBreak.argtypes = [HANDLE]
+
+LPCOMSTAT = POINTER(_COMSTAT)
+
+ClearCommError = _stdcall_libraries['kernel32'].ClearCommError
+ClearCommError.restype = BOOL
+ClearCommError.argtypes = [HANDLE, LPDWORD, LPCOMSTAT]
+
+SetupComm = _stdcall_libraries['kernel32'].SetupComm
+SetupComm.restype = BOOL
+SetupComm.argtypes = [HANDLE, DWORD, DWORD]
+
+EscapeCommFunction = _stdcall_libraries['kernel32'].EscapeCommFunction
+EscapeCommFunction.restype = BOOL
+EscapeCommFunction.argtypes = [HANDLE, DWORD]
+
+GetCommModemStatus = _stdcall_libraries['kernel32'].GetCommModemStatus
+GetCommModemStatus.restype = BOOL
+GetCommModemStatus.argtypes = [HANDLE, LPDWORD]
+
+LPDCB = POINTER(_DCB)
+
+GetCommState = _stdcall_libraries['kernel32'].GetCommState
+GetCommState.restype = BOOL
+GetCommState.argtypes = [HANDLE, LPDCB]
+
+LPCOMMTIMEOUTS = POINTER(_COMMTIMEOUTS)
+
+GetCommTimeouts = _stdcall_libraries['kernel32'].GetCommTimeouts
+GetCommTimeouts.restype = BOOL
+GetCommTimeouts.argtypes = [HANDLE, LPCOMMTIMEOUTS]
+
+PurgeComm = _stdcall_libraries['kernel32'].PurgeComm
+PurgeComm.restype = BOOL
+PurgeComm.argtypes = [HANDLE, DWORD]
+
+SetCommBreak = _stdcall_libraries['kernel32'].SetCommBreak
+SetCommBreak.restype = BOOL
+SetCommBreak.argtypes = [HANDLE]
+
+SetCommMask = _stdcall_libraries['kernel32'].SetCommMask
+SetCommMask.restype = BOOL
+SetCommMask.argtypes = [HANDLE, DWORD]
+
+SetCommState = _stdcall_libraries['kernel32'].SetCommState
+SetCommState.restype = BOOL
+SetCommState.argtypes = [HANDLE, LPDCB]
+
+SetCommTimeouts = _stdcall_libraries['kernel32'].SetCommTimeouts
+SetCommTimeouts.restype = BOOL
+SetCommTimeouts.argtypes = [HANDLE, LPCOMMTIMEOUTS]
+
+WaitForSingleObject = _stdcall_libraries['kernel32'].WaitForSingleObject
+WaitForSingleObject.restype = DWORD
+WaitForSingleObject.argtypes = [HANDLE, DWORD]
+
+WaitCommEvent = _stdcall_libraries['kernel32'].WaitCommEvent
+WaitCommEvent.restype = BOOL
+WaitCommEvent.argtypes = [HANDLE, LPDWORD, LPOVERLAPPED]
+
+CancelIoEx = _stdcall_libraries['kernel32'].CancelIoEx
+CancelIoEx.restype = BOOL
+CancelIoEx.argtypes = [HANDLE, LPOVERLAPPED]
+
+ONESTOPBIT = 0  # Variable c_int
+TWOSTOPBITS = 2  # Variable c_int
+ONE5STOPBITS = 1
+
+NOPARITY = 0  # Variable c_int
+ODDPARITY = 1  # Variable c_int
+EVENPARITY = 2  # Variable c_int
+MARKPARITY = 3
+SPACEPARITY = 4
+
+RTS_CONTROL_HANDSHAKE = 2  # Variable c_int
+RTS_CONTROL_DISABLE = 0  # Variable c_int
+RTS_CONTROL_ENABLE = 1  # Variable c_int
+RTS_CONTROL_TOGGLE = 3  # Variable c_int
+SETRTS = 3
+CLRRTS = 4
+
+DTR_CONTROL_HANDSHAKE = 2  # Variable c_int
+DTR_CONTROL_DISABLE = 0  # Variable c_int
+DTR_CONTROL_ENABLE = 1  # Variable c_int
+SETDTR = 5
+CLRDTR = 6
+
+MS_DSR_ON = 32  # Variable c_ulong
+EV_RING = 256  # Variable c_int
+EV_PERR = 512  # Variable c_int
+EV_ERR = 128  # Variable c_int
+SETXOFF = 1  # Variable c_int
+EV_RXCHAR = 1  # Variable c_int
+GENERIC_WRITE = 1073741824  # Variable c_long
+PURGE_TXCLEAR = 4  # Variable c_int
+FILE_FLAG_OVERLAPPED = 1073741824  # Variable c_int
+EV_DSR = 16  # Variable c_int
+MAXDWORD = 4294967295  # Variable c_uint
+EV_RLSD = 32  # Variable c_int
+
+ERROR_SUCCESS = 0
+ERROR_NOT_ENOUGH_MEMORY = 8
+ERROR_OPERATION_ABORTED = 995
+ERROR_IO_INCOMPLETE = 996
+ERROR_IO_PENDING = 997  # Variable c_long
+ERROR_INVALID_USER_BUFFER = 1784
+
+MS_CTS_ON = 16  # Variable c_ulong
+EV_EVENT1 = 2048  # Variable c_int
+EV_RX80FULL = 1024  # Variable c_int
+PURGE_RXABORT = 2  # Variable c_int
+FILE_ATTRIBUTE_NORMAL = 128  # Variable c_int
+PURGE_TXABORT = 1  # Variable c_int
+SETXON = 2  # Variable c_int
+OPEN_EXISTING = 3  # Variable c_int
+MS_RING_ON = 64  # Variable c_ulong
+EV_TXEMPTY = 4  # Variable c_int
+EV_RXFLAG = 2  # Variable c_int
+MS_RLSD_ON = 128  # Variable c_ulong
+GENERIC_READ = 2147483648  # Variable c_ulong
+EV_EVENT2 = 4096  # Variable c_int
+EV_CTS = 8  # Variable c_int
+EV_BREAK = 64  # Variable c_int
+PURGE_RXCLEAR = 8  # Variable c_int
+INFINITE = 0xFFFFFFFF
+
+CE_RXOVER = 0x0001
+CE_OVERRUN = 0x0002
+CE_RXPARITY = 0x0004
+CE_FRAME = 0x0008
+CE_BREAK = 0x0010
+
+
+class N11_OVERLAPPED4DOLLAR_48E(Union):
+    pass
+
+
+class N11_OVERLAPPED4DOLLAR_484DOLLAR_49E(Structure):
+    pass
+
+
+N11_OVERLAPPED4DOLLAR_484DOLLAR_49E._fields_ = [
+    ('Offset', DWORD),
+    ('OffsetHigh', DWORD),
+]
+
+PVOID = c_void_p
+
+N11_OVERLAPPED4DOLLAR_48E._anonymous_ = ['_0']
+N11_OVERLAPPED4DOLLAR_48E._fields_ = [
+    ('_0', N11_OVERLAPPED4DOLLAR_484DOLLAR_49E),
+    ('Pointer', PVOID),
+]
+_OVERLAPPED._anonymous_ = ['_0']
+_OVERLAPPED._fields_ = [
+    ('Internal', ULONG_PTR),
+    ('InternalHigh', ULONG_PTR),
+    ('_0', N11_OVERLAPPED4DOLLAR_48E),
+    ('hEvent', HANDLE),
+]
+_SECURITY_ATTRIBUTES._fields_ = [
+    ('nLength', DWORD),
+    ('lpSecurityDescriptor', LPVOID),
+    ('bInheritHandle', BOOL),
+]
+_COMSTAT._fields_ = [
+    ('fCtsHold', DWORD, 1),
+    ('fDsrHold', DWORD, 1),
+    ('fRlsdHold', DWORD, 1),
+    ('fXoffHold', DWORD, 1),
+    ('fXoffSent', DWORD, 1),
+    ('fEof', DWORD, 1),
+    ('fTxim', DWORD, 1),
+    ('fReserved', DWORD, 25),
+    ('cbInQue', DWORD),
+    ('cbOutQue', DWORD),
+]
+_DCB._fields_ = [
+    ('DCBlength', DWORD),
+    ('BaudRate', DWORD),
+    ('fBinary', DWORD, 1),
+    ('fParity', DWORD, 1),
+    ('fOutxCtsFlow', DWORD, 1),
+    ('fOutxDsrFlow', DWORD, 1),
+    ('fDtrControl', DWORD, 2),
+    ('fDsrSensitivity', DWORD, 1),
+    ('fTXContinueOnXoff', DWORD, 1),
+    ('fOutX', DWORD, 1),
+    ('fInX', DWORD, 1),
+    ('fErrorChar', DWORD, 1),
+    ('fNull', DWORD, 1),
+    ('fRtsControl', DWORD, 2),
+    ('fAbortOnError', DWORD, 1),
+    ('fDummy2', DWORD, 17),
+    ('wReserved', WORD),
+    ('XonLim', WORD),
+    ('XoffLim', WORD),
+    ('ByteSize', BYTE),
+    ('Parity', BYTE),
+    ('StopBits', BYTE),
+    ('XonChar', c_char),
+    ('XoffChar', c_char),
+    ('ErrorChar', c_char),
+    ('EofChar', c_char),
+    ('EvtChar', c_char),
+    ('wReserved1', WORD),
+]
+_COMMTIMEOUTS._fields_ = [
+    ('ReadIntervalTimeout', DWORD),
+    ('ReadTotalTimeoutMultiplier', DWORD),
+    ('ReadTotalTimeoutConstant', DWORD),
+    ('WriteTotalTimeoutMultiplier', DWORD),
+    ('WriteTotalTimeoutConstant', DWORD),
+]
+__all__ = ['GetLastError', 'MS_CTS_ON', 'FILE_ATTRIBUTE_NORMAL',
+           'DTR_CONTROL_ENABLE', '_COMSTAT', 'MS_RLSD_ON',
+           'GetOverlappedResult', 'SETXON', 'PURGE_TXABORT',
+           'PurgeComm', 'N11_OVERLAPPED4DOLLAR_48E', 'EV_RING',
+           'ONESTOPBIT', 'SETXOFF', 'PURGE_RXABORT', 'GetCommState',
+           'RTS_CONTROL_ENABLE', '_DCB', 'CreateEvent',
+           '_COMMTIMEOUTS', '_SECURITY_ATTRIBUTES', 'EV_DSR',
+           'EV_PERR', 'EV_RXFLAG', 'OPEN_EXISTING', 'DCB',
+           'FILE_FLAG_OVERLAPPED', 'EV_CTS', 'SetupComm',
+           'LPOVERLAPPED', 'EV_TXEMPTY', 'ClearCommBreak',
+           'LPSECURITY_ATTRIBUTES', 'SetCommBreak', 'SetCommTimeouts',
+           'COMMTIMEOUTS', 'ODDPARITY', 'EV_RLSD',
+           'GetCommModemStatus', 'EV_EVENT2', 'PURGE_TXCLEAR',
+           'EV_BREAK', 'EVENPARITY', 'LPCVOID', 'COMSTAT', 'ReadFile',
+           'PVOID', '_OVERLAPPED', 'WriteFile', 'GetCommTimeouts',
+           'ResetEvent', 'EV_RXCHAR', 'LPCOMSTAT', 'ClearCommError',
+           'ERROR_IO_PENDING', 'EscapeCommFunction', 'GENERIC_READ',
+           'RTS_CONTROL_HANDSHAKE', 'OVERLAPPED',
+           'DTR_CONTROL_HANDSHAKE', 'PURGE_RXCLEAR', 'GENERIC_WRITE',
+           'LPDCB', 'CreateEventW', 'SetCommMask', 'EV_EVENT1',
+           'SetCommState', 'LPVOID', 'CreateFileW', 'LPDWORD',
+           'EV_RX80FULL', 'TWOSTOPBITS', 'LPCOMMTIMEOUTS', 'MAXDWORD',
+           'MS_DSR_ON', 'MS_RING_ON',
+           'N11_OVERLAPPED4DOLLAR_484DOLLAR_49E', 'EV_ERR',
+           'ULONG_PTR', 'CreateFile', 'NOPARITY', 'CloseHandle']

tools/python/backports/__init__.py

@@ -0,0 +1,4 @@
+# See https://pypi.python.org/pypi/backports
+
+from pkgutil import extend_path
+__path__ = extend_path(__path__, __name__)

tools/python/backports/tempfile.py

@@ -0,0 +1,75 @@
+"""
+Partial backport of Python 3.5's tempfile module:
+
+    TemporaryDirectory
+
+Backport modifications are marked with marked with "XXX backport".
+"""
+from __future__ import absolute_import
+
+import sys
+import warnings as _warnings
+from shutil import rmtree as _rmtree
+
+from backports.weakref import finalize
+
+
+# XXX backport:  Rather than backporting all of mkdtemp(), we just create a
+# thin wrapper implementing its Python 3.5 signature.
+if sys.version_info < (3, 5):
+    from tempfile import mkdtemp as old_mkdtemp
+
+    def mkdtemp(suffix=None, prefix=None, dir=None):
+        """
+        Wrap `tempfile.mkdtemp()` to make the suffix and prefix optional (like Python 3.5).
+        """
+        kwargs = {k: v for (k, v) in
+                  dict(suffix=suffix, prefix=prefix, dir=dir).items()
+                  if v is not None}
+        return old_mkdtemp(**kwargs)
+
+else:
+    from tempfile import mkdtemp
+
+
+# XXX backport: ResourceWarning was added in Python 3.2.
+# For earlier versions, fall back to RuntimeWarning instead.
+_ResourceWarning = RuntimeWarning if sys.version_info < (3, 2) else ResourceWarning
+
+
+class TemporaryDirectory(object):
+    """Create and return a temporary directory.  This has the same
+    behavior as mkdtemp but can be used as a context manager.  For
+    example:
+
+        with TemporaryDirectory() as tmpdir:
+            ...
+
+    Upon exiting the context, the directory and everything contained
+    in it are removed.
+    """
+
+    def __init__(self, suffix=None, prefix=None, dir=None):
+        self.name = mkdtemp(suffix, prefix, dir)
+        self._finalizer = finalize(
+            self, self._cleanup, self.name,
+            warn_message="Implicitly cleaning up {!r}".format(self))
+
+    @classmethod
+    def _cleanup(cls, name, warn_message):
+        _rmtree(name)
+        _warnings.warn(warn_message, _ResourceWarning)
+
+
+    def __repr__(self):
+        return "<{} {!r}>".format(self.__class__.__name__, self.name)
+
+    def __enter__(self):
+        return self.name
+
+    def __exit__(self, exc, value, tb):
+        self.cleanup()
+
+    def cleanup(self):
+        if self._finalizer.detach():
+            _rmtree(self.name)

tools/python/backports/weakref.py

@@ -0,0 +1,151 @@
+"""
+Partial backport of Python 3.6's weakref module:
+
+    finalize (new in Python 3.4)
+
+Backport modifications are marked with "XXX backport".
+"""
+from __future__ import absolute_import
+
+import itertools
+import sys
+from weakref import ref
+
+__all__ = ['finalize']
+
+
+class finalize(object):
+    """Class for finalization of weakrefable objects
+
+    finalize(obj, func, *args, **kwargs) returns a callable finalizer
+    object which will be called when obj is garbage collected. The
+    first time the finalizer is called it evaluates func(*arg, **kwargs)
+    and returns the result. After this the finalizer is dead, and
+    calling it just returns None.
+
+    When the program exits any remaining finalizers for which the
+    atexit attribute is true will be run in reverse order of creation.
+    By default atexit is true.
+    """
+
+    # Finalizer objects don't have any state of their own.  They are
+    # just used as keys to lookup _Info objects in the registry.  This
+    # ensures that they cannot be part of a ref-cycle.
+
+    __slots__ = ()
+    _registry = {}
+    _shutdown = False
+    _index_iter = itertools.count()
+    _dirty = False
+    _registered_with_atexit = False
+
+    class _Info(object):
+        __slots__ = ("weakref", "func", "args", "kwargs", "atexit", "index")
+
+    def __init__(self, obj, func, *args, **kwargs):
+        if not self._registered_with_atexit:
+            # We may register the exit function more than once because
+            # of a thread race, but that is harmless
+            import atexit
+            atexit.register(self._exitfunc)
+            finalize._registered_with_atexit = True
+        info = self._Info()
+        info.weakref = ref(obj, self)
+        info.func = func
+        info.args = args
+        info.kwargs = kwargs or None
+        info.atexit = True
+        info.index = next(self._index_iter)
+        self._registry[self] = info
+        finalize._dirty = True
+
+    def __call__(self, _=None):
+        """If alive then mark as dead and return func(*args, **kwargs);
+        otherwise return None"""
+        info = self._registry.pop(self, None)
+        if info and not self._shutdown:
+            return info.func(*info.args, **(info.kwargs or {}))
+
+    def detach(self):
+        """If alive then mark as dead and return (obj, func, args, kwargs);
+        otherwise return None"""
+        info = self._registry.get(self)
+        obj = info and info.weakref()
+        if obj is not None and self._registry.pop(self, None):
+            return (obj, info.func, info.args, info.kwargs or {})
+
+    def peek(self):
+        """If alive then return (obj, func, args, kwargs);
+        otherwise return None"""
+        info = self._registry.get(self)
+        obj = info and info.weakref()
+        if obj is not None:
+            return (obj, info.func, info.args, info.kwargs or {})
+
+    @property
+    def alive(self):
+        """Whether finalizer is alive"""
+        return self in self._registry
+
+    @property
+    def atexit(self):
+        """Whether finalizer should be called at exit"""
+        info = self._registry.get(self)
+        return bool(info) and info.atexit
+
+    @atexit.setter
+    def atexit(self, value):
+        info = self._registry.get(self)
+        if info:
+            info.atexit = bool(value)
+
+    def __repr__(self):
+        info = self._registry.get(self)
+        obj = info and info.weakref()
+        if obj is None:
+            return '<%s object at %#x; dead>' % (type(self).__name__, id(self))
+        else:
+            return '<%s object at %#x; for %r at %#x>' % \
+                (type(self).__name__, id(self), type(obj).__name__, id(obj))
+
+    @classmethod
+    def _select_for_exit(cls):
+        # Return live finalizers marked for exit, oldest first
+        L = [(f,i) for (f,i) in cls._registry.items() if i.atexit]
+        L.sort(key=lambda item:item[1].index)
+        return [f for (f,i) in L]
+
+    @classmethod
+    def _exitfunc(cls):
+        # At shutdown invoke finalizers for which atexit is true.
+        # This is called once all other non-daemonic threads have been
+        # joined.
+        reenable_gc = False
+        try:
+            if cls._registry:
+                import gc
+                if gc.isenabled():
+                    reenable_gc = True
+                    gc.disable()
+                pending = None
+                while True:
+                    if pending is None or finalize._dirty:
+                        pending = cls._select_for_exit()
+                        finalize._dirty = False
+                    if not pending:
+                        break
+                    f = pending.pop()
+                    try:
+                        # gc is disabled, so (assuming no daemonic
+                        # threads) the following is the only line in
+                        # this function which might trigger creation
+                        # of a new finalizer
+                        f()
+                    except Exception:
+                        sys.excepthook(*sys.exc_info())
+                    assert f not in cls._registry
+        finally:
+            # prevent any more finalizers from executing during shutdown
+            finalize._shutdown = True
+            if reenable_gc:
+                gc.enable()

tools/python/esptool/__main__.py

@@ -0,0 +1,9 @@
+# SPDX-FileCopyrightText: 2014-2022 Fredrik Ahlberg, Angus Gratton,
+# Espressif Systems (Shanghai) CO LTD, other contributors as noted.
+#
+# SPDX-License-Identifier: GPL-2.0-or-later
+
+import esptool
+
+if __name__ == "__main__":
+    esptool._main()

tools/python/esptool/config.py

@@ -0,0 +1,92 @@
+# SPDX-FileCopyrightText: 2014-2023 Espressif Systems (Shanghai) CO LTD,
+# other contributors as noted.
+#
+# SPDX-License-Identifier: GPL-2.0-or-later
+
+import configparser
+import os
+
+CONFIG_OPTIONS = [
+    "timeout",
+    "chip_erase_timeout",
+    "max_timeout",
+    "sync_timeout",
+    "md5_timeout_per_mb",
+    "erase_region_timeout_per_mb",
+    "erase_write_timeout_per_mb",
+    "mem_end_rom_timeout",
+    "serial_write_timeout",
+    "connect_attempts",
+    "write_block_attempts",
+    "reset_delay",
+    "custom_reset_sequence",
+]
+
+
+def _validate_config_file(file_path, verbose=False):
+    if not os.path.exists(file_path):
+        return False
+
+    cfg = configparser.RawConfigParser()
+    try:
+        cfg.read(file_path, encoding="UTF-8")
+        # Only consider it a valid config file if it contains [esptool] section
+        if cfg.has_section("esptool"):
+            if verbose:
+                unknown_opts = list(set(cfg.options("esptool")) - set(CONFIG_OPTIONS))
+                unknown_opts.sort()
+                no_of_unknown_opts = len(unknown_opts)
+                if no_of_unknown_opts > 0:
+                    suffix = "s" if no_of_unknown_opts > 1 else ""
+                    print(
+                        "Ignoring unknown config file option{}: {}".format(
+                            suffix, ", ".join(unknown_opts)
+                        )
+                    )
+            return True
+    except (UnicodeDecodeError, configparser.Error) as e:
+        if verbose:
+            print(f"Ignoring invalid config file {file_path}: {e}")
+    return False
+
+
+def _find_config_file(dir_path, verbose=False):
+    for candidate in ("esptool.cfg", "setup.cfg", "tox.ini"):
+        cfg_path = os.path.join(dir_path, candidate)
+        if _validate_config_file(cfg_path, verbose):
+            return cfg_path
+    return None
+
+
+def load_config_file(verbose=False):
+    set_with_env_var = False
+    env_var_path = os.environ.get("ESPTOOL_CFGFILE")
+    if env_var_path is not None and _validate_config_file(env_var_path):
+        cfg_file_path = env_var_path
+        set_with_env_var = True
+    else:
+        home_dir = os.path.expanduser("~")
+        os_config_dir = (
+            f"{home_dir}/.config/esptool"
+            if os.name == "posix"
+            else f"{home_dir}/AppData/Local/esptool/"
+        )
+        # Search priority: 1) current dir, 2) OS specific config dir, 3) home dir
+        for dir_path in (os.getcwd(), os_config_dir, home_dir):
+            cfg_file_path = _find_config_file(dir_path, verbose)
+            if cfg_file_path:
+                break
+
+    cfg = configparser.ConfigParser()
+    cfg["esptool"] = {}  # Create an empty esptool config for when no file is found
+
+    if cfg_file_path is not None:
+        # If config file is found and validated, read and parse it
+        cfg.read(cfg_file_path)
+        if verbose:
+            msg = " (set with ESPTOOL_CFGFILE)" if set_with_env_var else ""
+            print(
+                f"Loaded custom configuration from "
+                f"{os.path.abspath(cfg_file_path)}{msg}"
+            )
+    return cfg, cfg_file_path

tools/python/esptool/reset.py

@@ -0,0 +1,178 @@
+# SPDX-FileCopyrightText: 2014-2023 Fredrik Ahlberg, Angus Gratton,
+# Espressif Systems (Shanghai) CO LTD, other contributors as noted.
+#
+# SPDX-License-Identifier: GPL-2.0-or-later
+
+import os
+import struct
+import time
+
+from util import FatalError
+
+# Used for resetting into bootloader on Unix-like systems
+if os.name != "nt":
+    import fcntl
+    import termios
+
+    # Constants used for terminal status lines reading/setting.
+    # Taken from pySerial's backend for IO:
+    # https://github.com/pyserial/pyserial/blob/master/serial/serialposix.py
+    TIOCMSET = getattr(termios, "TIOCMSET", 0x5418)
+    TIOCMGET = getattr(termios, "TIOCMGET", 0x5415)
+    TIOCM_DTR = getattr(termios, "TIOCM_DTR", 0x002)
+    TIOCM_RTS = getattr(termios, "TIOCM_RTS", 0x004)
+
+DEFAULT_RESET_DELAY = 0.05  # default time to wait before releasing boot pin after reset
+
+
+class ResetStrategy(object):
+    def __init__(self, port, reset_delay=DEFAULT_RESET_DELAY):
+        self.port = port
+        self.reset_delay = reset_delay
+
+    def __call__():
+        pass
+
+    def _setDTR(self, state):
+        self.port.setDTR(state)
+
+    def _setRTS(self, state):
+        self.port.setRTS(state)
+        # Work-around for adapters on Windows using the usbser.sys driver:
+        # generate a dummy change to DTR so that the set-control-line-state
+        # request is sent with the updated RTS state and the same DTR state
+        self.port.setDTR(self.port.dtr)
+
+    def _setDTRandRTS(self, dtr=False, rts=False):
+        status = struct.unpack(
+            "I", fcntl.ioctl(self.port.fileno(), TIOCMGET, struct.pack("I", 0))
+        )[0]
+        if dtr:
+            status |= TIOCM_DTR
+        else:
+            status &= ~TIOCM_DTR
+        if rts:
+            status |= TIOCM_RTS
+        else:
+            status &= ~TIOCM_RTS
+        fcntl.ioctl(self.port.fileno(), TIOCMSET, struct.pack("I", status))
+
+
+class ClassicReset(ResetStrategy):
+    """
+    Classic reset sequence, sets DTR and RTS lines sequentially.
+    """
+
+    def __call__(self):
+        self._setDTR(False)  # IO0=HIGH
+        self._setRTS(True)  # EN=LOW, chip in reset
+        time.sleep(0.1)
+        self._setDTR(True)  # IO0=LOW
+        self._setRTS(False)  # EN=HIGH, chip out of reset
+        time.sleep(self.reset_delay)
+        self._setDTR(False)  # IO0=HIGH, done
+
+
+class UnixTightReset(ResetStrategy):
+    """
+    UNIX-only reset sequence with custom implementation,
+    which allows setting DTR and RTS lines at the same time.
+    """
+
+    def __call__(self):
+        self._setDTRandRTS(False, False)
+        self._setDTRandRTS(True, True)
+        self._setDTRandRTS(False, True)  # IO0=HIGH & EN=LOW, chip in reset
+        time.sleep(0.1)
+        self._setDTRandRTS(True, False)  # IO0=LOW & EN=HIGH, chip out of reset
+        time.sleep(self.reset_delay)
+        self._setDTRandRTS(False, False)  # IO0=HIGH, done
+        self._setDTR(False)  # Needed in some environments to ensure IO0=HIGH
+
+
+class USBJTAGSerialReset(ResetStrategy):
+    """
+    Custom reset sequence, which is required when the device
+    is connecting via its USB-JTAG-Serial peripheral.
+    """
+
+    def __call__(self):
+        self._setRTS(False)
+        self._setDTR(False)  # Idle
+        time.sleep(0.1)
+        self._setDTR(True)  # Set IO0
+        self._setRTS(False)
+        time.sleep(0.1)
+        self._setRTS(True)  # Reset. Calls inverted to go through (1,1) instead of (0,0)
+        self._setDTR(False)
+        self._setRTS(True)  # RTS set as Windows only propagates DTR on RTS setting
+        time.sleep(0.1)
+        self._setDTR(False)
+        self._setRTS(False)  # Chip out of reset
+
+
+class HardReset(ResetStrategy):
+    """
+    Reset sequence for hard resetting the chip.
+    Can be used to reset out of the bootloader or to restart a running app.
+    """
+
+    def __init__(self, port, uses_usb_otg=False):
+        super().__init__(port)
+        self.uses_usb_otg = uses_usb_otg
+
+    def __call__(self):
+        self._setRTS(True)  # EN->LOW
+        if self.uses_usb_otg:
+            # Give the chip some time to come out of reset,
+            # to be able to handle further DTR/RTS transitions
+            time.sleep(0.2)
+            self._setRTS(False)
+            time.sleep(0.2)
+        else:
+            time.sleep(0.1)
+            self._setRTS(False)
+
+
+class CustomReset(ResetStrategy):
+    """
+    Custom reset strategy defined with a string.
+
+    CustomReset object is created as "rst = CustomReset(port, seq_str)"
+    and can be later executed simply with "rst()"
+
+    The seq_str input string consists of individual commands divided by "|".
+    Commands (e.g. R0) are defined by a code (R) and an argument (0).
+
+    The commands are:
+    D: setDTR - 1=True / 0=False
+    R: setRTS - 1=True / 0=False
+    U: setDTRandRTS (Unix-only) - 0,0 / 0,1 / 1,0 / or 1,1
+    W: Wait (time delay) - positive float number
+
+    e.g.
+    "D0|R1|W0.1|D1|R0|W0.05|D0" represents the ClassicReset strategy
+    "U1,1|U0,1|W0.1|U1,0|W0.05|U0,0" represents the UnixTightReset strategy
+    """
+
+    format_dict = {
+        "D": "self.port.setDTR({})",
+        "R": "self.port.setRTS({})",
+        "W": "time.sleep({})",
+        "U": "self._setDTRandRTS({})",
+    }
+
+    def __call__(self):
+        exec(self.constructed_strategy)
+
+    def __init__(self, port, seq_str):
+        super().__init__(port)
+        self.constructed_strategy = self._parse_string_to_seq(seq_str)
+
+    def _parse_string_to_seq(self, seq_str):
+        try:
+            cmds = seq_str.split("|")
+            fn_calls_list = [self.format_dict[cmd[0]].format(cmd[1:]) for cmd in cmds]
+        except Exception as e:
+            raise FatalError(f'Invalid "custom_reset_sequence" option format: {e}')
+        return "\n".join(fn_calls_list)

tools/python/esptool/targets/__init__.py

@@ -0,0 +1,31 @@
+from .esp32 import ESP32ROM
+from .esp32c2 import ESP32C2ROM
+from .esp32c3 import ESP32C3ROM
+from .esp32c6 import ESP32C6ROM
+from .esp32c6beta import ESP32C6BETAROM
+from .esp32h2 import ESP32H2ROM
+from .esp32h2beta1 import ESP32H2BETA1ROM
+from .esp32h2beta2 import ESP32H2BETA2ROM
+from .esp32s2 import ESP32S2ROM
+from .esp32s3 import ESP32S3ROM
+from .esp32s3beta2 import ESP32S3BETA2ROM
+from .esp8266 import ESP8266ROM
+
+
+CHIP_DEFS = {
+    "esp8266": ESP8266ROM,
+    "esp32": ESP32ROM,
+    "esp32s2": ESP32S2ROM,
+    "esp32s3beta2": ESP32S3BETA2ROM,
+    "esp32s3": ESP32S3ROM,
+    "esp32c3": ESP32C3ROM,
+    "esp32c6beta": ESP32C6BETAROM,
+    "esp32h2beta1": ESP32H2BETA1ROM,
+    "esp32h2beta2": ESP32H2BETA2ROM,
+    "esp32c2": ESP32C2ROM,
+    "esp32c6": ESP32C6ROM,
+    "esp32h2": ESP32H2ROM,
+}
+
+CHIP_LIST = list(CHIP_DEFS.keys())
+ROM_LIST = list(CHIP_DEFS.values())

tools/python/esptool/targets/esp32.py

@@ -0,0 +1,395 @@
+# SPDX-FileCopyrightText: 2014-2022 Fredrik Ahlberg, Angus Gratton,
+# Espressif Systems (Shanghai) CO LTD, other contributors as noted.
+#
+# SPDX-License-Identifier: GPL-2.0-or-later
+
+import struct
+import time
+
+from loader import ESPLoader
+from util import FatalError, NotSupportedError
+
+
+class ESP32ROM(ESPLoader):
+    """Access class for ESP32 ROM bootloader"""
+
+    CHIP_NAME = "ESP32"
+    IMAGE_CHIP_ID = 0
+    IS_STUB = False
+
+    FPGA_SLOW_BOOT = True
+
+    CHIP_DETECT_MAGIC_VALUE = [0x00F01D83]
+
+    IROM_MAP_START = 0x400D0000
+    IROM_MAP_END = 0x40400000
+
+    DROM_MAP_START = 0x3F400000
+    DROM_MAP_END = 0x3F800000
+
+    # ESP32 uses a 4 byte status reply
+    STATUS_BYTES_LENGTH = 4
+
+    SPI_REG_BASE = 0x3FF42000
+    SPI_USR_OFFS = 0x1C
+    SPI_USR1_OFFS = 0x20
+    SPI_USR2_OFFS = 0x24
+    SPI_MOSI_DLEN_OFFS = 0x28
+    SPI_MISO_DLEN_OFFS = 0x2C
+    EFUSE_RD_REG_BASE = 0x3FF5A000
+
+    EFUSE_DIS_DOWNLOAD_MANUAL_ENCRYPT_REG = EFUSE_RD_REG_BASE + 0x18
+    EFUSE_DIS_DOWNLOAD_MANUAL_ENCRYPT = 1 << 7  # EFUSE_RD_DISABLE_DL_ENCRYPT
+
+    EFUSE_SPI_BOOT_CRYPT_CNT_REG = EFUSE_RD_REG_BASE  # EFUSE_BLK0_WDATA0_REG
+    EFUSE_SPI_BOOT_CRYPT_CNT_MASK = 0x7F << 20  # EFUSE_FLASH_CRYPT_CNT
+
+    EFUSE_RD_ABS_DONE_REG = EFUSE_RD_REG_BASE + 0x018
+    EFUSE_RD_ABS_DONE_0_MASK = 1 << 4
+    EFUSE_RD_ABS_DONE_1_MASK = 1 << 5
+
+    DR_REG_SYSCON_BASE = 0x3FF66000
+    APB_CTL_DATE_ADDR = DR_REG_SYSCON_BASE + 0x7C
+    APB_CTL_DATE_V = 0x1
+    APB_CTL_DATE_S = 31
+
+    SPI_W0_OFFS = 0x80
+
+    UART_CLKDIV_REG = 0x3FF40014
+
+    XTAL_CLK_DIVIDER = 1
+
+    RTCCALICFG1 = 0x3FF5F06C
+    TIMERS_RTC_CALI_VALUE = 0x01FFFFFF
+    TIMERS_RTC_CALI_VALUE_S = 7
+
+    FLASH_SIZES = {
+        "1MB": 0x00,
+        "2MB": 0x10,
+        "4MB": 0x20,
+        "8MB": 0x30,
+        "16MB": 0x40,
+        "32MB": 0x50,
+        "64MB": 0x60,
+        "128MB": 0x70,
+    }
+
+    FLASH_FREQUENCY = {
+        "80m": 0xF,
+        "40m": 0x0,
+        "26m": 0x1,
+        "20m": 0x2,
+    }
+
+    BOOTLOADER_FLASH_OFFSET = 0x1000
+
+    OVERRIDE_VDDSDIO_CHOICES = ["1.8V", "1.9V", "OFF"]
+
+    MEMORY_MAP = [
+        [0x00000000, 0x00010000, "PADDING"],
+        [0x3F400000, 0x3F800000, "DROM"],
+        [0x3F800000, 0x3FC00000, "EXTRAM_DATA"],
+        [0x3FF80000, 0x3FF82000, "RTC_DRAM"],
+        [0x3FF90000, 0x40000000, "BYTE_ACCESSIBLE"],
+        [0x3FFAE000, 0x40000000, "DRAM"],
+        [0x3FFE0000, 0x3FFFFFFC, "DIRAM_DRAM"],
+        [0x40000000, 0x40070000, "IROM"],
+        [0x40070000, 0x40078000, "CACHE_PRO"],
+        [0x40078000, 0x40080000, "CACHE_APP"],
+        [0x40080000, 0x400A0000, "IRAM"],
+        [0x400A0000, 0x400BFFFC, "DIRAM_IRAM"],
+        [0x400C0000, 0x400C2000, "RTC_IRAM"],
+        [0x400D0000, 0x40400000, "IROM"],
+        [0x50000000, 0x50002000, "RTC_DATA"],
+    ]
+
+    FLASH_ENCRYPTED_WRITE_ALIGN = 32
+
+    """ Try to read the BLOCK1 (encryption key) and check if it is valid """
+
+    def is_flash_encryption_key_valid(self):
+        """Bit 0 of efuse_rd_disable[3:0] is mapped to BLOCK1
+        this bit is at position 16 in EFUSE_BLK0_RDATA0_REG"""
+        word0 = self.read_efuse(0)
+        rd_disable = (word0 >> 16) & 0x1
+
+        # reading of BLOCK1 is NOT ALLOWED so we assume valid key is programmed
+        if rd_disable:
+            return True
+        else:
+            # reading of BLOCK1 is ALLOWED so we will read and verify for non-zero.
+            # When ESP32 has not generated AES/encryption key in BLOCK1,
+            # the contents will be readable and 0.
+            # If the flash encryption is enabled it is expected to have a valid
+            # non-zero key. We break out on first occurance of non-zero value
+            key_word = [0] * 7
+            for i in range(len(key_word)):
+                key_word[i] = self.read_efuse(14 + i)
+                # key is non-zero so break & return
+                if key_word[i] != 0:
+                    return True
+            return False
+
+    def get_flash_crypt_config(self):
+        """For flash encryption related commands we need to make sure
+        user has programmed all the relevant efuse correctly so before
+        writing encrypted write_flash_encrypt esptool will verify the values
+        of flash_crypt_config to be non zero if they are not read
+        protected. If the values are zero a warning will be printed
+
+        bit 3 in efuse_rd_disable[3:0] is mapped to flash_crypt_config
+        this bit is at position 19 in EFUSE_BLK0_RDATA0_REG"""
+        word0 = self.read_efuse(0)
+        rd_disable = (word0 >> 19) & 0x1
+
+        if rd_disable == 0:
+            """we can read the flash_crypt_config efuse value
+            so go & read it (EFUSE_BLK0_RDATA5_REG[31:28])"""
+            word5 = self.read_efuse(5)
+            word5 = (word5 >> 28) & 0xF
+            return word5
+        else:
+            # if read of the efuse is disabled we assume it is set correctly
+            return 0xF
+
+    def get_encrypted_download_disabled(self):
+        return (
+            self.read_reg(self.EFUSE_DIS_DOWNLOAD_MANUAL_ENCRYPT_REG)
+            & self.EFUSE_DIS_DOWNLOAD_MANUAL_ENCRYPT
+        )
+
+    def get_flash_encryption_enabled(self):
+        flash_crypt_cnt = (
+            self.read_reg(self.EFUSE_SPI_BOOT_CRYPT_CNT_REG)
+            & self.EFUSE_SPI_BOOT_CRYPT_CNT_MASK
+        )
+        # Flash encryption enabled when odd number of bits are set
+        return bin(flash_crypt_cnt).count("1") & 1 != 0
+
+    def get_secure_boot_enabled(self):
+        efuses = self.read_reg(self.EFUSE_RD_ABS_DONE_REG)
+        rev = self.get_chip_revision()
+        return efuses & self.EFUSE_RD_ABS_DONE_0_MASK or (
+            rev >= 300 and efuses & self.EFUSE_RD_ABS_DONE_1_MASK
+        )
+
+    def get_pkg_version(self):
+        word3 = self.read_efuse(3)
+        pkg_version = (word3 >> 9) & 0x07
+        pkg_version += ((word3 >> 2) & 0x1) << 3
+        return pkg_version
+
+    def get_chip_revision(self):
+        return self.get_major_chip_version() * 100 + self.get_minor_chip_version()
+
+    def get_minor_chip_version(self):
+        return (self.read_efuse(5) >> 24) & 0x3
+
+    def get_major_chip_version(self):
+        rev_bit0 = (self.read_efuse(3) >> 15) & 0x1
+        rev_bit1 = (self.read_efuse(5) >> 20) & 0x1
+        apb_ctl_date = self.read_reg(self.APB_CTL_DATE_ADDR)
+        rev_bit2 = (apb_ctl_date >> self.APB_CTL_DATE_S) & self.APB_CTL_DATE_V
+        combine_value = (rev_bit2 << 2) | (rev_bit1 << 1) | rev_bit0
+
+        revision = {
+            0: 0,
+            1: 1,
+            3: 2,
+            7: 3,
+        }.get(combine_value, 0)
+        return revision
+
+    def get_chip_description(self):
+        pkg_version = self.get_pkg_version()
+        major_rev = self.get_major_chip_version()
+        minor_rev = self.get_minor_chip_version()
+        rev3 = major_rev == 3
+        single_core = self.read_efuse(3) & (1 << 0)  # CHIP_VER DIS_APP_CPU
+
+        chip_name = {
+            0: "ESP32-S0WDQ6" if single_core else "ESP32-D0WDQ6",
+            1: "ESP32-S0WD" if single_core else "ESP32-D0WD",
+            2: "ESP32-D2WD",
+            4: "ESP32-U4WDH",
+            5: "ESP32-PICO-V3" if rev3 else "ESP32-PICO-D4",
+            6: "ESP32-PICO-V3-02",
+            7: "ESP32-D0WDR2-V3",
+        }.get(pkg_version, "unknown ESP32")
+
+        # ESP32-D0WD-V3, ESP32-D0WDQ6-V3
+        if chip_name.startswith("ESP32-D0WD") and rev3:
+            chip_name += "-V3"
+
+        return f"{chip_name} (revision v{major_rev}.{minor_rev})"
+
+    def get_chip_features(self):
+        features = ["WiFi"]
+        word3 = self.read_efuse(3)
+
+        # names of variables in this section are lowercase
+        #  versions of EFUSE names as documented in TRM and
+        # ESP-IDF efuse_reg.h
+
+        chip_ver_dis_bt = word3 & (1 << 1)
+        if chip_ver_dis_bt == 0:
+            features += ["BT"]
+
+        chip_ver_dis_app_cpu = word3 & (1 << 0)
+        if chip_ver_dis_app_cpu:
+            features += ["Single Core"]
+        else:
+            features += ["Dual Core"]
+
+        chip_cpu_freq_rated = word3 & (1 << 13)
+        if chip_cpu_freq_rated:
+            chip_cpu_freq_low = word3 & (1 << 12)
+            if chip_cpu_freq_low:
+                features += ["160MHz"]
+            else:
+                features += ["240MHz"]
+
+        pkg_version = self.get_pkg_version()
+        if pkg_version in [2, 4, 5, 6]:
+            features += ["Embedded Flash"]
+
+        if pkg_version == 6:
+            features += ["Embedded PSRAM"]
+
+        word4 = self.read_efuse(4)
+        adc_vref = (word4 >> 8) & 0x1F
+        if adc_vref:
+            features += ["VRef calibration in efuse"]
+
+        blk3_part_res = word3 >> 14 & 0x1
+        if blk3_part_res:
+            features += ["BLK3 partially reserved"]
+
+        word6 = self.read_efuse(6)
+        coding_scheme = word6 & 0x3
+        features += [
+            "Coding Scheme %s"
+            % {0: "None", 1: "3/4", 2: "Repeat (UNSUPPORTED)", 3: "Invalid"}[
+                coding_scheme
+            ]
+        ]
+
+        return features
+
+    def read_efuse(self, n):
+        """Read the nth word of the ESP3x EFUSE region."""
+        return self.read_reg(self.EFUSE_RD_REG_BASE + (4 * n))
+
+    def chip_id(self):
+        raise NotSupportedError(self, "chip_id")
+
+    def read_mac(self, mac_type="BASE_MAC"):
+        """Read MAC from EFUSE region"""
+        if mac_type != "BASE_MAC":
+            return None
+        words = [self.read_efuse(2), self.read_efuse(1)]
+        bitstring = struct.pack(">II", *words)
+        bitstring = bitstring[2:8]  # trim the 2 byte CRC
+        return tuple(bitstring)
+
+    def get_erase_size(self, offset, size):
+        return size
+
+    def override_vddsdio(self, new_voltage):
+        new_voltage = new_voltage.upper()
+        if new_voltage not in self.OVERRIDE_VDDSDIO_CHOICES:
+            raise FatalError(
+                "The only accepted VDDSDIO overrides are '1.8V', '1.9V' and 'OFF'"
+            )
+        RTC_CNTL_SDIO_CONF_REG = 0x3FF48074
+        RTC_CNTL_XPD_SDIO_REG = 1 << 31
+        RTC_CNTL_DREFH_SDIO_M = 3 << 29
+        RTC_CNTL_DREFM_SDIO_M = 3 << 27
+        RTC_CNTL_DREFL_SDIO_M = 3 << 25
+        # RTC_CNTL_SDIO_TIEH = (1 << 23)
+        # not used here, setting TIEH=1 would set 3.3V output,
+        # not safe for esptool.py to do
+        RTC_CNTL_SDIO_FORCE = 1 << 22
+        RTC_CNTL_SDIO_PD_EN = 1 << 21
+
+        reg_val = RTC_CNTL_SDIO_FORCE  # override efuse setting
+        reg_val |= RTC_CNTL_SDIO_PD_EN
+        if new_voltage != "OFF":
+            reg_val |= RTC_CNTL_XPD_SDIO_REG  # enable internal LDO
+        if new_voltage == "1.9V":
+            reg_val |= (
+                RTC_CNTL_DREFH_SDIO_M | RTC_CNTL_DREFM_SDIO_M | RTC_CNTL_DREFL_SDIO_M
+            )  # boost voltage
+        self.write_reg(RTC_CNTL_SDIO_CONF_REG, reg_val)
+        print("VDDSDIO regulator set to %s" % new_voltage)
+
+    def read_flash_slow(self, offset, length, progress_fn):
+        BLOCK_LEN = 64  # ROM read limit per command (this limit is why it's so slow)
+
+        data = b""
+        while len(data) < length:
+            block_len = min(BLOCK_LEN, length - len(data))
+            r = self.check_command(
+                "read flash block",
+                self.ESP_READ_FLASH_SLOW,
+                struct.pack("<II", offset + len(data), block_len),
+            )
+            if len(r) < block_len:
+                raise FatalError(
+                    "Expected %d byte block, got %d bytes. Serial errors?"
+                    % (block_len, len(r))
+                )
+            # command always returns 64 byte buffer,
+            # regardless of how many bytes were actually read from flash
+            data += r[:block_len]
+            if progress_fn and (len(data) % 1024 == 0 or len(data) == length):
+                progress_fn(len(data), length)
+        return data
+
+    def get_rom_cal_crystal_freq(self):
+        """
+        Get the crystal frequency calculated by the ROM
+        """
+        # - Simulate the calculation in the ROM to get the XTAL frequency
+        #   calculated by the ROM
+
+        cali_val = (
+            self.read_reg(self.RTCCALICFG1) >> self.TIMERS_RTC_CALI_VALUE_S
+        ) & self.TIMERS_RTC_CALI_VALUE
+        clk_8M_freq = self.read_efuse(4) & (0xFF)  # EFUSE_RD_CK8M_FREQ
+        rom_calculated_freq = cali_val * 15625 * clk_8M_freq / 40
+        return rom_calculated_freq
+
+    def change_baud(self, baud):
+        # It's a workaround to avoid esp32 CK_8M frequency drift.
+        rom_calculated_freq = self.get_rom_cal_crystal_freq()
+        valid_freq = 40000000 if rom_calculated_freq > 33000000 else 26000000
+        false_rom_baud = int(baud * rom_calculated_freq // valid_freq)
+
+        print(f"Changing baud rate to {baud}")
+        self.command(self.ESP_CHANGE_BAUDRATE, struct.pack("<II", false_rom_baud, 0))
+        print("Changed.")
+        self._set_port_baudrate(baud)
+        time.sleep(0.05)  # get rid of garbage sent during baud rate change
+        self.flush_input()
+
+
+class ESP32StubLoader(ESP32ROM):
+    """Access class for ESP32 stub loader, runs on top of ROM."""
+
+    FLASH_WRITE_SIZE = 0x4000  # matches MAX_WRITE_BLOCK in stub_loader.c
+    STATUS_BYTES_LENGTH = 2  # same as ESP8266, different to ESP32 ROM
+    IS_STUB = True
+
+    def __init__(self, rom_loader):
+        self.secure_download_mode = rom_loader.secure_download_mode
+        self._port = rom_loader._port
+        self._trace_enabled = rom_loader._trace_enabled
+        self.cache = rom_loader.cache
+        self.flush_input()  # resets _slip_reader
+
+    def change_baud(self, baud):
+        ESPLoader.change_baud(self, baud)
+
+
+ESP32ROM.STUB_CLASS = ESP32StubLoader

tools/python/esptool/targets/esp32c2.py

@@ -0,0 +1,165 @@
+# SPDX-FileCopyrightText: 2014-2022 Fredrik Ahlberg, Angus Gratton,
+# Espressif Systems (Shanghai) CO LTD, other contributors as noted.
+#
+# SPDX-License-Identifier: GPL-2.0-or-later
+
+import struct
+import time
+
+from .esp32c3 import ESP32C3ROM
+from loader import ESPLoader
+
+
+class ESP32C2ROM(ESP32C3ROM):
+    CHIP_NAME = "ESP32-C2"
+    IMAGE_CHIP_ID = 12
+
+    IROM_MAP_START = 0x42000000
+    IROM_MAP_END = 0x42400000
+    DROM_MAP_START = 0x3C000000
+    DROM_MAP_END = 0x3C400000
+
+    # Magic value for ESP32C2 ECO0 and ECO1 respectively
+    CHIP_DETECT_MAGIC_VALUE = [0x6F51306F, 0x7C41A06F]
+
+    EFUSE_BASE = 0x60008800
+    EFUSE_BLOCK2_ADDR = EFUSE_BASE + 0x040
+    MAC_EFUSE_REG = EFUSE_BASE + 0x040
+
+    EFUSE_SECURE_BOOT_EN_REG = EFUSE_BASE + 0x30
+    EFUSE_SECURE_BOOT_EN_MASK = 1 << 21
+
+    EFUSE_SPI_BOOT_CRYPT_CNT_REG = EFUSE_BASE + 0x30
+    EFUSE_SPI_BOOT_CRYPT_CNT_MASK = 0x7 << 18
+
+    EFUSE_DIS_DOWNLOAD_MANUAL_ENCRYPT_REG = EFUSE_BASE + 0x30
+    EFUSE_DIS_DOWNLOAD_MANUAL_ENCRYPT = 1 << 6
+
+    EFUSE_XTS_KEY_LENGTH_256_REG = EFUSE_BASE + 0x30
+    EFUSE_XTS_KEY_LENGTH_256 = 1 << 10
+
+    EFUSE_BLOCK_KEY0_REG = EFUSE_BASE + 0x60
+
+    EFUSE_RD_DIS_REG = EFUSE_BASE + 0x30
+    EFUSE_RD_DIS = 3
+
+    FLASH_FREQUENCY = {
+        "60m": 0xF,
+        "30m": 0x0,
+        "20m": 0x1,
+        "15m": 0x2,
+    }
+
+    MEMORY_MAP = [
+        [0x00000000, 0x00010000, "PADDING"],
+        [0x3C000000, 0x3C400000, "DROM"],
+        [0x3FCA0000, 0x3FCE0000, "DRAM"],
+        [0x3FC88000, 0x3FD00000, "BYTE_ACCESSIBLE"],
+        [0x3FF00000, 0x3FF50000, "DROM_MASK"],
+        [0x40000000, 0x40090000, "IROM_MASK"],
+        [0x42000000, 0x42400000, "IROM"],
+        [0x4037C000, 0x403C0000, "IRAM"],
+    ]
+
+    def get_pkg_version(self):
+        num_word = 1
+        return (self.read_reg(self.EFUSE_BLOCK2_ADDR + (4 * num_word)) >> 22) & 0x07
+
+    def get_chip_description(self):
+        chip_name = {
+            0: "ESP32-C2",
+            1: "ESP32-C2",
+        }.get(self.get_pkg_version(), "unknown ESP32-C2")
+        major_rev = self.get_major_chip_version()
+        minor_rev = self.get_minor_chip_version()
+        return f"{chip_name} (revision v{major_rev}.{minor_rev})"
+
+    def get_minor_chip_version(self):
+        num_word = 1
+        return (self.read_reg(self.EFUSE_BLOCK2_ADDR + (4 * num_word)) >> 16) & 0xF
+
+    def get_major_chip_version(self):
+        num_word = 1
+        return (self.read_reg(self.EFUSE_BLOCK2_ADDR + (4 * num_word)) >> 20) & 0x3
+
+    def get_crystal_freq(self):
+        # The crystal detection algorithm of ESP32/ESP8266 works for ESP32-C2 as well.
+        return ESPLoader.get_crystal_freq(self)
+
+    def change_baud(self, baud):
+        rom_with_26M_XTAL = not self.IS_STUB and self.get_crystal_freq() == 26
+        if rom_with_26M_XTAL:
+            # The code is copied over from ESPLoader.change_baud().
+            # Probably this is just a temporary solution until the next chip revision.
+
+            # The ROM code thinks it uses a 40 MHz XTAL. Recompute the baud rate
+            # in order to trick the ROM code to set the correct baud rate for
+            # a 26 MHz XTAL.
+            false_rom_baud = baud * 40 // 26
+
+            print(f"Changing baud rate to {baud}")
+            self.command(
+                self.ESP_CHANGE_BAUDRATE, struct.pack("<II", false_rom_baud, 0)
+            )
+            print("Changed.")
+            self._set_port_baudrate(baud)
+            time.sleep(0.05)  # get rid of garbage sent during baud rate change
+            self.flush_input()
+        else:
+            ESPLoader.change_baud(self, baud)
+
+    def _post_connect(self):
+        # ESP32C2 ECO0 is no longer supported by the flasher stub
+        if self.get_chip_revision() == 0:
+            self.stub_is_disabled = True
+            self.IS_STUB = False
+
+    """ Try to read (encryption key) and check if it is valid """
+
+    def is_flash_encryption_key_valid(self):
+        key_len_256 = (
+            self.read_reg(self.EFUSE_XTS_KEY_LENGTH_256_REG)
+            & self.EFUSE_XTS_KEY_LENGTH_256
+        )
+
+        word0 = self.read_reg(self.EFUSE_RD_DIS_REG) & self.EFUSE_RD_DIS
+        rd_disable = word0 == 3 if key_len_256 else word0 == 1
+
+        # reading of BLOCK3 is NOT ALLOWED so we assume valid key is programmed
+        if rd_disable:
+            return True
+        else:
+            # reading of BLOCK3 is ALLOWED so we will read and verify for non-zero.
+            # When chip has not generated AES/encryption key in BLOCK3,
+            # the contents will be readable and 0.
+            # If the flash encryption is enabled it is expected to have a valid
+            # non-zero key. We break out on first occurance of non-zero value
+            key_word = [0] * 7 if key_len_256 else [0] * 3
+            for i in range(len(key_word)):
+                key_word[i] = self.read_reg(self.EFUSE_BLOCK_KEY0_REG + i * 4)
+                # key is non-zero so break & return
+                if key_word[i] != 0:
+                    return True
+            return False
+
+
+class ESP32C2StubLoader(ESP32C2ROM):
+    """Access class for ESP32C2 stub loader, runs on top of ROM.
+
+    (Basically the same as ESP32StubLoader, but different base class.
+    Can possibly be made into a mixin.)
+    """
+
+    FLASH_WRITE_SIZE = 0x4000  # matches MAX_WRITE_BLOCK in stub_loader.c
+    STATUS_BYTES_LENGTH = 2  # same as ESP8266, different to ESP32 ROM
+    IS_STUB = True
+
+    def __init__(self, rom_loader):
+        self.secure_download_mode = rom_loader.secure_download_mode
+        self._port = rom_loader._port
+        self._trace_enabled = rom_loader._trace_enabled
+        self.cache = rom_loader.cache
+        self.flush_input()  # resets _slip_reader
+
+
+ESP32C2ROM.STUB_CLASS = ESP32C2StubLoader

tools/python/esptool/targets/esp32c3.py

@@ -0,0 +1,228 @@
+# SPDX-FileCopyrightText: 2014-2022 Fredrik Ahlberg, Angus Gratton,
+# Espressif Systems (Shanghai) CO LTD, other contributors as noted.
+#
+# SPDX-License-Identifier: GPL-2.0-or-later
+
+import struct
+
+from .esp32 import ESP32ROM
+from loader import ESPLoader
+from util import FatalError, NotImplementedInROMError
+
+
+class ESP32C3ROM(ESP32ROM):
+    CHIP_NAME = "ESP32-C3"
+    IMAGE_CHIP_ID = 5
+
+    FPGA_SLOW_BOOT = False
+
+    IROM_MAP_START = 0x42000000
+    IROM_MAP_END = 0x42800000
+    DROM_MAP_START = 0x3C000000
+    DROM_MAP_END = 0x3C800000
+
+    SPI_REG_BASE = 0x60002000
+    SPI_USR_OFFS = 0x18
+    SPI_USR1_OFFS = 0x1C
+    SPI_USR2_OFFS = 0x20
+    SPI_MOSI_DLEN_OFFS = 0x24
+    SPI_MISO_DLEN_OFFS = 0x28
+    SPI_W0_OFFS = 0x58
+
+    BOOTLOADER_FLASH_OFFSET = 0x0
+
+    # Magic value for ESP32C3 eco 1+2 and ESP32C3 eco3 respectivly
+    CHIP_DETECT_MAGIC_VALUE = [0x6921506F, 0x1B31506F]
+
+    UART_DATE_REG_ADDR = 0x60000000 + 0x7C
+
+    UART_CLKDIV_REG = 0x60000014
+
+    EFUSE_BASE = 0x60008800
+    EFUSE_BLOCK1_ADDR = EFUSE_BASE + 0x044
+    MAC_EFUSE_REG = EFUSE_BASE + 0x044
+
+    EFUSE_RD_REG_BASE = EFUSE_BASE + 0x030  # BLOCK0 read base address
+
+    EFUSE_PURPOSE_KEY0_REG = EFUSE_BASE + 0x34
+    EFUSE_PURPOSE_KEY0_SHIFT = 24
+    EFUSE_PURPOSE_KEY1_REG = EFUSE_BASE + 0x34
+    EFUSE_PURPOSE_KEY1_SHIFT = 28
+    EFUSE_PURPOSE_KEY2_REG = EFUSE_BASE + 0x38
+    EFUSE_PURPOSE_KEY2_SHIFT = 0
+    EFUSE_PURPOSE_KEY3_REG = EFUSE_BASE + 0x38
+    EFUSE_PURPOSE_KEY3_SHIFT = 4
+    EFUSE_PURPOSE_KEY4_REG = EFUSE_BASE + 0x38
+    EFUSE_PURPOSE_KEY4_SHIFT = 8
+    EFUSE_PURPOSE_KEY5_REG = EFUSE_BASE + 0x38
+    EFUSE_PURPOSE_KEY5_SHIFT = 12
+
+    EFUSE_DIS_DOWNLOAD_MANUAL_ENCRYPT_REG = EFUSE_RD_REG_BASE
+    EFUSE_DIS_DOWNLOAD_MANUAL_ENCRYPT = 1 << 20
+
+    EFUSE_SPI_BOOT_CRYPT_CNT_REG = EFUSE_BASE + 0x034
+    EFUSE_SPI_BOOT_CRYPT_CNT_MASK = 0x7 << 18
+
+    EFUSE_SECURE_BOOT_EN_REG = EFUSE_BASE + 0x038
+    EFUSE_SECURE_BOOT_EN_MASK = 1 << 20
+
+    PURPOSE_VAL_XTS_AES128_KEY = 4
+
+    SUPPORTS_ENCRYPTED_FLASH = True
+
+    FLASH_ENCRYPTED_WRITE_ALIGN = 16
+
+    UARTDEV_BUF_NO = 0x3FCDF07C  # Variable in ROM .bss which indicates the port in use
+    UARTDEV_BUF_NO_USB_JTAG_SERIAL = 3  # The above var when USB-JTAG/Serial is used
+
+    RTCCNTL_BASE_REG = 0x60008000
+    RTC_CNTL_SWD_CONF_REG = RTCCNTL_BASE_REG + 0x00AC
+    RTC_CNTL_SWD_AUTO_FEED_EN = 1 << 31
+    RTC_CNTL_SWD_WPROTECT_REG = RTCCNTL_BASE_REG + 0x00B0
+    RTC_CNTL_SWD_WKEY = 0x8F1D312A
+
+    RTC_CNTL_WDTCONFIG0_REG = RTCCNTL_BASE_REG + 0x0090
+    RTC_CNTL_WDTWPROTECT_REG = RTCCNTL_BASE_REG + 0x00A8
+    RTC_CNTL_WDT_WKEY = 0x50D83AA1
+
+    MEMORY_MAP = [
+        [0x00000000, 0x00010000, "PADDING"],
+        [0x3C000000, 0x3C800000, "DROM"],
+        [0x3FC80000, 0x3FCE0000, "DRAM"],
+        [0x3FC88000, 0x3FD00000, "BYTE_ACCESSIBLE"],
+        [0x3FF00000, 0x3FF20000, "DROM_MASK"],
+        [0x40000000, 0x40060000, "IROM_MASK"],
+        [0x42000000, 0x42800000, "IROM"],
+        [0x4037C000, 0x403E0000, "IRAM"],
+        [0x50000000, 0x50002000, "RTC_IRAM"],
+        [0x50000000, 0x50002000, "RTC_DRAM"],
+        [0x600FE000, 0x60100000, "MEM_INTERNAL2"],
+    ]
+
+    def get_pkg_version(self):
+        num_word = 3
+        return (self.read_reg(self.EFUSE_BLOCK1_ADDR + (4 * num_word)) >> 21) & 0x07
+
+    def get_minor_chip_version(self):
+        hi_num_word = 5
+        hi = (self.read_reg(self.EFUSE_BLOCK1_ADDR + (4 * hi_num_word)) >> 23) & 0x01
+        low_num_word = 3
+        low = (self.read_reg(self.EFUSE_BLOCK1_ADDR + (4 * low_num_word)) >> 18) & 0x07
+        return (hi << 3) + low
+
+    def get_major_chip_version(self):
+        num_word = 5
+        return (self.read_reg(self.EFUSE_BLOCK1_ADDR + (4 * num_word)) >> 24) & 0x03
+
+    def get_chip_description(self):
+        chip_name = {
+            0: "ESP32-C3",
+        }.get(self.get_pkg_version(), "unknown ESP32-C3")
+        major_rev = self.get_major_chip_version()
+        minor_rev = self.get_minor_chip_version()
+        return f"{chip_name} (revision v{major_rev}.{minor_rev})"
+
+    def get_chip_features(self):
+        return ["WiFi", "BLE"]
+
+    def get_crystal_freq(self):
+        # ESP32C3 XTAL is fixed to 40MHz
+        return 40
+
+    def override_vddsdio(self, new_voltage):
+        raise NotImplementedInROMError(
+            "VDD_SDIO overrides are not supported for ESP32-C3"
+        )
+
+    def read_mac(self, mac_type="BASE_MAC"):
+        """Read MAC from EFUSE region"""
+        if mac_type != "BASE_MAC":
+            return None
+        mac0 = self.read_reg(self.MAC_EFUSE_REG)
+        mac1 = self.read_reg(self.MAC_EFUSE_REG + 4)  # only bottom 16 bits are MAC
+        bitstring = struct.pack(">II", mac1, mac0)[2:]
+        return tuple(bitstring)
+
+    def get_flash_crypt_config(self):
+        return None  # doesn't exist on ESP32-C3
+
+    def get_secure_boot_enabled(self):
+        return (
+            self.read_reg(self.EFUSE_SECURE_BOOT_EN_REG)
+            & self.EFUSE_SECURE_BOOT_EN_MASK
+        )
+
+    def get_key_block_purpose(self, key_block):
+        if key_block < 0 or key_block > 5:
+            raise FatalError("Valid key block numbers must be in range 0-5")
+
+        reg, shift = [
+            (self.EFUSE_PURPOSE_KEY0_REG, self.EFUSE_PURPOSE_KEY0_SHIFT),
+            (self.EFUSE_PURPOSE_KEY1_REG, self.EFUSE_PURPOSE_KEY1_SHIFT),
+            (self.EFUSE_PURPOSE_KEY2_REG, self.EFUSE_PURPOSE_KEY2_SHIFT),
+            (self.EFUSE_PURPOSE_KEY3_REG, self.EFUSE_PURPOSE_KEY3_SHIFT),
+            (self.EFUSE_PURPOSE_KEY4_REG, self.EFUSE_PURPOSE_KEY4_SHIFT),
+            (self.EFUSE_PURPOSE_KEY5_REG, self.EFUSE_PURPOSE_KEY5_SHIFT),
+        ][key_block]
+        return (self.read_reg(reg) >> shift) & 0xF
+
+    def is_flash_encryption_key_valid(self):
+        # Need to see an AES-128 key
+        purposes = [self.get_key_block_purpose(b) for b in range(6)]
+
+        return any(p == self.PURPOSE_VAL_XTS_AES128_KEY for p in purposes)
+
+    def change_baud(self, baud):
+        ESPLoader.change_baud(self, baud)
+
+    def uses_usb_jtag_serial(self):
+        """
+        Check the UARTDEV_BUF_NO register to see if USB-JTAG/Serial is being used
+        """
+        if self.secure_download_mode:
+            return False  # Can't detect USB-JTAG/Serial in secure download mode
+        return self.get_uart_no() == self.UARTDEV_BUF_NO_USB_JTAG_SERIAL
+
+    def disable_watchdogs(self):
+        # When USB-JTAG/Serial is used, the RTC WDT and SWD watchdog are not reset
+        # and can then reset the board during flashing. Disable or autofeed them.
+        if self.uses_usb_jtag_serial():
+            # Disable RTC WDT
+            self.write_reg(self.RTC_CNTL_WDTWPROTECT_REG, self.RTC_CNTL_WDT_WKEY)
+            self.write_reg(self.RTC_CNTL_WDTCONFIG0_REG, 0)
+            self.write_reg(self.RTC_CNTL_WDTWPROTECT_REG, 0)
+
+            # Automatically feed SWD
+            self.write_reg(self.RTC_CNTL_SWD_WPROTECT_REG, self.RTC_CNTL_SWD_WKEY)
+            self.write_reg(
+                self.RTC_CNTL_SWD_CONF_REG,
+                self.read_reg(self.RTC_CNTL_SWD_CONF_REG)
+                | self.RTC_CNTL_SWD_AUTO_FEED_EN,
+            )
+            self.write_reg(self.RTC_CNTL_SWD_WPROTECT_REG, 0)
+
+    def _post_connect(self):
+        if not self.sync_stub_detected:  # Don't run if stub is reused
+            self.disable_watchdogs()
+
+
+class ESP32C3StubLoader(ESP32C3ROM):
+    """Access class for ESP32C3 stub loader, runs on top of ROM.
+
+    (Basically the same as ESP32StubLoader, but different base class.
+    Can possibly be made into a mixin.)
+    """
+
+    FLASH_WRITE_SIZE = 0x4000  # matches MAX_WRITE_BLOCK in stub_loader.c
+    STATUS_BYTES_LENGTH = 2  # same as ESP8266, different to ESP32 ROM
+    IS_STUB = True
+
+    def __init__(self, rom_loader):
+        self.secure_download_mode = rom_loader.secure_download_mode
+        self._port = rom_loader._port
+        self._trace_enabled = rom_loader._trace_enabled
+        self.cache = rom_loader.cache
+        self.flush_input()  # resets _slip_reader
+
+
+ESP32C3ROM.STUB_CLASS = ESP32C3StubLoader

tools/python/esptool/targets/esp32c6.py

@@ -0,0 +1,203 @@
+# SPDX-FileCopyrightText: 2022 Fredrik Ahlberg, Angus Gratton,
+# Espressif Systems (Shanghai) CO LTD, other contributors as noted.
+#
+# SPDX-License-Identifier: GPL-2.0-or-later
+
+import struct
+
+from .esp32c3 import ESP32C3ROM
+from util import FatalError, NotImplementedInROMError
+
+
+class ESP32C6ROM(ESP32C3ROM):
+    CHIP_NAME = "ESP32-C6"
+    IMAGE_CHIP_ID = 13
+
+    FPGA_SLOW_BOOT = False
+
+    IROM_MAP_START = 0x42000000
+    IROM_MAP_END = 0x42800000
+    DROM_MAP_START = 0x42800000
+    DROM_MAP_END = 0x43000000
+
+    BOOTLOADER_FLASH_OFFSET = 0x0
+
+    # Magic value for ESP32C6
+    CHIP_DETECT_MAGIC_VALUE = [0x2CE0806F]
+
+    SPI_REG_BASE = 0x60003000
+    SPI_USR_OFFS = 0x18
+    SPI_USR1_OFFS = 0x1C
+    SPI_USR2_OFFS = 0x20
+    SPI_MOSI_DLEN_OFFS = 0x24
+    SPI_MISO_DLEN_OFFS = 0x28
+    SPI_W0_OFFS = 0x58
+
+    UART_DATE_REG_ADDR = 0x60000000 + 0x7C
+
+    EFUSE_BASE = 0x600B0800
+    EFUSE_BLOCK1_ADDR = EFUSE_BASE + 0x044
+    MAC_EFUSE_REG = EFUSE_BASE + 0x044
+
+    EFUSE_RD_REG_BASE = EFUSE_BASE + 0x030  # BLOCK0 read base address
+
+    EFUSE_PURPOSE_KEY0_REG = EFUSE_BASE + 0x34
+    EFUSE_PURPOSE_KEY0_SHIFT = 24
+    EFUSE_PURPOSE_KEY1_REG = EFUSE_BASE + 0x34
+    EFUSE_PURPOSE_KEY1_SHIFT = 28
+    EFUSE_PURPOSE_KEY2_REG = EFUSE_BASE + 0x38
+    EFUSE_PURPOSE_KEY2_SHIFT = 0
+    EFUSE_PURPOSE_KEY3_REG = EFUSE_BASE + 0x38
+    EFUSE_PURPOSE_KEY3_SHIFT = 4
+    EFUSE_PURPOSE_KEY4_REG = EFUSE_BASE + 0x38
+    EFUSE_PURPOSE_KEY4_SHIFT = 8
+    EFUSE_PURPOSE_KEY5_REG = EFUSE_BASE + 0x38
+    EFUSE_PURPOSE_KEY5_SHIFT = 12
+
+    EFUSE_DIS_DOWNLOAD_MANUAL_ENCRYPT_REG = EFUSE_RD_REG_BASE
+    EFUSE_DIS_DOWNLOAD_MANUAL_ENCRYPT = 1 << 20
+
+    EFUSE_SPI_BOOT_CRYPT_CNT_REG = EFUSE_BASE + 0x034
+    EFUSE_SPI_BOOT_CRYPT_CNT_MASK = 0x7 << 18
+
+    EFUSE_SECURE_BOOT_EN_REG = EFUSE_BASE + 0x038
+    EFUSE_SECURE_BOOT_EN_MASK = 1 << 20
+
+    PURPOSE_VAL_XTS_AES128_KEY = 4
+
+    SUPPORTS_ENCRYPTED_FLASH = True
+
+    FLASH_ENCRYPTED_WRITE_ALIGN = 16
+
+    UARTDEV_BUF_NO = 0x4087F580  # Variable in ROM .bss which indicates the port in use
+    UARTDEV_BUF_NO_USB_JTAG_SERIAL = 3  # The above var when USB-JTAG/Serial is used
+
+    DR_REG_LP_WDT_BASE = 0x600B1C00
+    RTC_CNTL_WDTCONFIG0_REG = DR_REG_LP_WDT_BASE + 0x0  # LP_WDT_RWDT_CONFIG0_REG
+    RTC_CNTL_WDTWPROTECT_REG = DR_REG_LP_WDT_BASE + 0x0018  # LP_WDT_RWDT_WPROTECT_REG
+
+    RTC_CNTL_SWD_CONF_REG = DR_REG_LP_WDT_BASE + 0x001C  # LP_WDT_SWD_CONFIG_REG
+    RTC_CNTL_SWD_AUTO_FEED_EN = 1 << 18
+    RTC_CNTL_SWD_WPROTECT_REG = DR_REG_LP_WDT_BASE + 0x0020  # LP_WDT_SWD_WPROTECT_REG
+    RTC_CNTL_SWD_WKEY = 0x50D83AA1  # LP_WDT_SWD_WKEY, same as WDT key in this case
+
+    FLASH_FREQUENCY = {
+        "80m": 0x0,  # workaround for wrong mspi HS div value in ROM
+        "40m": 0x0,
+        "20m": 0x2,
+    }
+
+    MEMORY_MAP = [
+        [0x00000000, 0x00010000, "PADDING"],
+        [0x42800000, 0x43000000, "DROM"],
+        [0x40800000, 0x40880000, "DRAM"],
+        [0x40800000, 0x40880000, "BYTE_ACCESSIBLE"],
+        [0x4004AC00, 0x40050000, "DROM_MASK"],
+        [0x40000000, 0x4004AC00, "IROM_MASK"],
+        [0x42000000, 0x42800000, "IROM"],
+        [0x40800000, 0x40880000, "IRAM"],
+        [0x50000000, 0x50004000, "RTC_IRAM"],
+        [0x50000000, 0x50004000, "RTC_DRAM"],
+        [0x600FE000, 0x60100000, "MEM_INTERNAL2"],
+    ]
+
+    def get_pkg_version(self):
+        num_word = 3
+        return (self.read_reg(self.EFUSE_BLOCK1_ADDR + (4 * num_word)) >> 24) & 0x07
+
+    def get_minor_chip_version(self):
+        num_word = 3
+        return (self.read_reg(self.EFUSE_BLOCK1_ADDR + (4 * num_word)) >> 18) & 0x0F
+
+    def get_major_chip_version(self):
+        num_word = 3
+        return (self.read_reg(self.EFUSE_BLOCK1_ADDR + (4 * num_word)) >> 22) & 0x03
+
+    def get_chip_description(self):
+        chip_name = {
+            0: "ESP32-C6 (QFN40)",
+            1: "ESP32-C6FH4 (QFN32)",
+        }.get(self.get_pkg_version(), "unknown ESP32-C6")
+        major_rev = self.get_major_chip_version()
+        minor_rev = self.get_minor_chip_version()
+        return f"{chip_name} (revision v{major_rev}.{minor_rev})"
+
+    def get_chip_features(self):
+        return ["WiFi 6", "BT 5", "IEEE802.15.4"]
+
+    def get_crystal_freq(self):
+        # ESP32C6 XTAL is fixed to 40MHz
+        return 40
+
+    def override_vddsdio(self, new_voltage):
+        raise NotImplementedInROMError(
+            "VDD_SDIO overrides are not supported for ESP32-C6"
+        )
+
+    def read_mac(self, mac_type="BASE_MAC"):
+        """Read MAC from EFUSE region"""
+        mac0 = self.read_reg(self.MAC_EFUSE_REG)
+        mac1 = self.read_reg(self.MAC_EFUSE_REG + 4)  # only bottom 16 bits are MAC
+        base_mac = struct.pack(">II", mac1, mac0)[2:]
+        ext_mac = struct.pack(">H", (mac1 >> 16) & 0xFFFF)
+        eui64 = base_mac[0:3] + ext_mac + base_mac[3:6]
+        # BASE MAC: 60:55:f9:f7:2c:a2
+        # EUI64 MAC: 60:55:f9:ff:fe:f7:2c:a2
+        # EXT_MAC: ff:fe
+        macs = {
+            "BASE_MAC": tuple(base_mac),
+            "EUI64": tuple(eui64),
+            "MAC_EXT": tuple(ext_mac),
+        }
+        return macs.get(mac_type, None)
+
+    def get_flash_crypt_config(self):
+        return None  # doesn't exist on ESP32-C6
+
+    def get_secure_boot_enabled(self):
+        return (
+            self.read_reg(self.EFUSE_SECURE_BOOT_EN_REG)
+            & self.EFUSE_SECURE_BOOT_EN_MASK
+        )
+
+    def get_key_block_purpose(self, key_block):
+        if key_block < 0 or key_block > 5:
+            raise FatalError("Valid key block numbers must be in range 0-5")
+
+        reg, shift = [
+            (self.EFUSE_PURPOSE_KEY0_REG, self.EFUSE_PURPOSE_KEY0_SHIFT),
+            (self.EFUSE_PURPOSE_KEY1_REG, self.EFUSE_PURPOSE_KEY1_SHIFT),
+            (self.EFUSE_PURPOSE_KEY2_REG, self.EFUSE_PURPOSE_KEY2_SHIFT),
+            (self.EFUSE_PURPOSE_KEY3_REG, self.EFUSE_PURPOSE_KEY3_SHIFT),
+            (self.EFUSE_PURPOSE_KEY4_REG, self.EFUSE_PURPOSE_KEY4_SHIFT),
+            (self.EFUSE_PURPOSE_KEY5_REG, self.EFUSE_PURPOSE_KEY5_SHIFT),
+        ][key_block]
+        return (self.read_reg(reg) >> shift) & 0xF
+
+    def is_flash_encryption_key_valid(self):
+        # Need to see an AES-128 key
+        purposes = [self.get_key_block_purpose(b) for b in range(6)]
+
+        return any(p == self.PURPOSE_VAL_XTS_AES128_KEY for p in purposes)
+
+
+class ESP32C6StubLoader(ESP32C6ROM):
+    """Access class for ESP32C6 stub loader, runs on top of ROM.
+
+    (Basically the same as ESP32StubLoader, but different base class.
+    Can possibly be made into a mixin.)
+    """
+
+    FLASH_WRITE_SIZE = 0x4000  # matches MAX_WRITE_BLOCK in stub_loader.c
+    STATUS_BYTES_LENGTH = 2  # same as ESP8266, different to ESP32 ROM
+    IS_STUB = True
+
+    def __init__(self, rom_loader):
+        self.secure_download_mode = rom_loader.secure_download_mode
+        self._port = rom_loader._port
+        self._trace_enabled = rom_loader._trace_enabled
+        self.cache = rom_loader.cache
+        self.flush_input()  # resets _slip_reader
+
+
+ESP32C6ROM.STUB_CLASS = ESP32C6StubLoader

tools/python/esptool/targets/esp32c6beta.py

@@ -0,0 +1,27 @@
+# SPDX-FileCopyrightText: 2014-2022 Fredrik Ahlberg, Angus Gratton,
+# Espressif Systems (Shanghai) CO LTD, other contributors as noted.
+#
+# SPDX-License-Identifier: GPL-2.0-or-later
+
+from .esp32c3 import ESP32C3ROM
+
+
+class ESP32C6BETAROM(ESP32C3ROM):
+    CHIP_NAME = "ESP32-C6(beta)"
+    IMAGE_CHIP_ID = 7
+
+    CHIP_DETECT_MAGIC_VALUE = [0x0DA1806F]
+
+    UART_DATE_REG_ADDR = 0x00000500
+
+    def get_chip_description(self):
+        chip_name = {
+            0: "ESP32-C6 (QFN40)",
+            1: "ESP32-C6FH4 (QFN32)",
+        }.get(self.get_pkg_version(), "unknown ESP32-C6")
+        major_rev = self.get_major_chip_version()
+        minor_rev = self.get_minor_chip_version()
+        return f"{chip_name} (revision v{major_rev}.{minor_rev})"
+
+    def _post_connect(self):
+        pass

tools/python/esptool/targets/esp32h2.py

@@ -0,0 +1,79 @@
+# SPDX-FileCopyrightText: 2022 Fredrik Ahlberg, Angus Gratton,
+# Espressif Systems (Shanghai) CO LTD, other contributors as noted.
+#
+# SPDX-License-Identifier: GPL-2.0-or-later
+
+from .esp32c6 import ESP32C6ROM
+
+
+class ESP32H2ROM(ESP32C6ROM):
+    CHIP_NAME = "ESP32-H2"
+    IMAGE_CHIP_ID = 16
+
+    # Magic value for ESP32H2
+    CHIP_DETECT_MAGIC_VALUE = [0xD7B73E80]
+
+    DR_REG_LP_WDT_BASE = 0x600B1C00
+    RTC_CNTL_WDTCONFIG0_REG = DR_REG_LP_WDT_BASE + 0x0  # LP_WDT_RWDT_CONFIG0_REG
+    RTC_CNTL_WDTWPROTECT_REG = DR_REG_LP_WDT_BASE + 0x001C  # LP_WDT_RWDT_WPROTECT_REG
+
+    RTC_CNTL_SWD_CONF_REG = DR_REG_LP_WDT_BASE + 0x0020  # LP_WDT_SWD_CONFIG_REG
+    RTC_CNTL_SWD_AUTO_FEED_EN = 1 << 18
+    RTC_CNTL_SWD_WPROTECT_REG = DR_REG_LP_WDT_BASE + 0x0024  # LP_WDT_SWD_WPROTECT_REG
+    RTC_CNTL_SWD_WKEY = 0x50D83AA1  # LP_WDT_SWD_WKEY, same as WDT key in this case
+
+    FLASH_FREQUENCY = {
+        "48m": 0xF,
+        "24m": 0x0,
+        "16m": 0x1,
+        "12m": 0x2,
+    }
+
+    def get_pkg_version(self):
+        num_word = 4
+        return (self.read_reg(self.EFUSE_BLOCK1_ADDR + (4 * num_word)) >> 0) & 0x07
+
+    def get_minor_chip_version(self):
+        num_word = 3
+        return (self.read_reg(self.EFUSE_BLOCK1_ADDR + (4 * num_word)) >> 18) & 0x07
+
+    def get_major_chip_version(self):
+        num_word = 3
+        return (self.read_reg(self.EFUSE_BLOCK1_ADDR + (4 * num_word)) >> 21) & 0x03
+
+    def get_chip_description(self):
+        chip_name = {
+            0: "ESP32-H2",
+        }.get(self.get_pkg_version(), "unknown ESP32-H2")
+        major_rev = self.get_major_chip_version()
+        minor_rev = self.get_minor_chip_version()
+        return f"{chip_name} (revision v{major_rev}.{minor_rev})"
+
+    def get_chip_features(self):
+        return ["BLE", "IEEE802.15.4"]
+
+    def get_crystal_freq(self):
+        # ESP32H2 XTAL is fixed to 32MHz
+        return 32
+
+
+class ESP32H2StubLoader(ESP32H2ROM):
+    """Access class for ESP32H2 stub loader, runs on top of ROM.
+
+    (Basically the same as ESP32StubLoader, but different base class.
+    Can possibly be made into a mixin.)
+    """
+
+    FLASH_WRITE_SIZE = 0x4000  # matches MAX_WRITE_BLOCK in stub_loader.c
+    STATUS_BYTES_LENGTH = 2  # same as ESP8266, different to ESP32 ROM
+    IS_STUB = True
+
+    def __init__(self, rom_loader):
+        self.secure_download_mode = rom_loader.secure_download_mode
+        self._port = rom_loader._port
+        self._trace_enabled = rom_loader._trace_enabled
+        self.cache = rom_loader.cache
+        self.flush_input()  # resets _slip_reader
+
+
+ESP32H2ROM.STUB_CLASS = ESP32H2StubLoader

tools/python/esptool/targets/esp32h2beta1.py

@@ -0,0 +1,164 @@
+# SPDX-FileCopyrightText: 2014-2022 Fredrik Ahlberg, Angus Gratton,
+# Espressif Systems (Shanghai) CO LTD, other contributors as noted.
+#
+# SPDX-License-Identifier: GPL-2.0-or-later
+
+import struct
+
+from .esp32c3 import ESP32C3ROM
+from util import FatalError, NotImplementedInROMError
+
+
+class ESP32H2BETA1ROM(ESP32C3ROM):
+    CHIP_NAME = "ESP32-H2(beta1)"
+    IMAGE_CHIP_ID = 10
+
+    IROM_MAP_START = 0x42000000
+    IROM_MAP_END = 0x42800000
+    DROM_MAP_START = 0x3C000000
+    DROM_MAP_END = 0x3C800000
+
+    SPI_REG_BASE = 0x60002000
+    SPI_USR_OFFS = 0x18
+    SPI_USR1_OFFS = 0x1C
+    SPI_USR2_OFFS = 0x20
+    SPI_MOSI_DLEN_OFFS = 0x24
+    SPI_MISO_DLEN_OFFS = 0x28
+    SPI_W0_OFFS = 0x58
+
+    BOOTLOADER_FLASH_OFFSET = 0x0
+
+    CHIP_DETECT_MAGIC_VALUE = [0xCA26CC22]
+
+    UART_DATE_REG_ADDR = 0x60000000 + 0x7C
+
+    EFUSE_BASE = 0x6001A000
+    EFUSE_BLOCK1_ADDR = EFUSE_BASE + 0x044
+    MAC_EFUSE_REG = EFUSE_BASE + 0x044
+
+    EFUSE_RD_REG_BASE = EFUSE_BASE + 0x030  # BLOCK0 read base address
+
+    EFUSE_PURPOSE_KEY0_REG = EFUSE_BASE + 0x34
+    EFUSE_PURPOSE_KEY0_SHIFT = 24
+    EFUSE_PURPOSE_KEY1_REG = EFUSE_BASE + 0x34
+    EFUSE_PURPOSE_KEY1_SHIFT = 28
+    EFUSE_PURPOSE_KEY2_REG = EFUSE_BASE + 0x38
+    EFUSE_PURPOSE_KEY2_SHIFT = 0
+    EFUSE_PURPOSE_KEY3_REG = EFUSE_BASE + 0x38
+    EFUSE_PURPOSE_KEY3_SHIFT = 4
+    EFUSE_PURPOSE_KEY4_REG = EFUSE_BASE + 0x38
+    EFUSE_PURPOSE_KEY4_SHIFT = 8
+    EFUSE_PURPOSE_KEY5_REG = EFUSE_BASE + 0x38
+    EFUSE_PURPOSE_KEY5_SHIFT = 12
+
+    EFUSE_DIS_DOWNLOAD_MANUAL_ENCRYPT_REG = EFUSE_RD_REG_BASE
+    EFUSE_DIS_DOWNLOAD_MANUAL_ENCRYPT = 1 << 20
+
+    EFUSE_SPI_BOOT_CRYPT_CNT_REG = EFUSE_BASE + 0x034
+    EFUSE_SPI_BOOT_CRYPT_CNT_MASK = 0x7 << 18
+
+    EFUSE_SECURE_BOOT_EN_REG = EFUSE_BASE + 0x038
+    EFUSE_SECURE_BOOT_EN_MASK = 1 << 20
+
+    PURPOSE_VAL_XTS_AES128_KEY = 4
+
+    SUPPORTS_ENCRYPTED_FLASH = True
+
+    FLASH_ENCRYPTED_WRITE_ALIGN = 16
+
+    MEMORY_MAP = []
+
+    FLASH_FREQUENCY = {
+        "48m": 0xF,
+        "24m": 0x0,
+        "16m": 0x1,
+        "12m": 0x2,
+    }
+
+    def get_pkg_version(self):
+        num_word = 4
+        return (self.read_reg(self.EFUSE_BLOCK1_ADDR + (4 * num_word)) >> 0) & 0x07
+
+    def get_minor_chip_version(self):
+        num_word = 3
+        return (self.read_reg(self.EFUSE_BLOCK1_ADDR + (4 * num_word)) >> 18) & 0x07
+
+    def get_major_chip_version(self):
+        num_word = 3
+        return (self.read_reg(self.EFUSE_BLOCK1_ADDR + (4 * num_word)) >> 21) & 0x03
+
+    def get_chip_description(self):
+        chip_name = {
+            0: "ESP32-H2",
+        }.get(self.get_pkg_version(), "unknown ESP32-H2")
+        major_rev = self.get_major_chip_version()
+        minor_rev = self.get_minor_chip_version()
+        return f"{chip_name} (revision v{major_rev}.{minor_rev})"
+
+    def get_chip_features(self):
+        return ["BLE", "IEEE802.15.4"]
+
+    def get_crystal_freq(self):
+        return 32
+
+    def override_vddsdio(self, new_voltage):
+        raise NotImplementedInROMError(
+            "VDD_SDIO overrides are not supported for ESP32-H2"
+        )
+
+    def read_mac(self, mac_type="BASE_MAC"):
+        """Read MAC from EFUSE region"""
+        if mac_type != "BASE_MAC":
+            return None
+        mac0 = self.read_reg(self.MAC_EFUSE_REG)
+        mac1 = self.read_reg(self.MAC_EFUSE_REG + 4)  # only bottom 16 bits are MAC
+        bitstring = struct.pack(">II", mac1, mac0)[2:]
+        return tuple(bitstring)
+
+    def get_flash_crypt_config(self):
+        return None  # doesn't exist on ESP32-H2
+
+    def get_key_block_purpose(self, key_block):
+        if key_block < 0 or key_block > 5:
+            raise FatalError("Valid key block numbers must be in range 0-5")
+
+        reg, shift = [
+            (self.EFUSE_PURPOSE_KEY0_REG, self.EFUSE_PURPOSE_KEY0_SHIFT),
+            (self.EFUSE_PURPOSE_KEY1_REG, self.EFUSE_PURPOSE_KEY1_SHIFT),
+            (self.EFUSE_PURPOSE_KEY2_REG, self.EFUSE_PURPOSE_KEY2_SHIFT),
+            (self.EFUSE_PURPOSE_KEY3_REG, self.EFUSE_PURPOSE_KEY3_SHIFT),
+            (self.EFUSE_PURPOSE_KEY4_REG, self.EFUSE_PURPOSE_KEY4_SHIFT),
+            (self.EFUSE_PURPOSE_KEY5_REG, self.EFUSE_PURPOSE_KEY5_SHIFT),
+        ][key_block]
+        return (self.read_reg(reg) >> shift) & 0xF
+
+    def is_flash_encryption_key_valid(self):
+        # Need to see an AES-128 key
+        purposes = [self.get_key_block_purpose(b) for b in range(6)]
+
+        return any(p == self.PURPOSE_VAL_XTS_AES128_KEY for p in purposes)
+
+    def _post_connect(self):
+        pass
+
+
+class ESP32H2BETA1StubLoader(ESP32H2BETA1ROM):
+    """Access class for ESP32H2BETA1 stub loader, runs on top of ROM.
+
+    (Basically the same as ESP32StubLoader, but different base class.
+    Can possibly be made into a mixin.)
+    """
+
+    FLASH_WRITE_SIZE = 0x4000  # matches MAX_WRITE_BLOCK in stub_loader.c
+    STATUS_BYTES_LENGTH = 2  # same as ESP8266, different to ESP32 ROM
+    IS_STUB = True
+
+    def __init__(self, rom_loader):
+        self.secure_download_mode = rom_loader.secure_download_mode
+        self._port = rom_loader._port
+        self._trace_enabled = rom_loader._trace_enabled
+        self.cache = rom_loader.cache
+        self.flush_input()  # resets _slip_reader
+
+
+ESP32H2BETA1ROM.STUB_CLASS = ESP32H2BETA1StubLoader

tools/python/esptool/targets/esp32h2beta2.py

@@ -0,0 +1,43 @@
+# SPDX-FileCopyrightText: 2014-2022 Fredrik Ahlberg, Angus Gratton,
+# Espressif Systems (Shanghai) CO LTD, other contributors as noted.
+#
+# SPDX-License-Identifier: GPL-2.0-or-later
+
+from .esp32h2beta1 import ESP32H2BETA1ROM
+
+
+class ESP32H2BETA2ROM(ESP32H2BETA1ROM):
+    CHIP_NAME = "ESP32-H2(beta2)"
+    IMAGE_CHIP_ID = 14
+
+    CHIP_DETECT_MAGIC_VALUE = [0x6881B06F]
+
+    def get_chip_description(self):
+        chip_name = {
+            1: "ESP32-H2(beta2)",
+        }.get(self.get_pkg_version(), "unknown ESP32-H2")
+        major_rev = self.get_major_chip_version()
+        minor_rev = self.get_minor_chip_version()
+        return f"{chip_name} (revision v{major_rev}.{minor_rev})"
+
+
+class ESP32H2BETA2StubLoader(ESP32H2BETA2ROM):
+    """Access class for ESP32H2BETA2 stub loader, runs on top of ROM.
+
+    (Basically the same as ESP32StubLoader, but different base class.
+    Can possibly be made into a mixin.)
+    """
+
+    FLASH_WRITE_SIZE = 0x4000  # matches MAX_WRITE_BLOCK in stub_loader.c
+    STATUS_BYTES_LENGTH = 2  # same as ESP8266, different to ESP32 ROM
+    IS_STUB = True
+
+    def __init__(self, rom_loader):
+        self.secure_download_mode = rom_loader.secure_download_mode
+        self._port = rom_loader._port
+        self._trace_enabled = rom_loader._trace_enabled
+        self.cache = rom_loader.cache
+        self.flush_input()  # resets _slip_reader
+
+
+ESP32H2BETA2ROM.STUB_CLASS = ESP32H2BETA2StubLoader

tools/python/esptool/targets/esp32s2.py

@@ -0,0 +1,308 @@
+# SPDX-FileCopyrightText: 2014-2023 Fredrik Ahlberg, Angus Gratton,
+# Espressif Systems (Shanghai) CO LTD, other contributors as noted.
+#
+# SPDX-License-Identifier: GPL-2.0-or-later
+
+import os
+import struct
+
+from .esp32 import ESP32ROM
+from loader import ESPLoader
+from reset import HardReset
+from util import FatalError, NotImplementedInROMError
+
+
+class ESP32S2ROM(ESP32ROM):
+    CHIP_NAME = "ESP32-S2"
+    IMAGE_CHIP_ID = 2
+
+    FPGA_SLOW_BOOT = False
+
+    IROM_MAP_START = 0x40080000
+    IROM_MAP_END = 0x40B80000
+    DROM_MAP_START = 0x3F000000
+    DROM_MAP_END = 0x3F3F0000
+
+    CHIP_DETECT_MAGIC_VALUE = [0x000007C6]
+
+    SPI_REG_BASE = 0x3F402000
+    SPI_USR_OFFS = 0x18
+    SPI_USR1_OFFS = 0x1C
+    SPI_USR2_OFFS = 0x20
+    SPI_MOSI_DLEN_OFFS = 0x24
+    SPI_MISO_DLEN_OFFS = 0x28
+    SPI_W0_OFFS = 0x58
+
+    MAC_EFUSE_REG = 0x3F41A044  # ESP32-S2 has special block for MAC efuses
+
+    UART_CLKDIV_REG = 0x3F400014
+
+    SUPPORTS_ENCRYPTED_FLASH = True
+
+    FLASH_ENCRYPTED_WRITE_ALIGN = 16
+
+    # todo: use espefuse APIs to get this info
+    EFUSE_BASE = 0x3F41A000
+    EFUSE_RD_REG_BASE = EFUSE_BASE + 0x030  # BLOCK0 read base address
+    EFUSE_BLOCK1_ADDR = EFUSE_BASE + 0x044
+    EFUSE_BLOCK2_ADDR = EFUSE_BASE + 0x05C
+
+    EFUSE_PURPOSE_KEY0_REG = EFUSE_BASE + 0x34
+    EFUSE_PURPOSE_KEY0_SHIFT = 24
+    EFUSE_PURPOSE_KEY1_REG = EFUSE_BASE + 0x34
+    EFUSE_PURPOSE_KEY1_SHIFT = 28
+    EFUSE_PURPOSE_KEY2_REG = EFUSE_BASE + 0x38
+    EFUSE_PURPOSE_KEY2_SHIFT = 0
+    EFUSE_PURPOSE_KEY3_REG = EFUSE_BASE + 0x38
+    EFUSE_PURPOSE_KEY3_SHIFT = 4
+    EFUSE_PURPOSE_KEY4_REG = EFUSE_BASE + 0x38
+    EFUSE_PURPOSE_KEY4_SHIFT = 8
+    EFUSE_PURPOSE_KEY5_REG = EFUSE_BASE + 0x38
+    EFUSE_PURPOSE_KEY5_SHIFT = 12
+
+    EFUSE_DIS_DOWNLOAD_MANUAL_ENCRYPT_REG = EFUSE_RD_REG_BASE
+    EFUSE_DIS_DOWNLOAD_MANUAL_ENCRYPT = 1 << 19
+
+    EFUSE_SPI_BOOT_CRYPT_CNT_REG = EFUSE_BASE + 0x034
+    EFUSE_SPI_BOOT_CRYPT_CNT_MASK = 0x7 << 18
+
+    EFUSE_SECURE_BOOT_EN_REG = EFUSE_BASE + 0x038
+    EFUSE_SECURE_BOOT_EN_MASK = 1 << 20
+
+    EFUSE_RD_REPEAT_DATA3_REG = EFUSE_BASE + 0x3C
+    EFUSE_RD_REPEAT_DATA3_REG_FLASH_TYPE_MASK = 1 << 9
+
+    PURPOSE_VAL_XTS_AES256_KEY_1 = 2
+    PURPOSE_VAL_XTS_AES256_KEY_2 = 3
+    PURPOSE_VAL_XTS_AES128_KEY = 4
+
+    UARTDEV_BUF_NO = 0x3FFFFD14  # Variable in ROM .bss which indicates the port in use
+    UARTDEV_BUF_NO_USB_OTG = 2  # Value of the above indicating that USB-OTG is in use
+
+    USB_RAM_BLOCK = 0x800  # Max block size USB-OTG is used
+
+    GPIO_STRAP_REG = 0x3F404038
+    GPIO_STRAP_SPI_BOOT_MASK = 0x8  # Not download mode
+    RTC_CNTL_OPTION1_REG = 0x3F408128
+    RTC_CNTL_FORCE_DOWNLOAD_BOOT_MASK = 0x1  # Is download mode forced over USB?
+
+    MEMORY_MAP = [
+        [0x00000000, 0x00010000, "PADDING"],
+        [0x3F000000, 0x3FF80000, "DROM"],
+        [0x3F500000, 0x3FF80000, "EXTRAM_DATA"],
+        [0x3FF9E000, 0x3FFA0000, "RTC_DRAM"],
+        [0x3FF9E000, 0x40000000, "BYTE_ACCESSIBLE"],
+        [0x3FF9E000, 0x40072000, "MEM_INTERNAL"],
+        [0x3FFB0000, 0x40000000, "DRAM"],
+        [0x40000000, 0x4001A100, "IROM_MASK"],
+        [0x40020000, 0x40070000, "IRAM"],
+        [0x40070000, 0x40072000, "RTC_IRAM"],
+        [0x40080000, 0x40800000, "IROM"],
+        [0x50000000, 0x50002000, "RTC_DATA"],
+    ]
+
+    def get_pkg_version(self):
+        num_word = 4
+        return (self.read_reg(self.EFUSE_BLOCK1_ADDR + (4 * num_word)) >> 0) & 0x0F
+
+    def get_minor_chip_version(self):
+        hi_num_word = 3
+        hi = (self.read_reg(self.EFUSE_BLOCK1_ADDR + (4 * hi_num_word)) >> 20) & 0x01
+        low_num_word = 4
+        low = (self.read_reg(self.EFUSE_BLOCK1_ADDR + (4 * low_num_word)) >> 4) & 0x07
+        return (hi << 3) + low
+
+    def get_major_chip_version(self):
+        num_word = 3
+        return (self.read_reg(self.EFUSE_BLOCK1_ADDR + (4 * num_word)) >> 18) & 0x03
+
+    def get_flash_version(self):
+        num_word = 3
+        return (self.read_reg(self.EFUSE_BLOCK1_ADDR + (4 * num_word)) >> 21) & 0x0F
+
+    def get_psram_version(self):
+        num_word = 3
+        return (self.read_reg(self.EFUSE_BLOCK1_ADDR + (4 * num_word)) >> 28) & 0x0F
+
+    def get_block2_version(self):
+        # BLK_VERSION_MINOR
+        num_word = 4
+        return (self.read_reg(self.EFUSE_BLOCK2_ADDR + (4 * num_word)) >> 4) & 0x07
+
+    def get_chip_description(self):
+        chip_name = {
+            0: "ESP32-S2",
+            1: "ESP32-S2FH2",
+            2: "ESP32-S2FH4",
+            102: "ESP32-S2FNR2",
+            100: "ESP32-S2R2",
+        }.get(
+            self.get_flash_version() + self.get_psram_version() * 100,
+            "unknown ESP32-S2",
+        )
+        major_rev = self.get_major_chip_version()
+        minor_rev = self.get_minor_chip_version()
+        return f"{chip_name} (revision v{major_rev}.{minor_rev})"
+
+    def get_chip_features(self):
+        features = ["WiFi"]
+
+        if self.secure_download_mode:
+            features += ["Secure Download Mode Enabled"]
+
+        flash_version = {
+            0: "No Embedded Flash",
+            1: "Embedded Flash 2MB",
+            2: "Embedded Flash 4MB",
+        }.get(self.get_flash_version(), "Unknown Embedded Flash")
+        features += [flash_version]
+
+        psram_version = {
+            0: "No Embedded PSRAM",
+            1: "Embedded PSRAM 2MB",
+            2: "Embedded PSRAM 4MB",
+        }.get(self.get_psram_version(), "Unknown Embedded PSRAM")
+        features += [psram_version]
+
+        block2_version = {
+            0: "No calibration in BLK2 of efuse",
+            1: "ADC and temperature sensor calibration in BLK2 of efuse V1",
+            2: "ADC and temperature sensor calibration in BLK2 of efuse V2",
+        }.get(self.get_block2_version(), "Unknown Calibration in BLK2")
+        features += [block2_version]
+
+        return features
+
+    def get_crystal_freq(self):
+        # ESP32-S2 XTAL is fixed to 40MHz
+        return 40
+
+    def override_vddsdio(self, new_voltage):
+        raise NotImplementedInROMError(
+            "VDD_SDIO overrides are not supported for ESP32-S2"
+        )
+
+    def read_mac(self, mac_type="BASE_MAC"):
+        """Read MAC from EFUSE region"""
+        if mac_type != "BASE_MAC":
+            return None
+        mac0 = self.read_reg(self.MAC_EFUSE_REG)
+        mac1 = self.read_reg(self.MAC_EFUSE_REG + 4)  # only bottom 16 bits are MAC
+        bitstring = struct.pack(">II", mac1, mac0)[2:]
+        return tuple(bitstring)
+
+    def flash_type(self):
+        return (
+            1
+            if self.read_reg(self.EFUSE_RD_REPEAT_DATA3_REG)
+            & self.EFUSE_RD_REPEAT_DATA3_REG_FLASH_TYPE_MASK
+            else 0
+        )
+
+    def get_flash_crypt_config(self):
+        return None  # doesn't exist on ESP32-S2
+
+    def get_secure_boot_enabled(self):
+        return (
+            self.read_reg(self.EFUSE_SECURE_BOOT_EN_REG)
+            & self.EFUSE_SECURE_BOOT_EN_MASK
+        )
+
+    def get_key_block_purpose(self, key_block):
+        if key_block < 0 or key_block > 5:
+            raise FatalError("Valid key block numbers must be in range 0-5")
+
+        reg, shift = [
+            (self.EFUSE_PURPOSE_KEY0_REG, self.EFUSE_PURPOSE_KEY0_SHIFT),
+            (self.EFUSE_PURPOSE_KEY1_REG, self.EFUSE_PURPOSE_KEY1_SHIFT),
+            (self.EFUSE_PURPOSE_KEY2_REG, self.EFUSE_PURPOSE_KEY2_SHIFT),
+            (self.EFUSE_PURPOSE_KEY3_REG, self.EFUSE_PURPOSE_KEY3_SHIFT),
+            (self.EFUSE_PURPOSE_KEY4_REG, self.EFUSE_PURPOSE_KEY4_SHIFT),
+            (self.EFUSE_PURPOSE_KEY5_REG, self.EFUSE_PURPOSE_KEY5_SHIFT),
+        ][key_block]
+        return (self.read_reg(reg) >> shift) & 0xF
+
+    def is_flash_encryption_key_valid(self):
+        # Need to see either an AES-128 key or two AES-256 keys
+        purposes = [self.get_key_block_purpose(b) for b in range(6)]
+
+        if any(p == self.PURPOSE_VAL_XTS_AES128_KEY for p in purposes):
+            return True
+
+        return any(p == self.PURPOSE_VAL_XTS_AES256_KEY_1 for p in purposes) and any(
+            p == self.PURPOSE_VAL_XTS_AES256_KEY_2 for p in purposes
+        )
+
+    def uses_usb_otg(self):
+        """
+        Check the UARTDEV_BUF_NO register to see if USB-OTG console is being used
+        """
+        if self.secure_download_mode:
+            return False  # can't detect native USB in secure download mode
+        return self.get_uart_no() == self.UARTDEV_BUF_NO_USB_OTG
+
+    def _post_connect(self):
+        if self.uses_usb_otg():
+            self.ESP_RAM_BLOCK = self.USB_RAM_BLOCK
+
+    def _check_if_can_reset(self):
+        """
+        Check the strapping register to see if we can reset out of download mode.
+        """
+        if os.getenv("ESPTOOL_TESTING") is not None:
+            print("ESPTOOL_TESTING is set, ignoring strapping mode check")
+            # Esptool tests over USB-OTG run with GPIO0 strapped low,
+            # don't complain in this case.
+            return
+        strap_reg = self.read_reg(self.GPIO_STRAP_REG)
+        force_dl_reg = self.read_reg(self.RTC_CNTL_OPTION1_REG)
+        if (
+            strap_reg & self.GPIO_STRAP_SPI_BOOT_MASK == 0
+            and force_dl_reg & self.RTC_CNTL_FORCE_DOWNLOAD_BOOT_MASK == 0
+        ):
+            print(
+                "WARNING: {} chip was placed into download mode using GPIO0.\n"
+                "esptool.py can not exit the download mode over USB. "
+                "To run the app, reset the chip manually.\n"
+                "To suppress this note, set --after option to 'no_reset'.".format(
+                    self.get_chip_description()
+                )
+            )
+            raise SystemExit(1)
+
+    def hard_reset(self):
+        uses_usb_otg = self.uses_usb_otg()
+        if uses_usb_otg:
+            self._check_if_can_reset()
+
+        print("Hard resetting via RTS pin...")
+        HardReset(self._port, uses_usb_otg)()
+
+    def change_baud(self, baud):
+        ESPLoader.change_baud(self, baud)
+
+
+class ESP32S2StubLoader(ESP32S2ROM):
+    """Access class for ESP32-S2 stub loader, runs on top of ROM.
+
+    (Basically the same as ESP32StubLoader, but different base class.
+    Can possibly be made into a mixin.)
+    """
+
+    FLASH_WRITE_SIZE = 0x4000  # matches MAX_WRITE_BLOCK in stub_loader.c
+    STATUS_BYTES_LENGTH = 2  # same as ESP8266, different to ESP32 ROM
+    IS_STUB = True
+
+    def __init__(self, rom_loader):
+        self.secure_download_mode = rom_loader.secure_download_mode
+        self._port = rom_loader._port
+        self._trace_enabled = rom_loader._trace_enabled
+        self.cache = rom_loader.cache
+        self.flush_input()  # resets _slip_reader
+
+        if rom_loader.uses_usb_otg():
+            self.ESP_RAM_BLOCK = self.USB_RAM_BLOCK
+            self.FLASH_WRITE_SIZE = self.USB_RAM_BLOCK
+
+
+ESP32S2ROM.STUB_CLASS = ESP32S2StubLoader

tools/python/esptool/targets/esp32s3.py

@@ -0,0 +1,333 @@
+# SPDX-FileCopyrightText: 2014-2023 Fredrik Ahlberg, Angus Gratton,
+# Espressif Systems (Shanghai) CO LTD, other contributors as noted.
+#
+# SPDX-License-Identifier: GPL-2.0-or-later
+
+import os
+import struct
+
+from .esp32 import ESP32ROM
+from loader import ESPLoader
+from reset import HardReset
+from util import FatalError, NotImplementedInROMError
+
+
+class ESP32S3ROM(ESP32ROM):
+    CHIP_NAME = "ESP32-S3"
+
+    IMAGE_CHIP_ID = 9
+
+    CHIP_DETECT_MAGIC_VALUE = [0x9]
+
+    FPGA_SLOW_BOOT = False
+
+    IROM_MAP_START = 0x42000000
+    IROM_MAP_END = 0x44000000
+    DROM_MAP_START = 0x3C000000
+    DROM_MAP_END = 0x3E000000
+
+    UART_DATE_REG_ADDR = 0x60000080
+
+    SPI_REG_BASE = 0x60002000
+    SPI_USR_OFFS = 0x18
+    SPI_USR1_OFFS = 0x1C
+    SPI_USR2_OFFS = 0x20
+    SPI_MOSI_DLEN_OFFS = 0x24
+    SPI_MISO_DLEN_OFFS = 0x28
+    SPI_W0_OFFS = 0x58
+
+    BOOTLOADER_FLASH_OFFSET = 0x0
+
+    SUPPORTS_ENCRYPTED_FLASH = True
+
+    FLASH_ENCRYPTED_WRITE_ALIGN = 16
+
+    # todo: use espefuse APIs to get this info
+    EFUSE_BASE = 0x60007000  # BLOCK0 read base address
+    EFUSE_BLOCK1_ADDR = EFUSE_BASE + 0x44
+    EFUSE_BLOCK2_ADDR = EFUSE_BASE + 0x5C
+    MAC_EFUSE_REG = EFUSE_BASE + 0x044
+
+    EFUSE_RD_REG_BASE = EFUSE_BASE + 0x030  # BLOCK0 read base address
+
+    EFUSE_PURPOSE_KEY0_REG = EFUSE_BASE + 0x34
+    EFUSE_PURPOSE_KEY0_SHIFT = 24
+    EFUSE_PURPOSE_KEY1_REG = EFUSE_BASE + 0x34
+    EFUSE_PURPOSE_KEY1_SHIFT = 28
+    EFUSE_PURPOSE_KEY2_REG = EFUSE_BASE + 0x38
+    EFUSE_PURPOSE_KEY2_SHIFT = 0
+    EFUSE_PURPOSE_KEY3_REG = EFUSE_BASE + 0x38
+    EFUSE_PURPOSE_KEY3_SHIFT = 4
+    EFUSE_PURPOSE_KEY4_REG = EFUSE_BASE + 0x38
+    EFUSE_PURPOSE_KEY4_SHIFT = 8
+    EFUSE_PURPOSE_KEY5_REG = EFUSE_BASE + 0x38
+    EFUSE_PURPOSE_KEY5_SHIFT = 12
+
+    EFUSE_DIS_DOWNLOAD_MANUAL_ENCRYPT_REG = EFUSE_RD_REG_BASE
+    EFUSE_DIS_DOWNLOAD_MANUAL_ENCRYPT = 1 << 20
+
+    EFUSE_SPI_BOOT_CRYPT_CNT_REG = EFUSE_BASE + 0x034
+    EFUSE_SPI_BOOT_CRYPT_CNT_MASK = 0x7 << 18
+
+    EFUSE_SECURE_BOOT_EN_REG = EFUSE_BASE + 0x038
+    EFUSE_SECURE_BOOT_EN_MASK = 1 << 20
+
+    EFUSE_RD_REPEAT_DATA3_REG = EFUSE_BASE + 0x3C
+    EFUSE_RD_REPEAT_DATA3_REG_FLASH_TYPE_MASK = 1 << 9
+
+    PURPOSE_VAL_XTS_AES256_KEY_1 = 2
+    PURPOSE_VAL_XTS_AES256_KEY_2 = 3
+    PURPOSE_VAL_XTS_AES128_KEY = 4
+
+    UARTDEV_BUF_NO = 0x3FCEF14C  # Variable in ROM .bss which indicates the port in use
+    UARTDEV_BUF_NO_USB_OTG = 3  # The above var when USB-OTG is used
+    UARTDEV_BUF_NO_USB_JTAG_SERIAL = 4  # The above var when USB-JTAG/Serial is used
+
+    RTCCNTL_BASE_REG = 0x60008000
+    RTC_CNTL_SWD_CONF_REG = RTCCNTL_BASE_REG + 0x00B4
+    RTC_CNTL_SWD_AUTO_FEED_EN = 1 << 31
+    RTC_CNTL_SWD_WPROTECT_REG = RTCCNTL_BASE_REG + 0x00B8
+    RTC_CNTL_SWD_WKEY = 0x8F1D312A
+
+    RTC_CNTL_WDTCONFIG0_REG = RTCCNTL_BASE_REG + 0x0098
+    RTC_CNTL_WDTWPROTECT_REG = RTCCNTL_BASE_REG + 0x00B0
+    RTC_CNTL_WDT_WKEY = 0x50D83AA1
+
+    USB_RAM_BLOCK = 0x800  # Max block size USB-OTG is used
+
+    GPIO_STRAP_REG = 0x60004038
+    GPIO_STRAP_SPI_BOOT_MASK = 0x8  # Not download mode
+    RTC_CNTL_OPTION1_REG = 0x6000812C
+    RTC_CNTL_FORCE_DOWNLOAD_BOOT_MASK = 0x1  # Is download mode forced over USB?
+
+    UART_CLKDIV_REG = 0x60000014
+
+    MEMORY_MAP = [
+        [0x00000000, 0x00010000, "PADDING"],
+        [0x3C000000, 0x3D000000, "DROM"],
+        [0x3D000000, 0x3E000000, "EXTRAM_DATA"],
+        [0x600FE000, 0x60100000, "RTC_DRAM"],
+        [0x3FC88000, 0x3FD00000, "BYTE_ACCESSIBLE"],
+        [0x3FC88000, 0x403E2000, "MEM_INTERNAL"],
+        [0x3FC88000, 0x3FD00000, "DRAM"],
+        [0x40000000, 0x4001A100, "IROM_MASK"],
+        [0x40370000, 0x403E0000, "IRAM"],
+        [0x600FE000, 0x60100000, "RTC_IRAM"],
+        [0x42000000, 0x42800000, "IROM"],
+        [0x50000000, 0x50002000, "RTC_DATA"],
+    ]
+
+    def get_pkg_version(self):
+        num_word = 3
+        return (self.read_reg(self.EFUSE_BLOCK1_ADDR + (4 * num_word)) >> 21) & 0x07
+
+    def is_eco0(self, minor_raw):
+        # Workaround: The major version field was allocated to other purposes
+        # when block version is v1.1.
+        # Luckily only chip v0.0 have this kind of block version and efuse usage.
+        return (
+            (minor_raw & 0x7) == 0
+            and self.get_blk_version_major() == 1
+            and self.get_blk_version_minor() == 1
+        )
+
+    def get_minor_chip_version(self):
+        minor_raw = self.get_raw_minor_chip_version()
+        if self.is_eco0(minor_raw):
+            return 0
+        return minor_raw
+
+    def get_raw_minor_chip_version(self):
+        hi_num_word = 5
+        hi = (self.read_reg(self.EFUSE_BLOCK1_ADDR + (4 * hi_num_word)) >> 23) & 0x01
+        low_num_word = 3
+        low = (self.read_reg(self.EFUSE_BLOCK1_ADDR + (4 * low_num_word)) >> 18) & 0x07
+        return (hi << 3) + low
+
+    def get_blk_version_major(self):
+        num_word = 4
+        return (self.read_reg(self.EFUSE_BLOCK2_ADDR + (4 * num_word)) >> 0) & 0x03
+
+    def get_blk_version_minor(self):
+        num_word = 3
+        return (self.read_reg(self.EFUSE_BLOCK1_ADDR + (4 * num_word)) >> 24) & 0x07
+
+    def get_major_chip_version(self):
+        minor_raw = self.get_raw_minor_chip_version()
+        if self.is_eco0(minor_raw):
+            return 0
+        return self.get_raw_major_chip_version()
+
+    def get_raw_major_chip_version(self):
+        num_word = 5
+        return (self.read_reg(self.EFUSE_BLOCK1_ADDR + (4 * num_word)) >> 24) & 0x03
+
+    def get_chip_description(self):
+        major_rev = self.get_major_chip_version()
+        minor_rev = self.get_minor_chip_version()
+        return f"{self.CHIP_NAME} (revision v{major_rev}.{minor_rev})"
+
+    def get_chip_features(self):
+        return ["WiFi", "BLE"]
+
+    def get_crystal_freq(self):
+        # ESP32S3 XTAL is fixed to 40MHz
+        return 40
+
+    def get_flash_crypt_config(self):
+        return None  # doesn't exist on ESP32-S3
+
+    def get_key_block_purpose(self, key_block):
+        if key_block < 0 or key_block > 5:
+            raise FatalError("Valid key block numbers must be in range 0-5")
+
+        reg, shift = [
+            (self.EFUSE_PURPOSE_KEY0_REG, self.EFUSE_PURPOSE_KEY0_SHIFT),
+            (self.EFUSE_PURPOSE_KEY1_REG, self.EFUSE_PURPOSE_KEY1_SHIFT),
+            (self.EFUSE_PURPOSE_KEY2_REG, self.EFUSE_PURPOSE_KEY2_SHIFT),
+            (self.EFUSE_PURPOSE_KEY3_REG, self.EFUSE_PURPOSE_KEY3_SHIFT),
+            (self.EFUSE_PURPOSE_KEY4_REG, self.EFUSE_PURPOSE_KEY4_SHIFT),
+            (self.EFUSE_PURPOSE_KEY5_REG, self.EFUSE_PURPOSE_KEY5_SHIFT),
+        ][key_block]
+        return (self.read_reg(reg) >> shift) & 0xF
+
+    def is_flash_encryption_key_valid(self):
+        # Need to see either an AES-128 key or two AES-256 keys
+        purposes = [self.get_key_block_purpose(b) for b in range(6)]
+
+        if any(p == self.PURPOSE_VAL_XTS_AES128_KEY for p in purposes):
+            return True
+
+        return any(p == self.PURPOSE_VAL_XTS_AES256_KEY_1 for p in purposes) and any(
+            p == self.PURPOSE_VAL_XTS_AES256_KEY_2 for p in purposes
+        )
+
+    def get_secure_boot_enabled(self):
+        return (
+            self.read_reg(self.EFUSE_SECURE_BOOT_EN_REG)
+            & self.EFUSE_SECURE_BOOT_EN_MASK
+        )
+
+    def override_vddsdio(self, new_voltage):
+        raise NotImplementedInROMError(
+            "VDD_SDIO overrides are not supported for ESP32-S3"
+        )
+
+    def read_mac(self, mac_type="BASE_MAC"):
+        """Read MAC from EFUSE region"""
+        if mac_type != "BASE_MAC":
+            return None
+        mac0 = self.read_reg(self.MAC_EFUSE_REG)
+        mac1 = self.read_reg(self.MAC_EFUSE_REG + 4)  # only bottom 16 bits are MAC
+        bitstring = struct.pack(">II", mac1, mac0)[2:]
+        return tuple(bitstring)
+
+    def flash_type(self):
+        return (
+            1
+            if self.read_reg(self.EFUSE_RD_REPEAT_DATA3_REG)
+            & self.EFUSE_RD_REPEAT_DATA3_REG_FLASH_TYPE_MASK
+            else 0
+        )
+
+    def uses_usb_otg(self):
+        """
+        Check the UARTDEV_BUF_NO register to see if USB-OTG console is being used
+        """
+        if self.secure_download_mode:
+            return False  # can't detect native USB in secure download mode
+        return self.get_uart_no() == self.UARTDEV_BUF_NO_USB_OTG
+
+    def uses_usb_jtag_serial(self):
+        """
+        Check the UARTDEV_BUF_NO register to see if USB-JTAG/Serial is being used
+        """
+        if self.secure_download_mode:
+            return False  # can't detect USB-JTAG/Serial in secure download mode
+        return self.get_uart_no() == self.UARTDEV_BUF_NO_USB_JTAG_SERIAL
+
+    def disable_watchdogs(self):
+        # When USB-JTAG/Serial is used, the RTC WDT and SWD watchdog are not reset
+        # and can then reset the board during flashing. Disable them.
+        if self.uses_usb_jtag_serial():
+            # Disable RTC WDT
+            self.write_reg(self.RTC_CNTL_WDTWPROTECT_REG, self.RTC_CNTL_WDT_WKEY)
+            self.write_reg(self.RTC_CNTL_WDTCONFIG0_REG, 0)
+            self.write_reg(self.RTC_CNTL_WDTWPROTECT_REG, 0)
+
+            # Automatically feed SWD
+            self.write_reg(self.RTC_CNTL_SWD_WPROTECT_REG, self.RTC_CNTL_SWD_WKEY)
+            self.write_reg(
+                self.RTC_CNTL_SWD_CONF_REG,
+                self.read_reg(self.RTC_CNTL_SWD_CONF_REG)
+                | self.RTC_CNTL_SWD_AUTO_FEED_EN,
+            )
+            self.write_reg(self.RTC_CNTL_SWD_WPROTECT_REG, 0)
+
+    def _post_connect(self):
+        if self.uses_usb_otg():
+            self.ESP_RAM_BLOCK = self.USB_RAM_BLOCK
+        if not self.sync_stub_detected:  # Don't run if stub is reused
+            self.disable_watchdogs()
+
+    def _check_if_can_reset(self):
+        """
+        Check the strapping register to see if we can reset out of download mode.
+        """
+        if os.getenv("ESPTOOL_TESTING") is not None:
+            print("ESPTOOL_TESTING is set, ignoring strapping mode check")
+            # Esptool tests over USB-OTG run with GPIO0 strapped low,
+            # don't complain in this case.
+            return
+        strap_reg = self.read_reg(self.GPIO_STRAP_REG)
+        force_dl_reg = self.read_reg(self.RTC_CNTL_OPTION1_REG)
+        if (
+            strap_reg & self.GPIO_STRAP_SPI_BOOT_MASK == 0
+            and force_dl_reg & self.RTC_CNTL_FORCE_DOWNLOAD_BOOT_MASK == 0
+        ):
+            print(
+                "WARNING: {} chip was placed into download mode using GPIO0.\n"
+                "esptool.py can not exit the download mode over USB. "
+                "To run the app, reset the chip manually.\n"
+                "To suppress this note, set --after option to 'no_reset'.".format(
+                    self.get_chip_description()
+                )
+            )
+            raise SystemExit(1)
+
+    def hard_reset(self):
+        uses_usb_otg = self.uses_usb_otg()
+        if uses_usb_otg:
+            self._check_if_can_reset()
+
+        print("Hard resetting via RTS pin...")
+        HardReset(self._port, uses_usb_otg)()
+
+    def change_baud(self, baud):
+        ESPLoader.change_baud(self, baud)
+
+
+class ESP32S3StubLoader(ESP32S3ROM):
+    """Access class for ESP32S3 stub loader, runs on top of ROM.
+
+    (Basically the same as ESP32StubLoader, but different base class.
+    Can possibly be made into a mixin.)
+    """
+
+    FLASH_WRITE_SIZE = 0x4000  # matches MAX_WRITE_BLOCK in stub_loader.c
+    STATUS_BYTES_LENGTH = 2  # same as ESP8266, different to ESP32 ROM
+    IS_STUB = True
+
+    def __init__(self, rom_loader):
+        self.secure_download_mode = rom_loader.secure_download_mode
+        self._port = rom_loader._port
+        self._trace_enabled = rom_loader._trace_enabled
+        self.cache = rom_loader.cache
+        self.flush_input()  # resets _slip_reader
+
+        if rom_loader.uses_usb_otg():
+            self.ESP_RAM_BLOCK = self.USB_RAM_BLOCK
+            self.FLASH_WRITE_SIZE = self.USB_RAM_BLOCK
+
+
+ESP32S3ROM.STUB_CLASS = ESP32S3StubLoader

tools/python/esptool/targets/esp32s3beta2.py

@@ -0,0 +1,42 @@
+# SPDX-FileCopyrightText: 2014-2022 Fredrik Ahlberg, Angus Gratton,
+# Espressif Systems (Shanghai) CO LTD, other contributors as noted.
+#
+# SPDX-License-Identifier: GPL-2.0-or-later
+
+from .esp32s3 import ESP32S3ROM
+
+
+class ESP32S3BETA2ROM(ESP32S3ROM):
+    CHIP_NAME = "ESP32-S3(beta2)"
+    IMAGE_CHIP_ID = 4
+
+    CHIP_DETECT_MAGIC_VALUE = [0xEB004136]
+
+    EFUSE_BASE = 0x6001A000  # BLOCK0 read base address
+
+    def get_chip_description(self):
+        major_rev = self.get_major_chip_version()
+        minor_rev = self.get_minor_chip_version()
+        return f"{self.CHIP_NAME} (revision v{major_rev}.{minor_rev})"
+
+
+class ESP32S3BETA2StubLoader(ESP32S3BETA2ROM):
+    """Access class for ESP32S3 stub loader, runs on top of ROM.
+
+    (Basically the same as ESP32StubLoader, but different base class.
+    Can possibly be made into a mixin.)
+    """
+
+    FLASH_WRITE_SIZE = 0x4000  # matches MAX_WRITE_BLOCK in stub_loader.c
+    STATUS_BYTES_LENGTH = 2  # same as ESP8266, different to ESP32 ROM
+    IS_STUB = True
+
+    def __init__(self, rom_loader):
+        self.secure_download_mode = rom_loader.secure_download_mode
+        self._port = rom_loader._port
+        self._trace_enabled = rom_loader._trace_enabled
+        self.cache = rom_loader.cache
+        self.flush_input()  # resets _slip_reader
+
+
+ESP32S3BETA2ROM.STUB_CLASS = ESP32S3BETA2StubLoader

tools/python/esptool/targets/esp8266.py

@@ -0,0 +1,193 @@
+# SPDX-FileCopyrightText: 2014-2022 Fredrik Ahlberg, Angus Gratton,
+# Espressif Systems (Shanghai) CO LTD, other contributors as noted.
+#
+# SPDX-License-Identifier: GPL-2.0-or-later
+
+from loader import ESPLoader
+from util import FatalError, NotImplementedInROMError
+
+
+class ESP8266ROM(ESPLoader):
+    """Access class for ESP8266 ROM bootloader"""
+
+    CHIP_NAME = "ESP8266"
+    IS_STUB = False
+
+    CHIP_DETECT_MAGIC_VALUE = [0xFFF0C101]
+
+    # OTP ROM addresses
+    ESP_OTP_MAC0 = 0x3FF00050
+    ESP_OTP_MAC1 = 0x3FF00054
+    ESP_OTP_MAC3 = 0x3FF0005C
+
+    SPI_REG_BASE = 0x60000200
+    SPI_USR_OFFS = 0x1C
+    SPI_USR1_OFFS = 0x20
+    SPI_USR2_OFFS = 0x24
+    SPI_MOSI_DLEN_OFFS = None
+    SPI_MISO_DLEN_OFFS = None
+    SPI_W0_OFFS = 0x40
+
+    UART_CLKDIV_REG = 0x60000014
+
+    XTAL_CLK_DIVIDER = 2
+
+    FLASH_SIZES = {
+        "512KB": 0x00,
+        "256KB": 0x10,
+        "1MB": 0x20,
+        "2MB": 0x30,
+        "4MB": 0x40,
+        "2MB-c1": 0x50,
+        "4MB-c1": 0x60,
+        "8MB": 0x80,
+        "16MB": 0x90,
+    }
+
+    FLASH_FREQUENCY = {
+        "80m": 0xF,
+        "40m": 0x0,
+        "26m": 0x1,
+        "20m": 0x2,
+    }
+
+    BOOTLOADER_FLASH_OFFSET = 0
+
+    MEMORY_MAP = [
+        [0x3FF00000, 0x3FF00010, "DPORT"],
+        [0x3FFE8000, 0x40000000, "DRAM"],
+        [0x40100000, 0x40108000, "IRAM"],
+        [0x40201010, 0x402E1010, "IROM"],
+    ]
+
+    def get_efuses(self):
+        # Return the 128 bits of ESP8266 efuse as a single Python integer
+        result = self.read_reg(0x3FF0005C) << 96
+        result |= self.read_reg(0x3FF00058) << 64
+        result |= self.read_reg(0x3FF00054) << 32
+        result |= self.read_reg(0x3FF00050)
+        return result
+
+    def _get_flash_size(self, efuses):
+        # rX_Y = EFUSE_DATA_OUTX[Y]
+        r0_4 = (efuses & (1 << 4)) != 0
+        r3_25 = (efuses & (1 << 121)) != 0
+        r3_26 = (efuses & (1 << 122)) != 0
+        r3_27 = (efuses & (1 << 123)) != 0
+
+        if r0_4 and not r3_25:
+            if not r3_27 and not r3_26:
+                return 1
+            elif not r3_27 and r3_26:
+                return 2
+        if not r0_4 and r3_25:
+            if not r3_27 and not r3_26:
+                return 2
+            elif not r3_27 and r3_26:
+                return 4
+        return -1
+
+    def get_chip_description(self):
+        efuses = self.get_efuses()
+        is_8285 = (
+            efuses & ((1 << 4) | 1 << 80)
+        ) != 0  # One or the other efuse bit is set for ESP8285
+        if is_8285:
+            flash_size = self._get_flash_size(efuses)
+            max_temp = (
+                efuses & (1 << 5)
+            ) != 0  # This efuse bit identifies the max flash temperature
+            chip_name = {
+                1: "ESP8285H08" if max_temp else "ESP8285N08",
+                2: "ESP8285H16" if max_temp else "ESP8285N16",
+            }.get(flash_size, "ESP8285")
+            return chip_name
+        return "ESP8266EX"
+
+    def get_chip_features(self):
+        features = ["WiFi"]
+        if "ESP8285" in self.get_chip_description():
+            features += ["Embedded Flash"]
+        return features
+
+    def flash_spi_attach(self, hspi_arg):
+        if self.IS_STUB:
+            super(ESP8266ROM, self).flash_spi_attach(hspi_arg)
+        else:
+            # ESP8266 ROM has no flash_spi_attach command in serial protocol,
+            # but flash_begin will do it
+            self.flash_begin(0, 0)
+
+    def flash_set_parameters(self, size):
+        # not implemented in ROM, but OK to silently skip for ROM
+        if self.IS_STUB:
+            super(ESP8266ROM, self).flash_set_parameters(size)
+
+    def chip_id(self):
+        """
+        Read Chip ID from efuse - the equivalent of the SDK system_get_chip_id() func
+        """
+        id0 = self.read_reg(self.ESP_OTP_MAC0)
+        id1 = self.read_reg(self.ESP_OTP_MAC1)
+        return (id0 >> 24) | ((id1 & 0xFFFFFF) << 8)
+
+    def read_mac(self, mac_type="BASE_MAC"):
+        """Read MAC from OTP ROM"""
+        if mac_type != "BASE_MAC":
+            return None
+        mac0 = self.read_reg(self.ESP_OTP_MAC0)
+        mac1 = self.read_reg(self.ESP_OTP_MAC1)
+        mac3 = self.read_reg(self.ESP_OTP_MAC3)
+        if mac3 != 0:
+            oui = ((mac3 >> 16) & 0xFF, (mac3 >> 8) & 0xFF, mac3 & 0xFF)
+        elif ((mac1 >> 16) & 0xFF) == 0:
+            oui = (0x18, 0xFE, 0x34)
+        elif ((mac1 >> 16) & 0xFF) == 1:
+            oui = (0xAC, 0xD0, 0x74)
+        else:
+            raise FatalError("Unknown OUI")
+        return oui + ((mac1 >> 8) & 0xFF, mac1 & 0xFF, (mac0 >> 24) & 0xFF)
+
+    def get_erase_size(self, offset, size):
+        """Calculate an erase size given a specific size in bytes.
+
+        Provides a workaround for the bootloader erase bug."""
+
+        sectors_per_block = 16
+        sector_size = self.FLASH_SECTOR_SIZE
+        num_sectors = (size + sector_size - 1) // sector_size
+        start_sector = offset // sector_size
+
+        head_sectors = sectors_per_block - (start_sector % sectors_per_block)
+        if num_sectors < head_sectors:
+            head_sectors = num_sectors
+
+        if num_sectors < 2 * head_sectors:
+            return (num_sectors + 1) // 2 * sector_size
+        else:
+            return (num_sectors - head_sectors) * sector_size
+
+    def override_vddsdio(self, new_voltage):
+        raise NotImplementedInROMError(
+            "Overriding VDDSDIO setting only applies to ESP32"
+        )
+
+
+class ESP8266StubLoader(ESP8266ROM):
+    """Access class for ESP8266 stub loader, runs on top of ROM."""
+
+    FLASH_WRITE_SIZE = 0x4000  # matches MAX_WRITE_BLOCK in stub_loader.c
+    IS_STUB = True
+
+    def __init__(self, rom_loader):
+        self.secure_download_mode = rom_loader.secure_download_mode
+        self._port = rom_loader._port
+        self._trace_enabled = rom_loader._trace_enabled
+        self.cache = rom_loader.cache
+        self.flush_input()  # resets _slip_reader
+
+    def get_erase_size(self, offset, size):
+        return size  # stub doesn't have same size bug as ROM loader
+
+
+ESP8266ROM.STUB_CLASS = ESP8266StubLoader

tools/python/esptool/targets/stub_flasher/stub_flasher_32.json

@@ -0,0 +1,7 @@
+{
+    "entry": 1074521560,
+    "text": "CAD0PxwA9D8AAPQ/AMD8PxAA9D82QQAh+v/AIAA4AkH5/8AgACgEICB0nOIGBQAAAEH1/4H2/8AgAKgEiAigoHTgCAALImYC54b0/yHx/8AgADkCHfAAAKDr/T8Ya/0/hIAAAEBAAABYq/0/pOv9PzZBALH5/yCgdBARIKXIAJYaBoH2/5KhAZCZEZqYwCAAuAmR8/+goHSaiMAgAJIYAJCQ9BvJwMD0wCAAwlgAmpvAIACiSQDAIACSGACB6v+QkPSAgPSHmUeB5f+SoQGQmRGamMAgAMgJoeX/seP/h5wXxgEAfOiHGt7GCADAIACJCsAgALkJRgIAwCAAuQrAIACJCZHX/5qIDAnAIACSWAAd8AAA+CD0P/gw9D82QQCR/f/AIACICYCAJFZI/5H6/8AgAIgJgIAkVkj/HfAAAAAQIPQ/ACD0PwAAAAg2QQAQESCl/P8h+v8MCMAgAIJiAJH6/4H4/8AgAJJoAMAgAJgIVnn/wCAAiAJ88oAiMCAgBB3wAAAAAEA2QQAQESDl+/8Wav+B7P+R+//AIACSaADAIACYCFZ5/x3wAAAMQP0/////AAQg9D82QQAh/P84QhaDBhARIGX4/xb6BQz4DAQ3qA2YIoCZEIKgAZBIg0BAdBARICX6/xARICXz/4giDBtAmBGQqwHMFICrAbHt/7CZELHs/8AgAJJrAJHO/8AgAKJpAMAgAKgJVnr/HAkMGkCag5AzwJqIOUKJIh3wAAAskgBANkEAoqDAgf3/4AgAHfAAADZBAIKgwK0Ch5IRoqDbgff/4AgAoqDcRgQAAAAAgqDbh5IIgfL/4AgAoqDdgfD/4AgAHfA2QQA6MsYCAACiAgAbIhARIKX7/zeS8R3wAAAAfNoFQNguBkCc2gVAHNsFQDYhIaLREIH6/+AIAEYLAAAADBRARBFAQ2PNBL0BrQKB9f/gCACgoHT8Ws0EELEgotEQgfH/4AgASiJAM8BWA/0iogsQIrAgoiCy0RCB7P/gCACtAhwLEBEgpff/LQOGAAAioGMd8AAA/GcAQNCSAEAIaABANkEhYqEHwGYRGmZZBiwKYtEQDAVSZhqB9//gCAAMGECIEUe4AkZFAK0GgdT/4AgAhjQAAJKkHVBzwOCZERqZQHdjiQnNB70BIKIggc3/4AgAkqQd4JkRGpmgoHSICYyqDAiCZhZ9CIYWAAAAkqQd4JkREJmAgmkAEBEgJer/vQetARARIKXt/xARICXp/80HELEgYKYggbv/4AgAkqQd4JkRGpmICXAigHBVgDe1sJKhB8CZERqZmAmAdcCXtwJG3P+G5v8MCIJGbKKkGxCqoIHK/+AIAFYK/7KiC6IGbBC7sBARIKWQAPfqEvZHD7KiDRC7sHq7oksAG3eG8f9867eawWZHCIImGje4Aoe1nCKiCxAisGC2IK0CgZv/4AgAEBEgpd//rQIcCxARICXj/xARIKXe/ywKgbH/4AgAHfAIIPQ/cOL6P0gkBkDwIgZANmEAEBEg5cr/EKEggfv/4AgAPQoMEvwqiAGSogCQiBCJARARIKXP/5Hy/6CiAcAgAIIpAKCIIMAgAIJpALIhAKHt/4Hu/+AIAKAjgx3wAAD/DwAANkEAgTv/DBmSSAAwnEGZKJH7/zkYKTgwMLSaIiozMDxBDAIpWDlIEBEgJfj/LQqMGiKgxR3wAABQLQZANkEAQSz/WDRQM2MWYwRYFFpTUFxBRgEAEBEgZcr/iESmGASIJIel7xARIKXC/xZq/6gUzQO9AoHx/+AIAKCgdIxKUqDEUmQFWBQ6VVkUWDQwVcBZNB3wAADA/D9PSEFJqOv9P3DgC0AU4AtADAD0PzhA9D///wAAjIAAABBAAACs6/0/vOv9PwTA/D8IwPw/BOz9PxQA9D/w//8AqOv9PwzA/D8kQP0/fGgAQOxnAEBYhgBAbCoGQDgyBkAULAZAzCwGQEwsBkA0hQBAzJAAQHguBkAw7wVAWJIAQEyCAEA2wQAh3v8MCiJhCEKgAIHu/+AIACHZ/zHa/8YAAEkCSyI3MvgQESBlw/8MS6LBIBARIOXG/yKhARARICXC/1GR/pAiESolMc//sc//wCAAWQIheP4MDAxaMmIAgdz/4AgAMcr/QqEBwCAAKAMsCkAiIMAgACkDgTH/4AgAgdX/4AgAIcP/wCAAKALMuhzDMCIQIsL4DBMgo4MMC4HO/+AIAPG8/wwdwqABsqAB4qEAQN0RAMwRgLsBoqAAgcf/4AgAIbX/YcT+KlVy1ivAIAAoBRZy/8AgADgFDAQMEsAgAEkFIkEQIgMBDCgiQRGCUQlJUSaSBxw0RxIdxgcAIgMDQgMCgCIRQCIgZkIQKCPAIAAoAilRBgEAHCIiUQkQESCls/8Mi6LBEBARIGW3/4IDAyIDAoCIESCIICGY/yAg9IeyHKKgwBARICWy/6Kg7hARIKWx/xARICWw/0bb/wAAIgMBHDQnNDT2IhhG2wAAACLCLyAgdPZCcEGJ/0AioCgCoAIAIsL+ICB0HBQntAJG0gBBhP9AIqAoAqACAELCMEBAdLZUyYbMACxJDAQioMCXGAKGygBJUQxyrQQQESDlqv+tBBARIGWq/xARIOWo/xARIKWo/wyLosEQIsL/EBEg5av/ViL9RigADBJWaC6CYQ+Bev/gCACI8aAog0a1ACaIBQwSRrMAAEgjKDMghCCAgLRWyP4QESBlx/8qRJwaxvf/AKCsQYFu/+AIAFYq/SLS8CCkwMwiBogAAKCA9FYY/oYEAKCg9YnxgWb/4AgAiPFW2vqAIsAMGACIESCkwCc44QYEAAAAoKxBgV3/4AgAVur4ItLwIKTAVqL+xnYAAAwEIqDAJogCBpUADAQtBEaTACa49QZpAAwSJrgCBo0AuDOoIwwEEBEgJaL/oCSDhogADBlmuFyIQyCpEQwEIqDCh7oCBoYAuFOiIwKSYQ4QESAlwf+Y4aCUg4YNAAwZZrgxiEMgqREMBCKgwoe6AkZ7ACgzuFOoIyBIgpnhEBEgJb7/ITT+DAiY4YliItIrSSKgmIMtCcZuAJEu/gwEogkAIqDGR5oCRm0ASCOCyPAioMCHlAEoWQwEkqDvRgIASqOiChgbRKCZMIck8oIDBUIDBICIEUCIIEIDBgBEEYBEIIIDB4CIAUCIIICZwIKgwQwEkCiTxlkAgRb+IqDGkggATQkWmRWYOAwEIqDIRxkCBlMAKFiSSABGTgAciQwEDBKXGAIGTgD4c+hj2FPIQ7gzqCOBCf/gCAAMCE0KoCiDBkcAAAAMEiZIAsZBAKgjDAuBAP/gCAAGIAAAAACAkDQMBCKgwEcZAgY9AICEQYuzfPzGDgCoO4nxmeG5wcnRgfr+4AgAuMGI8SgrSBuoC5jhyNFAQhAmAg3AIADYCiAsMNAiECBEIMAgAEkKG5myyxCHOcDGlP9mSAJGk/8MBCKgwIYmAAwSJrgCxiEAIdb+iFNII4kCIdX+SQIMAgYdALHR/gwE2AsMGoLI8J0ELQSAKpPQmoMgmRAioMZHmWDBy/5NCegMIqDJhz5TgPAUIqDAVq8ELQmGAgAAKpOYaUsimQSdCiD+wCpNhzLtFqnd+QxJC8Z0/wwSZogYIbv+giIAjBiCoMgMBEkCIbf+SQIMEoAkgwwERgEAAAwEIqD/IKB0EBEgZXj/QKB0EBEgpXf/EBEgZXb/VvK8IgMBHCQnNB/2MgJG8P4iwv0gIHQM9Ce0Asbs/kGm/kAioCgCoAIAAEKg0kcST0Kg1EcSdwbm/ogzoqJxwKoRSCOJ8YGq/uAIACGb/pGc/sAgACgCiPEgNDXAIhGQIhAgIyCAIoIMCkCywoGh/uAIAKKj6IGe/uAIAMbU/gAA2FPIQ7gzqCMQESCle/8G0P4AsgMDIgMCgLsRILsgssvwosMYEBEg5Zf/Bsn+ACIDA0IDAoAiEUAiIEGI/SLC8Ig0gCJjFpKwiBSKgoCMQUYCAInxEBEg5WD/iPGYRKYZBJgkl6jrEBEgJVn/Fmr/qBTNArLDGIGA/uAIAIw6MqDEOVQ4FCozORQ4NCAjwCk0hq/+IgMDggMCQsMYgCIRODaAIiAiwvBWwwn2UgKGJQAioMlGKgAxY/6BaP3oAylx4IjAiWGIJ60Jh7IBDDqZ4anR6cEQESDlWP+o0YFa/qkB6MGhWf7dCL0EwsEc8sEYifGBYv7gCAC4J80KqHGY4aC7wLknoCLAuAOqRKhhiPGquwwKuQPAqYOAu8Cg0HTMmuLbgK0N4KmDFuoBrQiJ8ZnhydEQESDlhv+I8ZjhyNGJA0YBAAAADBydDIyyODaMc8A/McAzwJaz9dZ8ACKgxylWBnv+VpyeKDYWQp4ioMgG+/+oI1aanYFB/uAIAKKiccCqEYE6/uAIAIE+/uAIAIZv/gAAKDMWcpsMCoE4/uAIAKKj6IEy/uAIAOACAAZo/h3wAAAANkEAnQKCoMAoA4eZD8wyDBKGBwAMAikDfOKGDwAmEgcmIhiGAwAAAIKg24ApI4eZKgwiKQN88kYIAAAAIqDcJ5kKDBIpAy0IBgQAAACCoN188oeZBgwSKQMioNsd8AAA",
+    "text_start": 1074520064,
+    "data": "DMD8P9jnC0Br6AtAA+0LQPLoC0CL6AtA8ugLQFHpC0Ae6gtAkOoLQDnqC0CB5wtAtukLQBDqC0B06QtAtOoLQJ7pC0C06gtAWegLQLboC0Dy6AtAUekLQHHoC0Bk6wtAxewLQKTmC0Dn7AtApOYLQKTmC0Ck5gtApOYLQKTmC0Ck5gtApOYLQKTmC0AL6wtApOYLQOXrC0DF7AtA",
+    "data_start": 1073605544
+}

tools/python/esptool/targets/stub_flasher/stub_flasher_32c2.json

@@ -0,0 +1,7 @@
+{
+    "entry": 1077413304,
+    "text": "ARG3BwBgTsaDqYcASsg3Sco/JspSxAbOIsy3BABgfVoTCQkAwEwTdPQ/DeDyQGJEI6g0AUJJ0kSySSJKBWGCgIhAgycJABN19Q+Cl30U4xlE/8m/EwcADJRBqodjGOUAhUeFxiOgBQB5VYKABUdjh+YACUZjjcYAfVWCgEIFEwewDUGFY5XnAolHnMH1t5MGwA1jFtUAmMETBQAMgoCTBtANfVVjldcAmMETBbANgoC3dcs/QRGThQW6BsZhP2NFBQa3d8s/k4eHsQOnBwgD1kcIE3X1D5MGFgDCBsGCI5LXCDKXIwCnAAPXRwiRZ5OHBwRjHvcCN/fKPxMHh7GhZ7qXA6YHCLc2yz+3d8s/k4eHsZOGhrVjH+YAI6bHCCOg1wgjkgcIIaD5V+MG9fyyQEEBgoAjptcII6DnCN23NycAYHxLnYv1/zc3AGB8S52L9f+CgEERBsbdN7cnAGAjpgcCNwcACJjDmEN9/8hXskATRfX/BYlBAYKAQREGxtk/fd03BwBAtycAYJjDNycAYBxD/f+yQEEBgoBBESLEN8TKP5MHxABKwAOpBwEGxibCYwoJBEU3OcW9RxMExACBRGPWJwEERL2Ik7QUAH03hT8cRDcGgAATl8cAmeA3BgABt/b/AHWPtyYAYNjCkMKYQn3/QUeR4AVHMwnpQLqXIygkARzEskAiRJJEAklBAYKAQREGxhMHAAxjEOUCEwWwDZcAyP/ngIDjEwXADbJAQQEXA8j/ZwCD4hMHsA3jGOX+lwDI/+eAgOETBdANxbdBESLEJsIGxiqEswS1AGMXlACyQCJEkkRBAYKAA0UEAAUERTfttxMFAAwXA8j/ZwAD3nVxJsPO3v10hWn9cpOEhPqThwkHIsVKwdLc1tqmlwbHFpGzhCcAKokmhS6ElzDI/+eAgJOThwkHBWqKl7OKR0Ep5AVnfXUTBIX5kwcHB6KXM4QnABMFhfqTBwcHqpeihTOFJwCXMMj/54CAkCKFwUW5PwFFhWIWkbpAKkSaRApJ9llmWtZaSWGCgKKJY3OKAIVpTobWhUqFlwDI/+eAQOITdfUPAe1OhtaFJoWXMMj/54DAi06ZMwQ0QVm3EwUwBlW/cXH9ck7PUs1Wy17HBtci1SbTStFayWLFZsNqwe7eqokWkRMFAAIuirKKtosCwpcAyP/ngEBIhWdj7FcRhWR9dBMEhPqThwQHopczhCcAIoWXMMj/54AghX17Eww7+ZMMi/kThwQHk4cEB2KX5pcBSTMMJwCzjCcAEk1je00JY3GpA3mgfTWmhYgYSTVdNSaGjBgihZcwyP/ngCCBppkmmWN1SQOzB6lBY/F3A7MEKkFj85oA1oQmhowYToWXAMj/54Dg0xN19Q9V3QLEgUR5XY1NowEBAGKFlwDI/+eAYMR9+QNFMQDmhS0xY04FAOPinf6FZ5OHBweml4qX2pcjiqf4hQT5t+MWpf2RR+OG9PYFZ311kwcHBxMEhfmilzOEJwATBYX6kwcHB6qXM4UnAKKFlyDI/+eAgHflOyKFwUXxM8U7EwUAApcAyP/ngOA2hWIWkbpQKlSaVApZ+klqStpKSku6SypMmkwKTfZdTWGCgAERBs4izFExNwTOP2wAEwVE/5cAyP/ngKDKqocFRZXnskeT9wcgPsZ5OTcnAGAcR7cGQAATBUT/1Y8cx7JFlwDI/+eAIMgzNaAA8kBiRAVhgoBBEbfHyj8GxpOHxwAFRyOA5wAT18UAmMcFZ30XzMPIx/mNOpWqlbGBjMsjqgcAQTcZwRMFUAyyQEEBgoABESLMN8TKP5MHxAAmysRHTsYGzkrIqokTBMQAY/OVAK6EqcADKUQAJpkTWckAHEhjVfAAHERjXvkC4T593UhAJobOhZcAyP/ngCC7E3X1DwHFkwdADFzIXECml1zAXESFj1zE8kBiRNJEQkmySQVhgoDdNm2/t1dBSRlxk4f3hAFFPs6G3qLcptrK2M7W0tTW0trQ3s7izObK6sjuxpcAyP/ngICtt0fKPzd3yz+ThwcAEweHumPg5xSlOZFFaAixMYU5t/fKP5OHh7EhZz6XIyD3CLcFOEC3BzhAAUaThwcLk4UFADdJyj8VRSMg+QCXAMj/54DgGzcHAGBcRxMFAAK3xMo/k+cXEFzHlwDI/+eAoBq3RwBgiF+BRbd5yz9xiWEVEzUVAJcAyP/ngOCwwWf9FxMHABCFZkFmtwUAAQFFk4TEALdKyj8NapcAyP/ngOCrk4mJsRMJCQATi8oAJpqDp8kI9d+Dq8kIhUcjpgkIIwLxAoPHGwAJRyMT4QKjAvECAtRNR2OL5wZRR2OJ5wYpR2Of5wCDxzsAA8crAKIH2Y8RR2OW5wCDp4sAnEM+1EE2oUVIEJE+g8c7AAPHKwCiB9mPEWdBB2N+9wITBbANlwDI/+eAQJQTBcANlwDI/+eAgJMTBeAOlwDI/+eAwJKBNr23I6AHAJEHbb3JRyMT8QJ9twPHGwDRRmPn5gKFRmPm5gABTBME8A+dqHkXE3f3D8lG4+jm/rd2yz8KB5OGxro2lxhDAoeTBgcDk/b2DxFG42nW/BMH9wITd/cPjUZj7uYIt3bLPwoHk4aGvzaXGEMChxMHQAJjmucQAtQdRAFFlwDI/+eAIIoBRYE8TTxFPKFFSBB9FEk0ffABTAFEE3X0DyU8E3X8Dw08UTzjEQTsg8cbAElHY2D3LglH43n36vUXk/f3Dz1H42P36jd3yz+KBxMHh8C6l5xDgocFRJ3rcBCBRQFFlwDI/+eAQIkd4dFFaBAVNAFEMagFRIHvlwDI/+eAwI0zNKAAKaAhR2OF5wAFRAFMYbcDrIsAA6TLALNnjADSB/X3mTll9cFsIpz9HH19MwWMQF3cs3eVAZXjwWwzBYxAY+aMAv18MwWMQF3QMYGXAMj/54Bgil35ZpT1tzGBlwDI/+eAYIld8WqU0bdBgZcAyP/ngKCIWfkzBJRBwbchR+OK5/ABTBMEAAw5t0FHzb9BRwVE453n9oOlywADpYsAVTK5v0FHBUTjk+f2A6cLAZFnY+PnHIOlSwEDpYsAMTGBt0FHBUTjlOf0g6cLARFnY2T3GgOnywCDpUsBA6WLADOE5wLdNiOsBAAjJIqwCb8DxwQAYw4HEAOniwDBFxMEAAxjE/cAwEgBR5MG8A5jRvcCg8dbAAPHSwABTKIH2Y8Dx2sAQgddj4PHewDiB9mP44T25hMEEAyFtTOG6wADRoYBBQexjuG3g8cEAPHD3ERjmAcSwEgjgAQAVb1hR2OW5wKDp8sBA6eLAYOmSwEDpgsBg6XLAAOliwCX8Mf/54BgeSqMMzSgAAG9AUwFRCm1EUcFROOd5+YDpYsAgUWX8Mf/54Dgeam1E/f3AOMcB+yT3EcAE4SLAAFMfV3jfJzdSESX8Mf/54BgZBhEVEAQQPmOYwenARxCE0f3/32P2Y4UwgUMQQTZvxFHWb1BRwVE45/n4IOniwADp0sBIyT5ACMi6QD1s4MlSQDBF5Hlic8BTBMEYAxJswMniQBjZvcGE/c3AOMQB+YDKIkAAUYBRzMF6ECzhuUAY2n3AOMMBtQjJKkAIyLZALGzM4brABBOEQeQwgVG6b8hRwVE45nn2gMkiQAZwBMEgAwjJAkAIyIJADM0gABhuwFMEwQgDCm7AUwTBIAMCbsBTBMEkAwpsxMHIA1jg+cMEwdADeOW57wDxDsAg8crACIEXYyX8Mf/54AAYgOsxABBFGNzhAEijOMEDLrAQGKUMYCcSGNV8ACcRGNa9Arv8C/kdd3IQGKGk4WLAZfwx//ngABeAcWTB0AM3MjcQOKX3MDcRLOHh0HcxJfwx//ngOBcub4JZRMFBXEDrMsAA6SLAJfwx//ngOBOtwcAYNhLtwYAAcEWk1dHARIHdY+9i9mPs4eHAwFFs9WHApfwx//ngIBPEwWAPpfwx//ngIBLAb6DpksBA6YLAYOlywADpYsA7/DP+e28g8U7AIPHKwAThYsBogXdjcEVUTLVtO/wj92BtwPEOwCDxysAE4yLASIEXYzcREEUxeORR4VLY/6HCJMHkAzcyGW8A6cNACLQBUizh+xAPtaDJ4qwY3P0AA1IQsY6xO/wD9kiRzJIN8XKP+KFfBCThsoAEBATBUUCl/DH/+eAgEw398o/kwjHAIJXA6eIsIOlDQAdjB2PPpyyVyOk6LCqi76VI6C9AJOHygCdjQHFoWdjlvUAWoXFMCOgbQEJxNxEmcPjQHD5Y98LAJMHcAyFv4VLt33LP7fMyj+TjY26k4zMAOm/45oLoNxE44cHoJMHgAyxt4OniwDjkAegAUWX8Mf/54BgPAllEwUFcZfwx//ngMA4l/DH/+eAgDzxugOkywDjCwScAUWX8Mf/54DAORMFgD6X8Mf/54BANgKUbbr2UGZU1lRGWbZZJlqWWgZb9ktmTNZMRk22TQlhgoAAAA==",
+    "text_start": 1077411840,
+    "data": "DEDKP+AIOEAsCThAhAk4QCgKOECUCjhAQgo4QKgHOEDkCThAJAo4QJgJOEBYBzhAzAk4QFgHOEC6CDhA/gg4QCwJOECECThAzAg4QBIIOEBCCDhAyAg4QOwMOEAsCThArAs4QKAMOECkBjhAygw4QKQGOECkBjhApAY4QKQGOECkBjhApAY4QKQGOECkBjhASAs4QKQGOEDICzhAoAw4QA==",
+    "data_start": 1070295976
+}

tools/python/esptool/targets/stub_flasher/stub_flasher_32c6.json

@@ -0,0 +1,7 @@
+{
+    "entry": 1082132112,
+    "text": "QREixCbCBsa39wBgEUc3BIRA2Mu39ABgEwQEANxAkYuR57JAIkSSREEBgoCIQBxAE3X1D4KX3bcBEbcHAGBOxoOphwBKyDcJhEAmylLEBs4izLcEAGB9WhMJCQDATBN09A8N4PJAYkQjqDQBQknSRLJJIkoFYYKAiECDJwkAE3X1D4KXfRTjGUT/yb8TBwAMlEGqh2MY5QCFR4XGI6AFAHlVgoAFR2OH5gAJRmONxgB9VYKAQgUTB7ANQYVjlecCiUecwfW3kwbADWMW1QCYwRMFAAyCgJMG0A19VWOV1wCYwRMFsA2CgLc1hUBBEZOFRboGxmE/Y0UFBrc3hUCTh8exA6cHCAPWRwgTdfUPkwYWAMIGwYIjktcIMpcjAKcAA9dHCJFnk4cHBGMe9wI3t4RAEwfHsaFnupcDpgcIt/aEQLc3hUCTh8exk4bGtWMf5gAjpscII6DXCCOSBwghoPlX4wb1/LJAQQGCgCOm1wgjoOcI3bc3NwBgfEudi/X/NycAYHxLnYv1/4KAQREGxt03tzcAYCOmBwI3BwAImMOYQ33/yFeyQBNF9f8FiUEBgoBBEQbG2T993TcHAEC3NwBgmMM3NwBgHEP9/7JAQQGCgEERIsQ3hIRAkwcEAUrAA6kHAQbGJsJjCgkERTc5xb1HEwQEAYFEY9YnAQREvYiTtBQAfTeFPxxENwaAABOXxwCZ4DcGAAG39v8AdY+3NgBg2MKQwphCff9BR5HgBUczCelAupcjKCQBHMSyQCJEkkQCSUEBgoABEQbOIswlNzcEzj9sABMFRP+XAID/54Cg8qqHBUWV57JHk/cHID7GiTc3NwBgHEe3BkAAEwVE/9WPHMeyRZcAgP/ngCDwMzWgAPJAYkQFYYKAQRG3h4RABsaThwcBBUcjgOcAE9fFAJjHBWd9F8zDyMf5jTqVqpWxgYzLI6oHAEE3GcETBVAMskBBAYKAAREizDeEhECTBwQBJsrER07GBs5KyKqJEwQEAWPzlQCuhKnAAylEACaZE1nJABxIY1XwABxEY175ArU9fd1IQCaGzoWXAID/54Ag4xN19Q8BxZMHQAxcyFxAppdcwFxEhY9cxPJAYkTSREJJskkFYYKAaTVtv0ERBsaXAID/54BA1gNFhQGyQHUVEzUVAEEBgoBBEQbGxTcNxbcHhECThwcA1EOZzjdnCWATBwcRHEM3Bv3/fRbxjzcGAwDxjtWPHMOyQEEBgoBBEQbGbTcRwQ1FskBBARcDgP9nAIPMQREGxpcAgP/ngEDKcTcBxbJAQQHZv7JAQQGCgEERBsYTBwAMYxrlABMFsA3RPxMFwA2yQEEB6bcTB7AN4xvl/sE3EwXQDfW3QREixCbCBsYqhLMEtQBjF5QAskAiRJJEQQGCgANFBAAFBE0/7bc1cSbLTsf9coVp/XQizUrJUsVWwwbPk4SE+haRk4cJB6aXGAizhOcAKokmhS6ElwCA/+eAwC+ThwkHGAgFarqXs4pHQTHkBWd9dZMFhfqTBwcHEwWF+RQIqpczhdcAkwcHB66Xs4XXACrGlwCA/+eAgCwyRcFFlTcBRYViFpH6QGpE2kRKSbpJKkqaSg1hgoCiiWNzigCFaU6G1oVKhZcAgP/ngADJE3X1DwHtTobWhSaFlwCA/+eAwCdOmTMENEFRtxMFMAZVvxMFAAzZtTFx/XIFZ07XUtVW017PBt8i3SbbStla0WLNZstqyW7H/XcWkRMHBwc+lxwIupc+xiOqB/iqiS6Ksoq2iwU1kwcAAhnBtwcCAD6FlwCA/+eAYCCFZ2PlVxMFZH15EwmJ+pMHBAfKlxgIM4nnAEqFlwCA/+eA4B59exMMO/mTDIv5EwcEB5MHBAcUCGKX5peBRDMM1wCzjNcAUk1jfE0JY/GkA0GomT+ihQgBjTW5NyKGDAFKhZcAgP/ngMAaopmilGP1RAOzh6RBY/F3AzMEmkBj84oAVoQihgwBToWXAID/54BAuBN19Q9V3QLMAUR5XY1NowkBAGKFlwCA/+eAgKd9+QNFMQHmhVE8Y08FAOPijf6FZ5OHBweilxgIupfalyOKp/gFBPG34xWl/ZFH4wX09gVnfXWTBwcHkwWF+hMFhfkUCKqXM4XXAJMHBweul7OF1wAqxpcAgP/ngOAQcT0yRcFFZTNRPdU5twcCABnhkwcAAj6FlwCA/+eA4A2FYhaR+lBqVNpUSlm6WSpamloKW/pLakzaTEpNuk0pYYKAt1dBSRlxk4f3hAFFht6i3KbaytjO1tLU1tLa0N7O4szmyurI7sY+zpcAgP/ngMCgcTENwTdnCWATBwcRHEO3BoRAI6L2ALcG/f/9FvWPwWbVjxzDpTEFzbcnC2A3R9hQk4aHwRMHF6qYwhOGB8AjIAYAI6AGAJOGB8KYwpOHx8GYQzcGBABRj5jDI6AGALcHhEA3N4VAk4cHABMHx7ohoCOgBwCRB+Pt5/5FO5FFaAh1OWUzt7eEQJOHx7EhZz6XIyD3CLcHgEA3CYRAk4eHDiMg+QC3OYVA1TYTCQkAk4nJsWMHBRC3BwFgRUcjoOcMhUVFRZcAgP/ngED5twWAQAFGk4UFAEVFlwCA/+eAQPo39wBgHEs3BQIAk+dHABzLlwCA/+eAQPm3FwlgiF+BRbeEhEBxiWEVEzUVAJcAgP/ngAChwWf9FxMHABCFZkFmtwUAAQFFk4QEAbcKhEANapcAgP/ngACXE4sKASaag6fJCPXfg6vJCIVHI6YJCCMC8QKDxxsACUcjE+ECowLxAgLUTUdjgecIUUdjj+cGKUdjn+cAg8c7AAPHKwCiB9mPEUdjlucAg6eLAJxDPtRxOaFFSBBlNoPHOwADxysAogfZjxFnQQdjdPcEEwWwDZk2EwXADYE2EwXgDi0+vTFBt7cFgEABRpOFhQMVRZcAgP/ngADrNwcAYFxHEwUAApPnFxBcxzG3yUcjE/ECTbcDxxsA0UZj5+YChUZj5uYAAUwTBPAPhah5FxN39w/JRuPo5v63NoVACgeThga7NpcYQwKHkwYHA5P29g8RRuNp1vwTB/cCE3f3D41GY+vmCLc2hUAKB5OGxr82lxhDAocTB0ACY5jnEALUHUQBRWE8AUVFPOE22TahRUgQfRTBPHX0AUwBRBN19A9hPBN1/A9JPG024x4E6oPHGwBJR2Nj9y4JR+N29+r1F5P39w89R+Ng9+o3N4VAigcTB8fAupecQ4KHBUSd63AQgUUBRZfwf//ngAB0HeHRRWgQjTwBRDGoBUSB75fwf//ngAB5MzSgACmgIUdjhecABUQBTGG3A6yLAAOkywCzZ4wA0gf19+/wv4h98cFsIpz9HH19MwWMQFXcs3eVAZXjwWwzBYxAY+aMAv18MwWMQFXQMYGX8H//54CAdVX5ZpT1tzGBl/B//+eAgHRV8WqU0bdBgZfwf//ngMBzUfkzBJRBwbchR+OJ5/ABTBMEAAwxt0FHzb9BRwVE45zn9oOlywADpYsA1TKxv0FHBUTjkuf2A6cLAZFnY+XnHIOlSwEDpYsA7/D/gzW/QUcFROOS5/SDpwsBEWdjZfcaA6fLAIOlSwEDpYsAM4TnAu/wf4EjrAQAIySKsDG3A8cEAGMOBxADp4sAwRcTBAAMYxP3AMBIAUeTBvAOY0b3AoPHWwADx0sAAUyiB9mPA8drAEIHXY+Dx3sA4gfZj+OB9uYTBBAMqb0zhusAA0aGAQUHsY7ht4PHBADxw9xEY5gHEsBII4AEAH21YUdjlucCg6fLAQOniwGDpksBA6YLAYOlywADpYsAl/B//+eAQGQqjDM0oAAptQFMBUQRtRFHBUTjmufmA6WLAIFFl/B//+eAwGmRtRP39wDjGgfsk9xHABOEiwABTH1d43mc3UhEl/B//+eAwE0YRFRAEED5jmMHpwEcQhNH9/99j9mOFMIFDEEE2b8RR0m9QUcFROOc5+CDp4sAA6dLASMm+QAjJOkA3bODJYkAwReR5YnPAUwTBGAMtbsDJ8kAY2b3BhP3NwDjHgfkAyjJAAFGAUczBehAs4blAGNp9wDjCQbUIyapACMk2QCZszOG6wAQThEHkMIFRum/IUcFROOW59oDJMkAGcATBIAMIyYJACMkCQAzNIAASbsBTBMEIAwRuwFMEwSADDGzAUwTBJAMEbMTByANY4PnDBMHQA3jkOe8A8Q7AIPHKwAiBF2Ml/B//+eA4EwDrMQAQRRjc4QBIozjDgy4wEBilDGAnEhjVfAAnERjW/QK7/BP0XXdyEBihpOFiwGX8H//54DgSAHFkwdADNzI3EDil9zA3ESzh4dB3MSX8H//54DAR4m+CWUTBQVxA6zLAAOkiwCX8H//54BAOLcHAGDYS7cGAAHBFpNXRwESB3WPvYvZj7OHhwMBRbPVhwKX8H//54BgORMFgD6X8H//54DgNBG2g6ZLAQOmCwGDpcsAA6WLAO/wT/79tIPFOwCDxysAE4WLAaIF3Y3BFe/wL9vZvO/wj8o9vwPEOwCDxysAE4yLASIEXYzcREEUzeORR4VLY/+HCJMHkAzcyG20A6cNACLQBUizh+xAPtaDJ4qwY3P0AA1IQsY6xO/wD8YiRzJIN4WEQOKFfBCThgoBEBATBYUCl/B//+eAwDY3t4RAkwgHAYJXA6eIsIOlDQAdjB2PPpyyVyOk6LCqi76VI6C9AJOHCgGdjQHFoWdjl/UAWoXv8M/QI6BtAQnE3ESZw+NPcPdj3wsAkwdwDL23hUu3PYVAt4yEQJONzbqTjAwB6b/jkgug3ETjjweekweADKm3g6eLAOOYB57v8M/YCWUTBQVxl/B//+eAQCLv8E/Ul/B//+eAgCb5sgOkywDjBASc7/BP1hMFgD6X8H//54DgH+/w79EClH2y7/Bv0fZQZlTWVEZZtlkmWpZaBlv2S2ZM1kxGTbZNCWGCgA==",
+    "text_start": 1082130432,
+    "data": "EACEQEIKgECSCoBA6gqAQI4LgED6C4BAqAuAQA4JgEBKC4BAiguAQP4KgEC+CIBAMguAQL4IgEAcCoBAYgqAQJIKgEDqCoBALgqAQHIJgECiCYBAKgqAQFIOgECSCoBAEg2AQAoOgED+B4BAMg6AQP4HgED+B4BA/geAQP4HgED+B4BA/geAQP4HgED+B4BArgyAQP4HgEAwDYBACg6AQA==",
+    "data_start": 1082469292
+}

tools/python/esptool/targets/stub_flasher/stub_flasher_32c6beta.json

@@ -0,0 +1,7 @@
+{
+    "entry": 1077413318,
+    "text": "ARG3BwBgTsaDqYcASsg3Scg/JspSxAbOIsy3BABgfVoTCQkAwEwTdPQ/DeDyQGJEI6g0AUJJ0kSySSJKBWGCgIhAgycJABN19Q+Cl30U4xlE/8m/EwcADJRBqodjGOUAhUeFxiOgBQB5VYKABUdjh+YACUZjjcYAfVWCgEIFEwewDUGFY5XnAolHnMH1t5MGwA1jFtUAmMETBQAMgoCTBtANfVVjldcAmMETBbANgoC3dck/QRGThQW6BsZhP2NFBQa3d8k/k4eHsQOnBwgD1kcIE3X1D5MGFgDCBsGCI5LXCDKXIwCnAAPXRwiRZ5OHBwRjHvcCN/fIPxMHh7GhZ7qXA6YHCLc2yT+3d8k/k4eHsZOGhrVjH+YAI6bHCCOg1wgjkgcIIaD5V+MG9fyyQEEBgoAjptcII6DnCN23NycAYHxLnYv1/zc3AGB8S52L9f+CgEERBsbdN7cnAGAjpgcCNwcACJjDmEN9/8hXskATRfX/BYlBAYKAQREGxtk/fd03BwBAtycAYJjDNycAYBxD/f+yQEEBgoBBESLEN8TIP5MHxABKwAOpBwEGxibCYwoJBEU3OcW9RxMExACBRGPWJwEERL2Ik7QUAH03hT8cRDcGgAATl8cAmeA3BgABt/b/AHWPtyYAYNjCkMKYQn3/QUeR4AVHMwnpQLqXIygkARzEskAiRJJEAklBAYKAQREGxhMHAAxjEOUCEwWwDZcAyP/ngMDjEwXADbJAQQEXA8j/ZwDD4hMHsA3jGOX+lwDI/+eAwOETBdANxbdBESLEJsIGxiqEswS1AGMXlACyQCJEkkRBAYKAA0UEAAUERTfttxMFAAwXA8j/ZwBD3jVxJstOx/1yhWn9dCLNSslSxVbDBs+ThIT6FpGThwkHppcYCLOE5wAqiSaFLoSXAMj/54AgNpOHCQcYCAVqupezikdBMeQFZ311kwWF+pMHBwcTBYX5FAiqlzOF1wCTBwcHrpezhdcAKsaXAMj/54DgMjJFwUWhPwFFhWIWkfpAakTaREpJukkqSppKDWGCgKKJY3OKAIVpTobWhUqFlwDI/+eA4OATdfUPAe1OhtaFJoWXAMj/54AgLk6ZMwQ0QVG3EwUwBlW/MXH9ck7XUtVW017PBt8i3SbbStla0WLNZstqyW7HqokWkRMFAAIuirKKtosCypcAyP/ngOAohWdj4FcThWR9dBMEhPqThwQHopcYCDOE5wAihZcAyP/ngGAnfXsTDDv5kwyL+ROHBAeThwQHFAhil+aXAUkzDNcAs4zXAFJNY3xNCWNxqQNBqFU1poUIAaU9cT0mhgwBIoWXAMj/54BAI6aZJpljdUkDswepQWPxdwOzBCpBY/OaANaEJoYMAU6FlwDI/+eAQNITdfUPVd0CzIFEeV2NTaMJAQBihZcAyP/ngADEffkDRTEB5oUFMWNPBQDj4p3+hWeThwcHppcYCLqX2pcjiqf4hQTxt+MVpf2RR+OF9PYFZ311kwcHB5MFhfoTBYX5FAiqlzOF1wCTBwcHrpezhdcAKsaXAMj/54BgGe0zMkXBRX07zTMTBQAClwDI/+eAABeFYhaR+lBqVNpUSlm6WSpamloKW/pLakzaTEpNuk0pYYKAAREGziLMnTk3BM4/bAATBUT/lwDI/+eAQMiqhwVFleeyR5P3ByA+xkE5NycAYBxHtwZAABMFRP/VjxzHskWXAMj/54DAxTM1oADyQGJEBWGCgEERt8fIPwbGk4fHAAVHI4DnABPXxQCYxwVnfRfMw8jH+Y06laqVsYGMyyOqBwBBNxnBEwVQDLJAQQGCgAERIsw3xMg/kwfEACbKxEdOxgbOSsiqiRMExABj85UAroSpwAMpRAAmmRNZyQAcSGNV8AAcRGNe+QLpNn3dSEAmhs6FlwDI/+eAQLkTdfUPAcWTB0AMXMhcQKaXXMBcRIWPXMTyQGJE0kRCSbJJBWGCgOE+bb+3V0FJGXGTh/eEAUU+zobeotym2srYztbS1NbS2tDezuLM5srqyO7GlwDI/+eAoKy3R8g/N3fJP5OHBwATB4e6Y+XnFK0xkUVoCD05jTG398g/k4eHsSFnPpcjIPcItwU4QLcHOECThwcLAUaThQUAN0nIPxVFIyD5AJcAyP/ngAD8NwcAYFxHEwUAArd5yT+T5xcQXMeXAMj/54DA+pcAyP/ngEALt0cAYJxfk4mJsRMJCQAJ5fGL4RcTtRcAgUWXAMj/54CgrcFnt8TIP/0XEwcAEIVmQWa3BQABAUWThMQAt0rIPw1qlwDI/+eAIKgTi8oAJpqDp8kI9d+Dq8kIhUcjpgkIIwLxAoPHGwAJRyMT4QKjAvECAtRNR2OL5wZRR2OJ5wYpR2Of5wCDxzsAA8crAKIH2Y8RR2OW5wCDp4sAnEM+1KU2oUVIEDU+g8c7AAPHKwCiB9mPEWdBB2N+9wITBbANlwDI/+eAAJMTBcANlwDI/+eAQJITBeAOlwDI/+eAgJElNr23I6AHAJEHRb3JRyMT8QJ9twPHGwDRRmPn5gKFRmPm5gABTBME8A+dqHkXE3f3D8lG4+jm/rd2yT8KB5OGxro2lxhDAoeTBgcDk/b2DxFG42nW/BMH9wITd/cPjUZj7uYIt3bJPwoHk4aGvzaXGEMChxMHQAJjmucQAtQdRAFFlwDI/+eA4IgBRSU8aTxhPKFFSBB9FK00ffABTAFEE3X0DwU0E3X8Dyk8tTzjEQTsg8cbAElHY2D3LglH43n36vUXk/f3Dz1H42P36jd3yT+KBxMHh8C6l5xDgocFRJ3rcBCBRQFFl7DM/+eA4JMd4dFFaBAxNAFEMagFRIHvlwDI/+eAAI0zNKAAKaAhR2OF5wAFRAFMYbcDrIsAA6TLALNnjADSB/X3sTFl9cFsIpz9HH19MwWMQF3cs3eVAZXjwWwzBYxAY+aMAv18MwWMQF3QMYGXAMj/54AgiF35ZpT1tzGBlwDI/+eAIIdd8WqU0bdBgZcAyP/ngOCFWfkzBJRBwbchR+OK5/ABTBMEAAw5t0FHzb9BRwVE453n9oOlywADpYsAcTK5v0FHBUTjk+f2A6cLAZFnY+PnHIOlSwEDpYsACTGBt0FHBUTjlOf0g6cLARFnY2T3GgOnywCDpUsBA6WLADOE5wLxPiOsBAAjJIqwCb8DxwQAYw4HEAOniwDBFxMEAAxjE/cAwEgBR5MG8A5jRvcCg8dbAAPHSwABTKIH2Y8Dx2sAQgddj4PHewDiB9mP44T25hMEEAyFtTOG6wADRoYBBQexjuG3g8cEAPHD3ERjmAcSwEgjgAQAVb1hR2OW5wKDp8sBA6eLAYOmSwEDpgsBg6XLAAOliwCX8Mf/54AgdiqMMzSgAAG9AUwFRCm1EUcFROOd5+YDpYsAgUWX8Mf/54Dgdqm1E/f3AOMcB+yT3EcAE4SLAAFMfV3jfJzdSESX8Mf/54DgYhhEVEAQQPmOYwenARxCE0f3/32P2Y4UwgUMQQTZvxFHWb1BRwVE45/n4IOniwADp0sBIyT5ACMi6QD1s4MlSQDBF5Hlic8BTBMEYAxJswMniQBjZvcGE/c3AOMQB+YDKIkAAUYBRzMF6ECzhuUAY2n3AOMMBtQjJKkAIyLZALGzM4brABBOEQeQwgVG6b8hRwVE45nn2gMkiQAZwBMEgAwjJAkAIyIJADM0gABhuwFMEwQgDCm7AUwTBIAMCbsBTBMEkAwpsxMHIA1jg+cMEwdADeOW57wDxDsAg8crACIEXYyX8Mf/54BAYQOsxABBFGNzhAEijOMEDLrAQGKUMYCcSGNV8ACcRGNa9Arv8K/idd3IQGKGk4WLAZfwx//ngEBdAcWTB0AM3MjcQOKX3MDcRLOHh0HcxJfwx//ngCBcub4JZRMFBXEDrMsAA6SLAJfwx//ngGBNtwcAYNhLtwYAAcEWk1dHARIHdY+9i9mPs4eHAwFFs9WHApfwx//ngEBOEwWAPpfwx//ngABKAb6DpksBA6YLAYOlywADpYsA7/Cv+O28g8U7AIPHKwAThYsBogXdjcEVrTrVtO/wD9yBtwPEOwCDxysAE4yLASIEXYzcREEUxeORR4VLY/6HCJMHkAzcyGW8A6cNACLQBUizh+xAPtaDJ4qwY3P0AA1IQsY6xO/wj9ciRzJIN8XIP+KFfBCThsoAEBATBUUCl/DH/+eAQEs398g/kwjHAIJXA6eIsIOlDQAdjB2PPpyyVyOk6LCqi76VI6C9AJOHygCdjQHFoWdjlvUAWoXZOCOgbQEJxNxEmcPjQHD5Y98LAJMHcAyFv4VLt33JP7fMyD+TjY26k4zMAOm/45oLoNxE44cHoJMHgAyxt4OniwDjkAegAUWX8Mf/54AgOwllEwUFcZfwx//ngEA3l/DH/+eAwDrxugOkywDjCwScAUWX8Mf/54CAOBMFgD6X8Mf/54DANAKUbbr2UGZU1lRGWbZZJlqWWgZb9ktmTNZMRk22TQlhgoAAAA==",
+    "text_start": 1077411840,
+    "data": "DEDIP/gIOEBECThAnAk4QEAKOECsCjhAWgo4QMAHOED8CThAPAo4QLAJOEBwBzhA5Ak4QHAHOEDSCDhAFgk4QEQJOECcCThA5Ag4QCoIOEBaCDhA4Ag4QAQNOEBECThAxAs4QLgMOEC8BjhA4gw4QLwGOEC8BjhAvAY4QLwGOEC8BjhAvAY4QLwGOEC8BjhAYAs4QLwGOEDgCzhAuAw4QA==",
+    "data_start": 1070164904
+}

tools/python/esptool/targets/stub_flasher/stub_flasher_32h2.json

@@ -0,0 +1,7 @@
+{
+    "entry": 1082132112,
+    "text": "QREixCbCBsa39wBgEUc3BINA2Mu39ABgEwQEANxAkYuR57JAIkSSREEBgoCIQBxAE3X1D4KX3bcBEbcHAGBOxoOphwBKyDcJg0AmylLEBs4izLcEAGB9WhMJCQDATBN09A8N4PJAYkQjqDQBQknSRLJJIkoFYYKAiECDJwkAE3X1D4KXfRTjGUT/yb8TBwAMlEGqh2MY5QCFR4XGI6AFAHlVgoAFR2OH5gAJRmONxgB9VYKAQgUTB7ANQYVjlecCiUecwfW3kwbADWMW1QCYwRMFAAyCgJMG0A19VWOV1wCYwRMFsA2CgLc1hEBBEZOFRboGxmE/Y0UFBrc3hECTh8exA6cHCAPWRwgTdfUPkwYWAMIGwYIjktcIMpcjAKcAA9dHCJFnk4cHBGMe9wI3t4NAEwfHsaFnupcDpgcIt/aDQLc3hECTh8exk4bGtWMf5gAjpscII6DXCCOSBwghoPlX4wb1/LJAQQGCgCOm1wgjoOcI3bc3NwBgfEudi/X/NycAYHxLnYv1/4KAQREGxt03tzcAYCOmBwI3BwAImMOYQ33/yFeyQBNF9f8FiUEBgoBBEQbG2T993TcHAEC3NwBgmMM3NwBgHEP9/7JAQQGCgEERIsQ3hINAkwcEAUrAA6kHAQbGJsJjCgkERTc5xb1HEwQEAYFEY9YnAQREvYiTtBQAfTeFPxxENwaAABOXxwCZ4DcGAAG39v8AdY+3NgBg2MKQwphCff9BR5HgBUczCelAupcjKCQBHMSyQCJEkkQCSUEBgoABEQbOIswlNzcEhUBsABMFBP+XAID/54Ag8qqHBUWV57JHk/cHID7GiTc3NwBgHEe3BkAAEwUE/9WPHMeyRZcAgP/ngKDvMzWgAPJAYkQFYYKAQRG3h4NABsaThwcBBUcjgOcAE9fFAJjHBWd9F8zDyMf5jTqVqpWxgYzLI6oHAEE3GcETBVAMskBBAYKAAREizDeEg0CTBwQBJsrER07GBs5KyKqJEwQEAWPzlQCuhKnAAylEACaZE1nJABxIY1XwABxEY175ArU9fd1IQCaGzoWXAID/54Cg4hN19Q8BxZMHQAxcyFxAppdcwFxEhY9cxPJAYkTSREJJskkFYYKAaTVtv0ERBsaXAID/54BA1gNFhQGyQHUVEzUVAEEBgoBBEQbGxTcNxbcHg0CThwcA1EOZzjdnCWATB8cQHEM3Bv3/fRbxjzcGAwDxjtWPHMOyQEEBgoBBEQbGbTcRwQ1FskBBARcDgP9nAIPMQREGxpcAgP/ngEDKcTcBxbJAQQHZv7JAQQGCgEERBsYTBwAMYxrlABMFsA3RPxMFwA2yQEEB6bcTB7AN4xvl/sE3EwXQDfW3QREixCbCBsYqhLMEtQBjF5QAskAiRJJEQQGCgANFBAAFBE0/7bc1cSbLTsf9coVp/XQizUrJUsVWwwbPk4SE+haRk4cJB6aXGAizhOcAKokmhS6ElwCA/+eAgCyThwkHGAgFarqXs4pHQTHkBWd9dZMFhfqTBwcHEwWF+RQIqpczhdcAkwcHB66Xs4XXACrGlwCA/+eAQCkyRcFFlTcBRYViFpH6QGpE2kRKSbpJKkqaSg1hgoCiiWNzigCFaU6G1oVKhZcAgP/ngIDIE3X1DwHtTobWhSaFlwCA/+eAgCROmTMENEFRtxMFMAZVvxMFAAzZtTFx/XIFZ07XUtVW017PBt8i3SbbStla0WLNZstqyW7H/XcWkRMHBwc+lxwIupc+xiOqB/iqiS6Ksoq2iwU1kwcAAhnBtwcCAD6FlwCA/+eAIB2FZ2PlVxMFZH15EwmJ+pMHBAfKlxgIM4nnAEqFlwCA/+eAoBt9exMMO/mTDIv5EwcEB5MHBAcUCGKX5peBRDMM1wCzjNcAUk1jfE0JY/GkA0GomT+ihQgBjTW5NyKGDAFKhZcAgP/ngIAXopmilGP1RAOzh6RBY/F3AzMEmkBj84oAVoQihgwBToWXAID/54DAtxN19Q9V3QLMAUR5XY1NowkBAGKFlwCA/+eAgKd9+QNFMQHmhVE8Y08FAOPijf6FZ5OHBweilxgIupfalyOKp/gFBPG34xWl/ZFH4wX09gVnfXWTBwcHkwWF+hMFhfkUCKqXM4XXAJMHBweul7OF1wAqxpcAgP/ngKANcT0yRcFFZTNRPdU5twcCABnhkwcAAj6FlwCA/+eAoAqFYhaR+lBqVNpUSlm6WSpamloKW/pLakzaTEpNuk0pYYKAt1dBSRlxk4f3hAFFht6i3KbaytjO1tLU1tLa0N7O4szmyurI7sY+zpcAgP/ngMCgcTENwTdnCWATB8cQHEO3BoNAI6L2ALcG/f/9FvWPwWbVjxzDpTEFzbcnC2A3R9hQk4bHwRMHF6qYwhOGB8AjIAYAI6AGAJOGR8KYwpOHB8KYQzcGBABRj5jDI6AGALcHg0A3N4RAk4cHABMHx7ohoCOgBwCRB+Pt5/5FO5FFaAh1OWUzt7eDQJOHx7EhZz6XIyD3CLcHgEA3CYNAk4eHDiMg+QC3OYRA1TYTCQkAk4nJsWMHBRC3BwFgRUcjqucIhUVFRZcAgP/ngAD2twWAQAFGk4UFAEVFlwCA/+eAAPc39wBgHEs3BQIAk+dHABzLlwCA/+eAAPa3FwlgiF+BRbeEg0BxiWEVEzUVAJcAgP/ngICgwWf9FxMHABCFZkFmtwUAAQFFk4QEAbcKg0ANapcAgP/ngICWE4sKASaag6fJCPXfg6vJCIVHI6YJCCMC8QKDxxsACUcjE+ECowLxAgLUTUdjgecIUUdjj+cGKUdjn+cAg8c7AAPHKwCiB9mPEUdjlucAg6eLAJxDPtRxOaFFSBBlNoPHOwADxysAogfZjxFnQQdjdPcEEwWwDZk2EwXADYE2EwXgDi0+vTFBt7cFgEABRpOFhQMVRZcAgP/ngMDnNwcAYFxHEwUAApPnFxBcxzG3yUcjE/ECTbcDxxsA0UZj5+YChUZj5uYAAUwTBPAPhah5FxN39w/JRuPo5v63NoRACgeThga7NpcYQwKHkwYHA5P29g8RRuNp1vwTB/cCE3f3D41GY+vmCLc2hEAKB5OGxr82lxhDAocTB0ACY5jnEALUHUQBRWE8AUVFPOE22TahRUgQfRTBPHX0AUwBRBN19A9hPBN1/A9JPG024x4E6oPHGwBJR2Nj9y4JR+N29+r1F5P39w89R+Ng9+o3N4RAigcTB8fAupecQ4KHBUSd63AQgUUBRZfwf//ngAB0HeHRRWgQjTwBRDGoBUSB75fwf//ngIB4MzSgACmgIUdjhecABUQBTGG3A6yLAAOkywCzZ4wA0gf19+/wv4h98cFsIpz9HH19MwWMQFXcs3eVAZXjwWwzBYxAY+aMAv18MwWMQFXQMYGX8H//54AAdVX5ZpT1tzGBl/B//+eAAHRV8WqU0bdBgZfwf//ngEBzUfkzBJRBwbchR+OJ5/ABTBMEAAwxt0FHzb9BRwVE45zn9oOlywADpYsA1TKxv0FHBUTjkuf2A6cLAZFnY+XnHIOlSwEDpYsA7/D/gzW/QUcFROOS5/SDpwsBEWdjZfcaA6fLAIOlSwEDpYsAM4TnAu/wf4EjrAQAIySKsDG3A8cEAGMOBxADp4sAwRcTBAAMYxP3AMBIAUeTBvAOY0b3AoPHWwADx0sAAUyiB9mPA8drAEIHXY+Dx3sA4gfZj+OB9uYTBBAMqb0zhusAA0aGAQUHsY7ht4PHBADxw9xEY5gHEsBII4AEAH21YUdjlucCg6fLAQOniwGDpksBA6YLAYOlywADpYsAl/B//+eAwGMqjDM0oAAptQFMBUQRtRFHBUTjmufmA6WLAIFFl/B//+eAQGmRtRP39wDjGgfsk9xHABOEiwABTH1d43mc3UhEl/B//+eAwE0YRFRAEED5jmMHpwEcQhNH9/99j9mOFMIFDEEE2b8RR0m9QUcFROOc5+CDp4sAA6dLASMm+QAjJOkA3bODJYkAwReR5YnPAUwTBGAMtbsDJ8kAY2b3BhP3NwDjHgfkAyjJAAFGAUczBehAs4blAGNp9wDjCQbUIyapACMk2QCZszOG6wAQThEHkMIFRum/IUcFROOW59oDJMkAGcATBIAMIyYJACMkCQAzNIAASbsBTBMEIAwRuwFMEwSADDGzAUwTBJAMEbMTByANY4PnDBMHQA3jkOe8A8Q7AIPHKwAiBF2Ml/B//+eAYEwDrMQAQRRjc4QBIozjDgy4wEBilDGAnEhjVfAAnERjW/QK7/BP0XXdyEBihpOFiwGX8H//54BgSAHFkwdADNzI3EDil9zA3ESzh4dB3MSX8H//54BAR4m+CWUTBQVxA6zLAAOkiwCX8H//54BAOLcHAGDYS7cGAAHBFpNXRwESB3WPvYvZj7OHhwMBRbPVhwKX8H//54BgORMFgD6X8H//54DgNBG2g6ZLAQOmCwGDpcsAA6WLAO/wT/79tIPFOwCDxysAE4WLAaIF3Y3BFe/wL9vZvO/wj8o9vwPEOwCDxysAE4yLASIEXYzcREEUzeORR4VLY/+HCJMHkAzcyG20A6cNACLQBUizh+xAPtaDJ4qwY3P0AA1IQsY6xO/wD8YiRzJIN4WDQOKFfBCThgoBEBATBYUCl/B//+eAwDY3t4NAkwgHAYJXA6eIsIOlDQAdjB2PPpyyVyOk6LCqi76VI6C9AJOHCgGdjQHFoWdjl/UAWoXv8M/QI6BtAQnE3ESZw+NPcPdj3wsAkwdwDL23hUu3PYRAt4yDQJONzbqTjAwB6b/jkgug3ETjjweekweADKm3g6eLAOOYB57v8M/YCWUTBQVxl/B//+eAQCLv8E/Ul/B//+eAgCb5sgOkywDjBASc7/BP1hMFgD6X8H//54DgH+/w79EClH2y7/Bv0fZQZlTWVEZZtlkmWpZaBlv2S2ZM1kxGTbZNCWGCgA==",
+    "text_start": 1082130432,
+    "data": "EACDQEIKgECSCoBA6gqAQI4LgED6C4BAqAuAQA4JgEBKC4BAiguAQP4KgEC+CIBAMguAQL4IgEAcCoBAYgqAQJIKgEDqCoBALgqAQHIJgECiCYBAKgqAQFIOgECSCoBAEg2AQAoOgED+B4BAMg6AQP4HgED+B4BA/geAQP4HgED+B4BA/geAQP4HgED+B4BArgyAQP4HgEAwDYBACg6AQA==",
+    "data_start": 1082403756
+}

tools/python/esptool/targets/stub_flasher/stub_flasher_32h2beta1.json

@@ -0,0 +1,7 @@
+{
+    "entry": 1077413318,
+    "text": "ARG3BwBgTsaDqYcASsg3Scg/JspSxAbOIsy3BABgfVoTCQkAwEwTdPQ/DeDyQGJEI6g0AUJJ0kSySSJKBWGCgIhAgycJABN19Q+Cl30U4xlE/8m/EwcADJRBqodjGOUAhUeFxiOgBQB5VYKABUdjh+YACUZjjcYAfVWCgEIFEwewDUGFY5XnAolHnMH1t5MGwA1jFtUAmMETBQAMgoCTBtANfVVjldcAmMETBbANgoC3dck/QRGThQW6BsZhP2NFBQa3d8k/k4eHsQOnBwgD1kcIE3X1D5MGFgDCBsGCI5LXCDKXIwCnAAPXRwiRZ5OHBwRjHvcCN/fIPxMHh7GhZ7qXA6YHCLc2yT+3d8k/k4eHsZOGhrVjH+YAI6bHCCOg1wgjkgcIIaD5V+MG9fyyQEEBgoAjptcII6DnCN23NycAYHxLnYv1/zc3AGB8S52L9f+CgEERBsbdN7cnAGAjpgcCNwcACJjDmEN9/8hXskATRfX/BYlBAYKAQREGxtk/fd03BwBAtycAYJjDNycAYBxD/f+yQEEBgoBBESLEN8TIP5MHxABKwAOpBwEGxibCYwoJBEU3OcW9RxMExACBRGPWJwEERL2Ik7QUAH03hT8cRDcGgAATl8cAmeA3BgABt/b/AHWPtyYAYNjCkMKYQn3/QUeR4AVHMwnpQLqXIygkARzEskAiRJJEAklBAYKAQREGxhMHAAxjEOUCEwWwDZcAyP/ngMDjEwXADbJAQQEXA8j/ZwDD4hMHsA3jGOX+lwDI/+eAwOETBdANxbdBESLEJsIGxiqEswS1AGMXlACyQCJEkkRBAYKAA0UEAAUERTfttxMFAAwXA8j/ZwBD3jVxJstOx/1yhWn9dCLNSslSxVbDBs+ThIT6FpGThwkHppcYCLOE5wAqiSaFLoSXAMj/54DgNpOHCQcYCAVqupezikdBMeQFZ311kwWF+pMHBwcTBYX5FAiqlzOF1wCTBwcHrpezhdcAKsaXAMj/54CgMzJFwUWhPwFFhWIWkfpAakTaREpJukkqSppKDWGCgKKJY3OKAIVpTobWhUqFlwDI/+eA4OATdfUPAe1OhtaFJoWXAMj/54DgLk6ZMwQ0QVG3EwUwBlW/MXH9ck7XUtVW017PBt8i3SbbStla0WLNZstqyW7HqokWkRMFAAIuirKKtosCypcAyP/ngKAphWdj4FcThWR9dBMEhPqThwQHopcYCDOE5wAihZcAyP/ngCAofXsTDDv5kwyL+ROHBAeThwQHFAhil+aXAUkzDNcAs4zXAFJNY3xNCWNxqQNBqFU1poUIAaU9cT0mhgwBIoWXAMj/54AAJKaZJpljdUkDswepQWPxdwOzBCpBY/OaANaEJoYMAU6FlwDI/+eAQNITdfUPVd0CzIFEeV2NTaMJAQBihZcAyP/ngADEffkDRTEB5oUFMWNPBQDj4p3+hWeThwcHppcYCLqX2pcjiqf4hQTxt+MVpf2RR+OF9PYFZ311kwcHB5MFhfoTBYX5FAiqlzOF1wCTBwcHrpezhdcAKsaXAMj/54AgGu0zMkXBRX07zTMTBQAClwDI/+eAwBeFYhaR+lBqVNpUSlm6WSpamloKW/pLakzaTEpNuk0pYYKAAREGziLMnTk3BM4/bAATBQT/lwDI/+eAQMiqhwVFleeyR5P3ByA+xkE5NycAYBxHtwZAABMFBP/VjxzHskWXAMj/54DAxTM1oADyQGJEBWGCgEERt8fIPwbGk4fHAAVHI4DnABPXxQCYxwVnfRfMw8jH+Y06laqVsYGMyyOqBwBBNxnBEwVQDLJAQQGCgAERIsw3xMg/kwfEACbKxEdOxgbOSsiqiRMExABj85UAroSpwAMpRAAmmRNZyQAcSGNV8AAcRGNe+QLpNn3dSEAmhs6FlwDI/+eAQLkTdfUPAcWTB0AMXMhcQKaXXMBcRIWPXMTyQGJE0kRCSbJJBWGCgOE+bb+3V0FJGXGTh/eEAUU+zobeotym2srYztbS1NbS2tDezuLM5srqyO7GlwDI/+eAoKy3R8g/N3fJP5OHBwATB4e6Y+XnFK0xkUVoCD05jTG398g/k4eHsSFnPpcjIPcItwU4QLcHOECThwcLAUaThQUAN0nIPxVFIyD5AJcAyP/ngMD8NwcAYFxHEwUAArd5yT+T5xcQXMeXAMj/54CA+5cAyP/ngAAMt0cAYJxfk4mJsRMJCQAJ5fGL4RcTtRcAgUWXAMj/54CgrcFnt8TIP/0XEwcAEIVmQWa3BQABAUWThMQAt0rIPw1qlwDI/+eAIKgTi8oAJpqDp8kI9d+Dq8kIhUcjpgkIIwLxAoPHGwAJRyMT4QKjAvECAtRNR2OL5wZRR2OJ5wYpR2Of5wCDxzsAA8crAKIH2Y8RR2OW5wCDp4sAnEM+1KU2oUVIEDU+g8c7AAPHKwCiB9mPEWdBB2N+9wITBbANlwDI/+eAAJMTBcANlwDI/+eAQJITBeAOlwDI/+eAgJElNr23I6AHAJEHRb3JRyMT8QJ9twPHGwDRRmPn5gKFRmPm5gABTBME8A+dqHkXE3f3D8lG4+jm/rd2yT8KB5OGxro2lxhDAoeTBgcDk/b2DxFG42nW/BMH9wITd/cPjUZj7uYIt3bJPwoHk4aGvzaXGEMChxMHQAJjmucQAtQdRAFFlwDI/+eA4IgBRSU8aTxhPKFFSBB9FK00ffABTAFEE3X0DwU0E3X8Dyk8tTzjEQTsg8cbAElHY2D3LglH43n36vUXk/f3Dz1H42P36jd3yT+KBxMHh8C6l5xDgocFRJ3rcBCBRQFFlyDJ/+eA4Icd4dFFaBAxNAFEMagFRIHvlwDI/+eAAI0zNKAAKaAhR2OF5wAFRAFMYbcDrIsAA6TLALNnjADSB/X3sTFl9cFsIpz9HH19MwWMQF3cs3eVAZXjwWwzBYxAY+aMAv18MwWMQF3QMYGXAMj/54AgiF35ZpT1tzGBlwDI/+eAIIdd8WqU0bdBgZcAyP/ngOCFWfkzBJRBwbchR+OK5/ABTBMEAAw5t0FHzb9BRwVE453n9oOlywADpYsAcTK5v0FHBUTjk+f2A6cLAZFnY+PnHIOlSwEDpYsACTGBt0FHBUTjlOf0g6cLARFnY2T3GgOnywCDpUsBA6WLADOE5wLxPiOsBAAjJIqwCb8DxwQAYw4HEAOniwDBFxMEAAxjE/cAwEgBR5MG8A5jRvcCg8dbAAPHSwABTKIH2Y8Dx2sAQgddj4PHewDiB9mP44T25hMEEAyFtTOG6wADRoYBBQexjuG3g8cEAPHD3ERjmAcSwEgjgAQAVb1hR2OW5wKDp8sBA6eLAYOmSwEDpgsBg6XLAAOliwCX8Mf/54AgdiqMMzSgAAG9AUwFRCm1EUcFROOd5+YDpYsAgUWX8Mf/54Dgdqm1E/f3AOMcB+yT3EcAE4SLAAFMfV3jfJzdSESX8Mf/54DgYhhEVEAQQPmOYwenARxCE0f3/32P2Y4UwgUMQQTZvxFHWb1BRwVE45/n4IOniwADp0sBIyT5ACMi6QD1s4MlSQDBF5Hlic8BTBMEYAxJswMniQBjZvcGE/c3AOMQB+YDKIkAAUYBRzMF6ECzhuUAY2n3AOMMBtQjJKkAIyLZALGzM4brABBOEQeQwgVG6b8hRwVE45nn2gMkiQAZwBMEgAwjJAkAIyIJADM0gABhuwFMEwQgDCm7AUwTBIAMCbsBTBMEkAwpsxMHIA1jg+cMEwdADeOW57wDxDsAg8crACIEXYyX8Mf/54BAYQOsxABBFGNzhAEijOMEDLrAQGKUMYCcSGNV8ACcRGNa9Arv8K/idd3IQGKGk4WLAZfwx//ngEBdAcWTB0AM3MjcQOKX3MDcRLOHh0HcxJfwx//ngCBcub4JZRMFBXEDrMsAA6SLAJfwx//ngGBNtwcAYNhLtwYAAcEWk1dHARIHdY+9i9mPs4eHAwFFs9WHApfwx//ngEBOEwWAPpfwx//ngABKAb6DpksBA6YLAYOlywADpYsA7/Cv+O28g8U7AIPHKwAThYsBogXdjcEVrTrVtO/wD9yBtwPEOwCDxysAE4yLASIEXYzcREEUxeORR4VLY/6HCJMHkAzcyGW8A6cNACLQBUizh+xAPtaDJ4qwY3P0AA1IQsY6xO/wj9ciRzJIN8XIP+KFfBCThsoAEBATBUUCl/DH/+eAQEs398g/kwjHAIJXA6eIsIOlDQAdjB2PPpyyVyOk6LCqi76VI6C9AJOHygCdjQHFoWdjlvUAWoXZOCOgbQEJxNxEmcPjQHD5Y98LAJMHcAyFv4VLt33JP7fMyD+TjY26k4zMAOm/45oLoNxE44cHoJMHgAyxt4OniwDjkAegAUWX8Mf/54AgOwllEwUFcZfwx//ngEA3l/DH/+eAwDrxugOkywDjCwScAUWX8Mf/54CAOBMFgD6X8Mf/54DANAKUbbr2UGZU1lRGWbZZJlqWWgZb9ktmTNZMRk22TQlhgoAAAA==",
+    "text_start": 1077411840,
+    "data": "DEDIP/gIOEBECThAnAk4QEAKOECsCjhAWgo4QMAHOED8CThAPAo4QLAJOEBwBzhA5Ak4QHAHOEDSCDhAFgk4QEQJOECcCThA5Ag4QCoIOEBaCDhA4Ag4QAQNOEBECThAxAs4QLgMOEC8BjhA4gw4QLwGOEC8BjhAvAY4QLwGOEC8BjhAvAY4QLwGOEC8BjhAYAs4QLwGOEDgCzhAuAw4QA==",
+    "data_start": 1070164904
+}

tools/python/esptool/targets/stub_flasher/stub_flasher_32h2beta2.json

@@ -0,0 +1,7 @@
+{
+    "entry": 1077413318,
+    "text": "ARG3BwBgTsaDqYcASsg3Scg/JspSxAbOIsy3BABgfVoTCQkAwEwTdPQ/DeDyQGJEI6g0AUJJ0kSySSJKBWGCgIhAgycJABN19Q+Cl30U4xlE/8m/EwcADJRBqodjGOUAhUeFxiOgBQB5VYKABUdjh+YACUZjjcYAfVWCgEIFEwewDUGFY5XnAolHnMH1t5MGwA1jFtUAmMETBQAMgoCTBtANfVVjldcAmMETBbANgoC3dck/QRGThQW6BsZhP2NFBQa3d8k/k4eHsQOnBwgD1kcIE3X1D5MGFgDCBsGCI5LXCDKXIwCnAAPXRwiRZ5OHBwRjHvcCN/fIPxMHh7GhZ7qXA6YHCLc2yT+3d8k/k4eHsZOGhrVjH+YAI6bHCCOg1wgjkgcIIaD5V+MG9fyyQEEBgoAjptcII6DnCN23NycAYHxLnYv1/zc3AGB8S52L9f+CgEERBsbdN7cnAGAjpgcCNwcACJjDmEN9/8hXskATRfX/BYlBAYKAQREGxtk/fd03BwBAtycAYJjDNycAYBxD/f+yQEEBgoBBESLEN8TIP5MHxABKwAOpBwEGxibCYwoJBEU3OcW9RxMExACBRGPWJwEERL2Ik7QUAH03hT8cRDcGgAATl8cAmeA3BgABt/b/AHWPtyYAYNjCkMKYQn3/QUeR4AVHMwnpQLqXIygkARzEskAiRJJEAklBAYKAQREGxhMHAAxjEOUCEwWwDZcAyP/ngIDjEwXADbJAQQEXA8j/ZwCD4hMHsA3jGOX+lwDI/+eAgOETBdANxbdBESLEJsIGxiqEswS1AGMXlACyQCJEkkRBAYKAA0UEAAUERTfttxMFAAwXA8j/ZwAD3jVxJstOx/1yhWn9dCLNSslSxVbDBs+ThIT6FpGThwkHppcYCLOE5wAqiSaFLoSXAMj/54BgWpOHCQcYCAVqupezikdBMeQFZ311kwWF+pMHBwcTBYX5FAiqlzOF1wCTBwcHrpezhdcAKsaXAMj/54AgVzJFwUWhPwFFhWIWkfpAakTaREpJukkqSppKDWGCgKKJY3OKAIVpTobWhUqFlwDI/+eA4OITdfUPAe1OhtaFJoWXAMj/54BgUk6ZMwQ0QVG3EwUwBlW/MXH9ck7XUtVW017PBt8i3SbbStla0WLNZstqyW7HqokWkRMFAAIuirKKtosCypcAyP/ngCBNhWdj4FcThWR9dBMEhPqThwQHopcYCDOE5wAihZcAyP/ngKBLfXsTDDv5kwyL+ROHBAeThwQHFAhil+aXAUkzDNcAs4zXAFJNY3xNCWNxqQNBqFU1poUIAaU9cT0mhgwBIoWXAMj/54CAR6aZJpljdUkDswepQWPxdwOzBCpBY/OaANaEJoYMAU6FlwDI/+eAQNQTdfUPVd0CzIFEeV2NTaMJAQBihZcAyP/ngMDDffkDRTEB5oUFMWNPBQDj4p3+hWeThwcHppcYCLqX2pcjiqf4hQTxt+MVpf2RR+OF9PYFZ311kwcHB5MFhfoTBYX5FAiqlzOF1wCTBwcHrpezhdcAKsaXAMj/54CgPe0zMkXBRX07zTMTBQAClwDI/+eAQDuFYhaR+lBqVNpUSlm6WSpamloKW/pLakzaTEpNuk0pYYKAAREGziLMnTk3BM4/bAATBQT/lwDI/+eAwMqqhwVFleeyR5P3ByA+xkE5NycAYBxHtwZAABMFBP/VjxzHskWXAMj/54BAyDM1oADyQGJEBWGCgEERt8fIPwbGk4fHAAVHI4DnABPXxQCYxwVnfRfMw8jH+Y06laqVsYGMyyOqBwBBNxnBEwVQDLJAQQGCgAERIsw3xMg/kwfEACbKxEdOxgbOSsiqiRMExABj85UAroSpwAMpRAAmmRNZyQAcSGNV8AAcRGNe+QLpNn3dSEAmhs6FlwDI/+eAQLsTdfUPAcWTB0AMXMhcQKaXXMBcRIWPXMTyQGJE0kRCSbJJBWGCgOE+bb+3V0FJGXGTh/eEAUU+zobeotym2srYztbS1NbS2tDezuLM5srqyO7GlwDI/+eAIK23R8g/N3fJP5OHBwATB4e6Y+XnFK0xkUVoCD05jTG398g/k4eHsSFnPpcjIPcItwU4QLcHOECThwcLAUaThQUAN0nIPxVFIyD5AJcAyP/ngEAgNwcAYFxHEwUAArd5yT+T5xcQXMeXAMj/54AAH5cAyP/ngAAwt0cAYJxfk4mJsRMJCQAJ5fGL4RcTtRcAgUWXAMj/54AgsMFnt8TIP/0XEwcAEIVmQWa3BQABAUWThMQAt0rIPw1qlwDI/+eA4KoTi8oAJpqDp8kI9d+Dq8kIhUcjpgkIIwLxAoPHGwAJRyMT4QKjAvECAtRNR2OL5wZRR2OJ5wYpR2Of5wCDxzsAA8crAKIH2Y8RR2OW5wCDp4sAnEM+1KU2oUVIEDU+g8c7AAPHKwCiB9mPEWdBB2N+9wITBbANlwDI/+eAwJITBcANlwDI/+eAAJITBeAOlwDI/+eAQJElNr23I6AHAJEHRb3JRyMT8QJ9twPHGwDRRmPn5gKFRmPm5gABTBME8A+dqHkXE3f3D8lG4+jm/rd2yT8KB5OGxro2lxhDAoeTBgcDk/b2DxFG42nW/BMH9wITd/cPjUZj7uYIt3bJPwoHk4aGvzaXGEMChxMHQAJjmucQAtQdRAFFlwDI/+eAoIgBRSU8aTxhPKFFSBB9FK00ffABTAFEE3X0DwU0E3X8Dyk8tTzjEQTsg8cbAElHY2D3LglH43n36vUXk/f3Dz1H42P36jd3yT+KBxMHh8C6l5xDgocFRJ3rcBCBRQFFlwDI/+eAQIgd4dFFaBAxNAFEMagFRIHvlwDI/+eAQI0zNKAAKaAhR2OF5wAFRAFMYbcDrIsAA6TLALNnjADSB/X3sTFl9cFsIpz9HH19MwWMQF3cs3eVAZXjwWwzBYxAY+aMAv18MwWMQF3QMYGXAMj/54DgiV35ZpT1tzGBlwDI/+eA4Ihd8WqU0bdBgZcAyP/ngCCIWfkzBJRBwbchR+OK5/ABTBMEAAw5t0FHzb9BRwVE453n9oOlywADpYsAcTK5v0FHBUTjk+f2A6cLAZFnY+PnHIOlSwEDpYsACTGBt0FHBUTjlOf0g6cLARFnY2T3GgOnywCDpUsBA6WLADOE5wLxPiOsBAAjJIqwCb8DxwQAYw4HEAOniwDBFxMEAAxjE/cAwEgBR5MG8A5jRvcCg8dbAAPHSwABTKIH2Y8Dx2sAQgddj4PHewDiB9mP44T25hMEEAyFtTOG6wADRoYBBQexjuG3g8cEAPHD3ERjmAcSwEgjgAQAVb1hR2OW5wKDp8sBA6eLAYOmSwEDpgsBg6XLAAOliwCX8Mf/54DgeCqMMzSgAAG9AUwFRCm1EUcFROOd5+YDpYsAgUWX8Mf/54Bgeam1E/f3AOMcB+yT3EcAE4SLAAFMfV3jfJzdSESX8Mf/54CgYhhEVEAQQPmOYwenARxCE0f3/32P2Y4UwgUMQQTZvxFHWb1BRwVE45/n4IOniwADp0sBIyT5ACMi6QD1s4MlSQDBF5Hlic8BTBMEYAxJswMniQBjZvcGE/c3AOMQB+YDKIkAAUYBRzMF6ECzhuUAY2n3AOMMBtQjJKkAIyLZALGzM4brABBOEQeQwgVG6b8hRwVE45nn2gMkiQAZwBMEgAwjJAkAIyIJADM0gABhuwFMEwQgDCm7AUwTBIAMCbsBTBMEkAwpsxMHIA1jg+cMEwdADeOW57wDxDsAg8crACIEXYyX8Mf/54CAYQOsxABBFGNzhAEijOMEDLrAQGKUMYCcSGNV8ACcRGNa9Arv8K/idd3IQGKGk4WLAZfwx//ngIBdAcWTB0AM3MjcQOKX3MDcRLOHh0HcxJfwx//ngGBcub4JZRMFBXEDrMsAA6SLAJfwx//ngCBNtwcAYNhLtwYAAcEWk1dHARIHdY+9i9mPs4eHAwFFs9WHApfwx//ngABOEwWAPpfwx//ngMBJAb6DpksBA6YLAYOlywADpYsA7/Cv+O28g8U7AIPHKwAThYsBogXdjcEVrTrVtO/wD9yBtwPEOwCDxysAE4yLASIEXYzcREEUxeORR4VLY/6HCJMHkAzcyGW8A6cNACLQBUizh+xAPtaDJ4qwY3P0AA1IQsY6xO/wj9ciRzJIN8XIP+KFfBCThsoAEBATBUUCl/DH/+eAgEs398g/kwjHAIJXA6eIsIOlDQAdjB2PPpyyVyOk6LCqi76VI6C9AJOHygCdjQHFoWdjlvUAWoXZOCOgbQEJxNxEmcPjQHD5Y98LAJMHcAyFv4VLt33JP7fMyD+TjY26k4zMAOm/45oLoNxE44cHoJMHgAyxt4OniwDjkAegAUWX8Mf/54DgOgllEwUFcZfwx//ngAA3l/DH/+eAADvxugOkywDjCwScAUWX8Mf/54BAOBMFgD6X8Mf/54CANAKUbbr2UGZU1lRGWbZZJlqWWgZb9ktmTNZMRk22TQlhgoAAAA==",
+    "text_start": 1077411840,
+    "data": "DEDIP/gIOEBECThAnAk4QEAKOECsCjhAWgo4QMAHOED8CThAPAo4QLAJOEBwBzhA5Ak4QHAHOEDSCDhAFgk4QEQJOECcCThA5Ag4QCoIOEBaCDhA4Ag4QAQNOEBECThAxAs4QLgMOEC8BjhA4gw4QLwGOEC8BjhAvAY4QLwGOEC8BjhAvAY4QLwGOEC8BjhAYAs4QLwGOEDgCzhAuAw4QA==",
+    "data_start": 1070164904
+}

tools/python/esptool/util.py

@@ -0,0 +1,184 @@
+# SPDX-FileCopyrightText: 2014-2022 Fredrik Ahlberg, Angus Gratton,
+# Espressif Systems (Shanghai) CO LTD, other contributors as noted.
+#
+# SPDX-License-Identifier: GPL-2.0-or-later
+
+import os
+import re
+import struct
+import sys
+
+
+def byte(bitstr, index):
+    return bitstr[index]
+
+
+def mask_to_shift(mask):
+    """Return the index of the least significant bit in the mask"""
+    shift = 0
+    while mask & 0x1 == 0:
+        shift += 1
+        mask >>= 1
+    return shift
+
+
+def div_roundup(a, b):
+    """Return a/b rounded up to nearest integer,
+    equivalent result to int(math.ceil(float(int(a)) / float(int(b))), only
+    without possible floating point accuracy errors.
+    """
+    return (int(a) + int(b) - 1) // int(b)
+
+
+def flash_size_bytes(size):
+    """Given a flash size of the type passed in args.flash_size
+    (ie 512KB or 1MB) then return the size in bytes.
+    """
+    if "MB" in size:
+        return int(size[: size.index("MB")]) * 1024 * 1024
+    elif "KB" in size:
+        return int(size[: size.index("KB")]) * 1024
+    else:
+        raise FatalError("Unknown size %s" % size)
+
+
+def hexify(s, uppercase=True):
+    format_str = "%02X" if uppercase else "%02x"
+    return "".join(format_str % c for c in s)
+
+
+def pad_to(data, alignment, pad_character=b"\xFF"):
+    """Pad to the next alignment boundary"""
+    pad_mod = len(data) % alignment
+    if pad_mod != 0:
+        data += pad_character * (alignment - pad_mod)
+    return data
+
+
+def print_overwrite(message, last_line=False):
+    """Print a message, overwriting the currently printed line.
+
+    If last_line is False, don't append a newline at the end
+    (expecting another subsequent call will overwrite this one.)
+
+    After a sequence of calls with last_line=False, call once with last_line=True.
+
+    If output is not a TTY (for example redirected a pipe),
+    no overwriting happens and this function is the same as print().
+    """
+    if sys.stdout.isatty():
+        print("\r%s" % message, end="\n" if last_line else "")
+    else:
+        print(message)
+
+
+def expand_chip_name(chip_name):
+    """Change chip name to official form, e.g. `esp32s3beta2` -> `ESP32-S3(beta2)`"""
+    # Put "-" after "esp32"
+    chip_name = re.sub(r"(esp32)(?!$)", r"\1-", chip_name)
+    # Put "()" around "betaN"
+    chip_name = re.sub(r"(beta\d*)", r"(\1)", chip_name)
+    # Uppercase everything before "(betaN)"
+    chip_name = re.sub(r"^[^\(]+", lambda x: x.group(0).upper(), chip_name)
+    return chip_name
+
+
+def strip_chip_name(chip_name):
+    """Strip chip name to normalized form, e.g. `ESP32-S3(beta2)` -> `esp32s3beta2`"""
+    return re.sub(r"[-()]", "", chip_name.lower())
+
+
+def get_file_size(path_to_file):
+    """Returns the file size in bytes"""
+    file_size = 0
+    with open(path_to_file, "rb") as f:
+        f.seek(0, os.SEEK_END)
+        file_size = f.tell()
+    return file_size
+
+
+class FatalError(RuntimeError):
+    """
+    Wrapper class for runtime errors that aren't caused by internal bugs, but by
+    ESP ROM responses or input content.
+    """
+
+    def __init__(self, message):
+        RuntimeError.__init__(self, message)
+
+    @staticmethod
+    def WithResult(message, result):
+        """
+        Return a fatal error object that appends the hex values of
+        'result' and its meaning as a string formatted argument.
+        """
+
+        err_defs = {
+            # ROM error codes
+            0x101: "Out of memory",
+            0x102: "Invalid argument",
+            0x103: "Invalid state",
+            0x104: "Invalid size",
+            0x105: "Requested resource not found",
+            0x106: "Operation or feature not supported",
+            0x107: "Operation timed out",
+            0x108: "Received response was invalid",
+            0x109: "CRC or checksum was invalid",
+            0x10A: "Version was invalid",
+            0x10B: "MAC address was invalid",
+            # Flasher stub error codes
+            0xC000: "Bad data length",
+            0xC100: "Bad data checksum",
+            0xC200: "Bad blocksize",
+            0xC300: "Invalid command",
+            0xC400: "Failed SPI operation",
+            0xC500: "Failed SPI unlock",
+            0xC600: "Not in flash mode",
+            0xC700: "Inflate error",
+            0xC800: "Not enough data",
+            0xC900: "Too much data",
+            0xFF00: "Command not implemented",
+        }
+
+        err_code = struct.unpack(">H", result[:2])
+        message += " (result was {}: {})".format(
+            hexify(result), err_defs.get(err_code[0], "Unknown result")
+        )
+        return FatalError(message)
+
+
+class NotImplementedInROMError(FatalError):
+    """
+    Wrapper class for the error thrown when a particular ESP bootloader function
+    is not implemented in the ROM bootloader.
+    """
+
+    def __init__(self, bootloader, func):
+        FatalError.__init__(
+            self,
+            "%s ROM does not support function %s."
+            % (bootloader.CHIP_NAME, func.__name__),
+        )
+
+
+class NotSupportedError(FatalError):
+    def __init__(self, esp, function_name):
+        FatalError.__init__(
+            self,
+            "Function %s is not supported for %s." % (function_name, esp.CHIP_NAME),
+        )
+
+
+class UnsupportedCommandError(RuntimeError):
+    """
+    Wrapper class for when ROM loader returns an invalid command response.
+
+    Usually this indicates the loader is running in Secure Download Mode.
+    """
+
+    def __init__(self, esp, op):
+        if esp.secure_download_mode:
+            msg = "This command (0x%x) is not supported in Secure Download Mode" % op
+        else:
+            msg = "Invalid (unsupported) command 0x%x" % op
+        RuntimeError.__init__(self, msg)

tools/python/serial/__init__.py

@@ -0,0 +1,91 @@
+#!/usr/bin/env python
+#
+# This is a wrapper module for different platform implementations
+#
+# This file is part of pySerial. https://github.com/pyserial/pyserial
+# (C) 2001-2020 Chris Liechti <cliechti@gmx.net>
+#
+# SPDX-License-Identifier:    BSD-3-Clause
+
+from __future__ import absolute_import
+
+import sys
+import importlib
+
+from serial.serialutil import *
+#~ SerialBase, SerialException, to_bytes, iterbytes
+
+__version__ = '3.5'
+
+VERSION = __version__
+
+# pylint: disable=wrong-import-position
+if sys.platform == 'cli':
+    from serial.serialcli import Serial
+else:
+    import os
+    # chose an implementation, depending on os
+    if os.name == 'nt':  # sys.platform == 'win32':
+        from serial.serialwin32 import Serial
+    elif os.name == 'posix':
+        from serial.serialposix import Serial, PosixPollSerial, VTIMESerial  # noqa
+    elif os.name == 'java':
+        from serial.serialjava import Serial
+    else:
+        raise ImportError("Sorry: no implementation for your platform ('{}') available".format(os.name))
+
+
+protocol_handler_packages = [
+    'serial.urlhandler',
+]
+
+
+def serial_for_url(url, *args, **kwargs):
+    """\
+    Get an instance of the Serial class, depending on port/url. The port is not
+    opened when the keyword parameter 'do_not_open' is true, by default it
+    is. All other parameters are directly passed to the __init__ method when
+    the port is instantiated.
+
+    The list of package names that is searched for protocol handlers is kept in
+    ``protocol_handler_packages``.
+
+    e.g. we want to support a URL ``foobar://``. A module
+    ``my_handlers.protocol_foobar`` is provided by the user. Then
+    ``protocol_handler_packages.append("my_handlers")`` would extend the search
+    path so that ``serial_for_url("foobar://"))`` would work.
+    """
+    # check and remove extra parameter to not confuse the Serial class
+    do_open = not kwargs.pop('do_not_open', False)
+    # the default is to use the native implementation
+    klass = Serial
+    try:
+        url_lowercase = url.lower()
+    except AttributeError:
+        # it's not a string, use default
+        pass
+    else:
+        # if it is an URL, try to import the handler module from the list of possible packages
+        if '://' in url_lowercase:
+            protocol = url_lowercase.split('://', 1)[0]
+            module_name = '.protocol_{}'.format(protocol)
+            for package_name in protocol_handler_packages:
+                try:
+                    importlib.import_module(package_name)
+                    handler_module = importlib.import_module(module_name, package_name)
+                except ImportError:
+                    continue
+                else:
+                    if hasattr(handler_module, 'serial_class_for_url'):
+                        url, klass = handler_module.serial_class_for_url(url)
+                    else:
+                        klass = handler_module.Serial
+                    break
+            else:
+                raise ValueError('invalid URL, protocol {!r} not known'.format(protocol))
+    # instantiate and open when desired
+    instance = klass(None, *args, **kwargs)
+    instance.port = url
+    if do_open:
+        instance.open()
+    return instance

tools/python/serial/__main__.py

@@ -0,0 +1,3 @@
+from .tools import miniterm
+
+miniterm.main()

tools/python/serial/rs485.py

@@ -0,0 +1,94 @@
+#!/usr/bin/env python
+
+# RS485 support
+#
+# This file is part of pySerial. https://github.com/pyserial/pyserial
+# (C) 2015 Chris Liechti <cliechti@gmx.net>
+#
+# SPDX-License-Identifier:    BSD-3-Clause
+
+"""\
+The settings for RS485 are stored in a dedicated object that can be applied to
+serial ports (where supported).
+NOTE: Some implementations may only support a subset of the settings.
+"""
+
+from __future__ import absolute_import
+
+import time
+import serial
+
+
+class RS485Settings(object):
+    def __init__(
+            self,
+            rts_level_for_tx=True,
+            rts_level_for_rx=False,
+            loopback=False,
+            delay_before_tx=None,
+            delay_before_rx=None):
+        self.rts_level_for_tx = rts_level_for_tx
+        self.rts_level_for_rx = rts_level_for_rx
+        self.loopback = loopback
+        self.delay_before_tx = delay_before_tx
+        self.delay_before_rx = delay_before_rx
+
+
+class RS485(serial.Serial):
+    """\
+    A subclass that replaces the write method with one that toggles RTS
+    according to the RS485 settings.
+
+    NOTE: This may work unreliably on some serial ports (control signals not
+          synchronized or delayed compared to data). Using delays may be
+          unreliable (varying times, larger than expected) as the OS may not
+          support very fine grained delays (no smaller than in the order of
+          tens of milliseconds).
+
+    NOTE: Some implementations support this natively. Better performance
+          can be expected when the native version is used.
+
+    NOTE: The loopback property is ignored by this implementation. The actual
+          behavior depends on the used hardware.
+
+    Usage:
+
+        ser = RS485(...)
+        ser.rs485_mode = RS485Settings(...)
+        ser.write(b'hello')
+    """
+
+    def __init__(self, *args, **kwargs):
+        super(RS485, self).__init__(*args, **kwargs)
+        self._alternate_rs485_settings = None
+
+    def write(self, b):
+        """Write to port, controlling RTS before and after transmitting."""
+        if self._alternate_rs485_settings is not None:
+            # apply level for TX and optional delay
+            self.setRTS(self._alternate_rs485_settings.rts_level_for_tx)
+            if self._alternate_rs485_settings.delay_before_tx is not None:
+                time.sleep(self._alternate_rs485_settings.delay_before_tx)
+            # write and wait for data to be written
+            super(RS485, self).write(b)
+            super(RS485, self).flush()
+            # optional delay and apply level for RX
+            if self._alternate_rs485_settings.delay_before_rx is not None:
+                time.sleep(self._alternate_rs485_settings.delay_before_rx)
+            self.setRTS(self._alternate_rs485_settings.rts_level_for_rx)
+        else:
+            super(RS485, self).write(b)
+
+    # redirect where the property stores the settings so that underlying Serial
+    # instance does not see them
+    @property
+    def rs485_mode(self):
+        """\
+        Enable RS485 mode and apply new settings, set to None to disable.
+        See serial.rs485.RS485Settings for more info about the value.
+        """
+        return self._alternate_rs485_settings
+
+    @rs485_mode.setter
+    def rs485_mode(self, rs485_settings):
+        self._alternate_rs485_settings = rs485_settings

tools/python/serial/serialcli.py

@@ -0,0 +1,253 @@
+#! python
+#
+# Backend for .NET/Mono (IronPython), .NET >= 2
+#
+# This file is part of pySerial. https://github.com/pyserial/pyserial
+# (C) 2008-2015 Chris Liechti <cliechti@gmx.net>
+#
+# SPDX-License-Identifier:    BSD-3-Clause
+
+from __future__ import absolute_import
+
+import System
+import System.IO.Ports
+from serial.serialutil import *
+
+# must invoke function with byte array, make a helper to convert strings
+# to byte arrays
+sab = System.Array[System.Byte]
+
+
+def as_byte_array(string):
+    return sab([ord(x) for x in string])  # XXX will require adaption when run with a 3.x compatible IronPython
+
+
+class Serial(SerialBase):
+    """Serial port implementation for .NET/Mono."""
+
+    BAUDRATES = (50, 75, 110, 134, 150, 200, 300, 600, 1200, 1800, 2400, 4800,
+                 9600, 19200, 38400, 57600, 115200)
+
+    def open(self):
+        """\
+        Open port with current settings. This may throw a SerialException
+        if the port cannot be opened.
+        """
+        if self._port is None:
+            raise SerialException("Port must be configured before it can be used.")
+        if self.is_open:
+            raise SerialException("Port is already open.")
+        try:
+            self._port_handle = System.IO.Ports.SerialPort(self.portstr)
+        except Exception as msg:
+            self._port_handle = None
+            raise SerialException("could not open port %s: %s" % (self.portstr, msg))
+
+        # if RTS and/or DTR are not set before open, they default to True
+        if self._rts_state is None:
+            self._rts_state = True
+        if self._dtr_state is None:
+            self._dtr_state = True
+
+        self._reconfigure_port()
+        self._port_handle.Open()
+        self.is_open = True
+        if not self._dsrdtr:
+            self._update_dtr_state()
+        if not self._rtscts:
+            self._update_rts_state()
+        self.reset_input_buffer()
+
+    def _reconfigure_port(self):
+        """Set communication parameters on opened port."""
+        if not self._port_handle:
+            raise SerialException("Can only operate on a valid port handle")
+
+        #~ self._port_handle.ReceivedBytesThreshold = 1
+
+        if self._timeout is None:
+            self._port_handle.ReadTimeout = System.IO.Ports.SerialPort.InfiniteTimeout
+        else:
+            self._port_handle.ReadTimeout = int(self._timeout * 1000)
+
+        # if self._timeout != 0 and self._interCharTimeout is not None:
+            # timeouts = (int(self._interCharTimeout * 1000),) + timeouts[1:]
+
+        if self._write_timeout is None:
+            self._port_handle.WriteTimeout = System.IO.Ports.SerialPort.InfiniteTimeout
+        else:
+            self._port_handle.WriteTimeout = int(self._write_timeout * 1000)
+
+        # Setup the connection info.
+        try:
+            self._port_handle.BaudRate = self._baudrate
+        except IOError as e:
+            # catch errors from illegal baudrate settings
+            raise ValueError(str(e))
+
+        if self._bytesize == FIVEBITS:
+            self._port_handle.DataBits = 5
+        elif self._bytesize == SIXBITS:
+            self._port_handle.DataBits = 6
+        elif self._bytesize == SEVENBITS:
+            self._port_handle.DataBits = 7
+        elif self._bytesize == EIGHTBITS:
+            self._port_handle.DataBits = 8
+        else:
+            raise ValueError("Unsupported number of data bits: %r" % self._bytesize)
+
+        if self._parity == PARITY_NONE:
+            self._port_handle.Parity = getattr(System.IO.Ports.Parity, 'None')  # reserved keyword in Py3k
+        elif self._parity == PARITY_EVEN:
+            self._port_handle.Parity = System.IO.Ports.Parity.Even
+        elif self._parity == PARITY_ODD:
+            self._port_handle.Parity = System.IO.Ports.Parity.Odd
+        elif self._parity == PARITY_MARK:
+            self._port_handle.Parity = System.IO.Ports.Parity.Mark
+        elif self._parity == PARITY_SPACE:
+            self._port_handle.Parity = System.IO.Ports.Parity.Space
+        else:
+            raise ValueError("Unsupported parity mode: %r" % self._parity)
+
+        if self._stopbits == STOPBITS_ONE:
+            self._port_handle.StopBits = System.IO.Ports.StopBits.One
+        elif self._stopbits == STOPBITS_ONE_POINT_FIVE:
+            self._port_handle.StopBits = System.IO.Ports.StopBits.OnePointFive
+        elif self._stopbits == STOPBITS_TWO:
+            self._port_handle.StopBits = System.IO.Ports.StopBits.Two
+        else:
+            raise ValueError("Unsupported number of stop bits: %r" % self._stopbits)
+
+        if self._rtscts and self._xonxoff:
+            self._port_handle.Handshake = System.IO.Ports.Handshake.RequestToSendXOnXOff
+        elif self._rtscts:
+            self._port_handle.Handshake = System.IO.Ports.Handshake.RequestToSend
+        elif self._xonxoff:
+            self._port_handle.Handshake = System.IO.Ports.Handshake.XOnXOff
+        else:
+            self._port_handle.Handshake = getattr(System.IO.Ports.Handshake, 'None')   # reserved keyword in Py3k
+
+    #~ def __del__(self):
+        #~ self.close()
+
+    def close(self):
+        """Close port"""
+        if self.is_open:
+            if self._port_handle:
+                try:
+                    self._port_handle.Close()
+                except System.IO.Ports.InvalidOperationException:
+                    # ignore errors. can happen for unplugged USB serial devices
+                    pass
+                self._port_handle = None
+            self.is_open = False
+
+    #  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -
+
+    @property
+    def in_waiting(self):
+        """Return the number of characters currently in the input buffer."""
+        if not self.is_open:
+            raise PortNotOpenError()
+        return self._port_handle.BytesToRead
+
+    def read(self, size=1):
+        """\
+        Read size bytes from the serial port. If a timeout is set it may
+        return less characters as requested. With no timeout it will block
+        until the requested number of bytes is read.
+        """
+        if not self.is_open:
+            raise PortNotOpenError()
+        # must use single byte reads as this is the only way to read
+        # without applying encodings
+        data = bytearray()
+        while size:
+            try:
+                data.append(self._port_handle.ReadByte())
+            except System.TimeoutException:
+                break
+            else:
+                size -= 1
+        return bytes(data)
+
+    def write(self, data):
+        """Output the given string over the serial port."""
+        if not self.is_open:
+            raise PortNotOpenError()
+        #~ if not isinstance(data, (bytes, bytearray)):
+            #~ raise TypeError('expected %s or bytearray, got %s' % (bytes, type(data)))
+        try:
+            # must call overloaded method with byte array argument
+            # as this is the only one not applying encodings
+            self._port_handle.Write(as_byte_array(data), 0, len(data))
+        except System.TimeoutException:
+            raise SerialTimeoutException('Write timeout')
+        return len(data)
+
+    def reset_input_buffer(self):
+        """Clear input buffer, discarding all that is in the buffer."""
+        if not self.is_open:
+            raise PortNotOpenError()
+        self._port_handle.DiscardInBuffer()
+
+    def reset_output_buffer(self):
+        """\
+        Clear output buffer, aborting the current output and
+        discarding all that is in the buffer.
+        """
+        if not self.is_open:
+            raise PortNotOpenError()
+        self._port_handle.DiscardOutBuffer()
+
+    def _update_break_state(self):
+        """
+        Set break: Controls TXD. When active, to transmitting is possible.
+        """
+        if not self.is_open:
+            raise PortNotOpenError()
+        self._port_handle.BreakState = bool(self._break_state)
+
+    def _update_rts_state(self):
+        """Set terminal status line: Request To Send"""
+        if not self.is_open:
+            raise PortNotOpenError()
+        self._port_handle.RtsEnable = bool(self._rts_state)
+
+    def _update_dtr_state(self):
+        """Set terminal status line: Data Terminal Ready"""
+        if not self.is_open:
+            raise PortNotOpenError()
+        self._port_handle.DtrEnable = bool(self._dtr_state)
+
+    @property
+    def cts(self):
+        """Read terminal status line: Clear To Send"""
+        if not self.is_open:
+            raise PortNotOpenError()
+        return self._port_handle.CtsHolding
+
+    @property
+    def dsr(self):
+        """Read terminal status line: Data Set Ready"""
+        if not self.is_open:
+            raise PortNotOpenError()
+        return self._port_handle.DsrHolding
+
+    @property
+    def ri(self):
+        """Read terminal status line: Ring Indicator"""
+        if not self.is_open:
+            raise PortNotOpenError()
+        #~ return self._port_handle.XXX
+        return False  # XXX an error would be better
+
+    @property
+    def cd(self):
+        """Read terminal status line: Carrier Detect"""
+        if not self.is_open:
+            raise PortNotOpenError()
+        return self._port_handle.CDHolding
+
+    # - - platform specific - - - -
+    # none