Skip to content

Commit

Permalink
Better error messages (#63)
Browse files Browse the repository at this point in the history
* Improve error messages

* Tweak error messages, fix types

* Use difflib, add help messages

* ruff + black --preview

* allow_abbrev=False in subparsers

* Fix exit codes

* Fix type errors

* Tune similarity thresholds

* Fix metavar ordering regression

* Tweak unrecognized argument filter

* Improve error messages for unrecognized arguments with subparsers (by a
lot!)

* Fix unrecognized argument false positives

* Python 3.7-compatible mypy

* Polish unrecognized argument error message when subcommands are present

* Fix for duplicate arguments

* Fix `same_exists` default

* Print helptext of similar arguments

* Refine helptext in error messages

* Remove delete

* Add tests + heuristics

* Tweaks for coverage

* Fix or suppress mypy error

* Fix test

* Bump version

* Error message cleanup, test fixes

* Handle flags / custom actions in unrecognized argument errors

* Fix Python 3.7

* Hack for jax error

* Revert jax hack

* Suppress long usage prints

* Error message consistency

* Test coverage

* Fix test typo
  • Loading branch information
brentyi authored Sep 1, 2023
1 parent 5340498 commit 83592d6
Show file tree
Hide file tree
Showing 13 changed files with 1,102 additions and 86 deletions.
Empty file modified examples/01_basics/01_functions.py
100755 → 100644
Empty file.
4 changes: 2 additions & 2 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ name = "tyro"
authors = [
{name = "brentyi", email = "[email protected]"},
]
version = "0.5.5"
version = "0.5.6"
description = "Strongly typed, zero-effort CLI interfaces"
readme = "README.md"
license = { text="MIT" }
Expand Down Expand Up @@ -41,7 +41,7 @@ dev = [
"attrs>=21.4.0",
"torch>=1.10.0",
"pyright>=1.1.264",
"mypy>=0.991",
"mypy>=1.4.1",
"numpy>=1.20.0",
# As of 7/27/2023, flax install fails for Python 3.7 without pinning to an
# old version. But doing so breaks other Python versions.
Expand Down
2 changes: 1 addition & 1 deletion tests/test_collections.py
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,7 @@ class A:
assert tyro.cli(A, args=[]) == A(x=(1, 2))

target = io.StringIO()
with pytest.raises(SystemExit), contextlib.redirect_stderr(target):
with pytest.raises(SystemExit), contextlib.redirect_stdout(target):
tyro.cli(A, args=["1", "2", "3", "4"])
assert "invalid choice" in target.getvalue()

Expand Down
315 changes: 315 additions & 0 deletions tests/test_errors.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
import contextlib
import dataclasses
import io
from typing import List, Tuple, TypeVar, Union

import pytest
Expand Down Expand Up @@ -148,3 +150,316 @@ def main(value: list) -> None:

with pytest.raises(tyro.UnsupportedTypeAnnotationError):
tyro.cli(main, args=["--help"])


def test_similar_arguments_basic() -> None:
@dataclasses.dataclass
class RewardConfig:
track: bool

@dataclasses.dataclass
class Class:
reward: RewardConfig

target = io.StringIO()
with pytest.raises(SystemExit), contextlib.redirect_stdout(target):
tyro.cli(Class, args="--reward.trac".split(" "))

error = target.getvalue()
assert "Unrecognized argument" in error
assert "Similar arguments" in error

# --reward.track should appear in both the usage string and as a similar argument.
assert error.count("--reward.track") == 2
assert error.count("--help") == 0


def test_similar_arguments_subcommands() -> None:
@dataclasses.dataclass
class RewardConfig:
track: bool

@dataclasses.dataclass
class ClassA:
reward: RewardConfig

@dataclasses.dataclass
class ClassB:
reward: RewardConfig

target = io.StringIO()
with pytest.raises(SystemExit), contextlib.redirect_stdout(target):
tyro.cli(Union[ClassA, ClassB], args="--reward.trac".split(" ")) # type: ignore

error = target.getvalue()
assert "Unrecognized argument" in error
assert "Similar arguments:" in error
assert error.count("--reward.track") == 1
assert error.count("--help") == 2


def test_similar_arguments_subcommands_multiple() -> None:
@dataclasses.dataclass
class RewardConfig:
track: bool
trace: int

@dataclasses.dataclass
class ClassA:
reward: RewardConfig

@dataclasses.dataclass
class ClassB:
reward: RewardConfig

target = io.StringIO()
with pytest.raises(SystemExit), contextlib.redirect_stdout(target):
tyro.cli(Union[ClassA, ClassB], args="--fjdkslaj --reward.trac".split(" ")) # type: ignore

error = target.getvalue()
assert "Unrecognized argument" in error
assert "Arguments similar to --reward.trac" in error
assert error.count("--reward.track {True,False}") == 1
assert error.count("--reward.trace INT") == 1
assert error.count("--help") == 4


def test_similar_arguments_subcommands_multiple_contains_match() -> None:
@dataclasses.dataclass
class RewardConfig:
track: bool
trace: int

@dataclasses.dataclass
class ClassA:
reward: RewardConfig

@dataclasses.dataclass
class ClassB:
reward: RewardConfig

target = io.StringIO()
with pytest.raises(SystemExit), contextlib.redirect_stdout(target):
tyro.cli(Union[ClassA, ClassB], args="--rd.trac".split(" ")) # type: ignore

error = target.getvalue()
assert "Unrecognized argument" in error
assert "Similar arguments" in error
assert error.count("--reward.track {True,False}") == 1
assert error.count("--reward.trace INT") == 1
assert error.count("--help") == 4 # 2 subcommands * 2 arguments.


def test_similar_arguments_subcommands_multiple_contains_match_alt() -> None:
@dataclasses.dataclass
class RewardConfig:
track: bool
trace: int

@dataclasses.dataclass
class ClassA:
reward: RewardConfig

@dataclasses.dataclass
class ClassB:
reward: RewardConfig

target = io.StringIO()
with pytest.raises(SystemExit), contextlib.redirect_stdout(target):
tyro.cli(Union[ClassA, ClassB], args="--track".split(" ")) # type: ignore

error = target.getvalue()
assert "Unrecognized argument" in error
assert "Similar arguments" in error
assert error.count("--reward.track {True,False}") == 1
assert error.count("--help") == 2 # Should show two possible subcommands.


def test_similar_arguments_subcommands_overflow_different() -> None:
@dataclasses.dataclass
class RewardConfig:
track0: bool
track1: bool
track2: bool
track3: bool
track4: bool
track5: bool
track6: bool
track7: bool
track8: bool
track9: bool
track10: bool
track11: bool
track12: bool
track13: bool
track14: bool
track15: bool
track16: bool

@dataclasses.dataclass
class ClassA:
reward: RewardConfig

@dataclasses.dataclass
class ClassB:
reward: RewardConfig

target = io.StringIO()
with pytest.raises(SystemExit), contextlib.redirect_stdout(target):
tyro.cli(Union[ClassA, ClassB], args="--track".split(" ")) # type: ignore

error = target.getvalue()
assert "Unrecognized argument" in error
assert "Similar arguments" in error
assert error.count("--reward.track") == 10
assert "[...]" not in error
assert error.count("--help") == 20

target = io.StringIO()
with pytest.raises(SystemExit), contextlib.redirect_stdout(target):
# --tracked is intentionally between 0.8 ~ 0.9 similarity to track{i} for test
# coverage.
tyro.cli(RewardConfig, args="--tracked".split(" ")) # type: ignore

# Usage print should be clipped.
error = target.getvalue()
assert "help:" in error


def test_similar_arguments_subcommands_overflow_same() -> None:
@dataclasses.dataclass
class RewardConfig:
track: bool

@dataclasses.dataclass
class ClassA:
reward: RewardConfig

@dataclasses.dataclass
class ClassB:
reward: RewardConfig

@dataclasses.dataclass
class ClassC:
reward: RewardConfig

@dataclasses.dataclass
class ClassD:
reward: RewardConfig

@dataclasses.dataclass
class ClassE:
reward: RewardConfig

@dataclasses.dataclass
class ClassF:
reward: RewardConfig

@dataclasses.dataclass
class ClassG:
reward: RewardConfig

@dataclasses.dataclass
class ClassH:
reward: RewardConfig

@dataclasses.dataclass
class ClassI:
reward: RewardConfig

target = io.StringIO()
with pytest.raises(SystemExit), contextlib.redirect_stdout(target):
tyro.cli( # type: ignore
Union[
ClassA, ClassB, ClassC, ClassD, ClassE, ClassF, ClassG, ClassH, ClassI
],
args="--track".split(" "),
)

error = target.getvalue()
assert "Unrecognized argument" in error
assert "Similar arguments" in error
assert error.count("--reward.track") == 1
assert "[...]" in error
assert error.count("--help") == 4


def test_similar_arguments_subcommands_overflow_same_startswith_multiple() -> None:
@dataclasses.dataclass
class RewardConfig:
track: bool

@dataclasses.dataclass
class ClassA:
reward: RewardConfig

@dataclasses.dataclass
class ClassB:
reward: RewardConfig

@dataclasses.dataclass
class ClassC:
reward: RewardConfig

@dataclasses.dataclass
class ClassD:
reward: RewardConfig

@dataclasses.dataclass
class ClassE:
reward: RewardConfig
rewarde: tyro.conf.Suppress[int] = 10

@dataclasses.dataclass
class ClassF:
reward: RewardConfig

@dataclasses.dataclass
class ClassG:
reward: RewardConfig

@dataclasses.dataclass
class ClassH:
reward: RewardConfig

@dataclasses.dataclass
class ClassI:
reward: RewardConfig

target = io.StringIO()
with pytest.raises(SystemExit), contextlib.redirect_stdout(target):
tyro.cli( # type: ignore
Union[
ClassA, ClassB, ClassC, ClassD, ClassE, ClassF, ClassG, ClassH, ClassI
],
args="--track --ffff".split(" "),
)

error = target.getvalue()
assert "Unrecognized argument" in error
assert "Arguments similar to --track" in error
assert error.count("--rewar") == 1
assert "rewarde" not in error
assert "[...]" in error
assert error.count("--help") == 4


def test_similar_flag() -> None:
@dataclasses.dataclass
class Args:
flag: bool = False

target = io.StringIO()
with pytest.raises(SystemExit), contextlib.redirect_stdout(target):
tyro.cli(
Args,
args="--lag".split(" "),
)

error = target.getvalue()

# Printed in the usage message.
assert error.count("--flag | --no-flag") == 1

# Printed in the similar argument list.
assert error.count("--flag, --no-flag") == 1
19 changes: 13 additions & 6 deletions tests/test_nested.py
Original file line number Diff line number Diff line change
Expand Up @@ -206,26 +206,33 @@ class ModelSettings:

assert tyro.cli(
ModelSettings,
args="output-head-settings:output-head-settings --output-head-settings.number-of-outputs 5 optimizer-settings:None".split(
" "
args=(
"output-head-settings:output-head-settings"
" --output-head-settings.number-of-outputs 5 optimizer-settings:None".split(
" "
)
),
) == ModelSettings(OutputHeadSettings(5), None)

assert tyro.cli(
tyro.conf.OmitSubcommandPrefixes[
tyro.conf.ConsolidateSubcommandArgs[ModelSettings]
],
args="output-head-settings:output-head-settings optimizer-settings:None --number-of-outputs 5".split(
" "
args=(
"output-head-settings:output-head-settings optimizer-settings:None"
" --number-of-outputs 5".split(" ")
),
) == ModelSettings(OutputHeadSettings(5), None)

assert tyro.cli(
tyro.conf.OmitSubcommandPrefixes[
tyro.conf.ConsolidateSubcommandArgs[ModelSettings]
],
args="output-head-settings:output-head-settings optimizer-settings:optimizer-settings --name sgd --number-of-outputs 5".split(
" "
args=(
"output-head-settings:output-head-settings"
" optimizer-settings:optimizer-settings --name sgd --number-of-outputs 5".split(
" "
)
),
) == ModelSettings(OutputHeadSettings(5), OptimizerSettings("sgd"))

Expand Down
Loading

0 comments on commit 83592d6

Please sign in to comment.