Skip to content

Commit

Permalink
Merge pull request #71 from 15r10nk/3.12
Browse files Browse the repository at this point in the history 
feat: 3.12 support for the PositionNodeFinder
  • Loading branch information
@alexmojaki
alexmojaki authored Sep 24, 2023
2 parents 0e5096a + 81888a3 commit 02ebf5b
Show file tree
Hide file tree
Showing 60 changed files with 64,057 additions and 191 deletions.
28 changes: 4 additions & 24 deletions .github/workflows/test.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ jobs:
runs-on: ubuntu-20.04
strategy:
matrix:
python-version: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9, '3.10', 3.11-dev, pypy2, pypy-3.6]
python-version: [3.5, 3.6, 3.7, 3.8, 3.9, '3.10', 3.11, 3.12-dev, pypy-3.6]

steps:
- uses: actions/checkout@v2
Expand All @@ -26,15 +26,15 @@ jobs:
pip install mypy==0.910
python -m mypy executing --exclude=executing/_position_node_finder.py
# fromJson because https://github.community/t/passing-an-array-literal-to-contains-function-causes-syntax-error/17213/3
if: ${{ !contains(fromJson('["2.7", "pypy2", "pypy-3.6", "3.11-dev"]'), matrix.python-version) }}
if: ${{ !contains(fromJson('["pypy-3.6", "3.11","3.12-dev"]'), matrix.python-version) }}
# pypy < 3.8 very doesn't work
# 2.7 is tested separately in mypy-py2, as we need to run mypy under Python 3.x
- name: Mypy testing (3.11)
run: |
pip install mypy==0.971
python -m mypy executing
# fromJson because https://github.community/t/passing-an-array-literal-to-contains-function-causes-syntax-error/17213/3
if: ${{ contains(fromJson('["3.11-dev"]'), matrix.python-version) }}
# TODO: enable typechecking for 3.12
if: ${{ contains(fromJson('["3.11"]'), matrix.python-version) }}
# only >=3.11 use _position_node_finder.py
- name: Test
env:
Expand All @@ -56,23 +56,3 @@ jobs:
uses: AndreMiras/coveralls-python-action@v20201129
with:
parallel-finished: true

# Can't run mypy on Python 2.7, but can run it in Python 2 mode
mypy-py2:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2

- name: Set up Python 3.9
uses: actions/setup-python@v2
with:
python-version: 3.9

- name: Install dependencies
run: |
pip install --upgrade setuptools setuptools_scm pep517
pip install .[tests]
pip install mypy[python2]==0.910
- name: Mypy testing for Python 2
run: |
python -m mypy --py2 executing --exclude=executing/_position_node_finder.py
225 changes: 212 additions & 13 deletions executing/_position_node_finder.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,8 +46,10 @@ def mangled_name(node: EnhancedAST) -> str:
elif isinstance(node, ast.ExceptHandler):
assert node.name
name = node.name
elif sys.version_info >= (3,12) and isinstance(node,ast.TypeVar):
name=node.name
else:
raise TypeError("no node to mangle")
raise TypeError("no node to mangle for type "+repr(type(node)))

if name.startswith("__") and not name.endswith("__"):

Expand Down Expand Up @@ -176,21 +178,24 @@ def test_for_decorator(self, node: EnhancedAST, index: int) -> None:

# index opname
# ------------------
# index-4 PRECALL
# index-4 PRECALL (only in 3.11)
# index-2 CACHE
# index CALL <- the call instruction
# ... CACHE some CACHE instructions

# maybe multiple other bytecode blocks for other decorators
# index-4 PRECALL
# index-4 PRECALL (only in 3.11)
# index-2 CACHE
# index CALL <- index of the next loop
# ... CACHE some CACHE instructions

# index+x STORE_* the ast-node of this instruction points to the decorated thing

if self.opname(index - 4) != "PRECALL" or self.opname(index) != "CALL": # pragma: no mutate
break # pragma: no mutate
if not (
(self.opname(index - 4) == "PRECALL" or sys.version_info >= (3, 12))
and self.opname(index) == "CALL"
): # pragma: no mutate
break # pragma: no mutate

index += 2

Expand All @@ -205,7 +210,8 @@ def test_for_decorator(self, node: EnhancedAST, index: int) -> None:
self.decorator = node
return

index += 4
if sys.version_info < (3, 12):
index += 4

def known_issues(self, node: EnhancedAST, instruction: dis.Instruction) -> None:
if instruction.opname in ("COMPARE_OP", "IS_OP", "CONTAINS_OP") and isinstance(
Expand Down Expand Up @@ -259,6 +265,37 @@ def known_issues(self, node: EnhancedAST, instruction: dis.Instruction) -> None:
# TODO: investigate
raise KnownIssue("pattern matching ranges seems to be wrong")

if (
sys.version_info >= (3, 12)
and isinstance(node, ast.Call)
and isinstance(node.func, ast.Name)
and node.func.id == "super"
):
# super is optimized to some instructions which do not map nicely to a Call

# find the enclosing function
func = node.parent
while hasattr(func, "parent") and not isinstance(
func, (ast.AsyncFunctionDef, ast.FunctionDef)
):

func = func.parent

# get the first function argument (self/cls)
first_arg = None

if hasattr(func, "args"):
args = [*func.args.posonlyargs, *func.args.args]
if args:
first_arg = args[0].arg

if (instruction.opname, instruction.argval) in [
("LOAD_DEREF", "__class__"),
("LOAD_FAST", first_arg),
("LOAD_DEREF", first_arg),
]:
raise KnownIssue("super optimization")

if self.is_except_cleanup(instruction, node):
raise KnownIssue("exeption cleanup does not belong to the last node in a except block")

Expand All @@ -279,6 +316,14 @@ def known_issues(self, node: EnhancedAST, instruction: dis.Instruction) -> None:

raise KnownIssue("store __classcell__")

if (
instruction.opname == "CALL"
and not isinstance(node,ast.Call)
and any(isinstance(p, ast.Assert) for p in parents(node))
and sys.version_info >= (3, 11, 2)
):
raise KnownIssue("exception generation maps to condition")

@staticmethod
def is_except_cleanup(inst: dis.Instruction, node: EnhancedAST) -> bool:
if inst.opname not in (
Expand Down Expand Up @@ -392,6 +437,13 @@ def node_match(node_type: Union[Type, Tuple[Type, ...]], **kwargs: Any) -> bool:
# call to the generator function
return

if (
sys.version_info >= (3, 12)
and inst_match(("LOAD_FAST_AND_CLEAR", "STORE_FAST"))
and node_match((ast.ListComp, ast.SetComp, ast.DictComp))
):
return

if inst_match(("CALL", "CALL_FUNCTION_EX")) and node_match(
(ast.ClassDef, ast.Call)
):
Expand All @@ -410,6 +462,7 @@ def node_match(node_type: Union[Type, Tuple[Type, ...]], **kwargs: Any) -> bool:
if (
(
inst_match("LOAD_METHOD", argval="join")
or inst_match("LOAD_ATTR", argval="join") # 3.12
or inst_match(("CALL", "BUILD_STRING"))
)
and node_match(ast.BinOp, left=ast.Constant, op=ast.Mod)
Expand Down Expand Up @@ -495,9 +548,14 @@ def node_match(node_type: Union[Type, Tuple[Type, ...]], **kwargs: Any) -> bool:
):
return

if inst_match(("JUMP_IF_TRUE_OR_POP", "JUMP_IF_FALSE_OR_POP")) and node_match(
ast.BoolOp
):
if inst_match(
(
"JUMP_IF_TRUE_OR_POP",
"JUMP_IF_FALSE_OR_POP",
"POP_JUMP_IF_TRUE",
"POP_JUMP_IF_FALSE",
)
) and node_match(ast.BoolOp):
# and/or short circuit
return

Expand All @@ -511,7 +569,15 @@ def node_match(node_type: Union[Type, Tuple[Type, ...]], **kwargs: Any) -> bool:
and isinstance(node.parent, ast.AugAssign)
)
) and inst_match(
("LOAD_NAME", "LOAD_FAST", "LOAD_GLOBAL", "LOAD_DEREF"), argval=mangled_name(node)
(
"LOAD_NAME",
"LOAD_FAST",
"LOAD_FAST_CHECK",
"LOAD_GLOBAL",
"LOAD_DEREF",
"LOAD_FROM_DICT_OR_DEREF",
),
argval=mangled_name(node),
):
return

Expand All @@ -520,6 +586,124 @@ def node_match(node_type: Union[Type, Tuple[Type, ...]], **kwargs: Any) -> bool:
):
return

if node_match(ast.Constant) and inst_match(
"LOAD_CONST", argval=cast(ast.Constant, node).value
):
return

if node_match(
(ast.ListComp, ast.SetComp, ast.DictComp, ast.GeneratorExp, ast.For)
) and inst_match(("GET_ITER", "FOR_ITER")):
return

if sys.version_info >= (3, 12):
if node_match(ast.UnaryOp, op=ast.UAdd) and inst_match(
"CALL_INTRINSIC_1", argrepr="INTRINSIC_UNARY_POSITIVE"
):
return

if node_match(ast.Subscript) and inst_match("BINARY_SLICE"):
return

if node_match(ast.ImportFrom) and inst_match(
"CALL_INTRINSIC_1", argrepr="INTRINSIC_IMPORT_STAR"
):
return

if (
node_match(ast.Yield) or isinstance(node.parent, ast.GeneratorExp)
) and inst_match("CALL_INTRINSIC_1", argrepr="INTRINSIC_ASYNC_GEN_WRAP"):
return

if node_match(ast.Name) and inst_match("LOAD_DEREF",argval="__classdict__"):
return

if node_match(ast.TypeVar) and (
inst_match("CALL_INTRINSIC_1", argrepr="INTRINSIC_TYPEVAR")
or inst_match(
"CALL_INTRINSIC_2", argrepr="INTRINSIC_TYPEVAR_WITH_BOUND"
)
or inst_match(
"CALL_INTRINSIC_2", argrepr="INTRINSIC_TYPEVAR_WITH_CONSTRAINTS"
)
or inst_match(("STORE_FAST", "STORE_DEREF"), argrepr=mangled_name(node))
):
return

if node_match(ast.TypeVarTuple) and (
inst_match("CALL_INTRINSIC_1", argrepr="INTRINSIC_TYPEVARTUPLE")
or inst_match(("STORE_FAST", "STORE_DEREF"), argrepr=node.name)
):
return

if node_match(ast.ParamSpec) and (
inst_match("CALL_INTRINSIC_1", argrepr="INTRINSIC_PARAMSPEC")

or inst_match(("STORE_FAST", "STORE_DEREF"), argrepr=node.name)):
return


if node_match(ast.TypeAlias):
if(
inst_match("CALL_INTRINSIC_1", argrepr="INTRINSIC_TYPEALIAS")
or inst_match(
("STORE_NAME", "STORE_FAST", "STORE_DEREF"), argrepr=node.name.id
)
or inst_match("CALL")
):
return


if node_match(ast.ClassDef) and node.type_params:
if inst_match(
("STORE_DEREF", "LOAD_DEREF", "LOAD_FROM_DICT_OR_DEREF"),
argrepr=".type_params",
):
return

if inst_match(("STORE_FAST", "LOAD_FAST"), argrepr=".generic_base"):
return

if inst_match(
"CALL_INTRINSIC_1", argrepr="INTRINSIC_SUBSCRIPT_GENERIC"
):
return

if inst_match("LOAD_DEREF",argval="__classdict__"):
return

if node_match((ast.FunctionDef,ast.AsyncFunctionDef)) and node.type_params:
if inst_match("CALL"):
return

if inst_match(
"CALL_INTRINSIC_2", argrepr="INTRINSIC_SET_FUNCTION_TYPE_PARAMS"
):
return

if inst_match("LOAD_FAST",argval=".defaults"):
return

if inst_match("LOAD_FAST",argval=".kwdefaults"):
return

if inst_match("STORE_NAME", argval="__classdictcell__"):
# this is a general thing
return


# f-strings

if node_match(ast.JoinedStr) and (
inst_match("LOAD_ATTR", argval="join")
or inst_match(("LIST_APPEND", "CALL"))
):
return

if node_match(ast.FormattedValue) and inst_match("FORMAT_VALUE"):
return


# old verifier

typ: Type = type(None)
Expand All @@ -541,7 +725,7 @@ def node_match(node_type: Union[Type, Tuple[Type, ...]], **kwargs: Any) -> bool:
UNARY_INVERT=ast.Invert,
)[op_name]
extra_filter = lambda e: isinstance(cast(ast.UnaryOp, e).op, op_type)
elif op_name in ("LOAD_ATTR", "LOAD_METHOD", "LOOKUP_METHOD"):
elif op_name in ("LOAD_ATTR", "LOAD_METHOD", "LOOKUP_METHOD","LOAD_SUPER_ATTR"):
typ = ast.Attribute
ctx = ast.Load
extra_filter = lambda e: mangled_name(e) == instruction.argval
Expand Down Expand Up @@ -593,11 +777,26 @@ def instruction(self, index: int) -> dis.Instruction:
def opname(self, index: int) -> str:
return self.instruction(index).opname

extra_node_types=()
if sys.version_info >= (3,12):
extra_node_types = (ast.type_param,)

def find_node(
self,
index: int,
match_positions: Sequence[str]=("lineno", "end_lineno", "col_offset", "end_col_offset"),
typ: tuple[Type, ...]=(ast.expr, ast.stmt, ast.excepthandler, ast.pattern),
match_positions: Sequence[str] = (
"lineno",
"end_lineno",
"col_offset",
"end_col_offset",
),
typ: tuple[Type, ...] = (
ast.expr,
ast.stmt,
ast.excepthandler,
ast.pattern,
*extra_node_types,
),
) -> EnhancedAST:
position = self.instruction(index).positions
assert position is not None and position.lineno is not None
Expand Down
Loading

0 comments on commit 02ebf5b

Please sign in to comment.