From 56644b3a9f80f72ed5ae3b474cef594bbb105fa5 Mon Sep 17 00:00:00 2001 From: Matt Davis Date: Wed, 22 Sep 2021 09:19:52 -0700 Subject: [PATCH 1/8] XXX - temporarily disable most CI --- .appveyor.yml => .appveyor_TEMP_DISABLED.yml | 0 .github/workflows/ci.yaml | 286 +++++++++---------- 2 files changed, 143 insertions(+), 143 deletions(-) rename .appveyor.yml => .appveyor_TEMP_DISABLED.yml (100%) diff --git a/.appveyor.yml b/.appveyor_TEMP_DISABLED.yml similarity index 100% rename from .appveyor.yml rename to .appveyor_TEMP_DISABLED.yml diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 777839cd..bb1b550b 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -12,59 +12,59 @@ env: LIBYAML_REF: 0.2.5 jobs: - python_sdist: - name: PyYAML sdist - runs-on: ubuntu-latest - steps: - - name: Checkout pyyaml - uses: actions/checkout@v2 - - - name: Install a python - uses: actions/setup-python@v2 - with: - python-version: 3.x - - - name: Install build deps - run: | - python -V - python -m pip install build - - - name: Build sdist - run: | - # We DO want to force Cythoning, at least until 6.0. - export PYYAML_FORCE_CYTHON=1 - # We don't actually want to build the lib. - export PYYAML_FORCE_LIBYAML=0 - - python -m build . - - # Ensure exactly one artifact was produced. - [[ $(shopt -s nullglob; ls dist/*.tar.gz | wc -w) == 1 ]] || { - echo "Unexpected content in dist dir: '$(ls dist/*.tar.gz)'." - exit 1 - } - - - name: Test sdist - run: | - # Install some libyaml headers. - # TODO Should we smoke test the sdist against the libyaml we built? - sudo apt update - sudo apt install libyaml-dev -y - - # Ensure Cython is not present so we use only what's in the sdist. - python -m pip uninstall Cython -y || true - - # Pass no extra args. - # We should auto-install with libyaml since it's present. - python -m pip install dist/*.tar.gz -v - - python packaging/build/smoketest.py - - - name: Upload sdist artifact - uses: actions/upload-artifact@v2 - with: - name: dist - path: dist/*.tar.gz +# python_sdist: +# name: PyYAML sdist +# runs-on: ubuntu-latest +# steps: +# - name: Checkout pyyaml +# uses: actions/checkout@v2 +# +# - name: Install a python +# uses: actions/setup-python@v2 +# with: +# python-version: 3.x +# +# - name: Install build deps +# run: | +# python -V +# python -m pip install build +# +# - name: Build sdist +# run: | +# # We DO want to force Cythoning, at least until 6.0. +# export PYYAML_FORCE_CYTHON=1 +# # We don't actually want to build the lib. +# export PYYAML_FORCE_LIBYAML=0 +# +# python -m build . +# +# # Ensure exactly one artifact was produced. +# [[ $(shopt -s nullglob; ls dist/*.tar.gz | wc -w) == 1 ]] || { +# echo "Unexpected content in dist dir: '$(ls dist/*.tar.gz)'." +# exit 1 +# } +# +# - name: Test sdist +# run: | +# # Install some libyaml headers. +# # TODO Should we smoke test the sdist against the libyaml we built? +# sudo apt update +# sudo apt install libyaml-dev -y +# +# # Ensure Cython is not present so we use only what's in the sdist. +# python -m pip uninstall Cython -y || true +# +# # Pass no extra args. +# # We should auto-install with libyaml since it's present. +# python -m pip install dist/*.tar.gz -v +# +# python packaging/build/smoketest.py +# +# - name: Upload sdist artifact +# uses: actions/upload-artifact@v2 +# with: +# name: dist +# path: dist/*.tar.gz linux_libyaml: @@ -74,8 +74,8 @@ jobs: matrix: cfg: - { platform: manylinux1, arch: x86_64 } - - { platform: manylinux2014, arch: aarch64 } - - { platform: manylinux2014, arch: s390x } +# - { platform: manylinux2014, arch: aarch64 } +# - { platform: manylinux2014, arch: s390x } env: DOCKER_IMAGE: quay.io/pypa/${{matrix.cfg.platform}}_${{matrix.cfg.arch}} steps: @@ -86,9 +86,9 @@ jobs: path: libyaml key: libyamlX_${{matrix.cfg.platform}}_${{matrix.cfg.arch}}_${{env.LIBYAML_REF}} - - name: configure docker foreign arch support - uses: docker/setup-qemu-action@v1 - if: matrix.cfg.arch != 'x86_64' && steps.cached_libyaml.outputs.cache-hit != 'true' +# - name: configure docker foreign arch support +# uses: docker/setup-qemu-action@v1 +# if: matrix.cfg.arch != 'x86_64' && steps.cached_libyaml.outputs.cache-hit != 'true' - name: Checkout pyyaml uses: actions/checkout@v2 @@ -118,18 +118,18 @@ jobs: strategy: matrix: cfg: - - { platform: manylinux1, arch: x86_64, python_tag: cp36-cp36m } - - { platform: manylinux1, arch: x86_64, python_tag: cp37-cp37m } - - { platform: manylinux1, arch: x86_64, python_tag: cp38-cp38 } +# - { platform: manylinux1, arch: x86_64, python_tag: cp36-cp36m } +# - { platform: manylinux1, arch: x86_64, python_tag: cp37-cp37m } +# - { platform: manylinux1, arch: x86_64, python_tag: cp38-cp38 } - { platform: manylinux1, arch: x86_64, python_tag: cp39-cp39 } - - { platform: manylinux2014, arch: aarch64, python_tag: cp36-cp36m } - - { platform: manylinux2014, arch: aarch64, python_tag: cp37-cp37m } - - { platform: manylinux2014, arch: aarch64, python_tag: cp38-cp38 } - - { platform: manylinux2014, arch: aarch64, python_tag: cp39-cp39 } - - { platform: manylinux2014, arch: s390x, python_tag: cp36-cp36m } - - { platform: manylinux2014, arch: s390x, python_tag: cp37-cp37m } - - { platform: manylinux2014, arch: s390x, python_tag: cp38-cp38 } - - { platform: manylinux2014, arch: s390x, python_tag: cp39-cp39 } +# - { platform: manylinux2014, arch: aarch64, python_tag: cp36-cp36m } +# - { platform: manylinux2014, arch: aarch64, python_tag: cp37-cp37m } +# - { platform: manylinux2014, arch: aarch64, python_tag: cp38-cp38 } +# - { platform: manylinux2014, arch: aarch64, python_tag: cp39-cp39 } +# - { platform: manylinux2014, arch: s390x, python_tag: cp36-cp36m } +# - { platform: manylinux2014, arch: s390x, python_tag: cp37-cp37m } +# - { platform: manylinux2014, arch: s390x, python_tag: cp38-cp38 } +# - { platform: manylinux2014, arch: s390x, python_tag: cp39-cp39 } env: AW_PLAT: ${{matrix.cfg.platform}}_${{matrix.cfg.arch}} DOCKER_IMAGE: quay.io/pypa/${{matrix.cfg.platform}}_${{matrix.cfg.arch}} @@ -176,78 +176,78 @@ jobs: name: dist path: dist/*.whl - macos_libyaml: - name: libyaml ${{matrix.arch}} ${{matrix.platform}} - runs-on: ${{matrix.platform}} - strategy: - matrix: - platform: - - macos-10.15 - arch: - - x86_64 - steps: - - name: Check cached libyaml state - id: cached_libyaml - uses: actions/cache@v2 - with: - path: libyaml - key: libyaml_${{matrix.platform}}_${{matrix.arch}}_${{env.LIBYAML_REF}} - - - name: Checkout pyyaml - uses: actions/checkout@v2 - if: steps.cached_libyaml.outputs.cache-hit != 'true' - - - name: Build libyaml - env: - MACOSX_DEPLOYMENT_TARGET: '10.9' - run: | - brew install automake coreutils - bash ./packaging/build/libyaml.sh - if: steps.cached_libyaml.outputs.cache-hit != 'true' - - macos_pyyaml: - needs: macos_libyaml - name: pyyaml ${{matrix.arch}} ${{matrix.platform}} ${{matrix.python_tag}} - runs-on: ${{matrix.platform}} - strategy: - matrix: - platform: - - macos-10.15 - arch: - - x86_64 - python_tag: - - cp36* - - cp37* - - cp38* - - cp39* - steps: - - name: Checkout pyyaml - uses: actions/checkout@v2 - - - name: Get cached libyaml state - id: cached_libyaml - uses: actions/cache@v2 - with: - path: libyaml - key: libyaml_${{matrix.platform}}_${{matrix.arch}}_${{env.LIBYAML_REF}} - - - name: Ensure libyaml fetched - run: exit 1 - if: steps.cached_libyaml.outputs.cache-hit != 'true' - - - name: Install a python - uses: actions/setup-python@v2 - with: - python-version: 3.x - - - name: Build/Test/Package - env: - CIBW_BUILD: ${{matrix.python_tag}} - CIBW_BUILD_VERBOSITY: 1 - run: bash ./packaging/build/macos.sh - - - uses: actions/upload-artifact@v2 - with: - name: dist - path: dist/*.whl +# macos_libyaml: +# name: libyaml ${{matrix.arch}} ${{matrix.platform}} +# runs-on: ${{matrix.platform}} +# strategy: +# matrix: +# platform: +# - macos-10.15 +# arch: +# - x86_64 +# steps: +# - name: Check cached libyaml state +# id: cached_libyaml +# uses: actions/cache@v2 +# with: +# path: libyaml +# key: libyaml_${{matrix.platform}}_${{matrix.arch}}_${{env.LIBYAML_REF}} +# +# - name: Checkout pyyaml +# uses: actions/checkout@v2 +# if: steps.cached_libyaml.outputs.cache-hit != 'true' +# +# - name: Build libyaml +# env: +# MACOSX_DEPLOYMENT_TARGET: '10.9' +# run: | +# brew install automake coreutils +# bash ./packaging/build/libyaml.sh +# if: steps.cached_libyaml.outputs.cache-hit != 'true' +# +# macos_pyyaml: +# needs: macos_libyaml +# name: pyyaml ${{matrix.arch}} ${{matrix.platform}} ${{matrix.python_tag}} +# runs-on: ${{matrix.platform}} +# strategy: +# matrix: +# platform: +# - macos-10.15 +# arch: +# - x86_64 +# python_tag: +# - cp36* +# - cp37* +# - cp38* +# - cp39* +# steps: +# - name: Checkout pyyaml +# uses: actions/checkout@v2 +# +# - name: Get cached libyaml state +# id: cached_libyaml +# uses: actions/cache@v2 +# with: +# path: libyaml +# key: libyaml_${{matrix.platform}}_${{matrix.arch}}_${{env.LIBYAML_REF}} +# +# - name: Ensure libyaml fetched +# run: exit 1 +# if: steps.cached_libyaml.outputs.cache-hit != 'true' +# +# - name: Install a python +# uses: actions/setup-python@v2 +# with: +# python-version: 3.x +# +# - name: Build/Test/Package +# env: +# CIBW_BUILD: ${{matrix.python_tag}} +# CIBW_BUILD_VERBOSITY: 1 +# run: bash ./packaging/build/macos.sh +# +# - uses: actions/upload-artifact@v2 +# with: +# name: dist +# path: dist/*.whl ... From 34e764fd9959513a1ce8a8ef984f15eaa0ec535a Mon Sep 17 00:00:00 2001 From: Thom Smith Date: Wed, 22 Sep 2021 11:51:14 -0400 Subject: [PATCH 2/8] Fix issue with representing Enum types --- lib/yaml/representer.py | 2 +- tests/data/construct-python-name-module.code | 2 +- tests/data/construct-python-name-module.data | 1 + tests/lib/test_constructor.py | 3 +++ 4 files changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/yaml/representer.py b/lib/yaml/representer.py index 3b0b192e..808ca06d 100644 --- a/lib/yaml/representer.py +++ b/lib/yaml/representer.py @@ -369,7 +369,7 @@ def represent_ordered_dict(self, data): Representer.add_representer(tuple, Representer.represent_tuple) -Representer.add_representer(type, +Representer.add_multi_representer(type, Representer.represent_name) Representer.add_representer(collections.OrderedDict, diff --git a/tests/data/construct-python-name-module.code b/tests/data/construct-python-name-module.code index 6f391488..b8a4b6f8 100644 --- a/tests/data/construct-python-name-module.code +++ b/tests/data/construct-python-name-module.code @@ -1 +1 @@ -[str, yaml.Loader, yaml.dump, abs, yaml.tokens] +[str, yaml.Loader, yaml.dump, abs, yaml.tokens, signal.Handlers] diff --git a/tests/data/construct-python-name-module.data b/tests/data/construct-python-name-module.data index f0c9712b..f1a2c24a 100644 --- a/tests/data/construct-python-name-module.data +++ b/tests/data/construct-python-name-module.data @@ -3,3 +3,4 @@ - !!python/name:yaml.dump - !!python/name:abs - !!python/module:yaml.tokens +- !!python/name:signal.Handlers diff --git a/tests/lib/test_constructor.py b/tests/lib/test_constructor.py index f9a50770..28987d29 100644 --- a/tests/lib/test_constructor.py +++ b/tests/lib/test_constructor.py @@ -5,6 +5,9 @@ import datetime import yaml.tokens +# Import any packages here that need to be referenced in .code files. +import signal + def execute(code): global value exec(code) From f8e89b3fefbda11932a2344fa1069a40b8c3726d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tina=20M=C3=BCller?= Date: Wed, 22 Sep 2021 01:18:03 +0200 Subject: [PATCH 3/8] Add a test for the YAML 1.1 types --- tests/data/yaml11.schema | 264 ++++++++++++++++++++++++++++++++++ tests/data/yaml11.schema-skip | 9 ++ tests/lib/test_schema.py | 152 ++++++++++++++++++++ tests/lib/test_yaml.py | 2 + 4 files changed, 427 insertions(+) create mode 100644 tests/data/yaml11.schema create mode 100644 tests/data/yaml11.schema-skip create mode 100644 tests/lib/test_schema.py diff --git a/tests/data/yaml11.schema b/tests/data/yaml11.schema new file mode 100644 index 00000000..e2791aa7 --- /dev/null +++ b/tests/data/yaml11.schema @@ -0,0 +1,264 @@ +# https://github.com/perlpunk/yaml-test-schema/blob/master/data/schema-yaml11.yaml +--- +'!!bool FALSE': ['bool', 'false()', 'false'] +'!!bool False': ['bool', 'false()', 'false'] +'!!bool N': ['bool', 'false()', "false"] +'!!bool NO': ['bool', 'false()', "false"] +'!!bool No': ['bool', 'false()', "false"] +'!!bool OFF': ['bool', 'false()', "false"] +'!!bool ON': ['bool', 'true()', "true"] +'!!bool Off': ['bool', 'false()', "false"] +'!!bool On': ['bool', 'true()', "true"] +'!!bool TRUE': ['bool', 'true()', 'true'] +'!!bool True': ['bool', 'true()', 'true'] +'!!bool Y': ['bool', 'true()', "true"] +'!!bool YES': ['bool', 'true()', "true"] +'!!bool Yes': ['bool', 'true()', "true"] +'!!bool false': ['bool', 'false()', 'false'] +'!!bool n': ['bool', 'false()', "false"] +'!!bool no': ['bool', 'false()', "false"] +'!!bool off': ['bool', 'false()', "false"] +'!!bool on': ['bool', 'true()', "true"] +'!!bool true': ['bool', 'true()', 'true'] +'!!bool y': ['bool', 'true()', "true"] +'!!bool yes': ['bool', 'true()', "true"] +'!!float +.INF': ['inf', 'inf()', '.inf'] +'!!float +.Inf': ['inf', 'inf()', '.inf'] +'!!float +.inf': ['inf', 'inf()', '.inf'] +'!!float +0.3e+3': ['float', '300.0', '300.0'] +'!!float -.INF': ['inf', 'inf-neg()', '-.inf'] +'!!float -.Inf': ['inf', 'inf-neg()', '-.inf'] +'!!float -.inf': ['inf', 'inf-neg()', '-.inf'] +'!!float -3.14': ['float', '-3.14', '-3.14'] +'!!float .0': ['float', '0.0', '0.0'] +'!!float .14': ['float', '0.14', '0.14'] +'!!float .1_4': ['float', '0.14', '0.14'] +'!!float .3E-1': ['float', '0.03', '0.03'] +'!!float .3e+3': ['float', '300.0', '300.0'] +'!!float .INF': ['inf', 'inf()', '.inf'] +'!!float .Inf': ['inf', 'inf()', '.inf'] +'!!float .NAN': ['nan', 'nan()', '.nan'] +'!!float .NaN': ['nan', 'nan()', '.nan'] +'!!float .inf': ['inf', 'inf()', '.inf'] +'!!float .nan': ['nan', 'nan()', '.nan'] +'!!float 0.0': ['float', '0.0', '0.0'] +'!!float 001.23': ['float', '1.23', '1.23'] +'!!float 190:20:30.15': ['float', '685230.15', '685230.15'] +'!!float 3.': ['float', '3.0', '3.0'] +'!!float 3.14': ['float', '3.14', '3.14'] +'!!float 3.3e+3': ['float', '3300.0', '3300.0'] +'!!float 85.230_15e+03': ['float', '85230.15', '85230.15'] +'!!float 85_230.15': ['float', '85230.15', '85230.15'] +'!!int +0': ['int', '0', '0'] +'!!int +0100_200': ['int', '32896', '32896'] +'!!int +0b100': ['int', '4', '4'] +'!!int +190:20:30': ['int', '685230', '685230'] +'!!int +23': ['int', '23', '23'] +'!!int -0': ['int', '0', '0'] +'!!int -0100_200': ['int', '-32896', '-32896'] +'!!int -0b101': ['int', '-5', '-5'] +'!!int -0x30': ['int', '-48', '-48'] +'!!int -190:20:30': ['int', '-685230', '-685230'] +'!!int -23': ['int', '-23', '-23'] +'!!int 0': ['int', '0', '0'] +'!!int 00': ['int', '0', '0'] +'!!int 0011': ['int', '9', '9'] +'!!int 010': ['int', '8', '8'] +'!!int 02_0': ['int', '16', '16'] +'!!int 07': ['int', '7', '7'] +'!!int 0b0': ['int', '0', '0'] +'!!int 0b100_101': ['int', '37', '37'] +'!!int 0x0': ['int', '0', '0'] +'!!int 0x10': ['int', '16', '16'] +'!!int 0x2_0': ['int', '32', '32'] +'!!int 0x42': ['int', '66', '66'] +'!!int 0xa': ['int', '10', '10'] +'!!int 100_000': ['int', '100000', '100000'] +'!!int 190:20:30': ['int', '685230', '685230'] +'!!int 23': ['int', '23', '23'] +'!!null #empty': ['null', 'null()', "null"] +'!!null NULL': ['null', 'null()', "null"] +'!!null Null': ['null', 'null()', "null"] +'!!null null': ['null', 'null()', 'null'] +'!!null ~': ['null', 'null()', 'null'] +'!!str #empty': ['str', '', "''"] +'!!str +.INF': ['str', '+.INF', "'+.INF'"] +'!!str +.Inf': ['str', '+.Inf', "'+.Inf'"] +'!!str +.inf': ['str', '+.inf', "'+.inf'"] +'!!str +0': ['str', '+0', "'+0'"] +'!!str +0.3e+3': ['str', '+0.3e+3', "'+0.3e+3'"] +'!!str +0.3e3': ['str', '+0.3e3', "+0.3e3"] +'!!str +0100_200': ['str', '+0100_200', "'+0100_200'"] +'!!str +0b100': ['str', '+0b100', "'+0b100'"] +'!!str +190:20:30': ['str', '+190:20:30', "'+190:20:30'"] +'!!str +23': ['str', '+23', "'+23'"] +'!!str -.INF': ['str', '-.INF', "'-.INF'"] +'!!str -.Inf': ['str', '-.Inf', "'-.Inf'"] +'!!str -.inf': ['str', '-.inf', "'-.inf'"] +'!!str -0': ['str', '-0', "'-0'"] +'!!str -0100_200': ['str', '-0100_200', "'-0100_200'"] +'!!str -0b101': ['str', '-0b101', "'-0b101'"] +'!!str -0x30': ['str', '-0x30', "'-0x30'"] +'!!str -190:20:30': ['str', '-190:20:30', "'-190:20:30'"] +'!!str -23': ['str', '-23', "'-23'"] +'!!str -3.14': ['str', '-3.14', "'-3.14'"] +'!!str .': ['str', '.', '.'] +'!!str .0': ['str', '.0', "'.0'"] +'!!str .14': ['str', '.14', "'.14'"] +'!!str .1_4': ['str', '.1_4', "'.1_4'"] +'!!str .3E-1': ['str', '.3E-1', "'.3E-1'"] +'!!str .3e+3': ['str', '.3e+3', "'.3e+3'"] +'!!str .3e3': ['str', '.3e3', ".3e3"] +'!!str .INF': ['str', '.INF', "'.INF'"] +'!!str .Inf': ['str', '.Inf', "'.Inf'"] +'!!str .NAN': ['str', '.NAN', "'.NAN'"] +'!!str .NaN': ['str', '.NaN', "'.NaN'"] +'!!str ._': ['str', '._', '._'] +'!!str ._14': ['str', '._14', '._14'] +'!!str .inf': ['str', '.inf', "'.inf'"] +'!!str .nan': ['str', '.nan', "'.nan'"] +'!!str 0': ['str', '0', "'0'"] +'!!str 0.0': ['str', '0.0', "'0.0'"] +'!!str 0.3e3': ['str', '0.3e3', "0.3e3"] +'!!str 00': ['str', '00', "'00'"] +'!!str 001.23': ['str', '001.23', "'001.23'"] +'!!str 0011': ['str', '0011', "'0011'"] +'!!str 010': ['str', '010', "'010'"] +'!!str 02_0': ['str', '02_0', "'02_0'"] +'!!str 07': ['str', '07', "'07'"] +'!!str 0b0': ['str', '0b0', "'0b0'"] +'!!str 0b100_101': ['str', '0b100_101', "'0b100_101'"] +'!!str 0o0': ['str', '0o0', "0o0"] +'!!str 0o10': ['str', '0o10', "0o10"] +'!!str 0o7': ['str', '0o7', "0o7"] +'!!str 0x0': ['str', '0x0', "'0x0'"] +'!!str 0x2_0': ['str', '0x2_0', "'0x2_0'"] +'!!str 0xa': ['str', '0xa', "'0xa'"] +'!!str 100_000': ['str', '100_000', "'100_000'"] +'!!str 190:20:30': ['str', '190:20:30', "'190:20:30'"] +'!!str 190:20:30.15': ['str', '190:20:30.15', "'190:20:30.15'"] +'!!str 23': ['str', '23', "'23'"] +'!!str 3.': ['str', '3.', "'3.'"] +'!!str 3.14': ['str', '3.14', "'3.14'"] +'!!str 3.3e+3': ['str', '3.3e+3', "'3.3e+3'"] +'!!str 85.230_15e+03': ['str', '85.230_15e+03', "'85.230_15e+03'"] +'!!str 85_230.15': ['str', '85_230.15', "'85_230.15'"] +'!!str FALSE': ['str', 'FALSE', "'FALSE'"] +'!!str False': ['str', 'False', "'False'"] +'!!str N': ['str', 'N', "'N'"] +'!!str NO': ['str', 'NO', "'NO'"] +'!!str NULL': ['str', 'NULL', "'NULL'"] +'!!str Null': ['str', 'Null', "'Null'"] +'!!str OFF': ['str', 'OFF', "'OFF'"] +'!!str ON': ['str', 'ON', "'ON'"] +'!!str Off': ['str', 'Off', "'Off'"] +'!!str On': ['str', 'On', "'On'"] +'!!str TRUE': ['str', 'TRUE', "'TRUE'"] +'!!str True': ['str', 'True', "'True'"] +'!!str Y': ['str', 'Y', "'Y'"] +'!!str YES': ['str', 'YES', "'YES'"] +'!!str Yes': ['str', 'Yes', "'Yes'"] +'!!str _._': ['str', '_._', '_._'] +'!!str false': ['str', 'false', "'false'"] +'!!str n': ['str', 'n', "'n'"] +'!!str no': ['str', 'no', "'no'"] +'!!str null': ['str', 'null', "'null'"] +'!!str off': ['str', 'off', "'off'"] +'!!str on': ['str', 'on', "'on'"] +'!!str true': ['str', 'true', "'true'"] +'!!str y': ['str', 'y', "'y'"] +'!!str yes': ['str', 'yes', "'yes'"] +'!!str ~': ['str', '~', "'~'"] +'#empty': ['null', 'null()', "null"] +'+.INF': ['inf', 'inf()', '.inf'] +'+.Inf': ['inf', 'inf()', '.inf'] +'+.inf': ['inf', 'inf()', '.inf'] +'+0': ['int', '0', '0'] +'+0.3e+3': ['float', '300.0', '300.0'] +'+0.3e3': ['str', '+0.3e3', '+0.3e3'] +'+0100_200': ['int', '32896', '32896'] +'+0b100': ['int', '4', '4'] +'+190:20:30': ['int', '685230', '685230'] +'+23': ['int', '23', '23'] +'+3.14': ['float', '3.14', '3.14'] +'-.INF': ['inf', 'inf-neg()', '-.inf'] +'-.Inf': ['inf', 'inf-neg()', '-.inf'] +'-.inf': ['inf', 'inf-neg()', '-.inf'] +'-0': ['int', '0', '0'] +'-0100_200': ['int', '-32896', '-32896'] +'-0b101': ['int', '-5', '-5'] +'-0x30': ['int', '-48', '-48'] +'-190:20:30': ['int', '-685230', '-685230'] +'-23': ['int', '-23', '-23'] +'-3.14': ['float', '-3.14', '-3.14'] +'.': ['str', '.', '.'] +'.0': ['float', '0.0', '0.0'] +'.14': ['float', '0.14', '0.14'] +'.1_4': ['float', '0.14', '0.14'] +'.3E-1': ['float', '0.03', '0.03'] +'.3e+3': ['float', '300.0', '300.0'] +'.3e3': ['str', '.3e3', '.3e3'] +'.INF': ['inf', 'inf()', '.inf'] +'.Inf': ['inf', 'inf()', '.inf'] +'.NAN': ['nan', 'nan()', '.nan'] +'.NaN': ['nan', 'nan()', '.nan'] +'._': ['str', '._', '._'] +'._14': ['str', '._14', '._14'] +'.inf': ['inf', 'inf()', '.inf'] +'.nan': ['nan', 'nan()', '.nan'] +'0': ['int', '0', '0'] +'0.0': ['float', '0.0', '0.0'] +'0.3e3': ['str', '0.3e3', '0.3e3'] +'00': ['int', '0', '0'] +'001.23': ['float', '1.23', '1.23'] +'0011': ['int', '9', '9'] +'010': ['int', '8', '8'] +'02_0': ['int', '16', '16'] +'07': ['int', '7', '7'] +'08': ['str', '08', '08'] +'0b0': ['int', '0', '0'] +'0b100_101': ['int', '37', '37'] +'0o0': ['str', '0o0', '0o0'] +'0o10': ['str', '0o10', '0o10'] +'0o7': ['str', '0o7', '0o7'] +'0x0': ['int', '0', '0'] +'0x10': ['int', '16', '16'] +'0x2_0': ['int', '32', '32'] +'0x42': ['int', '66', '66'] +'0xa': ['int', '10', '10'] +'100_000': ['int', '100000', '100000'] +'190:20:30': ['int', '685230', '685230'] +'190:20:30.15': ['float', '685230.15', '685230.15'] +'23': ['int', '23', '23'] +'3.': ['float', '3.0', '3.0'] +'3.14': ['float', '3.14', '3.14'] +'3.3e+3': ['float', '3300', '3300.0'] +'3e3': ['str', '3e3', '3e3'] +'85.230_15e+03': ['float', '85230.15', '85230.15'] +'85_230.15': ['float', '85230.15', '85230.15'] +'FALSE': ['bool', 'false()', 'false'] +'False': ['bool', 'false()', 'false'] +'N': ['bool', 'false()', "false"] +'NO': ['bool', 'false()', "false"] +'NULL': ['null', 'null()', "null"] +'Null': ['null', 'null()', "null"] +'OFF': ['bool', 'false()', "false"] +'ON': ['bool', 'true()', "true"] +'Off': ['bool', 'false()', "false"] +'On': ['bool', 'true()', "true"] +'TRUE': ['bool', 'true()', 'true'] +'True': ['bool', 'true()', 'true'] +'Y': ['bool', 'true()', "true"] +'YES': ['bool', 'true()', "true"] +'Yes': ['bool', 'true()', "true"] +'_._': ['str', '_._', '_._'] +'false': ['bool', 'false()', 'false'] +'n': ['bool', 'false()', "false"] +'no': ['bool', 'false()', "false"] +'null': ['null', 'null()', "null"] +'off': ['bool', 'false()', "false"] +'on': ['bool', 'true()', "true"] +'true': ['bool', 'true()', 'true'] +'y': ['bool', 'true()', "true"] +'yes': ['bool', 'true()', "true"] +'~': ['null', 'null()', "null"] diff --git a/tests/data/yaml11.schema-skip b/tests/data/yaml11.schema-skip new file mode 100644 index 00000000..4fe0f0bf --- /dev/null +++ b/tests/data/yaml11.schema-skip @@ -0,0 +1,9 @@ +load: { + 'Y': 1, 'y': 1, 'N': 1, 'n': 1, + '!!bool Y': 1, '!!bool N': 1, '!!bool n': 1, '!!bool y': 1, + '._', '!!str ._', + '._14', '!!str ._14' + } +dump: { + '!!str N': 1, '!!str Y': 1, '!!str n': 1, '!!str y': 1, + } diff --git a/tests/lib/test_schema.py b/tests/lib/test_schema.py new file mode 100644 index 00000000..f3370ec1 --- /dev/null +++ b/tests/lib/test_schema.py @@ -0,0 +1,152 @@ +import yaml +import sys +import pprint +import math + +def check_bool(value, expected): + if expected == 'false()' and value is False: + return 1 + if expected == 'true()' and value is True: + return 1 + print(value) + print(expected) + return 0 + +def check_int(value, expected): + if (int(expected) == value): + return 1 + print(value) + print(expected) + return 0 + +def check_float(value, expected): + if expected == 'inf()': + if value == math.inf: + return 1 + elif expected == 'inf-neg()': + if value == -math.inf: + return 1 + elif expected == 'nan()': + if math.isnan(value): + return 1 + elif (float(expected) == value): + return 1 + else: + print(value) + print(expected) + return 0 + +def check_str(value, expected): + if value == expected: + return 1 + print(value) + print(expected) + return 0 + + +def _fail(input, test): + print("Input: >>" + input + "<<") + print(test) + +# The tests/data/yaml11.schema file is copied from +# https://github.com/perlpunk/yaml-test-schema/blob/master/data/schema-yaml11.yaml +def test_implicit_resolver(data_filename, skip_filename, verbose=False): + types = { + 'str': [str, check_str], + 'int': [int, check_int], + 'float': [float, check_float], + 'inf': [float, check_float], + 'nan': [float, check_float], + 'bool': [bool, check_bool], + } + with open(skip_filename, 'rb') as file: + skipdata = yaml.load(file, Loader=yaml.SafeLoader) + skip_load = skipdata['load'] + skip_dump = skipdata['dump'] + if verbose: + print(skip_load) + with open(data_filename, 'rb') as file: + tests = yaml.load(file, Loader=yaml.SafeLoader) + + i = 0 + fail = 0 + for i, (input, test) in enumerate(sorted(tests.items())): + if verbose: + print('-------------------- ' + str(i)) + + # Skip known loader bugs + if input in skip_load: + continue + + exp_type = test[0] + data = test[1] + exp_dump = test[2] + + # Test loading + try: + loaded = yaml.safe_load(input) + except: + print("Error:", sys.exc_info()[0], '(', sys.exc_info()[1], ')') + fail+=1 + _fail(input, test) + continue + + if verbose: + print(input) + print(test) + print(loaded) + print(type(loaded)) + + if exp_type == 'null': + if loaded is None: + pass + else: + fail+=1 + _fail(input, test) + else: + t = types[exp_type][0] + code = types[exp_type][1] + if isinstance(loaded, t): + if code(loaded, data): + pass + else: + fail+=1 + _fail(input, test) + else: + fail+=1 + _fail(input, test) + + # Skip known dumper bugs + if input in skip_dump: + continue + + dump = yaml.safe_dump(loaded, explicit_end=False) + # strip trailing newlines and footers + if dump.endswith('\n...\n'): + dump = dump[:-5] + if dump.endswith('\n'): + dump = dump[:-1] + if dump == exp_dump: + pass + else: + print("Compare: >>" + dump + "<< >>" + exp_dump + "<<") + fail+=1 + _fail(input, test) + +# if i >= 80: +# break + + if fail > 0: + print("Failed " + str(fail) + " / " + str(i) + " tests") + assert(False) + else: + print("Passed " + str(i) + " tests") + print("Skipped " + str(len(skip_load)) + " load tests") + print("Skipped " + str(len(skip_dump)) + " dump tests") + +test_implicit_resolver.unittest = ['.schema', '.schema-skip'] + +if __name__ == '__main__': + import test_appliance + test_appliance.run(globals()) + diff --git a/tests/lib/test_yaml.py b/tests/lib/test_yaml.py index 352cd8d1..7b3d8f9d 100644 --- a/tests/lib/test_yaml.py +++ b/tests/lib/test_yaml.py @@ -14,6 +14,8 @@ from test_sort_keys import * from test_multi_constructor import * +from test_schema import * + if __name__ == '__main__': import test_appliance test_appliance.run(globals()) From 35f04177a325c43fc6cefd52fa36d7bf272a0f5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ingy=20d=C3=B6t=20Net?= Date: Wed, 22 Sep 2021 11:25:30 -0700 Subject: [PATCH 4/8] XXX - remove appveyor.yml file for now --- .appveyor_TEMP_DISABLED.yml | 33 --------------------------------- 1 file changed, 33 deletions(-) delete mode 100644 .appveyor_TEMP_DISABLED.yml diff --git a/.appveyor_TEMP_DISABLED.yml b/.appveyor_TEMP_DISABLED.yml deleted file mode 100644 index 65358bdb..00000000 --- a/.appveyor_TEMP_DISABLED.yml +++ /dev/null @@ -1,33 +0,0 @@ -# TODO: update this from inside the build to use branch current version -version: '{build}' - -image: -- Visual Studio 2015 - -#cache: -#- 'C:\Python38\' -#- 'C:\Python38-x64' - -environment: - libyaml_repo_url: https://github.com/yaml/libyaml.git - libyaml_refspec: 0.2.5 - PYYAML_TEST_GROUP: all - -# matrix: -# - PYTHON_VER: Python36 -# - PYTHON_VER: Python36-x64 -# - PYTHON_VER: Python37 -# - PYTHON_VER: Python37-x64 -# - PYTHON_VER: Python38 -# - PYTHON_VER: Python38-x64 -# - PYTHON_VER: Python39 -# - PYTHON_VER: Python39-x64 - -#init: -#- ps: iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1')) - -build_script: -- ps: packaging\build\appveyor.ps1 - -#on_finish: -#- ps: $blockRdp = $true; iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1')) From 623a36e081aa827e5530f9b9ac49b72639ccc526 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tina=20M=C3=BCller?= Date: Wed, 22 Sep 2021 22:34:37 +0200 Subject: [PATCH 5/8] Move around methods for better inheritance so that other classes inheriting from it can use them * Move methods from SafeConstructor to BaseConstructor * Move methods from SafeRepresenter to BaseRepresenter --- lib/yaml/constructor.py | 83 +++++++++++++++++++++-------------------- lib/yaml/representer.py | 76 ++++++++++++++++++------------------- 2 files changed, 80 insertions(+), 79 deletions(-) diff --git a/lib/yaml/constructor.py b/lib/yaml/constructor.py index 619acd30..a3547a1c 100644 --- a/lib/yaml/constructor.py +++ b/lib/yaml/constructor.py @@ -18,6 +18,12 @@ class ConstructorError(MarkedYAMLError): class BaseConstructor: + inf_value = 1e300 + while inf_value != inf_value*inf_value: + inf_value *= inf_value + nan_value = -inf_value/inf_value # Trying to make a quiet NaN (like C99). + + yaml_constructors = {} yaml_multi_constructors = {} @@ -156,6 +162,42 @@ def construct_pairs(self, node, deep=False): pairs.append((key, value)) return pairs + def construct_yaml_null(self, node): + self.construct_scalar(node) + return None + + bool_values = { + 'yes': True, + 'no': False, + 'true': True, + 'false': False, + 'on': True, + 'off': False, + } + + def construct_yaml_bool(self, node): + value = self.construct_scalar(node) + return self.bool_values[value.lower()] + + def construct_yaml_str(self, node): + return self.construct_scalar(node) + + def construct_yaml_seq(self, node): + data = [] + yield data + data.extend(self.construct_sequence(node)) + + def construct_yaml_map(self, node): + data = {} + yield data + value = self.construct_mapping(node) + data.update(value) + + def construct_undefined(self, node): + raise ConstructorError(None, None, + "could not determine a constructor for the tag %r" % node.tag, + node.start_mark) + @classmethod def add_constructor(cls, tag, constructor): if not 'yaml_constructors' in cls.__dict__: @@ -217,23 +259,6 @@ def construct_mapping(self, node, deep=False): self.flatten_mapping(node) return super().construct_mapping(node, deep=deep) - def construct_yaml_null(self, node): - self.construct_scalar(node) - return None - - bool_values = { - 'yes': True, - 'no': False, - 'true': True, - 'false': False, - 'on': True, - 'off': False, - } - - def construct_yaml_bool(self, node): - value = self.construct_scalar(node) - return self.bool_values[value.lower()] - def construct_yaml_int(self, node): value = self.construct_scalar(node) value = value.replace('_', '') @@ -262,11 +287,6 @@ def construct_yaml_int(self, node): else: return sign*int(value) - inf_value = 1e300 - while inf_value != inf_value*inf_value: - inf_value *= inf_value - nan_value = -inf_value/inf_value # Trying to make a quiet NaN (like C99). - def construct_yaml_float(self, node): value = self.construct_scalar(node) value = value.replace('_', '').lower() @@ -399,20 +419,6 @@ def construct_yaml_set(self, node): value = self.construct_mapping(node) data.update(value) - def construct_yaml_str(self, node): - return self.construct_scalar(node) - - def construct_yaml_seq(self, node): - data = [] - yield data - data.extend(self.construct_sequence(node)) - - def construct_yaml_map(self, node): - data = {} - yield data - value = self.construct_mapping(node) - data.update(value) - def construct_yaml_object(self, node, cls): data = cls.__new__(cls) yield data @@ -423,11 +429,6 @@ def construct_yaml_object(self, node, cls): state = self.construct_mapping(node) data.__dict__.update(state) - def construct_undefined(self, node): - raise ConstructorError(None, None, - "could not determine a constructor for the tag %r" % node.tag, - node.start_mark) - SafeConstructor.add_constructor( 'tag:yaml.org,2002:null', SafeConstructor.construct_yaml_null) diff --git a/lib/yaml/representer.py b/lib/yaml/representer.py index 808ca06d..24418779 100644 --- a/lib/yaml/representer.py +++ b/lib/yaml/representer.py @@ -131,29 +131,30 @@ def represent_mapping(self, tag, mapping, flow_style=None): def ignore_aliases(self, data): return False -class SafeRepresenter(BaseRepresenter): + def represent_str(self, data): + return self.represent_scalar('tag:yaml.org,2002:str', data) - def ignore_aliases(self, data): - if data is None: - return True - if isinstance(data, tuple) and data == (): - return True - if isinstance(data, (str, bytes, bool, int, float)): - return True + def represent_list(self, data): + #pairs = (len(data) > 0 and isinstance(data, list)) + #if pairs: + # for item in data: + # if not isinstance(item, tuple) or len(item) != 2: + # pairs = False + # break + #if not pairs: + return self.represent_sequence('tag:yaml.org,2002:seq', data) + #value = [] + #for item_key, item_value in data: + # value.append(self.represent_mapping(u'tag:yaml.org,2002:map', + # [(item_key, item_value)])) + #return SequenceNode(u'tag:yaml.org,2002:pairs', value) + + def represent_dict(self, data): + return self.represent_mapping('tag:yaml.org,2002:map', data) def represent_none(self, data): return self.represent_scalar('tag:yaml.org,2002:null', 'null') - def represent_str(self, data): - return self.represent_scalar('tag:yaml.org,2002:str', data) - - def represent_binary(self, data): - if hasattr(base64, 'encodebytes'): - data = base64.encodebytes(data).decode('ascii') - else: - data = base64.encodestring(data).decode('ascii') - return self.represent_scalar('tag:yaml.org,2002:binary', data, style='|') - def represent_bool(self, data): if data: value = 'true' @@ -188,23 +189,12 @@ def represent_float(self, data): value = value.replace('e', '.0e', 1) return self.represent_scalar('tag:yaml.org,2002:float', value) - def represent_list(self, data): - #pairs = (len(data) > 0 and isinstance(data, list)) - #if pairs: - # for item in data: - # if not isinstance(item, tuple) or len(item) != 2: - # pairs = False - # break - #if not pairs: - return self.represent_sequence('tag:yaml.org,2002:seq', data) - #value = [] - #for item_key, item_value in data: - # value.append(self.represent_mapping(u'tag:yaml.org,2002:map', - # [(item_key, item_value)])) - #return SequenceNode(u'tag:yaml.org,2002:pairs', value) - - def represent_dict(self, data): - return self.represent_mapping('tag:yaml.org,2002:map', data) + def represent_binary(self, data): + if hasattr(base64, 'encodebytes'): + data = base64.encodebytes(data).decode('ascii') + else: + data = base64.encodestring(data).decode('ascii') + return self.represent_scalar('tag:yaml.org,2002:binary', data, style='|') def represent_set(self, data): value = {} @@ -220,6 +210,19 @@ def represent_datetime(self, data): value = data.isoformat(' ') return self.represent_scalar('tag:yaml.org,2002:timestamp', value) + def represent_undefined(self, data): + raise RepresenterError("cannot represent an object", data) + +class SafeRepresenter(BaseRepresenter): + + def ignore_aliases(self, data): + if data is None: + return True + if isinstance(data, tuple) and data == (): + return True + if isinstance(data, (str, bytes, bool, int, float)): + return True + def represent_yaml_object(self, tag, data, cls, flow_style=None): if hasattr(data, '__getstate__'): state = data.__getstate__() @@ -227,9 +230,6 @@ def represent_yaml_object(self, tag, data, cls, flow_style=None): state = data.__dict__.copy() return self.represent_mapping(tag, state, flow_style=flow_style) - def represent_undefined(self, data): - raise RepresenterError("cannot represent an object", data) - SafeRepresenter.add_representer(type(None), SafeRepresenter.represent_none) From 31cd170e50748b74425334584bf4440be4d8d9f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tina=20M=C3=BCller?= Date: Wed, 22 Sep 2021 21:56:21 +0200 Subject: [PATCH 6/8] Add support for YAML 1.2 schemas More and more YAML libraries are implementing YAML 1.2, either new ones simply starting with 1.2 or older ones adding support for it. While also the syntax was changed in YAML 1.2, this pull request is about the schema changes. As an example, in 1.1, Y, yes, NO, on etc. are resolved as booleans in 1.1. This sounds convenient, but also means that all these 22 different strings must be quoted if they are not meant as booleans. A very common obstacle is the country code for Norway, NO ("Norway Problem"). In YAML 1.2 this was improved by reducing the list of boolean representations. Also other types have been improved. The 1.1 regular expression for float allows . and ._ as floats, although there isn't a single digit in these strings. While the 1.2 Core Schema, the recommended default for 1.2, still allows a few variations (true, True and TRUE, etc.), the 1.2 JSON Schema is there to match JSON behaviour regarding types, so it allows only true and false. Note that this implementation of the YAML JSON Schema might not be exactly like the spec defines it (all plain scalars not resolving to numbers, null or booleans would be an error). Short usage example: class MyCoreLoader(yaml.BaseLoader): pass class MyCoreDumper(yaml.CommonDumper): pass MyCoreLoader.init_tags('core') MyCoreDumper.init_tags('core') data = yaml.load(input, Loader=MyCoreLoader) output = yaml.dump(data, Dumper=MyCoreDumper) Detailed example code to play with: import yaml class MyCoreLoader(yaml.BaseLoader): pass MyCoreLoader.init_tags('core') class MyJSONLoader(yaml.BaseLoader): pass MyJSONLoader.init_tags('json') class MyCoreDumper(yaml.CommonDumper): pass MyCoreDumper.init_tags('core') class MyJSONDumper(yaml.CommonDumper): pass MyJSONDumper.init_tags('json') input = """ - TRUE - yes - ~ - true #- .inf #- 23 #- #empty #- !!str #empty #- 010 #- 0o10 #- 0b100 #- 0x20 #- -0x20 #- 1_000 #- 3:14 #- 0011 #- +0 #- 0001.23 #- !!str +0.3e3 #- +0.3e3 #- &x foo #- *x #- 1e27 #- 1x+27 """ print('--------------------------------------------- BaseLoader') data = yaml.load(input, Loader=yaml.BaseLoader) print(data) print('--------------------------------------------- SafeLoader') data = yaml.load(input, Loader=yaml.SafeLoader) print(data) print('--------------------------------------------- CoreLoader') data = yaml.load(input, Loader=MyCoreLoader) print(data) print('--------------------------------------------- JSONLoader') data = yaml.load(input, Loader=MyJSONLoader) print(data) print('--------------------------------------------- SafeDumper') out = yaml.dump(data, Dumper=yaml.SafeDumper) print(out) print('--------------------------------------------- MyCoreDumper') out = yaml.dump(data, Dumper=MyCoreDumper) print(out) print('--------------------------------------------- MyJSONDumper') out = yaml.dump(data, Dumper=MyJSONDumper) print(out) --- lib/yaml/constructor.py | 333 +++++++++++++++++++++--------------- lib/yaml/dumper.py | 25 ++- lib/yaml/loader.py | 8 +- lib/yaml/representer.py | 102 +++++++---- lib/yaml/resolver.py | 116 ++++++++----- tests/data/core.schema | 235 +++++++++++++++++++++++++ tests/data/core.schema-skip | 4 + tests/data/json.schema | 195 +++++++++++++++++++++ tests/data/json.schema-skip | 6 + tests/lib/test_schema.py | 40 ++++- 10 files changed, 844 insertions(+), 220 deletions(-) create mode 100644 tests/data/core.schema create mode 100644 tests/data/core.schema-skip create mode 100644 tests/data/json.schema create mode 100644 tests/data/json.schema-skip diff --git a/lib/yaml/constructor.py b/lib/yaml/constructor.py index a3547a1c..f26e0cdd 100644 --- a/lib/yaml/constructor.py +++ b/lib/yaml/constructor.py @@ -5,7 +5,7 @@ 'FullConstructor', 'UnsafeConstructor', 'Constructor', - 'ConstructorError' + 'ConstructorError', ] from .error import * @@ -198,98 +198,26 @@ def construct_undefined(self, node): "could not determine a constructor for the tag %r" % node.tag, node.start_mark) - @classmethod - def add_constructor(cls, tag, constructor): - if not 'yaml_constructors' in cls.__dict__: - cls.yaml_constructors = cls.yaml_constructors.copy() - cls.yaml_constructors[tag] = constructor - - @classmethod - def add_multi_constructor(cls, tag_prefix, multi_constructor): - if not 'yaml_multi_constructors' in cls.__dict__: - cls.yaml_multi_constructors = cls.yaml_multi_constructors.copy() - cls.yaml_multi_constructors[tag_prefix] = multi_constructor - -class SafeConstructor(BaseConstructor): - - def construct_scalar(self, node): - if isinstance(node, MappingNode): - for key_node, value_node in node.value: - if key_node.tag == 'tag:yaml.org,2002:value': - return self.construct_scalar(value_node) - return super().construct_scalar(node) - - def flatten_mapping(self, node): - merge = [] - index = 0 - while index < len(node.value): - key_node, value_node = node.value[index] - if key_node.tag == 'tag:yaml.org,2002:merge': - del node.value[index] - if isinstance(value_node, MappingNode): - self.flatten_mapping(value_node) - merge.extend(value_node.value) - elif isinstance(value_node, SequenceNode): - submerge = [] - for subnode in value_node.value: - if not isinstance(subnode, MappingNode): - raise ConstructorError("while constructing a mapping", - node.start_mark, - "expected a mapping for merging, but found %s" - % subnode.id, subnode.start_mark) - self.flatten_mapping(subnode) - submerge.append(subnode.value) - submerge.reverse() - for value in submerge: - merge.extend(value) - else: - raise ConstructorError("while constructing a mapping", node.start_mark, - "expected a mapping or list of mappings for merging, but found %s" - % value_node.id, value_node.start_mark) - elif key_node.tag == 'tag:yaml.org,2002:value': - key_node.tag = 'tag:yaml.org,2002:str' - index += 1 - else: - index += 1 - if merge: - node.value = merge + node.value - - def construct_mapping(self, node, deep=False): - if isinstance(node, MappingNode): - self.flatten_mapping(node) - return super().construct_mapping(node, deep=deep) - - def construct_yaml_int(self, node): + def construct_yaml_int_core(self, node): value = self.construct_scalar(node) - value = value.replace('_', '') sign = +1 if value[0] == '-': sign = -1 if value[0] in '+-': value = value[1:] + if value == '0': return 0 - elif value.startswith('0b'): - return sign*int(value[2:], 2) + elif value.startswith('0o'): + return sign*int(value[2:], 8) elif value.startswith('0x'): return sign*int(value[2:], 16) - elif value[0] == '0': - return sign*int(value, 8) - elif ':' in value: - digits = [int(part) for part in value.split(':')] - digits.reverse() - base = 1 - value = 0 - for digit in digits: - value += digit*base - base *= 60 - return sign*value else: return sign*int(value) - def construct_yaml_float(self, node): + def construct_yaml_float_core(self, node): value = self.construct_scalar(node) - value = value.replace('_', '').lower() + value = value.lower() sign = +1 if value[0] == '-': sign = -1 @@ -299,18 +227,60 @@ def construct_yaml_float(self, node): return sign*self.inf_value elif value == '.nan': return self.nan_value - elif ':' in value: - digits = [float(part) for part in value.split(':')] - digits.reverse() - base = 1 - value = 0.0 - for digit in digits: - value += digit*base - base *= 60 - return sign*value else: return sign*float(value) + def construct_yaml_int_json(self, node): + value = self.construct_scalar(node) + sign = +1 + if value[0] == '-': + sign = -1 + if value[0] in '+-': + value = value[1:] + + if value == '0': + return 0 + else: + return sign*int(value) + + def construct_yaml_float_json(self, node): + value = self.construct_scalar(node) + value = value.lower() + sign = +1 + if value[0] == '-': + sign = -1 + if value[0] in '+-': + value = value[1:] + return sign*float(value) + + @classmethod + def add_constructor(cls, tag, constructor): + if not 'yaml_constructors' in cls.__dict__: + cls.yaml_constructors = cls.yaml_constructors.copy() + cls.yaml_constructors[tag] = constructor + + @classmethod + def add_multi_constructor(cls, tag_prefix, multi_constructor): + if not 'yaml_multi_constructors' in cls.__dict__: + cls.yaml_multi_constructors = cls.yaml_multi_constructors.copy() + cls.yaml_multi_constructors[tag_prefix] = multi_constructor + + + @classmethod + def init_constructors(cls, tagset): + if tagset not in _constructors: + return + for key in _constructors[tagset]: + callback = _constructors[tagset][key] + if (key is None): + cls.add_constructor(key, callback) + else: + cls.add_constructor('tag:yaml.org,2002:' + key, callback) + + +# SafeConstructor implements YAML 1.1 +class SafeConstructor(BaseConstructor): + def construct_yaml_binary(self, node): try: value = self.construct_scalar(node).encode('ascii') @@ -419,6 +389,105 @@ def construct_yaml_set(self, node): value = self.construct_mapping(node) data.update(value) + def construct_scalar(self, node): + if isinstance(node, MappingNode): + for key_node, value_node in node.value: + if key_node.tag == 'tag:yaml.org,2002:value': + return self.construct_scalar(value_node) + return super().construct_scalar(node) + + def flatten_mapping(self, node): + merge = [] + index = 0 + while index < len(node.value): + key_node, value_node = node.value[index] + if key_node.tag == 'tag:yaml.org,2002:merge': + del node.value[index] + if isinstance(value_node, MappingNode): + self.flatten_mapping(value_node) + merge.extend(value_node.value) + elif isinstance(value_node, SequenceNode): + submerge = [] + for subnode in value_node.value: + if not isinstance(subnode, MappingNode): + raise ConstructorError("while constructing a mapping", + node.start_mark, + "expected a mapping for merging, but found %s" + % subnode.id, subnode.start_mark) + self.flatten_mapping(subnode) + submerge.append(subnode.value) + submerge.reverse() + for value in submerge: + merge.extend(value) + else: + raise ConstructorError("while constructing a mapping", node.start_mark, + "expected a mapping or list of mappings for merging, but found %s" + % value_node.id, value_node.start_mark) + elif key_node.tag == 'tag:yaml.org,2002:value': + key_node.tag = 'tag:yaml.org,2002:str' + index += 1 + else: + index += 1 + if merge: + node.value = merge + node.value + + def construct_yaml_int(self, node): + value = self.construct_scalar(node) + value = value.replace('_', '') + sign = +1 + if value[0] == '-': + sign = -1 + if value[0] in '+-': + value = value[1:] + if value == '0': + return 0 + elif value.startswith('0b'): + return sign*int(value[2:], 2) + elif value.startswith('0x'): + return sign*int(value[2:], 16) + elif value[0] == '0': + return sign*int(value, 8) + elif ':' in value: + digits = [int(part) for part in value.split(':')] + digits.reverse() + base = 1 + value = 0 + for digit in digits: + value += digit*base + base *= 60 + return sign*value + else: + return sign*int(value) + + def construct_yaml_float(self, node): + value = self.construct_scalar(node) + value = value.replace('_', '').lower() + sign = +1 + if value[0] == '-': + sign = -1 + if value[0] in '+-': + value = value[1:] + if value == '.inf': + return sign*self.inf_value + elif value == '.nan': + return self.nan_value + elif ':' in value: + digits = [float(part) for part in value.split(':')] + digits.reverse() + base = 1 + value = 0.0 + for digit in digits: + value += digit*base + base *= 60 + return sign*value + else: + return sign*float(value) + + def construct_mapping(self, node, deep=False): + if isinstance(node, MappingNode): + self.flatten_mapping(node) + return super().construct_mapping(node, deep=deep) + def construct_yaml_object(self, node, cls): data = cls.__new__(cls) yield data @@ -429,56 +498,46 @@ def construct_yaml_object(self, node, cls): state = self.construct_mapping(node) data.__dict__.update(state) -SafeConstructor.add_constructor( - 'tag:yaml.org,2002:null', - SafeConstructor.construct_yaml_null) - -SafeConstructor.add_constructor( - 'tag:yaml.org,2002:bool', - SafeConstructor.construct_yaml_bool) - -SafeConstructor.add_constructor( - 'tag:yaml.org,2002:int', - SafeConstructor.construct_yaml_int) - -SafeConstructor.add_constructor( - 'tag:yaml.org,2002:float', - SafeConstructor.construct_yaml_float) - -SafeConstructor.add_constructor( - 'tag:yaml.org,2002:binary', - SafeConstructor.construct_yaml_binary) - -SafeConstructor.add_constructor( - 'tag:yaml.org,2002:timestamp', - SafeConstructor.construct_yaml_timestamp) - -SafeConstructor.add_constructor( - 'tag:yaml.org,2002:omap', - SafeConstructor.construct_yaml_omap) - -SafeConstructor.add_constructor( - 'tag:yaml.org,2002:pairs', - SafeConstructor.construct_yaml_pairs) - -SafeConstructor.add_constructor( - 'tag:yaml.org,2002:set', - SafeConstructor.construct_yaml_set) - -SafeConstructor.add_constructor( - 'tag:yaml.org,2002:str', - SafeConstructor.construct_yaml_str) - -SafeConstructor.add_constructor( - 'tag:yaml.org,2002:seq', - SafeConstructor.construct_yaml_seq) - -SafeConstructor.add_constructor( - 'tag:yaml.org,2002:map', - SafeConstructor.construct_yaml_map) -SafeConstructor.add_constructor(None, - SafeConstructor.construct_undefined) +_constructors = { + 'yaml11': { + 'str': BaseConstructor.construct_yaml_str, + 'seq': BaseConstructor.construct_yaml_seq, + 'map': BaseConstructor.construct_yaml_map, + None: BaseConstructor.construct_undefined, + 'int': SafeConstructor.construct_yaml_int, + 'float': SafeConstructor.construct_yaml_float, + 'null': BaseConstructor.construct_yaml_null, + 'bool': BaseConstructor.construct_yaml_bool, + 'binary': SafeConstructor.construct_yaml_binary, + 'timestamp': SafeConstructor.construct_yaml_timestamp, + 'omap': SafeConstructor.construct_yaml_omap, + 'pairs': SafeConstructor.construct_yaml_pairs, + 'set': SafeConstructor.construct_yaml_set, + }, + 'core': { + 'str': BaseConstructor.construct_yaml_str, + 'seq': BaseConstructor.construct_yaml_seq, + 'map': BaseConstructor.construct_yaml_map, + None: BaseConstructor.construct_undefined, + 'int': BaseConstructor.construct_yaml_int_core, + 'float': BaseConstructor.construct_yaml_float_core, + 'null': BaseConstructor.construct_yaml_null, + 'bool': BaseConstructor.construct_yaml_bool, + }, + 'json': { + 'str': BaseConstructor.construct_yaml_str, + 'seq': BaseConstructor.construct_yaml_seq, + 'map': BaseConstructor.construct_yaml_map, + None: BaseConstructor.construct_undefined, + 'int': BaseConstructor.construct_yaml_int_json, + 'float': BaseConstructor.construct_yaml_float_json, + 'null': BaseConstructor.construct_yaml_null, + 'bool': BaseConstructor.construct_yaml_bool, + }, +} + +SafeConstructor.init_constructors('yaml11') class FullConstructor(SafeConstructor): # 'extend' is blacklisted because it is used by diff --git a/lib/yaml/dumper.py b/lib/yaml/dumper.py index 6aadba55..e78e749e 100644 --- a/lib/yaml/dumper.py +++ b/lib/yaml/dumper.py @@ -1,5 +1,5 @@ -__all__ = ['BaseDumper', 'SafeDumper', 'Dumper'] +__all__ = ['BaseDumper', 'SafeDumper', 'Dumper', 'CommonDumper'] from .emitter import * from .serializer import * @@ -42,6 +42,29 @@ def __init__(self, stream, default_flow_style=default_flow_style, sort_keys=sort_keys) Resolver.__init__(self) +class CommonDumper(Emitter, Serializer, CommonRepresenter, BaseResolver): + + def __init__(self, stream, + default_style=None, default_flow_style=False, + canonical=None, indent=None, width=None, + allow_unicode=None, line_break=None, + encoding=None, explicit_start=None, explicit_end=None, + version=None, tags=None, sort_keys=True): + Emitter.__init__(self, stream, canonical=canonical, + indent=indent, width=width, + allow_unicode=allow_unicode, line_break=line_break) + Serializer.__init__(self, encoding=encoding, + explicit_start=explicit_start, explicit_end=explicit_end, + version=version, tags=tags) + CommonRepresenter.__init__(self, default_style=default_style, + default_flow_style=default_flow_style, sort_keys=sort_keys) + BaseResolver.__init__(self) + + @classmethod + def init_tags(cls, tagset): + cls.init_representers(tagset) + cls.init_resolvers(tagset) + class Dumper(Emitter, Serializer, Representer, Resolver): def __init__(self, stream, diff --git a/lib/yaml/loader.py b/lib/yaml/loader.py index e90c1122..8e2dc6f9 100644 --- a/lib/yaml/loader.py +++ b/lib/yaml/loader.py @@ -1,5 +1,6 @@ -__all__ = ['BaseLoader', 'FullLoader', 'SafeLoader', 'Loader', 'UnsafeLoader'] +__all__ = ['BaseLoader', 'FullLoader', 'SafeLoader', 'Loader', 'UnsafeLoader', + ] from .reader import * from .scanner import * @@ -18,6 +19,11 @@ def __init__(self, stream): BaseConstructor.__init__(self) BaseResolver.__init__(self) + @classmethod + def init_tags(cls, tagset): + cls.init_constructors(tagset) + cls.init_resolvers(tagset) + class FullLoader(Reader, Scanner, Parser, Composer, FullConstructor, Resolver): def __init__(self, stream): diff --git a/lib/yaml/representer.py b/lib/yaml/representer.py index 24418779..aa9360d7 100644 --- a/lib/yaml/representer.py +++ b/lib/yaml/representer.py @@ -1,6 +1,6 @@ __all__ = ['BaseRepresenter', 'SafeRepresenter', 'Representer', - 'RepresenterError'] + 'RepresenterError', 'CommonRepresenter'] from .error import * from .nodes import * @@ -185,6 +185,7 @@ def represent_float(self, data): # Unfortunately, this is not a valid float representation according # to the definition of the `!!float` tag. We fix this by adding # '.0' before the 'e' symbol. + # TODO (In YAML 1.2 Core, 1e17 would be a valid float though) if '.' not in value and 'e' in value: value = value.replace('e', '.0e', 1) return self.represent_scalar('tag:yaml.org,2002:float', value) @@ -213,7 +214,13 @@ def represent_datetime(self, data): def represent_undefined(self, data): raise RepresenterError("cannot represent an object", data) -class SafeRepresenter(BaseRepresenter): + @classmethod + def init_representers(cls, name): + for key in _representers[name]: + callback = _representers[name][key] + cls.add_representer(key, callback) + +class CommonRepresenter(BaseRepresenter): def ignore_aliases(self, data): if data is None: @@ -230,44 +237,65 @@ def represent_yaml_object(self, tag, data, cls, flow_style=None): state = data.__dict__.copy() return self.represent_mapping(tag, state, flow_style=flow_style) -SafeRepresenter.add_representer(type(None), - SafeRepresenter.represent_none) - -SafeRepresenter.add_representer(str, - SafeRepresenter.represent_str) - -SafeRepresenter.add_representer(bytes, - SafeRepresenter.represent_binary) - -SafeRepresenter.add_representer(bool, - SafeRepresenter.represent_bool) - -SafeRepresenter.add_representer(int, - SafeRepresenter.represent_int) - -SafeRepresenter.add_representer(float, - SafeRepresenter.represent_float) - -SafeRepresenter.add_representer(list, - SafeRepresenter.represent_list) - -SafeRepresenter.add_representer(tuple, - SafeRepresenter.represent_list) - -SafeRepresenter.add_representer(dict, - SafeRepresenter.represent_dict) - -SafeRepresenter.add_representer(set, - SafeRepresenter.represent_set) +class SafeRepresenter(BaseRepresenter): -SafeRepresenter.add_representer(datetime.date, - SafeRepresenter.represent_date) + def ignore_aliases(self, data): + if data is None: + return True + if isinstance(data, tuple) and data == (): + return True + if isinstance(data, (str, bytes, bool, int, float)): + return True -SafeRepresenter.add_representer(datetime.datetime, - SafeRepresenter.represent_datetime) + def represent_yaml_object(self, tag, data, cls, flow_style=None): + if hasattr(data, '__getstate__'): + state = data.__getstate__() + else: + state = data.__dict__.copy() + return self.represent_mapping(tag, state, flow_style=flow_style) -SafeRepresenter.add_representer(None, - SafeRepresenter.represent_undefined) +_representers = { + 'yaml11': { + bool: BaseRepresenter.represent_bool, + dict: BaseRepresenter.represent_dict, + list: BaseRepresenter.represent_list, + tuple: BaseRepresenter.represent_list, + str: BaseRepresenter.represent_str, + float: BaseRepresenter.represent_float, + int: BaseRepresenter.represent_int, + type(None): BaseRepresenter.represent_none, + bytes: BaseRepresenter.represent_binary, + set: BaseRepresenter.represent_set, + datetime.date: BaseRepresenter.represent_date, + datetime.datetime: BaseRepresenter.represent_datetime, + None: BaseRepresenter.represent_undefined, + }, + 'core': { + bool: BaseRepresenter.represent_bool, + dict: BaseRepresenter.represent_dict, + list: BaseRepresenter.represent_list, + tuple: BaseRepresenter.represent_list, + str: BaseRepresenter.represent_str, + float: BaseRepresenter.represent_float, + int: BaseRepresenter.represent_int, + type(None): BaseRepresenter.represent_none, + None: BaseRepresenter.represent_undefined, + }, + 'json': { + bool: BaseRepresenter.represent_bool, + dict: BaseRepresenter.represent_dict, + list: BaseRepresenter.represent_list, + tuple: BaseRepresenter.represent_list, + str: BaseRepresenter.represent_str, + float: BaseRepresenter.represent_float, + int: BaseRepresenter.represent_int, + type(None): BaseRepresenter.represent_none, + None: BaseRepresenter.represent_undefined, + }, +} + + +SafeRepresenter.init_representers('yaml11') class Representer(SafeRepresenter): diff --git a/lib/yaml/resolver.py b/lib/yaml/resolver.py index 013896d2..11248ca6 100644 --- a/lib/yaml/resolver.py +++ b/lib/yaml/resolver.py @@ -1,5 +1,5 @@ -__all__ = ['BaseResolver', 'Resolver'] +__all__ = ['BaseResolver', 'Resolver' ] from .error import * from .nodes import * @@ -164,64 +164,96 @@ def resolve(self, kind, value, implicit): elif kind is MappingNode: return self.DEFAULT_MAPPING_TAG -class Resolver(BaseResolver): - pass + @classmethod + def init_resolvers(cls, key): + for args in _resolvers[key]: + cls.add_implicit_resolver( + 'tag:yaml.org,2002:' + args[0], + args[1], args[2] + ) -Resolver.add_implicit_resolver( - 'tag:yaml.org,2002:bool', - re.compile(r'''^(?:yes|Yes|YES|no|No|NO +_resolvers = { + 'yaml11': [ + ['bool', + re.compile(r'''^(?:yes|Yes|YES|no|No|NO |true|True|TRUE|false|False|FALSE |on|On|ON|off|Off|OFF)$''', re.X), - list('yYnNtTfFoO')) - -Resolver.add_implicit_resolver( - 'tag:yaml.org,2002:float', - re.compile(r'''^(?:[-+]?(?:[0-9][0-9_]*)\.[0-9_]*(?:[eE][-+][0-9]+)? + list('yYnNtTfFoO')], + ['float', + re.compile(r'''^(?:[-+]?(?:[0-9][0-9_]*)\.[0-9_]*(?:[eE][-+][0-9]+)? |\.[0-9_]+(?:[eE][-+][0-9]+)? |[-+]?[0-9][0-9_]*(?::[0-5]?[0-9])+\.[0-9_]* |[-+]?\.(?:inf|Inf|INF) |\.(?:nan|NaN|NAN))$''', re.X), - list('-+0123456789.')) - -Resolver.add_implicit_resolver( - 'tag:yaml.org,2002:int', - re.compile(r'''^(?:[-+]?0b[0-1_]+ + list('-+0123456789.')], + ['int', + re.compile(r'''^(?:[-+]?0b[0-1_]+ |[-+]?0[0-7_]+ |[-+]?(?:0|[1-9][0-9_]*) |[-+]?0x[0-9a-fA-F_]+ |[-+]?[1-9][0-9_]*(?::[0-5]?[0-9])+)$''', re.X), - list('-+0123456789')) - -Resolver.add_implicit_resolver( - 'tag:yaml.org,2002:merge', - re.compile(r'^(?:<<)$'), - ['<']) - -Resolver.add_implicit_resolver( - 'tag:yaml.org,2002:null', - re.compile(r'''^(?: ~ + list('-+0123456789')], + ['merge', + re.compile(r'^(?:<<)$'), + ['<']], + ['null', + re.compile(r'''^(?: ~ |null|Null|NULL | )$''', re.X), - ['~', 'n', 'N', '']) - -Resolver.add_implicit_resolver( - 'tag:yaml.org,2002:timestamp', - re.compile(r'''^(?:[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9] + ['~', 'n', 'N', '']], + ['timestamp', + re.compile(r'''^(?:[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9] |[0-9][0-9][0-9][0-9] -[0-9][0-9]? -[0-9][0-9]? (?:[Tt]|[ \t]+)[0-9][0-9]? :[0-9][0-9] :[0-9][0-9] (?:\.[0-9]*)? (?:[ \t]*(?:Z|[-+][0-9][0-9]?(?::[0-9][0-9])?))?)$''', re.X), - list('0123456789')) - -Resolver.add_implicit_resolver( - 'tag:yaml.org,2002:value', - re.compile(r'^(?:=)$'), - ['=']) - + list('0123456789')], + ['value', + re.compile(r'^(?:=)$'), + ['=']], # The following resolver is only for documentation purposes. It cannot work # because plain scalars cannot start with '!', '&', or '*'. -Resolver.add_implicit_resolver( - 'tag:yaml.org,2002:yaml', - re.compile(r'^(?:!|&|\*)$'), - list('!&*')) + ['yaml', + re.compile(r'^(?:!|&|\*)$'), + list('!&*')], + ], + 'core': [ + ['bool', + re.compile(r'''^(?:|true|True|TRUE|false|False|FALSE)$''', re.X), + list('tTfF')], + ['int', + re.compile(r'''^(?: + |0o[0-7]+ + |[-+]?(?:[0-9]+) + |0x[0-9a-fA-F]+ + )$''', re.X), + list('-+0123456789')], + ['float', + re.compile(r'''^(?:[-+]?(?:\.[0-9]+|[0-9]+(\.[0-9]*)?)(?:[eE][-+]?[0-9]+)? + |[-+]?\.(?:inf|Inf|INF) + |\.(?:nan|NaN|NAN))$''', re.X), + list('-+0123456789.')], + ['null', + re.compile(r'''^(?:~||null|Null|NULL)$''', re.X), + ['~', 'n', 'N', '']], + ], + 'json': [ + ['bool', + re.compile(r'''^(?:true|false)$''', re.X), + list('tf')], + ['int', + re.compile(r'''^-?(?:0|[1-9][0-9]*)$''', re.X), + list('-0123456789')], + ['float', + re.compile(r'''^-?(?:0|[1-9][0-9]*)(\.[0-9]*)?(?:[eE][-+]?[0-9]+)?$''', re.X), + list('-0123456789.')], + ['null', + re.compile(r'''^null$''', re.X), + ['n']], + ], +} + +class Resolver(BaseResolver): pass + +Resolver.init_resolvers('yaml11') diff --git a/tests/data/core.schema b/tests/data/core.schema new file mode 100644 index 00000000..f830a277 --- /dev/null +++ b/tests/data/core.schema @@ -0,0 +1,235 @@ +# https://github.com/perlpunk/yaml-test-schema/blob/master/data/schema-core.yaml +--- +'!!bool FALSE': ['bool', 'false()', 'false'] +'!!bool False': ['bool', 'false()', 'false'] +'!!bool TRUE': ['bool', 'true()', 'true'] +'!!bool True': ['bool', 'true()', 'true'] +'!!bool false': ['bool', 'false()', 'false'] +'!!bool true': ['bool', 'true()', 'true'] +'!!float +.INF': ['inf', 'inf()', '.inf'] +'!!float +.Inf': ['inf', 'inf()', '.inf'] +'!!float +.inf': ['inf', 'inf()', '.inf'] +'!!float +0.3e+3': ['float', '300.0', '300.0'] +'!!float +0.3e3': ['float', '300.0', '300.0'] +'!!float -.INF': ['inf', 'inf-neg()', '-.inf'] +'!!float -.Inf': ['inf', 'inf-neg()', '-.inf'] +'!!float -.inf': ['inf', 'inf-neg()', '-.inf'] +'!!float -3.14': ['float', '-3.14', '-3.14'] +'!!float .0': ['float', '0.0', '0.0'] +'!!float .14': ['float', '0.14', '0.14'] +'!!float .3E-1': ['float', '0.03', '0.03'] +'!!float .3e+3': ['float', '300.0', '300.0'] +'!!float .3e3': ['float', '300.0', '300.0'] +'!!float .INF': ['inf', 'inf()', '.inf'] +'!!float .Inf': ['inf', 'inf()', '.inf'] +'!!float .NAN': ['nan', 'nan()', '.nan'] +'!!float .NaN': ['nan', 'nan()', '.nan'] +'!!float .inf': ['inf', 'inf()', '.inf'] +'!!float .nan': ['nan', 'nan()', '.nan'] +'!!float 0.0': ['float', '0.0', '0.0'] +'!!float 0.3e3': ['float', '300.0', '300.0'] +'!!float 001.23': ['float', '1.23', '1.23'] +'!!float 3.': ['float', '3.0', '3.0'] +'!!float 3.14': ['float', '3.14', '3.14'] +'!!float 3.3e+3': ['float', '3300.0', '3300.0'] +'!!int +0': ['int', '0', '0'] +'!!int +23': ['int', '23', '23'] +'!!int -0': ['int', '0', '0'] +'!!int -23': ['int', '-23', '-23'] +'!!int 0': ['int', '0', '0'] +'!!int 0011': ['int', '11', '11'] +'!!int 07': ['int', '7', '7'] +'!!int 0o0': ['int', '0', '0'] +'!!int 0o10': ['int', '8', '8'] +'!!int 0o7': ['int', '7', '7'] +'!!int 0x0': ['int', '0', '0'] +'!!int 0x10': ['int', '16', '16'] +'!!int 0x42': ['int', '66', '66'] +'!!int 0xa': ['int', '10', '10'] +'!!int 23': ['int', '23', '23'] +'!!null #empty': ['null', 'null()', "null"] +'!!null NULL': ['null', 'null()', "null"] +'!!null Null': ['null', 'null()', "null"] +'!!null null': ['null', 'null()', 'null'] +'!!null ~': ['null', 'null()', 'null'] +'!!str #empty': ['str', '', "''"] +'!!str +.INF': ['str', '+.INF', "'+.INF'"] +'!!str +.Inf': ['str', '+.Inf', "'+.Inf'"] +'!!str +.inf': ['str', '+.inf', "'+.inf'"] +'!!str +0': ['str', '+0', "'+0'"] +'!!str +0.3e+3': ['str', '+0.3e+3', "'+0.3e+3'"] +'!!str +0.3e3': ['str', '+0.3e3', "'+0.3e3'"] +'!!str +0100_200': ['str', '+0100_200', "+0100_200"] +'!!str +0b100': ['str', '+0b100', "+0b100"] +'!!str +190:20:30': ['str', '+190:20:30', "+190:20:30"] +'!!str +23': ['str', '+23', "'+23'"] +'!!str -.INF': ['str', '-.INF', "'-.INF'"] +'!!str -.Inf': ['str', '-.Inf', "'-.Inf'"] +'!!str -.inf': ['str', '-.inf', "'-.inf'"] +'!!str -0': ['str', '-0', "'-0'"] +'!!str -0100_200': ['str', '-0100_200', "-0100_200"] +'!!str -0b101': ['str', '-0b101', "-0b101"] +'!!str -0x30': ['str', '-0x30', "-0x30"] +'!!str -190:20:30': ['str', '-190:20:30', "-190:20:30"] +'!!str -23': ['str', '-23', "'-23'"] +'!!str -3.14': ['str', '-3.14', "'-3.14'"] +'!!str .': ['str', '.', '.'] +'!!str .0': ['str', '.0', "'.0'"] +'!!str .14': ['str', '.14', "'.14'"] +'!!str .1_4': ['str', '.1_4', '.1_4'] +'!!str .3E-1': ['str', '.3E-1', "'.3E-1'"] +'!!str .3e+3': ['str', '.3e+3', "'.3e+3'"] +'!!str .3e3': ['str', '.3e3', "'.3e3'"] +'!!str .INF': ['str', '.INF', "'.INF'"] +'!!str .Inf': ['str', '.Inf', "'.Inf'"] +'!!str .NAN': ['str', '.NAN', "'.NAN'"] +'!!str .NaN': ['str', '.NaN', "'.NaN'"] +'!!str ._': ['str', '._', '._'] +'!!str ._14': ['str', '._14', '._14'] +'!!str .inf': ['str', '.inf', "'.inf'"] +'!!str .nan': ['str', '.nan', "'.nan'"] +'!!str 0': ['str', '0', "'0'"] +'!!str 0.0': ['str', '0.0', "'0.0'"] +'!!str 0.3e3': ['str', '0.3e3', "'0.3e3'"] +'!!str 00': ['str', '00', "'00'"] +'!!str 001.23': ['str', '001.23', "'001.23'"] +'!!str 0011': ['str', '0011', "'0011'"] +'!!str 010': ['str', '010', "'010'"] +'!!str 02_0': ['str', '02_0', "02_0"] +'!!str 07': ['str', '07', "'07'"] +'!!str 0b0': ['str', '0b0', "0b0"] +'!!str 0b100_101': ['str', '0b100_101', "0b100_101"] +'!!str 0o0': ['str', '0o0', "'0o0'"] +'!!str 0o10': ['str', '0o10', "'0o10'"] +'!!str 0o7': ['str', '0o7', "'0o7'"] +'!!str 0x0': ['str', '0x0', "'0x0'"] +'!!str 0x2_0': ['str', '0x2_0', "0x2_0"] +'!!str 0xa': ['str', '0xa', "'0xa'"] +'!!str 100_000': ['str', '100_000', "100_000"] +'!!str 190:20:30': ['str', '190:20:30', "190:20:30"] +'!!str 190:20:30.15': ['str', '190:20:30.15', "190:20:30.15"] +'!!str 23': ['str', '23', "'23'"] +'!!str 3.': ['str', '3.', "'3.'"] +'!!str 3.14': ['str', '3.14', "'3.14'"] +'!!str 3.3e+3': ['str', '3.3e+3', "'3.3e+3'"] +'!!str 85.230_15e+03': ['str', '85.230_15e+03', "85.230_15e+03"] +'!!str 85_230.15': ['str', '85_230.15', "85_230.15"] +'!!str FALSE': ['str', 'FALSE', "'FALSE'"] +'!!str False': ['str', 'False', "'False'"] +'!!str N': ['str', 'N', "N"] +'!!str NO': ['str', 'NO', "NO"] +'!!str NULL': ['str', 'NULL', "'NULL'"] +'!!str Null': ['str', 'Null', "'Null'"] +'!!str OFF': ['str', 'OFF', "OFF"] +'!!str ON': ['str', 'ON', "ON"] +'!!str Off': ['str', 'Off', "Off"] +'!!str On': ['str', 'On', "On"] +'!!str TRUE': ['str', 'TRUE', "'TRUE'"] +'!!str True': ['str', 'True', "'True'"] +'!!str Y': ['str', 'Y', "Y"] +'!!str YES': ['str', 'YES', "YES"] +'!!str Yes': ['str', 'Yes', "Yes"] +'!!str _._': ['str', '_._', '_._'] +'!!str false': ['str', 'false', "'false'"] +'!!str n': ['str', 'n', "n"] +'!!str no': ['str', 'no', "no"] +'!!str null': ['str', 'null', "'null'"] +'!!str off': ['str', 'off', "off"] +'!!str on': ['str', 'on', "on"] +'!!str true': ['str', 'true', "'true'"] +'!!str y': ['str', 'y', "y"] +'!!str yes': ['str', 'yes', "yes"] +'!!str ~': ['str', '~', "'~'"] +'#empty': ['null', 'null()', "null"] +'+.INF': ['inf', 'inf()', '.inf'] +'+.Inf': ['inf', 'inf()', '.inf'] +'+.inf': ['inf', 'inf()', '.inf'] +'+0': ['int', '0', '0'] +'+0.3e+3': ['float', '300.0', '300.0'] +'+0.3e3': ['float', '300.0', '300.0'] +'+0100_200': ['str', '+0100_200', '+0100_200'] +'+0b100': ['str', '+0b100', '+0b100'] +'+190:20:30': ['str', '+190:20:30', '+190:20:30'] +'+23': ['int', '23', '23'] +'+3.14': ['float', '3.14', '3.14'] +'-.INF': ['inf', 'inf-neg()', '-.inf'] +'-.Inf': ['inf', 'inf-neg()', '-.inf'] +'-.inf': ['inf', 'inf-neg()', '-.inf'] +'-0': ['int', '0', '0'] +'-0100_200': ['str', '-0100_200', '-0100_200'] +'-0b101': ['str', '-0b101', '-0b101'] +'-0x30': ['str', '-0x30', '-0x30'] +'-190:20:30': ['str', '-190:20:30', '-190:20:30'] +'-23': ['int', '-23', '-23'] +'-3.14': ['float', '-3.14', '-3.14'] +'.': ['str', '.', '.'] +'.0': ['float', '0.0', '0.0'] +'.14': ['float', '0.14', '0.14'] +'.1_4': ['str', '.1_4', '.1_4'] +'.3E-1': ['float', '0.03', '0.03'] +'.3e+3': ['float', '300.0', '300.0'] +'.3e3': ['float', '300.0', '300.0'] +'.INF': ['inf', 'inf()', '.inf'] +'.Inf': ['inf', 'inf()', '.inf'] +'.NAN': ['nan', 'nan()', '.nan'] +'.NaN': ['nan', 'nan()', '.nan'] +'._': ['str', '._', '._'] +'._14': ['str', '._14', '._14'] +'.inf': ['inf', 'inf()', '.inf'] +'.nan': ['nan', 'nan()', '.nan'] +'0': ['int', '0', '0'] +'0.0': ['float', '0.0', '0.0'] +'0.3e3': ['float', '300.0', '300.0'] +'00': ['int', '0', '0'] +'001.23': ['float', '1.23', '1.23'] +'0011': ['int', '11', '11'] +'010': ['int', '10', '10'] +'02_0': ['str', '02_0', '02_0'] +'07': ['int', '7', '7'] +'08': ['int', '8', '8'] +'0b0': ['str', '0b0', '0b0'] +'0b100_101': ['str', '0b100_101', '0b100_101'] +'0o0': ['int', '0', '0'] +'0o10': ['int', '8', '8'] +'0o7': ['int', '7', '7'] +'0x0': ['int', '0', '0'] +'0x10': ['int', '16', '16'] +'0x2_0': ['str', '0x2_0', '0x2_0'] +'0x42': ['int', '66', '66'] +'0xa': ['int', '10', '10'] +'100_000': ['str', '100_000', '100_000'] +'190:20:30': ['str', '190:20:30', '190:20:30'] +'190:20:30.15': ['str', '190:20:30.15', '190:20:30.15'] +'23': ['int', '23', '23'] +'3.': ['float', '3.0', '3.0'] +'3.14': ['float', '3.14', '3.14'] +'3.3e+3': ['float', '3300', '3300.0'] +'3e3': ['float', '3000', '3000.0'] +'85.230_15e+03': ['str', '85.230_15e+03', '85.230_15e+03'] +'85_230.15': ['str', '85_230.15', '85_230.15'] +'FALSE': ['bool', 'false()', 'false'] +'False': ['bool', 'false()', 'false'] +'N': ['str', 'N', "N"] +'NO': ['str', 'NO', "NO"] +'NULL': ['null', 'null()', "null"] +'Null': ['null', 'null()', "null"] +'OFF': ['str', 'OFF', "OFF"] +'ON': ['str', 'ON', "ON"] +'Off': ['str', 'Off', "Off"] +'On': ['str', 'On', "On"] +'TRUE': ['bool', 'true()', 'true'] +'True': ['bool', 'true()', 'true'] +'Y': ['str', 'Y', "Y"] +'YES': ['str', 'YES', "YES"] +'Yes': ['str', 'Yes', "Yes"] +'_._': ['str', '_._', '_._'] +'false': ['bool', 'false()', 'false'] +'n': ['str', 'n', "n"] +'no': ['str', 'no', "no"] +'null': ['null', 'null()', "null"] +'off': ['str', 'off', "off"] +'on': ['str', 'on', "on"] +'true': ['bool', 'true()', 'true'] +'y': ['str', 'y', "y"] +'yes': ['str', 'yes', "yes"] +'~': ['null', 'null()', "null"] diff --git a/tests/data/core.schema-skip b/tests/data/core.schema-skip new file mode 100644 index 00000000..d412d083 --- /dev/null +++ b/tests/data/core.schema-skip @@ -0,0 +1,4 @@ +load: { + } +dump: { + } diff --git a/tests/data/json.schema b/tests/data/json.schema new file mode 100644 index 00000000..676f2909 --- /dev/null +++ b/tests/data/json.schema @@ -0,0 +1,195 @@ +# https://github.com/perlpunk/yaml-test-schema/blob/master/data/schema-json.yaml +--- +'!!bool false': ['bool', 'false()', 'false'] +'!!bool true': ['bool', 'true()', 'true'] +'!!float -3.14': ['float', '-3.14', '-3.14'] +'!!float 0.0': ['float', '0.0', '0.0'] +'!!float 0.3e3': ['float', '300.0', '300.0'] +'!!float 3.': ['float', '3.0', '3.0'] +'!!float 3.14': ['float', '3.14', '3.14'] +'!!int -0': ['int', '0', '0'] +'!!int -23': ['int', '-23', '-23'] +'!!int 0': ['int', '0', '0'] +'!!int 23': ['int', '23', '23'] +'!!null null': ['null', 'null()', 'null'] +'!!str #empty': ['str', '', "''"] +'!!str +.INF': ['str', '+.INF', "+.INF"] +'!!str +.Inf': ['str', '+.Inf', "+.Inf"] +'!!str +.inf': ['str', '+.inf', "+.inf"] +'!!str +0': ['str', '+0', "+0"] +'!!str +0.3e+3': ['str', '+0.3e+3', "+0.3e+3"] +'!!str +0.3e3': ['str', '+0.3e3', "+0.3e3"] +'!!str +0100_200': ['str', '+0100_200', "+0100_200"] +'!!str +0b100': ['str', '+0b100', "+0b100"] +'!!str +190:20:30': ['str', '+190:20:30', "+190:20:30"] +'!!str +23': ['str', '+23', "+23"] +'!!str -.INF': ['str', '-.INF', "-.INF"] +'!!str -.Inf': ['str', '-.Inf', "-.Inf"] +'!!str -.inf': ['str', '-.inf', "-.inf"] +'!!str -0': ['str', '-0', "'-0'"] +'!!str -0100_200': ['str', '-0100_200', "-0100_200"] +'!!str -0b101': ['str', '-0b101', "-0b101"] +'!!str -0x30': ['str', '-0x30', "-0x30"] +'!!str -190:20:30': ['str', '-190:20:30', "-190:20:30"] +'!!str -23': ['str', '-23', "'-23'"] +'!!str -3.14': ['str', '-3.14', "'-3.14'"] +'!!str .': ['str', '.', '.'] +'!!str .0': ['str', '.0', ".0"] +'!!str .14': ['str', '.14', '.14'] +'!!str .1_4': ['str', '.1_4', '.1_4'] +'!!str .3E-1': ['str', '.3E-1', ".3E-1"] +'!!str .3e+3': ['str', '.3e+3', ".3e+3"] +'!!str .3e3': ['str', '.3e3', ".3e3"] +'!!str .INF': ['str', '.INF', ".INF"] +'!!str .Inf': ['str', '.Inf', ".Inf"] +'!!str .NAN': ['str', '.NAN', ".NAN"] +'!!str .NaN': ['str', '.NaN', ".NaN"] +'!!str ._': ['str', '._', '._'] +'!!str ._14': ['str', '._14', '._14'] +'!!str .inf': ['str', '.inf', ".inf"] +'!!str .nan': ['str', '.nan', ".nan"] +'!!str 0': ['str', '0', "'0'"] +'!!str 0.0': ['str', '0.0', "'0.0'"] +'!!str 0.3e3': ['str', '0.3e3', "'0.3e3'"] +'!!str 00': ['str', '00', "00"] +'!!str 001.23': ['str', '001.23', "001.23"] +'!!str 0011': ['str', '0011', "0011"] +'!!str 010': ['str', '010', "010"] +'!!str 02_0': ['str', '02_0', "02_0"] +'!!str 07': ['str', '07', "07"] +'!!str 0b0': ['str', '0b0', "0b0"] +'!!str 0b100_101': ['str', '0b100_101', "0b100_101"] +'!!str 0o0': ['str', '0o0', "0o0"] +'!!str 0o10': ['str', '0o10', "0o10"] +'!!str 0o7': ['str', '0o7', "0o7"] +'!!str 0x0': ['str', '0x0', "0x0"] +'!!str 0x2_0': ['str', '0x2_0', "0x2_0"] +'!!str 0xa': ['str', '0xa', "0xa"] +'!!str 100_000': ['str', '100_000', "100_000"] +'!!str 190:20:30': ['str', '190:20:30', "190:20:30"] +'!!str 190:20:30.15': ['str', '190:20:30.15', "190:20:30.15"] +'!!str 23': ['str', '23', "'23'"] +'!!str 3.': ['str', '3.', "'3.'"] +'!!str 3.14': ['str', '3.14', "'3.14'"] +'!!str 3.3e+3': ['str', '3.3e+3', "'3.3e+3'"] +'!!str 85.230_15e+03': ['str', '85.230_15e+03', "85.230_15e+03"] +'!!str 85_230.15': ['str', '85_230.15', "85_230.15"] +'!!str FALSE': ['str', 'FALSE', "FALSE"] +'!!str False': ['str', 'False', "False"] +'!!str N': ['str', 'N', "N"] +'!!str NO': ['str', 'NO', "NO"] +'!!str NULL': ['str', 'NULL', "NULL"] +'!!str Null': ['str', 'Null', "Null"] +'!!str OFF': ['str', 'OFF', "OFF"] +'!!str ON': ['str', 'ON', "ON"] +'!!str Off': ['str', 'Off', "Off"] +'!!str On': ['str', 'On', "On"] +'!!str TRUE': ['str', 'TRUE', 'TRUE'] +'!!str True': ['str', 'True', 'True'] +'!!str Y': ['str', 'Y', "Y"] +'!!str YES': ['str', 'YES', "YES"] +'!!str Yes': ['str', 'Yes', "Yes"] +'!!str _._': ['str', '_._', '_._'] +'!!str false': ['str', 'false', "'false'"] +'!!str n': ['str', 'n', "n"] +'!!str no': ['str', 'no', "no"] +'!!str null': ['str', 'null', "'null'"] +'!!str off': ['str', 'off', "off"] +'!!str on': ['str', 'on', "on"] +'!!str true': ['str', 'true', "'true'"] +'!!str y': ['str', 'y', "y"] +'!!str yes': ['str', 'yes', "yes"] +'!!str ~': ['str', '~', '~'] +'#empty': ['str', '', "''"] +'+.INF': ['str', '+.INF', '+.INF'] +'+.Inf': ['str', '+.Inf', '+.Inf'] +'+.inf': ['str', '+.inf', '+.inf'] +'+0': ['str', '+0', '+0'] +'+0.3e+3': ['str', '+0.3e+3', '+0.3e+3'] +'+0.3e3': ['str', '+0.3e3', '+0.3e3'] +'+0100_200': ['str', '+0100_200', '+0100_200'] +'+0b100': ['str', '+0b100', '+0b100'] +'+190:20:30': ['str', '+190:20:30', '+190:20:30'] +'+23': ['str', '+23', '+23'] +'+3.14': ['str', '+3.14', '+3.14'] +'-.INF': ['str', '-.INF', '-.INF'] +'-.Inf': ['str', '-.Inf', '-.Inf'] +'-.inf': ['str', '-.inf', '-.inf'] +'-0': ['int', '0', '0'] +'-0100_200': ['str', '-0100_200', '-0100_200'] +'-0b101': ['str', '-0b101', '-0b101'] +'-0x30': ['str', '-0x30', '-0x30'] +'-190:20:30': ['str', '-190:20:30', '-190:20:30'] +'-23': ['int', '-23', '-23'] +'-3.14': ['float', '-3.14', '-3.14'] +'.': ['str', '.', '.'] +'.0': ['str', '.0', '.0'] +'.14': ['str', '.14', '.14'] +'.1_4': ['str', '.1_4', '.1_4'] +'.3E-1': ['str', '.3E-1', '.3E-1'] +'.3e+3': ['str', '.3e+3', '.3e+3'] +'.3e3': ['str', '.3e3', '.3e3'] +'.INF': ['str', '.INF', '.INF'] +'.Inf': ['str', '.Inf', '.Inf'] +'.NAN': ['str', '.NAN', '.NAN'] +'.NaN': ['str', '.NaN', '.NaN'] +'._': ['str', '._', '._'] +'._14': ['str', '._14', '._14'] +'.inf': ['str', '.inf', '.inf'] +'.nan': ['str', '.nan', '.nan'] +'0': ['int', '0', '0'] +'0.0': ['float', '0.0', '0.0'] +'0.3e3': ['float', '300.0', '300.0'] +'00': ['str', '00', '00'] +'001.23': ['str', '001.23', '001.23'] +'0011': ['str', '0011', '0011'] +'010': ['str', '010', '010'] +'02_0': ['str', '02_0', '02_0'] +'07': ['str', '07', '07'] +'08': ['str', '08', '08'] +'0b0': ['str', '0b0', '0b0'] +'0b100_101': ['str', '0b100_101', '0b100_101'] +'0o0': ['str', '0o0', '0o0'] +'0o10': ['str', '0o10', '0o10'] +'0o7': ['str', '0o7', '0o7'] +'0x0': ['str', '0x0', '0x0'] +'0x10': ['str', '0x10', '0x10'] +'0x2_0': ['str', '0x2_0', '0x2_0'] +'0x42': ['str', '0x42', '0x42'] +'0xa': ['str', '0xa', '0xa'] +'100_000': ['str', '100_000', '100_000'] +'190:20:30': ['str', '190:20:30', '190:20:30'] +'190:20:30.15': ['str', '190:20:30.15', '190:20:30.15'] +'23': ['int', '23', '23'] +'3.': ['float', '3.0', '3.0'] +'3.14': ['float', '3.14', '3.14'] +'3.3e+3': ['float', '3300', '3300.0'] +'3e3': ['float', '3000', '3000.0'] +'85.230_15e+03': ['str', '85.230_15e+03', '85.230_15e+03'] +'85_230.15': ['str', '85_230.15', '85_230.15'] +'FALSE': ['str', 'FALSE', 'FALSE'] +'False': ['str', 'False', 'False'] +'N': ['str', 'N', "N"] +'NO': ['str', 'NO', "NO"] +'NULL': ['str', 'NULL', 'NULL'] +'Null': ['str', 'Null', 'Null'] +'OFF': ['str', 'OFF', "OFF"] +'ON': ['str', 'ON', "ON"] +'Off': ['str', 'Off', "Off"] +'On': ['str', 'On', "On"] +'TRUE': ['str', 'TRUE', 'TRUE'] +'True': ['str', 'True', 'True'] +'Y': ['str', 'Y', "Y"] +'YES': ['str', 'YES', "YES"] +'Yes': ['str', 'Yes', "Yes"] +'_._': ['str', '_._', '_._'] +'false': ['bool', 'false()', 'false'] +'n': ['str', 'n', "n"] +'no': ['str', 'no', "no"] +'null': ['null', 'null()', "null"] +'off': ['str', 'off', "off"] +'on': ['str', 'on', "on"] +'true': ['bool', 'true()', 'true'] +'y': ['str', 'y', "y"] +'yes': ['str', 'yes', "yes"] +'~': ['str', '~', '~'] diff --git a/tests/data/json.schema-skip b/tests/data/json.schema-skip new file mode 100644 index 00000000..d2fe12e4 --- /dev/null +++ b/tests/data/json.schema-skip @@ -0,0 +1,6 @@ +load: { + } +dump: { + '#empty': 1, + '!!str #empty': 1, + } diff --git a/tests/lib/test_schema.py b/tests/lib/test_schema.py index f3370ec1..7bc84b7f 100644 --- a/tests/lib/test_schema.py +++ b/tests/lib/test_schema.py @@ -1,4 +1,5 @@ import yaml +import os import sys import pprint import math @@ -48,6 +49,17 @@ def _fail(input, test): print("Input: >>" + input + "<<") print(test) +class MyCoreLoader(yaml.BaseLoader): pass +class MyJSONLoader(yaml.BaseLoader): pass +class MyCoreDumper(yaml.CommonDumper): pass +class MyJSONDumper(yaml.CommonDumper): pass + +MyCoreLoader.init_tags('core') +MyJSONLoader.init_tags('json') + +MyCoreDumper.init_tags('core') +MyJSONDumper.init_tags('json') + # The tests/data/yaml11.schema file is copied from # https://github.com/perlpunk/yaml-test-schema/blob/master/data/schema-yaml11.yaml def test_implicit_resolver(data_filename, skip_filename, verbose=False): @@ -59,6 +71,19 @@ def test_implicit_resolver(data_filename, skip_filename, verbose=False): 'nan': [float, check_float], 'bool': [bool, check_bool], } + loaders = { + 'yaml11': yaml.SafeLoader, + 'core': MyCoreLoader, + 'json': MyJSONLoader, + } + dumpers = { + 'yaml11': yaml.SafeDumper, + 'core': MyCoreDumper, + 'json': MyJSONDumper, + } + loadername = os.path.splitext(os.path.basename(data_filename))[0] + print('==================') + print(loadername) with open(skip_filename, 'rb') as file: skipdata = yaml.load(file, Loader=yaml.SafeLoader) skip_load = skipdata['load'] @@ -84,7 +109,8 @@ def test_implicit_resolver(data_filename, skip_filename, verbose=False): # Test loading try: - loaded = yaml.safe_load(input) + doc_input = """---\n""" + input + loaded = yaml.load(doc_input, Loader=loaders[loadername]) except: print("Error:", sys.exc_info()[0], '(', sys.exc_info()[1], ')') fail+=1 @@ -106,22 +132,27 @@ def test_implicit_resolver(data_filename, skip_filename, verbose=False): else: t = types[exp_type][0] code = types[exp_type][1] + if isinstance(loaded, t): if code(loaded, data): pass else: fail+=1 + print("Expected data: >>" + str(data) + "<< Got: >>" + str(loaded) + "<<") _fail(input, test) else: fail+=1 + print("Expected type: >>" + exp_type + "<< Got: >>" + str(loaded) + "<<") _fail(input, test) # Skip known dumper bugs if input in skip_dump: continue - dump = yaml.safe_dump(loaded, explicit_end=False) + dump = yaml.dump(loaded, explicit_end=False, Dumper=dumpers[loadername]) # strip trailing newlines and footers + if (dump == '...\n'): + dump = '' if dump.endswith('\n...\n'): dump = dump[:-5] if dump.endswith('\n'): @@ -130,6 +161,11 @@ def test_implicit_resolver(data_filename, skip_filename, verbose=False): pass else: print("Compare: >>" + dump + "<< >>" + exp_dump + "<<") + print(skip_dump) + print(input) + print(test) + print(loaded) + print(type(loaded)) fail+=1 _fail(input, test) From b88bd448ffb645875f1fe9973b4c335966b2328e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tina=20M=C3=BCller?= Date: Thu, 23 Sep 2021 18:47:31 +0200 Subject: [PATCH 7/8] Add experimental wrappers around YAML 1.2 tags code This way people can play with it, and we don't promise this wrapper will stay around forever, and newly created classes CommonDumper/CommonRepresenter aren't exposed. MyCoreLoader = yaml.experimental_12_Core_loader() data = yaml.load(input, Loader=MyCoreLoader) MyCoreDumper = yaml.experimental_12_Core_dumper() out = yaml.dump(data, Dumper=MyCoreDumper) --- lib/yaml/__init__.py | 10 ++++++++++ lib/yaml/dumper.py | 5 +++++ lib/yaml/loader.py | 6 ++++++ 3 files changed, 21 insertions(+) diff --git a/lib/yaml/__init__.py b/lib/yaml/__init__.py index 86d07b55..7ebb5b13 100644 --- a/lib/yaml/__init__.py +++ b/lib/yaml/__init__.py @@ -381,7 +381,17 @@ def add_multi_representer(data_type, multi_representer, Dumper=Dumper): """ Dumper.add_multi_representer(data_type, multi_representer) +def experimental_12_Core_loader(): + return loader._12_CoreLoader +def experimental_12_JSON_loader(): + return loader._12_JSONLoader + +def experimental_12_Core_dumper(): + return dumper._12_CoreDumper +def experimental_12_JSON_dumper(): + return dumper._12_JSONDumper class YAMLObjectMetaclass(type): + """ The metaclass for YAMLObject. """ diff --git a/lib/yaml/dumper.py b/lib/yaml/dumper.py index e78e749e..2b148181 100644 --- a/lib/yaml/dumper.py +++ b/lib/yaml/dumper.py @@ -83,3 +83,8 @@ def __init__(self, stream, default_flow_style=default_flow_style, sort_keys=sort_keys) Resolver.__init__(self) +class _12_CoreDumper(CommonDumper): pass +_12_CoreDumper.init_tags('core') +class _12_JSONDumper(CommonDumper): pass +_12_JSONDumper.init_tags('json') + diff --git a/lib/yaml/loader.py b/lib/yaml/loader.py index 8e2dc6f9..bd18f518 100644 --- a/lib/yaml/loader.py +++ b/lib/yaml/loader.py @@ -67,3 +67,9 @@ def __init__(self, stream): Composer.__init__(self) Constructor.__init__(self) Resolver.__init__(self) + +class _12_CoreLoader(BaseLoader): pass +_12_CoreLoader.init_tags('core') +class _12_JSONLoader(BaseLoader): pass +_12_JSONLoader.init_tags('json') + From 5e566d87c6c04b7ec73d44c0417a48369d2be148 Mon Sep 17 00:00:00 2001 From: Matt Davis Date: Tue, 14 Feb 2023 20:03:56 -0800 Subject: [PATCH 8/8] WIP tagset config * Loader/Dumper config mixins to create dynamic types and configure them at instantiation with generated partials * New `FastestBaseLoader`/`FastestBaseDumper` base classes to auto-select C-back impl if available --- lib/yaml/__init__.py | 33 +++++--- lib/yaml/config.py | 75 +++++++++++++++++ lib/yaml/constructor.py | 93 ++++++++++----------- lib/yaml/cyaml.py | 7 +- lib/yaml/dumper.py | 44 +++++++--- lib/yaml/loader.py | 47 +++++++---- lib/yaml/representer.py | 10 +-- lib/yaml/resolver.py | 177 +++++++++++++++++++++------------------- lib/yaml/tagset.py | 37 +++++++++ yaml/_yaml.pyx | 8 ++ 10 files changed, 352 insertions(+), 179 deletions(-) create mode 100644 lib/yaml/config.py create mode 100644 lib/yaml/tagset.py diff --git a/lib/yaml/__init__.py b/lib/yaml/__init__.py index 7ebb5b13..856caf52 100644 --- a/lib/yaml/__init__.py +++ b/lib/yaml/__init__.py @@ -202,6 +202,7 @@ def emit(events, stream=None, Dumper=Dumper, if stream is None: stream = io.StringIO() getvalue = stream.getvalue + # FIXME: redefine these defaults with sentinels to allow preservation of wrapped config defaults dumper = Dumper(stream, canonical=canonical, indent=indent, width=width, allow_unicode=allow_unicode, line_break=line_break) try: @@ -228,6 +229,7 @@ def serialize_all(nodes, stream=None, Dumper=Dumper, else: stream = io.BytesIO() getvalue = stream.getvalue + # FIXME: redefine these defaults with sentinels to allow preservation of wrapped config defaults dumper = Dumper(stream, canonical=canonical, indent=indent, width=width, allow_unicode=allow_unicode, line_break=line_break, encoding=encoding, version=version, tags=tags, @@ -250,28 +252,35 @@ def serialize(node, stream=None, Dumper=Dumper, **kwds): return serialize_all([node], stream, Dumper=Dumper, **kwds) def dump_all(documents, stream=None, Dumper=Dumper, - default_style=None, default_flow_style=False, - canonical=None, indent=None, width=None, - allow_unicode=None, line_break=None, - encoding=None, explicit_start=None, explicit_end=None, - version=None, tags=None, sort_keys=True): + default_style=..., default_flow_style=..., + canonical=..., indent=..., width=..., + allow_unicode=..., line_break=..., + encoding=..., explicit_start=..., explicit_end=..., + version=..., tags=..., sort_keys=..., **kwargs): """ Serialize a sequence of Python objects into a YAML stream. If stream is None, return the produced string instead. """ getvalue = None if stream is None: - if encoding is None: + if encoding is None or encoding is ...: stream = io.StringIO() else: stream = io.BytesIO() getvalue = stream.getvalue - dumper = Dumper(stream, default_style=default_style, - default_flow_style=default_flow_style, - canonical=canonical, indent=indent, width=width, - allow_unicode=allow_unicode, line_break=line_break, - encoding=encoding, version=version, tags=tags, - explicit_start=explicit_start, explicit_end=explicit_end, sort_keys=sort_keys) + + # preserve wrapped config defaults for values where we didn't get a default + # FIXME: share this code with the one in config mixin + dumper_init_kwargs = dict( + default_style=default_style, + default_flow_style=default_flow_style, + canonical=canonical, indent=indent, width=width, + allow_unicode=allow_unicode, line_break=line_break, + encoding=encoding, version=version, tags=tags, + explicit_start=explicit_start, explicit_end=explicit_end, sort_keys=sort_keys, **kwargs) + + dumper_init_kwargs = {k: v for k, v in dumper_init_kwargs.items() if v is not ...} + dumper = Dumper(stream, **dumper_init_kwargs) try: dumper.open() for data in documents: diff --git a/lib/yaml/config.py b/lib/yaml/config.py new file mode 100644 index 00000000..df8aeee8 --- /dev/null +++ b/lib/yaml/config.py @@ -0,0 +1,75 @@ +from __future__ import annotations + +import typing as t + +from functools import partialmethod +from .tagset import TagSet + +T = t.TypeVar('T') + + +class LoaderConfigMixin: + @classmethod + # FIXME: fix tagset type to use DataClasses, at least externally? + def config(cls: type[T], type_name: str | None = None, tagset: TagSet | ... = ..., **kwargs) -> type[T]: + if not type_name: + # FIXME: hash the inputs for a dynamic type name and cache it? + type_name = f'abcd_from_{cls.__name__}' + + new_type = t.cast(cls, type(type_name, (cls, ), {})) + + # FIXME: add support for arbitrary kwargs passthru ala dumper? + + if tagset is not ...: + # FIXME: provide a base class hook/method for this reset + new_type.yaml_implicit_resolvers = {} + new_type.init_resolvers(tagset.resolvers) + new_type.yaml_constructors = {} + new_type.init_constructors(tagset.constructors) + + return new_type + + +class DumperConfigMixin: + @classmethod + def config(cls: type[T], type_name: str | None = None, + tagset: TagSet | ... = ..., + # FIXME: make some of the more obscure style things "nicer" (eg enums?) or just pass through existing values? + default_style: str | ... = ..., default_flow_style: bool | ... = ..., + # FIXME: properly type-annotate the rest of these + canonical=..., indent=..., width=..., + allow_unicode=..., line_break=..., + encoding=..., explicit_start=..., explicit_end=..., + version=..., tags=..., sort_keys=..., + **kwargs) -> type[T]: + + if not type_name: + # FIXME: hash the inputs for a dynamic type name and cache it? + type_name = f'abcd_from_{cls.__name__}' + + # preserve wrapped config defaults for values where we didn't get a default + # FIXME: share this code with the one in __init__.dump_all (and implement on others) + dumper_init_kwargs = dict( + default_style=default_style, + default_flow_style=default_flow_style, + canonical=canonical, indent=indent, width=width, + allow_unicode=allow_unicode, line_break=line_break, + encoding=encoding, version=version, tags=tags, + explicit_start=explicit_start, explicit_end=explicit_end, sort_keys=sort_keys, **kwargs) + + dumper_init_kwargs = {k: v for k, v in dumper_init_kwargs.items() if v is not ...} + + patched_init = partialmethod(cls.__init__, + **dumper_init_kwargs) + + new_type = t.cast(cls, type(type_name, (cls, ), {'__init__': patched_init})) + + # FIXME: support all the dynamic dispatch types (multi*, etc) + if tagset is not ...: + # FIXME: provide a base class hook/method for this reset + new_type.yaml_implicit_resolvers = {} + new_type.init_resolvers(tagset.resolvers) + new_type.yaml_representers = {} + new_type.init_representers(tagset.representers) + + return new_type diff --git a/lib/yaml/constructor.py b/lib/yaml/constructor.py index f26e0cdd..73f421d4 100644 --- a/lib/yaml/constructor.py +++ b/lib/yaml/constructor.py @@ -12,6 +12,7 @@ from .nodes import * import collections.abc, datetime, base64, binascii, re, sys, types +import typing as t class ConstructorError(MarkedYAMLError): pass @@ -267,15 +268,11 @@ def add_multi_constructor(cls, tag_prefix, multi_constructor): @classmethod - def init_constructors(cls, tagset): - if tagset not in _constructors: - return - for key in _constructors[tagset]: - callback = _constructors[tagset][key] - if (key is None): - cls.add_constructor(key, callback) - else: - cls.add_constructor('tag:yaml.org,2002:' + key, callback) + def init_constructors(cls, tagset: dict[str, t.Callable]): + for type_name, constructor in tagset.items(): + # FIXME: encode full tag names in TagSets to avoid this logic and make user-definable types easier + tag_name = f'tag:yaml.org,2002:{type_name}' if type_name else None + cls.add_constructor(tag_name, constructor) # SafeConstructor implements YAML 1.1 @@ -499,45 +496,45 @@ def construct_yaml_object(self, node, cls): data.__dict__.update(state) -_constructors = { - 'yaml11': { - 'str': BaseConstructor.construct_yaml_str, - 'seq': BaseConstructor.construct_yaml_seq, - 'map': BaseConstructor.construct_yaml_map, - None: BaseConstructor.construct_undefined, - 'int': SafeConstructor.construct_yaml_int, - 'float': SafeConstructor.construct_yaml_float, - 'null': BaseConstructor.construct_yaml_null, - 'bool': BaseConstructor.construct_yaml_bool, - 'binary': SafeConstructor.construct_yaml_binary, - 'timestamp': SafeConstructor.construct_yaml_timestamp, - 'omap': SafeConstructor.construct_yaml_omap, - 'pairs': SafeConstructor.construct_yaml_pairs, - 'set': SafeConstructor.construct_yaml_set, - }, - 'core': { - 'str': BaseConstructor.construct_yaml_str, - 'seq': BaseConstructor.construct_yaml_seq, - 'map': BaseConstructor.construct_yaml_map, - None: BaseConstructor.construct_undefined, - 'int': BaseConstructor.construct_yaml_int_core, - 'float': BaseConstructor.construct_yaml_float_core, - 'null': BaseConstructor.construct_yaml_null, - 'bool': BaseConstructor.construct_yaml_bool, - }, - 'json': { - 'str': BaseConstructor.construct_yaml_str, - 'seq': BaseConstructor.construct_yaml_seq, - 'map': BaseConstructor.construct_yaml_map, - None: BaseConstructor.construct_undefined, - 'int': BaseConstructor.construct_yaml_int_json, - 'float': BaseConstructor.construct_yaml_float_json, - 'null': BaseConstructor.construct_yaml_null, - 'bool': BaseConstructor.construct_yaml_bool, - }, -} - -SafeConstructor.init_constructors('yaml11') +_yaml11_constructors = { + 'str': BaseConstructor.construct_yaml_str, + 'seq': BaseConstructor.construct_yaml_seq, + 'map': BaseConstructor.construct_yaml_map, + None: BaseConstructor.construct_undefined, + 'int': SafeConstructor.construct_yaml_int, + 'float': SafeConstructor.construct_yaml_float, + 'null': BaseConstructor.construct_yaml_null, + 'bool': BaseConstructor.construct_yaml_bool, + 'binary': SafeConstructor.construct_yaml_binary, + 'timestamp': SafeConstructor.construct_yaml_timestamp, + 'omap': SafeConstructor.construct_yaml_omap, + 'pairs': SafeConstructor.construct_yaml_pairs, + 'set': SafeConstructor.construct_yaml_set, + } + +_core_constructors = { + 'str': BaseConstructor.construct_yaml_str, + 'seq': BaseConstructor.construct_yaml_seq, + 'map': BaseConstructor.construct_yaml_map, + None: BaseConstructor.construct_undefined, + 'int': BaseConstructor.construct_yaml_int_core, + 'float': BaseConstructor.construct_yaml_float_core, + 'null': BaseConstructor.construct_yaml_null, + 'bool': BaseConstructor.construct_yaml_bool, + } + +_json_constructors = { + 'str': BaseConstructor.construct_yaml_str, + 'seq': BaseConstructor.construct_yaml_seq, + 'map': BaseConstructor.construct_yaml_map, + None: BaseConstructor.construct_undefined, + 'int': BaseConstructor.construct_yaml_int_json, + 'float': BaseConstructor.construct_yaml_float_json, + 'null': BaseConstructor.construct_yaml_null, + 'bool': BaseConstructor.construct_yaml_bool, + } + +SafeConstructor.init_constructors(_yaml11_constructors) class FullConstructor(SafeConstructor): # 'extend' is blacklisted because it is used by diff --git a/lib/yaml/cyaml.py b/lib/yaml/cyaml.py index 0c213458..9019209e 100644 --- a/lib/yaml/cyaml.py +++ b/lib/yaml/cyaml.py @@ -6,6 +6,7 @@ from yaml._yaml import CParser, CEmitter +from .config import LoaderConfigMixin, DumperConfigMixin from .constructor import * from .serializer import * @@ -13,7 +14,7 @@ from .resolver import * -class CBaseLoader(CParser, BaseConstructor, BaseResolver): +class CBaseLoader(CParser, BaseConstructor, BaseResolver, LoaderConfigMixin): def __init__(self, stream): CParser.__init__(self, stream) @@ -41,7 +42,7 @@ def __init__(self, stream): UnsafeConstructor.__init__(self) Resolver.__init__(self) -class CLoader(CParser, Constructor, Resolver): +class CLoader(CParser, Constructor, Resolver, LoaderConfigMixin): def __init__(self, stream): CParser.__init__(self, stream) @@ -82,7 +83,7 @@ def __init__(self, stream, default_flow_style=default_flow_style, sort_keys=sort_keys) Resolver.__init__(self) -class CDumper(CEmitter, Serializer, Representer, Resolver): +class CDumper(CEmitter, Serializer, Representer, Resolver, DumperConfigMixin): def __init__(self, stream, default_style=None, default_flow_style=False, diff --git a/lib/yaml/dumper.py b/lib/yaml/dumper.py index 2b148181..1c5c236e 100644 --- a/lib/yaml/dumper.py +++ b/lib/yaml/dumper.py @@ -1,12 +1,21 @@ -__all__ = ['BaseDumper', 'SafeDumper', 'Dumper', 'CommonDumper'] +__all__ = ['BaseDumper', 'SafeDumper', 'Dumper', 'CommonDumper', 'FastestBaseDumper'] +from . import tagset + +from .config import DumperConfigMixin from .emitter import * from .serializer import * from .representer import * from .resolver import * -class BaseDumper(Emitter, Serializer, BaseRepresenter, BaseResolver): +try: + from .cyaml import CDumper as FastestBaseDumper +except ImportError as ie: + FastestBaseDumper = None + + +class BaseDumper(Emitter, Serializer, BaseRepresenter, BaseResolver, DumperConfigMixin): def __init__(self, stream, default_style=None, default_flow_style=False, @@ -24,7 +33,17 @@ def __init__(self, stream, default_flow_style=default_flow_style, sort_keys=sort_keys) Resolver.__init__(self) -class SafeDumper(Emitter, Serializer, SafeRepresenter, Resolver): + +if not FastestBaseDumper: + # fall back to pure-Python if CBaseDumper is unavailable + FastestBaseDumper = BaseDumper + +# FIXME: reimplement all these as config calls, eg: +# SafeDumper = FastestBaseDumper.config(type_name='SafeDumper', tagset=tagset.yaml11) + + + +class SafeDumper(Emitter, Serializer, SafeRepresenter, Resolver, DumperConfigMixin): def __init__(self, stream, default_style=None, default_flow_style=False, @@ -42,7 +61,7 @@ def __init__(self, stream, default_flow_style=default_flow_style, sort_keys=sort_keys) Resolver.__init__(self) -class CommonDumper(Emitter, Serializer, CommonRepresenter, BaseResolver): +class CommonDumper(Emitter, Serializer, CommonRepresenter, BaseResolver, DumperConfigMixin): def __init__(self, stream, default_style=None, default_flow_style=False, @@ -61,11 +80,12 @@ def __init__(self, stream, BaseResolver.__init__(self) @classmethod - def init_tags(cls, tagset): - cls.init_representers(tagset) - cls.init_resolvers(tagset) + def init_tags(cls, tagset: tagset.TagSet): + cls.init_representers(tagset.representers) + cls.init_resolvers(tagset.resolvers) + -class Dumper(Emitter, Serializer, Representer, Resolver): +class Dumper(Emitter, Serializer, Representer, Resolver, DumperConfigMixin): def __init__(self, stream, default_style=None, default_flow_style=False, @@ -83,8 +103,8 @@ def __init__(self, stream, default_flow_style=default_flow_style, sort_keys=sort_keys) Resolver.__init__(self) -class _12_CoreDumper(CommonDumper): pass -_12_CoreDumper.init_tags('core') -class _12_JSONDumper(CommonDumper): pass -_12_JSONDumper.init_tags('json') + +_12_CoreDumper = CommonDumper.config(type_name='_12_CoreDumper', tagset=tagset.core) +_12_JSONDumper = CommonDumper.config(type_name='_12_JSONDumper', tagset=tagset.json) + diff --git a/lib/yaml/loader.py b/lib/yaml/loader.py index bd18f518..61ab4c94 100644 --- a/lib/yaml/loader.py +++ b/lib/yaml/loader.py @@ -1,16 +1,24 @@ +from __future__ import annotations -__all__ = ['BaseLoader', 'FullLoader', 'SafeLoader', 'Loader', 'UnsafeLoader', - ] +__all__ = ['BaseLoader', 'FullLoader', 'SafeLoader', 'Loader', 'UnsafeLoader', 'FastestBaseLoader'] +from .config import LoaderConfigMixin from .reader import * from .scanner import * from .parser import * from .composer import * from .constructor import * from .resolver import * +from . import tagset -class BaseLoader(Reader, Scanner, Parser, Composer, BaseConstructor, BaseResolver): +# FIXME: defer setup to fix circular import +try: + from .cyaml import CBaseLoader as FastestBaseLoader +except ImportError as ie: + FastestBaseLoader = None + +class BaseLoader(Reader, Scanner, Parser, Composer, BaseConstructor, BaseResolver, LoaderConfigMixin): def __init__(self, stream): Reader.__init__(self, stream) Scanner.__init__(self) @@ -20,11 +28,23 @@ def __init__(self, stream): BaseResolver.__init__(self) @classmethod - def init_tags(cls, tagset): - cls.init_constructors(tagset) - cls.init_resolvers(tagset) + def init_tags(cls, tagset: tagset.TagSet): + cls.init_constructors(tagset.constructors) + cls.init_resolvers(tagset.resolvers) + + +if not FastestBaseLoader: + # fall back to pure Python if CBaseLoader is unavailable + FastestBaseLoader = BaseLoader -class FullLoader(Reader, Scanner, Parser, Composer, FullConstructor, Resolver): + +# FIXME: once we fully implement the config stuff, these can all be collapsed to a config call, eg: +# FullLoader = FastestBaseLoader.config(type_name='FullLoader', tagset=tagset.yaml11 | tagset.python_full) +# SafeLoader = FastestBaseLoader.config(type_name='SafeLoader', tagset=tagset.yaml11) +# UnsafeLoader = FastestBaseLoader.config(type_name='UnsafeLoader', tagset=tagset.yaml11 | tagset.python_unsafe) +# this pattern will also allow a much easier path for users to bolt on default behavior to any old loader + +class FullLoader(Reader, Scanner, Parser, Composer, FullConstructor, Resolver, LoaderConfigMixin): def __init__(self, stream): Reader.__init__(self, stream) @@ -34,7 +54,7 @@ def __init__(self, stream): FullConstructor.__init__(self) Resolver.__init__(self) -class SafeLoader(Reader, Scanner, Parser, Composer, SafeConstructor, Resolver): +class SafeLoader(Reader, Scanner, Parser, Composer, SafeConstructor, Resolver, LoaderConfigMixin): def __init__(self, stream): Reader.__init__(self, stream) @@ -44,7 +64,7 @@ def __init__(self, stream): SafeConstructor.__init__(self) Resolver.__init__(self) -class Loader(Reader, Scanner, Parser, Composer, Constructor, Resolver): +class Loader(Reader, Scanner, Parser, Composer, Constructor, Resolver, LoaderConfigMixin): def __init__(self, stream): Reader.__init__(self, stream) @@ -58,7 +78,7 @@ def __init__(self, stream): # untrusted input). Use of either Loader or UnsafeLoader should be rare, since # FullLoad should be able to load almost all YAML safely. Loader is left intact # to ensure backwards compatibility. -class UnsafeLoader(Reader, Scanner, Parser, Composer, Constructor, Resolver): +class UnsafeLoader(Reader, Scanner, Parser, Composer, Constructor, Resolver, LoaderConfigMixin): def __init__(self, stream): Reader.__init__(self, stream) @@ -68,8 +88,7 @@ def __init__(self, stream): Constructor.__init__(self) Resolver.__init__(self) -class _12_CoreLoader(BaseLoader): pass -_12_CoreLoader.init_tags('core') -class _12_JSONLoader(BaseLoader): pass -_12_JSONLoader.init_tags('json') + +_12_CoreLoader = BaseLoader.config(type_name='_12_CoreLoader', tagset=tagset.core) +_12_JSONLoader = BaseLoader.config(type_name='_12_JSONLoader', tagset=tagset.json) diff --git a/lib/yaml/representer.py b/lib/yaml/representer.py index aa9360d7..70905a3c 100644 --- a/lib/yaml/representer.py +++ b/lib/yaml/representer.py @@ -6,6 +6,7 @@ from .nodes import * import datetime, copyreg, types, base64, collections +import typing as t class RepresenterError(YAMLError): pass @@ -215,10 +216,9 @@ def represent_undefined(self, data): raise RepresenterError("cannot represent an object", data) @classmethod - def init_representers(cls, name): - for key in _representers[name]: - callback = _representers[name][key] - cls.add_representer(key, callback) + def init_representers(cls, tagset_representers: dict[str, t.Callable]): + for type_name, representer in tagset_representers.items(): + cls.add_representer(type_name, representer) class CommonRepresenter(BaseRepresenter): @@ -295,7 +295,7 @@ def represent_yaml_object(self, tag, data, cls, flow_style=None): } -SafeRepresenter.init_representers('yaml11') +SafeRepresenter.init_representers(_representers['yaml11']) class Representer(SafeRepresenter): diff --git a/lib/yaml/resolver.py b/lib/yaml/resolver.py index 11248ca6..e85500d7 100644 --- a/lib/yaml/resolver.py +++ b/lib/yaml/resolver.py @@ -1,10 +1,11 @@ -__all__ = ['BaseResolver', 'Resolver' ] +__all__ = ['BaseResolver', 'Resolver'] + +import re from .error import * from .nodes import * -import re class ResolverError(YAMLError): pass @@ -165,95 +166,101 @@ def resolve(self, kind, value, implicit): return self.DEFAULT_MAPPING_TAG @classmethod - def init_resolvers(cls, key): - for args in _resolvers[key]: + def init_resolvers(cls, implicit_resolvers: list[list]): + # FIXME: factor this out as a config mixin with dataclasses + for args in implicit_resolvers: cls.add_implicit_resolver( 'tag:yaml.org,2002:' + args[0], args[1], args[2] ) -_resolvers = { - 'yaml11': [ - ['bool', - re.compile(r'''^(?:yes|Yes|YES|no|No|NO - |true|True|TRUE|false|False|FALSE - |on|On|ON|off|Off|OFF)$''', re.X), - list('yYnNtTfFoO')], - ['float', - re.compile(r'''^(?:[-+]?(?:[0-9][0-9_]*)\.[0-9_]*(?:[eE][-+][0-9]+)? - |\.[0-9_]+(?:[eE][-+][0-9]+)? - |[-+]?[0-9][0-9_]*(?::[0-5]?[0-9])+\.[0-9_]* - |[-+]?\.(?:inf|Inf|INF) - |\.(?:nan|NaN|NAN))$''', re.X), - list('-+0123456789.')], - ['int', - re.compile(r'''^(?:[-+]?0b[0-1_]+ - |[-+]?0[0-7_]+ - |[-+]?(?:0|[1-9][0-9_]*) - |[-+]?0x[0-9a-fA-F_]+ - |[-+]?[1-9][0-9_]*(?::[0-5]?[0-9])+)$''', re.X), - list('-+0123456789')], - ['merge', - re.compile(r'^(?:<<)$'), - ['<']], - ['null', - re.compile(r'''^(?: ~ - |null|Null|NULL - | )$''', re.X), - ['~', 'n', 'N', '']], - ['timestamp', - re.compile(r'''^(?:[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9] - |[0-9][0-9][0-9][0-9] -[0-9][0-9]? -[0-9][0-9]? - (?:[Tt]|[ \t]+)[0-9][0-9]? - :[0-9][0-9] :[0-9][0-9] (?:\.[0-9]*)? - (?:[ \t]*(?:Z|[-+][0-9][0-9]?(?::[0-9][0-9])?))?)$''', re.X), - list('0123456789')], - ['value', - re.compile(r'^(?:=)$'), - ['=']], -# The following resolver is only for documentation purposes. It cannot work -# because plain scalars cannot start with '!', '&', or '*'. - ['yaml', - re.compile(r'^(?:!|&|\*)$'), - list('!&*')], - ], - 'core': [ - ['bool', - re.compile(r'''^(?:|true|True|TRUE|false|False|FALSE)$''', re.X), - list('tTfF')], - ['int', - re.compile(r'''^(?: - |0o[0-7]+ - |[-+]?(?:[0-9]+) - |0x[0-9a-fA-F]+ - )$''', re.X), - list('-+0123456789')], - ['float', - re.compile(r'''^(?:[-+]?(?:\.[0-9]+|[0-9]+(\.[0-9]*)?)(?:[eE][-+]?[0-9]+)? - |[-+]?\.(?:inf|Inf|INF) - |\.(?:nan|NaN|NAN))$''', re.X), - list('-+0123456789.')], - ['null', - re.compile(r'''^(?:~||null|Null|NULL)$''', re.X), - ['~', 'n', 'N', '']], - ], - 'json': [ - ['bool', - re.compile(r'''^(?:true|false)$''', re.X), - list('tf')], - ['int', - re.compile(r'''^-?(?:0|[1-9][0-9]*)$''', re.X), - list('-0123456789')], - ['float', - re.compile(r'''^-?(?:0|[1-9][0-9]*)(\.[0-9]*)?(?:[eE][-+]?[0-9]+)?$''', re.X), - list('-0123456789.')], - ['null', - re.compile(r'''^null$''', re.X), - ['n']], - ], -} + + +# FIXME: come up with a better way to lazy init these so we can define all the well-known stuff in one place +# avoid circular import by defining here for now +_yaml11_resolvers = [ + ['bool', + re.compile(r'''^(?:yes|Yes|YES|no|No|NO + |true|True|TRUE|false|False|FALSE + |on|On|ON|off|Off|OFF)$''', re.X), + list('yYnNtTfFoO')], + ['float', + re.compile(r'''^(?:[-+]?(?:[0-9][0-9_]*)\.[0-9_]*(?:[eE][-+][0-9]+)? + |\.[0-9_]+(?:[eE][-+][0-9]+)? + |[-+]?[0-9][0-9_]*(?::[0-5]?[0-9])+\.[0-9_]* + |[-+]?\.(?:inf|Inf|INF) + |\.(?:nan|NaN|NAN))$''', re.X), + list('-+0123456789.')], + ['int', + re.compile(r'''^(?:[-+]?0b[0-1_]+ + |[-+]?0[0-7_]+ + |[-+]?(?:0|[1-9][0-9_]*) + |[-+]?0x[0-9a-fA-F_]+ + |[-+]?[1-9][0-9_]*(?::[0-5]?[0-9])+)$''', re.X), + list('-+0123456789')], + ['merge', + re.compile(r'^(?:<<)$'), + ['<']], + ['null', + re.compile(r'''^(?: ~ + |null|Null|NULL + | )$''', re.X), + ['~', 'n', 'N', '']], + ['timestamp', + re.compile(r'''^(?:[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9] + |[0-9][0-9][0-9][0-9] -[0-9][0-9]? -[0-9][0-9]? + (?:[Tt]|[ \t]+)[0-9][0-9]? + :[0-9][0-9] :[0-9][0-9] (?:\.[0-9]*)? + (?:[ \t]*(?:Z|[-+][0-9][0-9]?(?::[0-9][0-9])?))?)$''', re.X), + list('0123456789')], + ['value', + re.compile(r'^(?:=)$'), + ['=']], + # The following resolver is only for documentation purposes. It cannot work + # because plain scalars cannot start with '!', '&', or '*'. + ['yaml', + re.compile(r'^(?:!|&|\*)$'), + list('!&*')], +] + +_core_resolvers = [ + ['bool', + re.compile(r'''^(?:|true|True|TRUE|false|False|FALSE)$''', re.X), + list('tTfF')], + ['int', + re.compile(r'''^(?: + |0o[0-7]+ + |[-+]?(?:[0-9]+) + |0x[0-9a-fA-F]+ + )$''', re.X), + list('-+0123456789')], + ['float', + re.compile(r'''^(?:[-+]?(?:\.[0-9]+|[0-9]+(\.[0-9]*)?)(?:[eE][-+]?[0-9]+)? + |[-+]?\.(?:inf|Inf|INF) + |\.(?:nan|NaN|NAN))$''', re.X), + list('-+0123456789.')], + ['null', + re.compile(r'''^(?:~||null|Null|NULL)$''', re.X), + ['~', 'n', 'N', '']], + ] + +_json_resolvers = [ + ['bool', + re.compile(r'''^(?:true|false)$''', re.X), + list('tf')], + ['int', + re.compile(r'''^-?(?:0|[1-9][0-9]*)$''', re.X), + list('-0123456789')], + ['float', + re.compile(r'''^-?(?:0|[1-9][0-9]*)(\.[0-9]*)?(?:[eE][-+]?[0-9]+)?$''', re.X), + list('-0123456789.')], + ['null', + re.compile(r'''^null$''', re.X), + ['n']], + ] class Resolver(BaseResolver): pass -Resolver.init_resolvers('yaml11') +# FIXME: config mixin or defer this to loader setup via config? +Resolver.init_resolvers(_yaml11_resolvers) diff --git a/lib/yaml/tagset.py b/lib/yaml/tagset.py new file mode 100644 index 00000000..f04d78aa --- /dev/null +++ b/lib/yaml/tagset.py @@ -0,0 +1,37 @@ +import re +import typing as t + +from dataclasses import dataclass +# FIXME: disentangle various class inits so we can define these all together +from .constructor import _core_constructors, _json_constructors, _yaml11_constructors +from .representer import _representers # FIXME make consistent +from .resolver import _core_resolvers, _json_resolvers, _yaml11_resolvers + +# FIXME: restructure these as dataclasses for each type that are individually accessible to make it easier to build up a custom tag set with well-known types + +# FIXME: add Python tags here as well + +@dataclass +class TagSet: + name: str + constructors: dict[str, t.Callable] # FIXME: implement constructor dataclass? + representers: dict[str, t.Callable] # FIXME: implement representer dataclass? + resolvers: list[list] # FIXME: implement ImplicitResolver dataclass + # FIXME: add support for multi/implicit constructors, representers, resolvers, etc? + + +yaml11 = TagSet(name='yaml11', + constructors=_yaml11_constructors, + representers=_representers['yaml11'], + resolvers=_yaml11_resolvers) + +core = TagSet(name='core', + constructors=_core_constructors, + representers=_representers['core'], + resolvers=_core_resolvers) + +json = TagSet(name='json', + constructors=_json_constructors, + representers=_representers['json'], + resolvers=_json_resolvers) + diff --git a/yaml/_yaml.pyx b/yaml/_yaml.pyx index ff4efe80..47dedab5 100644 --- a/yaml/_yaml.pyx +++ b/yaml/_yaml.pyx @@ -1,6 +1,14 @@ import yaml +# FIXME: change all the below stuff to from X import Y to avoid this +# import submodules explicitly since we're getting loaded earlier +import yaml.emitter +import yaml.parser +import yaml.reader +import yaml.scanner +import yaml.serializer + def get_version_string(): cdef char *value value = yaml_get_version_string()