From bcb02dc61812f159aee7b0327e92fabc5d79214d Mon Sep 17 00:00:00 2001 From: Joseph Brill <48932340+jcbrill@users.noreply.github.com> Date: Sat, 14 Oct 2023 17:19:59 -0400 Subject: [PATCH 1/3] Replace process_path with normalize_path in MSCommon/MSVC/Util.py Changes: * Replace process_path function with resolve_path and normalize_path functions in MSCommon/MSVC/Util.py. * Replace process_path invocations with normalize_path invocations. * Protect against inadvertent resolve/realpath for windows drive specifications in resolve_path and normalize_path. * Additional options for normalize_path with defaults consistent with process_path. --- SCons/Tool/MSCommon/MSVC/Registry.py | 4 +- SCons/Tool/MSCommon/MSVC/Util.py | 87 +++++++++++++++++++++- SCons/Tool/MSCommon/MSVC/UtilTests.py | 103 +++++++++++++++++++++++--- SCons/Tool/MSCommon/MSVC/WinSDK.py | 4 +- 4 files changed, 180 insertions(+), 18 deletions(-) diff --git a/SCons/Tool/MSCommon/MSVC/Registry.py b/SCons/Tool/MSCommon/MSVC/Registry.py index eee20ccbc7..f9e544c6fc 100644 --- a/SCons/Tool/MSCommon/MSVC/Registry.py +++ b/SCons/Tool/MSCommon/MSVC/Registry.py @@ -62,7 +62,7 @@ def registry_query_path(key, val, suffix, expand: bool=True): extval = val + '\\' + suffix if suffix else val qpath = read_value(key, extval, expand=expand) if qpath and os.path.exists(qpath): - qpath = Util.process_path(qpath) + qpath = Util.normalize_path(qpath) else: qpath = None return (qpath, key, val, extval) @@ -81,7 +81,7 @@ def microsoft_query_paths(suffix, usrval=None, expand: bool=True): extval = val + '\\' + suffix if suffix else val qpath = read_value(key, extval, expand=expand) if qpath and os.path.exists(qpath): - qpath = Util.process_path(qpath) + qpath = Util.normalize_path(qpath) if qpath not in paths: paths.append(qpath) records.append((qpath, key, val, extval, usrval)) diff --git a/SCons/Tool/MSCommon/MSVC/Util.py b/SCons/Tool/MSCommon/MSVC/Util.py index 64b8d673eb..8afe784ac5 100644 --- a/SCons/Tool/MSCommon/MSVC/Util.py +++ b/SCons/Tool/MSCommon/MSVC/Util.py @@ -26,16 +26,25 @@ """ import os +import pathlib import re from collections import ( namedtuple, ) +from ..common import debug + from . import Config # path utilities +# windows drive specification (e.g., 'C:') +_RE_DRIVESPEC = re.compile(r'^[A-Za-z][:]$', re.IGNORECASE) + +# windows path separators +_OS_PATH_SEPS = ('\\', '/') + def listdir_dirs(p): """ Return a list of tuples for each subdirectory of the given directory path. @@ -57,22 +66,92 @@ def listdir_dirs(p): dirs.append((dir_name, dir_path)) return dirs -def process_path(p): +def resolve_path(p, ignore_drivespec=True): """ - Normalize a system path + Make path absolute resolving any symlinks Args: p: str system path + ignore_drivespec: bool + ignore drive specifications when True Returns: - str: normalized system path + str: absolute path with symlinks resolved """ + if p: + + if ignore_drivespec and _RE_DRIVESPEC.match(p): + # don't attempt to resolve drive specification (e.g., C:) + pass + else: + # both abspath and resolve necessary to produce identical path + # when (unqualified) file name is on a mapped network drive for + # python 3.6 and 3.11 + p = os.path.abspath(p) + try: + p = str(pathlib.Path(p).resolve()) + except OSError as e: + debug( + 'caught exception: path=%s, exception=%s(%s)', + repr(p), type(e).__name__, repr(str(e)) + ) + + return p + +def normalize_path( + p, + strip=True, + preserve_trailing=False, + expand=False, + realpath=True, + ignore_drivespec=True, +): + """ + Normalize path + + Args: + p: str + system path + strip: bool + remove leading and trailing whitespace when True + preserve_trailing: bool + preserve trailing path separator when True + expand: bool + apply expanduser and expandvars when True + realpath: bool + make the path absolute resolving any symlinks when True + ignore_drivespec: bool + ignore drive specifications for realpath when True + + Returns: + str: normalized path + + """ + + if p and strip: + p = p.strip() + + if p: + + trailing = bool(preserve_trailing and p.endswith(_OS_PATH_SEPS)) + + if expand: + p = os.path.expanduser(p) + p = os.path.expandvars(p) + p = os.path.normpath(p) - p = os.path.realpath(p) + + if realpath: + p = resolve_path(p, ignore_drivespec=ignore_drivespec) + p = os.path.normcase(p) + + if trailing: + p += os.path.sep + return p # msvc version and msvc toolset version regexes diff --git a/SCons/Tool/MSCommon/MSVC/UtilTests.py b/SCons/Tool/MSCommon/MSVC/UtilTests.py index 36e08f5eb1..d92d2ca5c8 100644 --- a/SCons/Tool/MSCommon/MSVC/UtilTests.py +++ b/SCons/Tool/MSCommon/MSVC/UtilTests.py @@ -28,14 +28,46 @@ import unittest import os import re +import sys +import pathlib from SCons.Tool.MSCommon.MSVC import Config from SCons.Tool.MSCommon.MSVC import Util from SCons.Tool.MSCommon.MSVC import WinSDK +def resolve(p): + p = os.path.abspath(p) + p = str(pathlib.Path(p).resolve()) + return p + +def normalize(*comps): + p = os.path.join(*comps) + p = os.path.normpath(p) + p = os.path.normcase(p) + return os.path.normcase(p) + class Data: - UTIL_PARENT_DIR = os.path.join(os.path.dirname(Util.__file__), os.pardir) + IS_WINDOWS = sys.platform == 'win32' + + CWD = os.getcwd() + + UTIL_MODULE = os.path.dirname(Util.__file__) + UTIL_MODULE_PARENT = os.path.join(UTIL_MODULE, os.pardir) + + HOME = pathlib.Path.home() + HOMEDRIVE = HOME.drive + HOMEPATH = str(HOME) + + REALPATH_CWD = resolve(CWD) + + REALPATH_UTIL_MODULE = resolve(UTIL_MODULE) + REALPATH_UTIL_MODULE_PARENT = resolve(UTIL_MODULE_PARENT) + + REALPATH_HOMEPATH = resolve(HOMEPATH) + REALPATH_HOMEPATH_PARENT = resolve(os.path.join(HOMEPATH, os.pardir)) + REALPATH_HOMEDRIVE = resolve(HOMEDRIVE) + REALPATH_HOMEDRIVE_CWD = resolve(HOMEDRIVE) class UtilTests(unittest.TestCase): @@ -43,21 +75,72 @@ def test_listdir_dirs(self) -> None: func = Util.listdir_dirs for dirname, expect in [ (None, False), ('', False), ('doesnotexist.xyz.abc', False), - (Data.UTIL_PARENT_DIR, True), + (Data.UTIL_MODULE_PARENT, True), ]: dirs = func(dirname) self.assertTrue((len(dirs) > 0) == expect, "{}({}): {}".format( func.__name__, repr(dirname), 'list is empty' if expect else 'list is not empty' )) - def test_process_path(self) -> None: - func = Util.process_path - for p, expect in [ - (None, True), ('', True), - ('doesnotexist.xyz.abc', False), (Data.UTIL_PARENT_DIR, False), - ]: - rval = func(p) - self.assertTrue((p == rval) == expect, "{}({}): {}".format( + def test_resolve_path(self) -> None: + func = Util.resolve_path + # default kwargs: + # ignore_drivespec=True + test_list = [ + (Data.UTIL_MODULE, Data.REALPATH_UTIL_MODULE, {}), + (os.path.join(Data.UTIL_MODULE, os.pardir), Data.REALPATH_UTIL_MODULE_PARENT, {}), + (Data.HOMEPATH, Data.REALPATH_HOMEPATH, {}), + (os.path.join(Data.HOMEPATH, os.pardir), Data.REALPATH_HOMEPATH_PARENT, {}), + ] + if Data.IS_WINDOWS: + test_list.extend([ + (Data.HOMEDRIVE, Data.HOMEDRIVE, {}), + (Data.HOMEDRIVE, Data.HOMEDRIVE, {'ignore_drivespec': True}), + (Data.HOMEDRIVE, Data.REALPATH_HOMEDRIVE_CWD, {'ignore_drivespec': False}), + ]) + for p, expect, kwargs in test_list: + rval = func(p, **kwargs) + # print(repr(p), repr(expect), repr(rval)) + self.assertTrue(rval == expect, "{}({}): {}".format( + func.__name__, repr(p), repr(rval) + )) + + def test_normalize_path(self) -> None: + func = Util.normalize_path + # default kwargs: + # strip=True + # preserve_trailing=False + # expand=False + # realpath=True + # ignore_drivespec=True + test_list = [ + (Data.UTIL_MODULE, normalize(Data.REALPATH_UTIL_MODULE), {}), + (os.path.join(Data.UTIL_MODULE, os.pardir), normalize(Data.REALPATH_UTIL_MODULE_PARENT), {}), + (None, None, {}), + ('', '', {'realpath': False}), + ('', '', {'realpath': True}), + ('DoesNotExist.xyz.abc', normalize('DoesNotExist.xyz.abc'), {'realpath': False}), + ('DoesNotExist.xyz.abc', normalize(Data.REALPATH_CWD, 'DoesNotExist.xyz.abc'), {'realpath': True}), + (' DoesNotExist.xyz.abc ', normalize(Data.REALPATH_CWD, 'DoesNotExist.xyz.abc'), {'realpath': True}), + (' ~ ', '~', {'realpath': False, 'expand': False}), + (' ~ ', normalize(Data.REALPATH_HOMEPATH), {'realpath': True, 'expand': True}), + ] + if Data.IS_WINDOWS: + test_list.extend([ + ('DoesNotExist.xyz.abc/', normalize('DoesNotExist.xyz.abc') + os.path.sep, {'realpath': False, 'preserve_trailing': True}), + (' DoesNotExist.xyz.abc\\ ', normalize('DoesNotExist.xyz.abc') + os.path.sep, {'realpath': False, 'preserve_trailing': True}), + (' ~/ ', normalize(Data.REALPATH_HOMEPATH) + os.path.sep, {'realpath': True, 'expand': True, 'preserve_trailing': True}), + (' ~\\ ', normalize(Data.REALPATH_HOMEPATH) + os.path.sep, {'realpath': True, 'expand': True, 'preserve_trailing': True}), + (' ~/ ', normalize(Data.REALPATH_CWD, '~') + os.path.sep, {'realpath': True, 'expand': False, 'preserve_trailing': True}), + (' ~\\ ', normalize(Data.REALPATH_CWD, '~') + os.path.sep, {'realpath': True, 'expand': False, 'preserve_trailing': True}), + (Data.HOMEDRIVE, normalize(Data.HOMEDRIVE), {}), + (Data.HOMEDRIVE, normalize(Data.HOMEDRIVE), {'ignore_drivespec': True}), + (Data.HOMEDRIVE, normalize(Data.REALPATH_HOMEDRIVE_CWD), {'ignore_drivespec': False}), + ]) + for p, expect, kwargs in test_list: + rval = func(p, **kwargs) + # print(repr(p), repr(expect), repr(rval)) + self.assertTrue(rval == expect, "{}({}): {}".format( func.__name__, repr(p), repr(rval) )) diff --git a/SCons/Tool/MSCommon/MSVC/WinSDK.py b/SCons/Tool/MSCommon/MSVC/WinSDK.py index 39617b16cc..7115d505ee 100644 --- a/SCons/Tool/MSCommon/MSVC/WinSDK.py +++ b/SCons/Tool/MSCommon/MSVC/WinSDK.py @@ -83,7 +83,7 @@ def _sdk_10_layout(version): if not version_nbr.startswith(folder_prefix): continue - sdk_inc_path = Util.process_path(os.path.join(version_nbr_path, 'um')) + sdk_inc_path = Util.normalize_path(os.path.join(version_nbr_path, 'um')) if not os.path.exists(sdk_inc_path): continue @@ -127,7 +127,7 @@ def _sdk_81_layout(version): # msvc does not check for existence of root or other files - sdk_inc_path = Util.process_path(os.path.join(sdk_root, r'include\um')) + sdk_inc_path = Util.normalize_path(os.path.join(sdk_root, r'include\um')) if not os.path.exists(sdk_inc_path): continue From 4c5a68ff03ff907c8a646492e3ca4997c9e1dceb Mon Sep 17 00:00:00 2001 From: Joseph Brill <48932340+jcbrill@users.noreply.github.com> Date: Sun, 15 Oct 2023 12:43:23 -0400 Subject: [PATCH 2/3] Internal updates to MSCommon/MSVC/Util.py and MSCommon/MSVC/Config.py. Changes: * Adjust os path separators in MSCommon/MSVC/Util.py * Revise commend inside resolve_path method in in MSCommon/MSVC/Util.py * Move Util import inside verify method in MSCommon/MSVC/Config.py (prevent import dependency loops). --- SCons/Tool/MSCommon/MSVC/Config.py | 3 +-- SCons/Tool/MSCommon/MSVC/Util.py | 8 ++++---- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/SCons/Tool/MSCommon/MSVC/Config.py b/SCons/Tool/MSCommon/MSVC/Config.py index 29d6f246f2..7c0f1fe6ff 100644 --- a/SCons/Tool/MSCommon/MSVC/Config.py +++ b/SCons/Tool/MSCommon/MSVC/Config.py @@ -29,8 +29,6 @@ namedtuple, ) -from . import Util - from .Exceptions import ( MSVCInternalError, ) @@ -319,6 +317,7 @@ def verify(): + from . import Util from .. import vc for msvc_version in vc._VCVER: if msvc_version not in MSVC_VERSION_SUFFIX: diff --git a/SCons/Tool/MSCommon/MSVC/Util.py b/SCons/Tool/MSCommon/MSVC/Util.py index 8afe784ac5..6fd188bc35 100644 --- a/SCons/Tool/MSCommon/MSVC/Util.py +++ b/SCons/Tool/MSCommon/MSVC/Util.py @@ -43,7 +43,7 @@ _RE_DRIVESPEC = re.compile(r'^[A-Za-z][:]$', re.IGNORECASE) # windows path separators -_OS_PATH_SEPS = ('\\', '/') +_OS_PATH_SEPS = (os.path.sep, os.path.altsep) if os.path.altsep else (os.path.sep,) def listdir_dirs(p): """ @@ -87,9 +87,9 @@ def resolve_path(p, ignore_drivespec=True): # don't attempt to resolve drive specification (e.g., C:) pass else: - # both abspath and resolve necessary to produce identical path - # when (unqualified) file name is on a mapped network drive for - # python 3.6 and 3.11 + # both abspath and resolve necessary for an unqualified file name + # on a mapped network drive in order to return a mapped drive letter + # path rather than a UNC path. p = os.path.abspath(p) try: p = str(pathlib.Path(p).resolve()) From 42828672b33f3d0fa495b95f02422dafcd8b4993 Mon Sep 17 00:00:00 2001 From: Joseph Brill <48932340+jcbrill@users.noreply.github.com> Date: Sun, 15 Oct 2023 13:17:11 -0400 Subject: [PATCH 3/3] Remove redundant os.path.normcase invocation in SCons/Tool/MSCommon/MSVC/UtilTests.py. --- SCons/Tool/MSCommon/MSVC/UtilTests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SCons/Tool/MSCommon/MSVC/UtilTests.py b/SCons/Tool/MSCommon/MSVC/UtilTests.py index d92d2ca5c8..86ea58d875 100644 --- a/SCons/Tool/MSCommon/MSVC/UtilTests.py +++ b/SCons/Tool/MSCommon/MSVC/UtilTests.py @@ -44,7 +44,7 @@ def normalize(*comps): p = os.path.join(*comps) p = os.path.normpath(p) p = os.path.normcase(p) - return os.path.normcase(p) + return p class Data: