diff --git a/.github/workflows/runtest.yml b/.github/workflows/runtest.yml index 02be7c1140..6b7c036b85 100644 --- a/.github/workflows/runtest.yml +++ b/.github/workflows/runtest.yml @@ -40,6 +40,9 @@ jobs: python -m pip install --upgrade pip setuptools wheel python -m pip install -r requirements-dev.txt # sudo apt-get update + sudo apt-get install gcc-12 g++-12 + sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 100 + sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-12 100 - name: runtest ${{ matrix.os }} run: | diff --git a/SCons/Tool/cxx.py b/SCons/Tool/cxx.py index 2cf3299fb3..1f05687a37 100644 --- a/SCons/Tool/cxx.py +++ b/SCons/Tool/cxx.py @@ -29,9 +29,11 @@ """ import os.path +import re import SCons.Defaults import SCons.Util +import SCons.Scanner compilers = ['CC', 'c++'] @@ -50,6 +52,89 @@ def iscplusplus(source) -> bool: return True return False +def gen_module_map_file(root, module_map): + module_map_text = '$$root ' + str(root) + '\n' + for module, file in module_map.items(): + module_map_text += str(module) + ' ' + str(file) + '\n' + return module_map_text + +#TODO filter out C++ whitespace and comments +module_decl_re = re.compile("(module;)?(.|\n)*export module (.*);") + +def module_emitter(target, source, env): + if("CXXMODULEPATH" in env): + env["__CXXMODULEINIT__"](env) + + export = module_decl_re.match(source[0].get_text_contents()) + if export: + modulename = export[3].strip() + if modulename not in env["CXXMODULEMAP"]: + env["CXXMODULEMAP"][modulename] = modulename + env["CXXMODULESUFFIX"] + target.append(env.File("$CXXMODULEPATH/" + env["CXXMODULEMAP"][modulename])) + return (target, source, env) + +def module_emitter_static(target, source, env): + import SCons.Defaults + return SCons.Defaults.StaticObjectEmitter(*module_emitter(target, source, env)) + +def module_emitter_shared(target, source, env): + import SCons.Defaults + return SCons.Defaults.SharedObjectEmitter(*module_emitter(target, source, env)) + +#TODO filter out C++ whitespace and comments +module_import_re = re.compile(r"\s*(?:(?:export)?\s*import\s*(\S*)\s*;)|(?:(export)?\s*module\s*(\S*)\s*;)") + + +class CxxModuleScanner(SCons.Scanner.Current): + def scan(self, node, env, path): + result = self.c_scanner(node, env, path) + + if not env.get("CXXMODULEPATH"): + return result + + imports = module_import_re.findall(node.get_text_contents()) + for module, export, impl in imports: + if not module: + if not export and impl: # module implementation unit depends on module interface + module = impl + else: + continue + is_header_unit = False + if(module[0] == "<" or module[0] == '"'): + module_id_prefix = "@system-header/" if module[0] == "<" else "@header/" + cmi = module_id_prefix + module[1:-1] + "$CXXMODULESUFFIX" + is_header_unit = True + else: + cmi = env["CXXMODULEMAP"].get( + module, module+"$CXXMODULESUFFIX") + cmi = env.File("$CXXMODULEPATH/" + cmi) + + if(is_header_unit and not cmi.has_builder()): + source = self.c_scanner.find_include( + (module[0], module[1:-1]), node.dir, path) + if source[0]: + source = source[0] + else: + source = env.Value(module[1:-1]) + env.CxxHeaderUnit( + cmi, source, + CXXCOMOUTPUTSPEC="$CXXSYSTEMHEADERFLAGS" if module[0] == "<" else "$CXXUSERHEADERFLAGS", + CPPPATH = [node.dir, "$CPPPATH"] if module[0] == '"' else "$CPPPATH", + CXXMODULEIDPREFIX=module_id_prefix + ) + result.append(cmi) + return result + + def __init__(self, *args, **kwargs): + from SCons.Scanner import FindPathDirs + super().__init__(self.scan, recursive = True, path_function = FindPathDirs("CPPPATH"), *args, **kwargs) + from SCons.Tool import CScanner + self.c_scanner = CScanner + + +header_unit = SCons.Builder.Builder(action="$CXXCOM", + source_scanner=CxxModuleScanner()) + def generate(env) -> None: """ Add Builders and construction variables for Visual Age C++ compilers @@ -58,22 +143,28 @@ def generate(env) -> None: import SCons.Tool import SCons.Tool.cc static_obj, shared_obj = SCons.Tool.createObjBuilders(env) + from SCons.Tool import SourceFileScanner for suffix in CXXSuffixes: static_obj.add_action(suffix, SCons.Defaults.CXXAction) shared_obj.add_action(suffix, SCons.Defaults.ShCXXAction) - static_obj.add_emitter(suffix, SCons.Defaults.StaticObjectEmitter) - shared_obj.add_emitter(suffix, SCons.Defaults.SharedObjectEmitter) + static_obj.add_emitter(suffix, module_emitter_static) + shared_obj.add_emitter(suffix, module_emitter_shared) + SourceFileScanner.add_scanner(suffix, CxxModuleScanner()) + + env['BUILDERS']['CxxHeaderUnit'] = header_unit SCons.Tool.cc.add_common_cc_variables(env) if 'CXX' not in env: env['CXX'] = env.Detect(compilers) or compilers[0] env['CXXFLAGS'] = SCons.Util.CLVar('') - env['CXXCOM'] = '$CXX -o $TARGET -c $CXXFLAGS $CCFLAGS $_CCCOMCOM $SOURCES' + env['CXXCOMOUTPUTSPEC'] = '-o $TARGET' + env['CXXCOM'] = '$CXX $CXXCOMOUTPUTSPEC -c ${ CXXMODULEFLAGS if CXXMODULEPATH else "" } $CXXFLAGS $CCFLAGS $_CCCOMCOM $SOURCES' + env['CXXMODULEMAP'] = {} env['SHCXX'] = '$CXX' env['SHCXXFLAGS'] = SCons.Util.CLVar('$CXXFLAGS') - env['SHCXXCOM'] = '$SHCXX -o $TARGET -c $SHCXXFLAGS $SHCCFLAGS $_CCCOMCOM $SOURCES' + env['SHCXXCOM'] = '$SHCXX -o $TARGET -c ${ CXXMODULEFLAGS if CXXMODULEPATH else "" } $SHCXXFLAGS $SHCCFLAGS $_CCCOMCOM $SOURCES' env['CPPDEFPREFIX'] = '-D' env['CPPDEFSUFFIX'] = '' diff --git a/SCons/Tool/gcc.py b/SCons/Tool/gcc.py index d564f9cb2a..2f03d7f576 100644 --- a/SCons/Tool/gcc.py +++ b/SCons/Tool/gcc.py @@ -61,7 +61,6 @@ def generate(env) -> None: env["NINJA_DEPFILE_PARSE_FORMAT"] = 'gcc' - def exists(env): # is executable, and is a GNU compiler (or accepts '--version' at least) return detect_version(env, env.Detect(env.get('CC', compilers))) diff --git a/SCons/Tool/gxx.py b/SCons/Tool/gxx.py index e788381b75..7bf141eaaa 100644 --- a/SCons/Tool/gxx.py +++ b/SCons/Tool/gxx.py @@ -32,6 +32,10 @@ """ +import os, sys +import threading +import asyncio +import atexit import SCons.Tool import SCons.Util @@ -41,6 +45,159 @@ compilers = ['g++'] +def CODY_decode(input): + quoted = False + backslashed = False + result = [] + output = "" + + for c in input: + if quoted: + if backslashed: + output.append(c) + backslashed = False + continue + if c == "'": + quoted = False + continue + if c == "\\": + backslashed = True + continue + output += c + continue + + if c == "'": + quoted = True + continue + if c == ' ' and output: + result.append(output) + output = "" + continue + output += c + if output: + result.append(output) + + return result + + +class module_mapper(threading.Thread): + def __init__(self, env, *args, **kw): + super().__init__(*args, **kw) + self.daemon = True + self.loop = asyncio.new_event_loop() + + self.env = env + self.load_map() + atexit.register(self.save_map) + + self.request_dispatch = {} + self.request_dispatch["HELLO"] = self.hello_response + self.request_dispatch["INCLUDE-TRANSLATE"] = self.include_translate_response + self.request_dispatch["MODULE-REPO"] = self.module_repo_response + self.request_dispatch["MODULE-EXPORT"] = self.module_export_response + self.request_dispatch["MODULE-COMPILED"] = self.module_compiled_response + self.request_dispatch["MODULE-IMPORT"] = self.module_import_response + + def save_map(self): + save = open(self.env.subst("$CXXMODULEPATH/module.map"), "w") + + save.writelines( + [self.env.subst("$$root $CXXMODULEPATH") + '\n'] + + [' '.join(pair) + '\n' for pair in self.env["CXXMODULEMAP"].items()]) + + def load_map(self): + try: + load = open(self.env.subst("$CXXMODULEPATH/module.map"), "r") + except FileNotFoundError: + return + + saved_map = dict([tuple(line.rstrip("\n").split(maxsplit=1)) + for line in load.readlines() if not line[0] == '$']) + saved_map.update(self.env["CXXMODULEMAP"]) + self.env["CXXMODULEMAP"] = saved_map + + def run(self): + self.loop.run_forever() + + def hello_response(self, request, sourcefile): + if(sourcefile != ""): + return ("ERROR", "'Unexpected handshake'") + if len(request) == 4 and request[1] == "1": + return (request[3], "HELLO", "1", "SCONS", "''") + else: + return ("ERROR", "'Invalid handshake'") + + def module_repo_response(self, request, sourcefile): + return ("PATHNAME", self.env["CXXMODULEPATH"]) + + def include_translate_response(self, request, sourcefile): + return ("BOOL", "TRUE") + + def module_export_response(self, request, sourcefile): + if sourcefile[0] == '@': + cmi = self.env.subst(sourcefile + "$CXXMODULESUFFIX") + self.env["CXXMODULEMAP"][request[1]] = cmi + return ("PATHNAME", cmi) + else: + return ("PATHNAME", self.env["CXXMODULEMAP"].get(request[1], self.env.subst(request[1] + "$CXXMODULESUFFIX"))) + + def module_compiled_response(self, request, sourcefile): + return ("OK") + + def module_import_response(self, request, sourcefile): + return ("PATHNAME", self.env["CXXMODULEMAP"].get(request[1], self.env.subst(request[1] + "$CXXMODULESUFFIX"))) + + def default_response(self, request, sourcefile): + return ("ERROR", "'Unknown CODY request {}'".format(request[0])) + + async def handle_connect(self, reader, writer): + sourcefile = "" + while True: + try: + request = await reader.readuntil() + except EOFError: + return + + request = request.decode("utf-8") + + separator = '' + request = request.rstrip('\n') + if request[-1] == ';': + request = request.rstrip(';') + separator = ' ;' + + request = CODY_decode(request) + + response = self.request_dispatch.get( + request[0], self.default_response)(request, sourcefile) + if(request[0] == "HELLO" and response[0] != "ERROR"): + sourcefile = response[0] + response = response[1:] + response = " ".join(response) + separator + "\n" + + writer.write(response.encode()) + await writer.drain() + + async def listen(self, path): + if self.env["__GCCMAPPERMODE__"] == "domain_socket": + await asyncio.start_unix_server(self.handle_connect, path=path) + else: + srv = await asyncio.start_server(self.handle_connect, host="::1", port=0) + self.env["__GCCMAPPERPORT__"] = srv.sockets[0].getsockname()[1] + + +def init_mapper(env): + if env.get("__GCCMODULEMAPPER__"): + return + + mapper = module_mapper(env) + mapper.start() + os.makedirs(env.subst("$CXXMODULEPATH"), exist_ok=True) + asyncio.run_coroutine_threadsafe(mapper.listen( + env.subst("$CXXMODULEPATH/socket")), mapper.loop) + + env["__GCCMODULEMAPPER__"] = mapper + def generate(env) -> None: """Add Builders and construction variables for g++ to an Environment.""" static_obj, shared_obj = SCons.Tool.createObjBuilders(env) @@ -67,6 +224,17 @@ def generate(env) -> None: env['CCDEPFLAGS'] = '-MMD -MF ${TARGET}.d' env["NINJA_DEPFILE_PARSE_FORMAT"] = 'gcc' + env['__CXXMODULEINIT__'] = init_mapper + if sys.platform != "win32": + env['__GCCMAPPERMODE__'] = "domain_socket" + env['__GCCMAPPERFLAGS__'] = "=$CXXMODULEPATH/socket" + else: + env['__GCCMAPPERMODE__'] = "tcp" + env['__GCCMAPPERFLAGS__'] = "::1:$($__GCCMAPPERPORT__$)" + env['CXXMODULEFLAGS'] = '-fmodules-ts -fmodule-mapper=$__GCCMAPPERFLAGS__?$CXXMODULEIDPREFIX$SOURCE' + env['CXXMODULESUFFIX'] = '.gcm' + env['CXXUSERHEADERFLAGS'] = '-x c++-user-header' + env['CXXSYSTEMHEADERFLAGS'] = '-x c++-system-header' def exists(env): diff --git a/test/CXX/CXX-modules-fixture/SConstruct b/test/CXX/CXX-modules-fixture/SConstruct new file mode 100644 index 0000000000..30afd5525b --- /dev/null +++ b/test/CXX/CXX-modules-fixture/SConstruct @@ -0,0 +1,9 @@ +env = Environment(tools = ["link"]) +try: + env.Tool(ARGUMENTS["toolset"]) +except KeyError: + pass + +env.Append(CXXFLAGS = ["-std=c++20"]) +env['CXXMODULEPATH'] = "cxx-scons-modules" +env.Program("scons-module-test", ["itest.cpp", "test.cpp", "main.cpp"]) diff --git a/test/CXX/CXX-modules-fixture/incl.h b/test/CXX/CXX-modules-fixture/incl.h new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/CXX/CXX-modules-fixture/itest.cpp b/test/CXX/CXX-modules-fixture/itest.cpp new file mode 100644 index 0000000000..81feabf28c --- /dev/null +++ b/test/CXX/CXX-modules-fixture/itest.cpp @@ -0,0 +1,3 @@ +module test; + +int i = 42; diff --git a/test/CXX/CXX-modules-fixture/main.cpp b/test/CXX/CXX-modules-fixture/main.cpp new file mode 100644 index 0000000000..5425b02b33 --- /dev/null +++ b/test/CXX/CXX-modules-fixture/main.cpp @@ -0,0 +1,12 @@ +import test; + +import ; + +int main() +{ + std::cout << i << std::endl; + test(); + int i = fact<4>::value; + std::cout << i << std::endl; + return 0; +} diff --git a/test/CXX/CXX-modules-fixture/test.cpp b/test/CXX/CXX-modules-fixture/test.cpp new file mode 100644 index 0000000000..3848c43356 --- /dev/null +++ b/test/CXX/CXX-modules-fixture/test.cpp @@ -0,0 +1,24 @@ +module; + +#include +#include "incl.h" + +export module test; +export void test() +{ + std::cout << "hello, world\n"; +} + +export template struct fact; + +export extern int i; + +template +struct fact { + static constexpr unsigned int value = n * fact::value; +}; + +template<> +struct fact<0> { + static constexpr unsigned int value = 1; +}; diff --git a/test/CXX/CXXMODULES-gcc.py b/test/CXX/CXXMODULES-gcc.py new file mode 100644 index 0000000000..f8d7a4c830 --- /dev/null +++ b/test/CXX/CXXMODULES-gcc.py @@ -0,0 +1,21 @@ +import TestSCons + +test = TestSCons.TestSCons() + +from SCons.Tool import gxx +from SCons.Script import DefaultEnvironment + +if not gxx.exists(DefaultEnvironment()): + test.skip_test('g++ not found, skipping test\n') + +test.dir_fixture("CXX-modules-fixture") + +test.run(arguments = ". toolset=g++") + +test.pass_test() + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: