From d150cb5c36b278dcd3cff371fec48a16c81a1472 Mon Sep 17 00:00:00 2001 From: German Diago Gomez Date: Wed, 3 Sep 2025 16:15:11 +0700 Subject: [PATCH 1/2] DRAFT: Add modules support via clang-scan-deps. Tested on Clang 19 on MacOS. BEWARE, WARNING: This is a drafty implementation whose only goal was to see module resolution working in a projet. This is not representative of the quality of the code I would be willing to submit, but just a fast test. That said: support for modules has been added consisting of: - a global (not per target) scan step via clang-scan-deps. - the scan + accumulate step are done at once via clang-scan-deps. - dependency order resolution for module imports should work. - the strategy to activate scanning right now is global, even subprojects are scanned, based on compile_commands.json generated. You need clang 17 and above though only Clang 19 has been tested. The script depscanaccumulate.py *currently has the path to clang-scan-deps hardcoded* and will not work unless you fix it to point to your clang-scan-deps until I do some cleanups. Please find attached a small project to work with that I converted in the Github issue: https://github.com/mesonbuild/meson/pull/14989. The project is a NES emulator that *does almost nothing* since it was an abandoned side-project, but for the sake of converting to modules it has been invaluable. There are many strategies to do the scanning: - per file. - per target. - globally (as it is currently done with this prototype) Also, we could make a differentiation to split the files that are modules from the normal source files, but I would not recommend it. I think doing per-target scans all-or-nothing per target should be enough. Currently, the .pcm files are put at the top-level of the build directory. This should go in some global per-project cache to the best of my understanding. There is a small trick now, which is not optimal, but it does the job: the cpp_COMPILE rule is added -fmodule-output={the-pcm-file}.pcm -fprebuilt-module-path={the-build-dir} and since not all files are modules, this is surrounded by --start-no-unused-arguments and --end-no-unused-arguments. This avoids the need to generate more fancy or dynamic stuff, which I do not know even if it is possible at all at the moment. This method of compiling generates a .o file and a .pcm as a side-effect. There is another way, potentially more parallel, to build separately (via --precompile flag) a .pcm and a .o, but again, this will do as a first step I guess... I would expect that for big projects, unless clang-scan-deps is really fast, there might be other strategies that are better for incremental builds but something like this should do for now. I would even expect that Meson could choose in a smart way what strategy to adopt? (maybe) Your targets are what you are already familiar with, this is a transparent feature except for: 1. the name of your build target should have the name of the module. 2. you need to have a module interface file called target.name.cppm. Control to opt-out from module scanning, when the strategy is not global scanning, should probably be added, but since I used compile_commands.json + global scanning I did not get bothered, which whould have deviated me from my goal. I updated and activated the rule for should_use_dyndeps() in the ninja backend that already existed in the code and generated the depscanaccumulate rule. Check the build.ninja file to see what it is generated. - I generate every target with a dependency on std (using cpp_import_std feature, which is active). - A single deps.dd file is generated every time a file is touched. - any file you touch should trigger a deps.dd scan. We could make a differentiation to split the files that are modules from the normal source files to do less agressive scanning, but I would not recommend it. I think doing per-target scans all-or-nothing per target should be enough since these tools should do bulk updates and not be invoked once per file anyway. Currently, the .pcm files are put at the top-level of the build directory. This should go in some global per-project cache to the best of my understanding. There is a small trick now, which is not optimal, but it does the job: the cpp_COMPILE rule is added -fmodule-output={the-pcm-file}.pcm -fprebuilt-module-path={the-build-dir} and since not all files are modules, this is surrounded by --start-no-unused-arguments and --end-no-unused-arguments. If the file is just a normal translation unit, a .pcm will not be generated even if it was specified. This avoids the need to generate more fancy or dynamic stuff, which I do not know even if it is possible at all at the moment. Please find attached a small project to work with that I converted. The project is a NES emulator that *does almost nothing* since it was an abandoned side-project, but for the sake of converting to modules has been invaluable. ------------------------------------------------------------------------- Also add 'import std' support for Meson Clang/GCC. This commit adds the ability for Meson targets for: - clang (tested in clang 19 homebrew MacOS) - gcc (tested in a Docker container with Linx GCC15) The feature can be used by setting a new option 'cpp_import_std' globally, which can be true or false. By default the option is conservatively set to false. If enabled, during the configuration phase, Meson checks that the compiler supports building the std library as a module via compiler version checks. At build time, it adds a custom target to build the std bmi. This bmi is wrapped in a dependency and included in build targets that are executables, shared libraries and libraries if the target uses c++ language and it is not in a subproject. See build_target function. A best effort has been done to align debug/release flags with the built version of the std library, though this needs review. Note that in order to use the feature, the minimum version is C++23, since import std is a C++23 feature. An example of how to use it can be found in test '1 import std'. Smoke-tested in: - Clang 19 homebrew, in my Mac. - Gcc 15 in Docker container. - Tested with a multi-file project that uses subprojects by replacing all std library with 'import std'. ROADMAP: - Add override to build targets to avoid using 'import std'? --- mesonbuild/backend/ninjabackend.py | 194 +++++++++++++----- mesonbuild/build.py | 3 + mesonbuild/compilers/cpp.py | 115 ++++++++++- mesonbuild/compilers/mixins/clike.py | 3 + mesonbuild/interpreter/interpreter.py | 53 +++++ mesonbuild/options.py | 6 + mesonbuild/scripts/depscan.py | 38 +++- mesonbuild/scripts/depscanaccumulate.py | 99 +++++++++ pyproject.toml | 29 ++- test cases/cpp/1 import std/import_std.cpp | 12 ++ test cases/cpp/1 import std/meson.build | 14 ++ .../meson/native/clang19_modules.ini | 10 + 12 files changed, 512 insertions(+), 64 deletions(-) create mode 100644 mesonbuild/scripts/depscanaccumulate.py create mode 100644 test cases/cpp/1 import std/import_std.cpp create mode 100644 test cases/cpp/1 import std/meson.build create mode 100644 test cases/cpp/1 import std/meson/native/clang19_modules.ini diff --git a/mesonbuild/backend/ninjabackend.py b/mesonbuild/backend/ninjabackend.py index d93e8c99cfd2..332727139e11 100644 --- a/mesonbuild/backend/ninjabackend.py +++ b/mesonbuild/backend/ninjabackend.py @@ -36,6 +36,7 @@ from .backends import CleanTrees from ..build import GeneratedList, InvalidArguments + if T.TYPE_CHECKING: from typing_extensions import Literal @@ -53,6 +54,9 @@ RUST_EDITIONS = Literal['2015', '2018', '2021'] +EXPERIMENTAL_CPP_MODULES_FEATURE :T.Literal['disabled', 'global_scan', 'per_target_scan'] = 'global_scan' + + FORTRAN_INCLUDE_PAT = r"^\s*#?include\s*['\"](\w+\.\w+)['\"]" FORTRAN_MODULE_PAT = r"^\s*\bmodule\b\s+(\w+)\s*(?:!+.*)*$" FORTRAN_SUBMOD_PAT = r"^\s*\bsubmodule\b\s*\((\w+:?\w+)\)\s*(\w+)" @@ -493,6 +497,8 @@ def __init__(self, build: T.Optional[build.Build], interpreter: T.Optional[Inter self.implicit_meson_outs: T.List[str] = [] self._uses_dyndeps = False self._generated_header_cache: T.Dict[str, T.List[FileOrString]] = {} + self._first_deps_dd_rule_generated = False + self._all_scan_sources = [] # nvcc chokes on thin archives: # nvlink fatal : Could not open input file 'libfoo.a.p' # nvlink fatal : elfLink internal error @@ -645,6 +651,10 @@ def generate(self, capture: bool = False, vslite_ctx: T.Optional[T.Dict] = None) for t in ProgressBar(self.build.get_targets().values(), desc='Generating targets'): self.generate_target(t) + + if EXPERIMENTAL_CPP_MODULES_FEATURE == 'global_scan': + self.generate_global_dependency_scan_target() + mlog.log_timestamp("Targets generated") self.add_build_comment(NinjaComment('Test rules')) self.generate_tests() @@ -1069,6 +1079,9 @@ def generate_target(self, target) -> None: compiled_sources.append(s) source2object[s] = o + # if EXPERIMENTAL_CPP_MODULES_FEATURE == 'global_scan': + #self.scan_build + if is_unity: for src in self.generate_unity_files(target, unity_src): o, s = self.generate_single_compile(target, src, True, unity_deps + header_deps + d_generated_deps, @@ -1088,7 +1101,8 @@ def generate_target(self, target) -> None: else: final_obj_list = obj_list - self.generate_dependency_scan_target(target, compiled_sources, source2object, fortran_order_deps) + if self.should_use_dyndeps_for_target(target): + self._all_scan_sources.extend(compiled_sources) if target.uses_rust(): self.generate_rust_target(target, outname, final_obj_list, fortran_order_deps) @@ -1114,6 +1128,20 @@ def should_use_dyndeps_for_target(self, target: 'build.BuildTarget') -> bool: return True # Currently only the preview version of Visual Studio is supported. cpp = target.compilers['cpp'] + + target_has_primary_module_interface = any(f for f in self.get_target_sources(target).keys() if f.endswith('.cppm')) + global_scan_enabled = EXPERIMENTAL_CPP_MODULES_FEATURE == 'global_scan' + if cpp.get_id() == 'clang': + clang_version_ok = mesonlib.version_compare(cpp.version, '>=17') + if not clang_version_ok: + raise MesonException('Tried to compile a library that uses modules. Modules feature is available in clang startint at version 17.') + return (target_has_primary_module_interface or global_scan_enabled) and clang_version_ok and EXPERIMENTAL_CPP_MODULES_FEATURE != 'disabled' + if cpp.get_id() == 'gcc': + gcc_version_ok = mesonlib.version_compare(cpp.version, '>=14') + if not gcc_version_ok: + raise MesonException('Tried to compile a library that uses modules. Modules feature is available in gcc startint at version 14.') + return (target_has_primary_module_interface or global_scan_enabled) and clang_version_ok and EXPERIMENTAL_CPP_MODULES_FEATURE != 'disabled' + if cpp.get_id() != 'msvc': return False cppversion = self.get_target_option(target, OptionKey('cpp_std', @@ -1125,7 +1153,20 @@ def should_use_dyndeps_for_target(self, target: 'build.BuildTarget') -> bool: return False if mesonlib.version_compare(cpp.version, '<19.28.28617'): return False - return True + return (target_has_primary_module_interface or global_scan_enabled) and clang_version_ok and EXPERIMENTAL_CPP_MODULES_FEATURE != 'disabled' + + def generate_project_wide_cpp_scanner_rules(self) -> None: + rulename = 'depscanaccumulate' + if rulename in self.ruledict: + # Scanning command is the same for native and cross compilation. + return + + command = self.environment.get_build_command() + \ + ['--internal', 'depscanaccumulate'] + args = ['$in', 'deps.json', '$out'] + description = 'Scanning project for modules' + rule = NinjaRule(rulename, command, args, description) + self.add_rule(rule) def generate_dependency_scan_target(self, target: build.BuildTarget, compiled_sources: T.List[str], @@ -1134,46 +1175,72 @@ def generate_dependency_scan_target(self, target: build.BuildTarget, if not self.should_use_dyndeps_for_target(target): return self._uses_dyndeps = True - json_file, depscan_file = self.get_dep_scan_file_for(target) - pickle_base = target.name + '.dat' - pickle_file = os.path.join(self.get_target_private_dir(target), pickle_base).replace('\\', '/') - pickle_abs = os.path.join(self.get_target_private_dir_abs(target), pickle_base).replace('\\', '/') - rule_name = 'depscan' - scan_sources = list(self.select_sources_to_scan(compiled_sources)) - - scaninfo = TargetDependencyScannerInfo( - self.get_target_private_dir(target), source2object, scan_sources) - - write = True - if os.path.exists(pickle_abs): - with open(pickle_abs, 'rb') as p: - old = pickle.load(p) - write = old != scaninfo - - if write: - with open(pickle_abs, 'wb') as p: - pickle.dump(scaninfo, p) - - elem = NinjaBuildElement(self.all_outputs, json_file, rule_name, pickle_file) - # A full dependency is required on all scanned sources, if any of them - # are updated we need to rescan, as they may have changed the modules - # they use or export. - for s in scan_sources: - elem.deps.add(s[0]) - elem.orderdeps.update(object_deps) - elem.add_item('name', target.name) + if EXPERIMENTAL_CPP_MODULES_FEATURE == 'global_scan' and not self._first_deps_dd_rule_generated: + self._first_deps_dd_rule_generated = True + self.generate_project_wide_cpp_scanner_rules() + rule_name = 'depscanaccumulate' + elem = NinjaBuildElement(self.all_outputs, "deps.dd", rule_name, "compile_commands.json") + raise MesonException('asklfjasl') + self.add_build(elem) + + def generate_global_dependency_scan_target(self) -> None: + self._uses_dyndeps = True + self.generate_project_wide_cpp_scanner_rules() + rule_name = 'depscanaccumulate' + elem = NinjaBuildElement(self.all_outputs, "deps.dd", rule_name, "compile_commands.json") + elem.add_dep(self._all_scan_sources) self.add_build(elem) - infiles: T.Set[str] = set() - for t in target.get_all_linked_targets(): - if self.should_use_dyndeps_for_target(t): - infiles.add(self.get_dep_scan_file_for(t)[0]) - _, od = self.flatten_object_list(target) - infiles.update({self.get_dep_scan_file_for(t)[0] for t in od if t.uses_fortran()}) - elem = NinjaBuildElement(self.all_outputs, depscan_file, 'depaccumulate', [json_file] + sorted(infiles)) - elem.add_item('name', target.name) - self.add_build(elem) + + # def generate_dependency_scan_target(self, target: build.BuildTarget, + # compiled_sources: T.List[str], + # source2object: T.Dict[str, str], + # object_deps: T.List[FileOrString]) -> None: + # if not self.should_use_dyndeps_for_target(target): + # return + # self.generate_project_wide_cpp_scanner_rules() + # self._uses_dyndeps = True + # json_file, depscan_file = self.get_dep_scan_file_for(target) + # pickle_base = target.name + '.dat' + # pickle_file = os.path.join(self.get_target_private_dir(target), pickle_base).replace('\\', '/') + # pickle_abs = os.path.join(self.get_target_private_dir_abs(target), pickle_base).replace('\\', '/') + # rule_name = 'depscan' + # scan_sources = list(self.select_sources_to_scan(compiled_sources)) + + # scaninfo = TargetDependencyScannerInfo( + # self.get_target_private_dir(target), source2object, scan_sources) + + # write = True + # if os.path.exists(pickle_abs): + # with open(pickle_abs, 'rb') as p: + # old = pickle.load(p) + # write = old != scaninfo + + # if write: + # with open(pickle_abs, 'wb') as p: + # pickle.dump(scaninfo, p) + + # elem = NinjaBuildElement(self.all_outputs, json_file, rule_name, pickle_file) + # A full dependency is required on all scanned sources, if any of them + # are updated we need to rescan, as they may have changed the modules + # they use or export. + # for s in scan_sources: + # elem.deps.add(s[0]) + # elem.orderdeps.update(object_deps) + # elem.add_item('name', target.name) + # self.add_build(elem) + + # infiles: T.Set[str] = set() + # for t in target.get_all_linked_targets(): + # if self.should_use_dyndeps_for_target(t): + # infiles.add(self.get_dep_scan_file_for(t)[0]) + # _, od = self.flatten_object_list(target) + # infiles.update({self.get_dep_scan_file_for(t)[0] for t in od if t.uses_fortran()}) + + # elem = NinjaBuildElement(self.all_outputs, depscan_file, 'depaccumulate', [json_file] + sorted(infiles)) + # elem.add_item('name', target.name) + # self.add_build(elem) def select_sources_to_scan(self, compiled_sources: T.List[str], ) -> T.Iterable[T.Tuple[str, Literal['cpp', 'fortran']]]: @@ -2687,20 +2754,23 @@ def generate_scanner_rules(self) -> None: # Scanning command is the same for native and cross compilation. return - command = self.environment.get_build_command() + \ - ['--internal', 'depscan'] - args = ['$picklefile', '$out', '$in'] - description = 'Scanning target $name for modules' - rule = NinjaRule(rulename, command, args, description) - self.add_rule(rule) - - rulename = 'depaccumulate' - command = self.environment.get_build_command() + \ - ['--internal', 'depaccumulate'] - args = ['$out', '$in'] - description = 'Generating dynamic dependency information for target $name' - rule = NinjaRule(rulename, command, args, description) - self.add_rule(rule) + if EXPERIMENTAL_CPP_MODULES_FEATURE == 'per_target_scan': + command = self.environment.get_build_command() + \ + ['--internal', 'depscan'] + args = ['$picklefile', '$out', '$in'] + description = 'Scanning target $name for modules' + rule = NinjaRule(rulename, command, args, description) + self.add_rule(rule) + + rulename = 'depaccumulate' + command = self.environment.get_build_command() + \ + ['--internal', 'depaccumulate'] + args = ['$out', '$in'] + description = 'Generating dynamic dependency information for target $name' + rule = NinjaRule(rulename, command, args, description) + self.add_rule(rule) + elif EXPERIMENTAL_CPP_MODULES_FEATURE == 'global_scan': + self.generate_project_wide_cpp_scanner_rules() def generate_compile_rules(self) -> None: for for_machine in MachineChoice: @@ -3244,6 +3314,12 @@ def quote_make_target(targetName: str) -> str: result += c return result element.add_item('CUDA_ESCAPED_TARGET', quote_make_target(rel_obj)) + + if self.should_use_dyndeps_for_target(target) and compiler.get_language() == 'cpp' and compiler.get_id() == 'clang': + commands.extend(['--start-no-unused-arguments', + f'-fmodule-output={target.name}{compiler.get_cpp20_module_bmi_extension()}', + f'-fprebuilt-module-path={self.environment.get_build_dir()}', + '--end-no-unused-arguments']) element.add_item('ARGS', commands) self.add_dependency_scanner_entries_to_element(target, compiler, element, src) @@ -3262,9 +3338,15 @@ def add_dependency_scanner_entries_to_element(self, target: build.BuildTarget, c extension = extension.lower() if not (extension in compilers.lang_suffixes['fortran'] or extension in compilers.lang_suffixes['cpp']): return - dep_scan_file = self.get_dep_scan_file_for(target)[1] - element.add_item('dyndep', dep_scan_file) - element.add_orderdep(dep_scan_file) + + if EXPERIMENTAL_CPP_MODULES_FEATURE == 'per_target_scan': + dep_scan_file = self.get_dep_scan_file_for(target)[1] + element.add_item('dyndep', dep_scan_file) + element.add_orderdep(dep_scan_file) + elif EXPERIMENTAL_CPP_MODULES_FEATURE == 'global_scan': + dep_scan_file = 'deps.dd' + element.add_item('dyndep', dep_scan_file) + element.add_orderdep(dep_scan_file) def get_dep_scan_file_for(self, target: build.BuildTarget) -> T.Tuple[str, str]: priv = self.get_target_private_dir(target) diff --git a/mesonbuild/build.py b/mesonbuild/build.py index b43532006023..3ab53653a796 100644 --- a/mesonbuild/build.py +++ b/mesonbuild/build.py @@ -1704,6 +1704,9 @@ def get_used_stdlib_args(self, link_language: str) -> T.List[str]: stdlib_args.extend(all_compilers[dl].language_stdlib_only_link_flags(self.environment)) return stdlib_args + def uses_cpp(self) -> bool: + return 'cpp' in self.compilers + def uses_rust(self) -> bool: return 'rust' in self.compilers diff --git a/mesonbuild/compilers/cpp.py b/mesonbuild/compilers/cpp.py index fa032ec7968f..d5731e0dcacd 100644 --- a/mesonbuild/compilers/cpp.py +++ b/mesonbuild/compilers/cpp.py @@ -4,18 +4,22 @@ from __future__ import annotations import functools +import os import os.path +import re import typing as T from .. import options from .. import mlog -from ..mesonlib import MesonException, version_compare +from ..mesonlib import (File, MesonException, MesonBugException, Popen_safe_logged, + version_compare) from .compilers import ( gnu_winlibs, msvc_winlibs, Compiler, CompileCheckMode, + CompileResult ) from .c_function_attributes import CXX_FUNC_ATTRIBUTES, C_FUNC_ATTRIBUTES from .mixins.apple import AppleCompilerMixin, AppleCPPStdsMixin @@ -88,7 +92,60 @@ def get_no_stdlib_link_args(self) -> T.List[str]: def sanity_check(self, work_dir: str, environment: 'Environment') -> None: code = 'class breakCCompiler;int main(void) { return 0; }\n' - return self._sanity_check_impl(work_dir, environment, 'sanitycheckcpp.cc', code) + self._sanity_check_impl(work_dir, environment, 'sanitycheckcpp.cc', code) + if environment.coredata.optstore.get_value('cpp_import_std'): + self._import_cpp_std_sanity_check(work_dir, environment) + + def compile_import_std_module(self, + env: 'Environment', + code: File): + cpp_std = env.coredata.optstore.get_value('cpp_std') + srcname = code.fname + # Construct the compiler command-line + commands = self.compiler_args() + commands.append(f"-std={cpp_std}") + commands.extend(['-Wno-reserved-identifier', '-Wno-reserved-module-identifier']) + commands.append("--precompile") + + all_lists_to_add = [self.get_always_args(), self.get_debug_args(env.coredata.optstore.get_value('buildtype') == 'debug'), + self.get_assert_args(disable=env.coredata.optstore.get_value('b_ndebug') in ['if-release', 'true'], + env=env)] + for args_list in all_lists_to_add: + for arg in args_list: + commands.append(arg) + commands.append(srcname) + tmpdirname = env.build_dir + + # Preprocess mode outputs to stdout, so no output args + print(f"***{self.get_exelist()}") + output = f'std{self.get_cpp20_module_bmi_extension()}' + commands += self.get_output_args(output) + no_ccache = True + os_env = os.environ.copy() + os_env['LC_ALL'] = 'C' + os_env['CCACHE_DISABLE'] = '1' + command_list = self.get_exelist(ccache=not no_ccache) + commands.to_native() + p, stdo, stde = Popen_safe_logged(command_list, msg="Command line for compiling 'import std' feature", cwd=tmpdirname, env=os_env) + if p.returncode != 0: + raise MesonException("Could not compile library for use with 'import std'") + + def get_import_std_lib_source_args(self, env: Environment) -> T.List[str]: + raise MesonException("Your compiler does not support 'import std' feature or it has not been implemented") + + def get_import_std_lib_source_file(self) -> str: + raise MesonException("Your compiler does not support 'import std' feature or it has not been implemented") + + def get_cpp20_module_bmi_extension(self) -> str: + raise MesonException("Your compiler does not support 'import std' feature or it has not been implemented") + + def get_import_std_compile_args(self, environment: 'Environment') -> T.List[str]: + raise MesonException("Your compiler does not support 'import std' feature or it has not been implemented") + + def check_cpp_import_std_support(self): + raise MesonException("Your compiler does not support 'import std' feature or it has not been implemented") + + def _import_cpp_std_sanity_check(self, work_dir: str, environment: 'Environment') -> None: + self.check_cpp_import_std_support() def get_compiler_check_args(self, mode: CompileCheckMode) -> T.List[str]: # -fpermissive allows non-conforming code to compile which is necessary @@ -174,9 +231,13 @@ def _find_best_cpp_std(self, cpp_std: str) -> str: def get_options(self) -> 'MutableKeyedOptionDictType': opts = super().get_options() key = self.form_compileropt_key('std') + import_std_key = self.form_compileropt_key('import_std') + opts.update({ key: options.UserStdOption('cpp', ALL_STDS), + import_std_key: options.UseImportStd('cpp') }) + return opts @@ -235,6 +296,36 @@ def __init__(self, ccache: T.List[str], exelist: T.List[str], version: str, for_ '3': default_warn_args + ['-Wextra', '-Wpedantic'], 'everything': ['-Weverything']} + def check_cpp_import_std_support(self): + if int(self.version.split('.')[0]) < 17: + raise MesonException('Your compiler does not support import std feature. Clang support starts at version >= 17') + + def get_import_std_compile_args(self, env: 'Environment') -> T.List[str]: + bmi_path = f'{env.get_build_dir()}/std{self.get_cpp20_module_bmi_extension()}' + return [f'-fmodule-file=std={bmi_path}'] + + def get_cpp20_module_bmi_extension(self) -> str: + return '.pcm' + + def get_import_std_lib_source_args(self, env: Environment) -> T.List[str]: + cpp_std = env.coredata.optstore.get_value('cpp_std') + return [f'-std={cpp_std}', + '-Wno-reserved-identifier', + '-Wno-reserved-module-identifier', + '--precompile'] + + llvm_dir_re = re.compile(r'(/\D*/(?:\.?\d+)+)/.*') + + def get_import_std_lib_source_file(self) -> str: + dirs = [d for d in self.get_preprocessor().get_default_include_dirs() if 'llvm' in d and not '..' in d] + for d in dirs: + if m := type(self).llvm_dir_re.match(d): + break + if not m: + raise MesonBugException('Could not find import std lib source file. This should work') + llvm_dir = str(m[1]) + return f'{llvm_dir}/share/libc++/v1/std.cppm' + def get_options(self) -> 'MutableKeyedOptionDictType': opts = super().get_options() @@ -460,6 +551,26 @@ def __init__(self, ccache: T.List[str], exelist: T.List[str], version: str, for_ self.supported_warn_args(gnu_common_warning_args) + self.supported_warn_args(gnu_cpp_warning_args))} + def get_import_std_lib_source_args(self, env: Environment) -> T.List[str]: + cpp_std = env.coredata.optstore.get_value('cpp_std') + return [f"-std={cpp_std}", '-fmodules', '-fsearch-include-path', '-fmodule-only'] + self.get_compile_only_args() + + def get_import_std_lib_source_file(self) -> str: + std_lib_dir = [d for d in self.get_preprocessor().get_default_include_dirs() + if "c++" in d and '..' not in d and d[-1].isdigit()][0] + return f'{std_lib_dir}/bits/std.cc' + + def get_cpp20_module_bmi_extension(self) -> str: + return '.gcm' + + def get_import_std_compile_args(self, env: 'Environment'): + bmi_path = f'{env.get_build_dir()}/std{self.get_cpp20_module_bmi_extension()}' + return ['-fmodules'] + + def check_cpp_import_std_support(self): + if int(self.version.split('.')[0]) < 15: + raise MesonException('Your compiler does not support import std feature. GCC support starts at version >= 15') + def get_options(self) -> 'MutableKeyedOptionDictType': opts = super().get_options() diff --git a/mesonbuild/compilers/mixins/clike.py b/mesonbuild/compilers/mixins/clike.py index d2eb6119bcd0..f67a23515e90 100644 --- a/mesonbuild/compilers/mixins/clike.py +++ b/mesonbuild/compilers/mixins/clike.py @@ -287,6 +287,9 @@ def _sanity_check_impl(self, work_dir: str, environment: 'Environment', mode = CompileCheckMode.COMPILE cargs, largs = self._get_basic_compiler_args(environment, mode) extra_flags = cargs + self.linker_to_compiler_args(largs) + if environment.coredata.optstore.get_value('cpp_import_std'): + # extra_flags.extend(self.get_import_std) + pass # Is a valid executable output for all toolchains and platforms binname += '.exe' diff --git a/mesonbuild/interpreter/interpreter.py b/mesonbuild/interpreter/interpreter.py index 2cf5b7a5bf10..0ec3cb284f37 100644 --- a/mesonbuild/interpreter/interpreter.py +++ b/mesonbuild/interpreter/interpreter.py @@ -309,6 +309,54 @@ def __init__( self.compilers: PerMachine[T.Dict[str, 'compilers.Compiler']] = PerMachine({}, {}) self.parse_project() self._redetect_machines() + self._cpp_import_std_bmi_dep = None + + if self.coredata.optstore.get_value('cpp_import_std') and self.subproject == "": + self._cpp_import_std_bmi_dep = self._create_cpp_import_std_dep(self.environment) + + def _create_cpp_import_std_dep(self, env: environment.Environment): + compiler_to_use: T.Optional[compilers.cpp.CPPCompiler] = None + for comp_lang, compiler in self.compilers.host.items(): + if comp_lang == 'cpp': + compiler_to_use = T.cast(compilers.cpp.CPPCompiler, compiler) + if not compiler_to_use: + raise MesonException('cpp_import_std option is set to true but no cpp compiler could be found.' + ' Enable cpp language in your project to use this feature.') + # Construct the compiler command-line + commands = compiler_to_use.compiler_args() + commands.extend(compiler_to_use.get_import_std_lib_source_args(self.environment)) + all_lists_to_add = [compiler_to_use.get_always_args(), compiler_to_use.get_debug_args(env.coredata.optstore.get_value('buildtype') == 'debug'), + compiler_to_use.get_assert_args(disable=env.coredata.optstore.get_value('b_ndebug') in ['if-release', 'true'], + env=env)] + for args_list in all_lists_to_add: + for arg in args_list: + commands.append(arg) + commands.append("-o") + commands.append("@OUTPUT@") + commands.append("@INPUT@") + no_ccache = True + command_list = compiler_to_use.get_exelist(ccache=not no_ccache) + commands.to_native() + tgt = build.CustomTarget('', + '', '', self.environment, command_list, + sources=[compiler_to_use.get_import_std_lib_source_file()], + outputs=[f'std{compiler_to_use.get_cpp20_module_bmi_extension()}']) + self.add_target('_cpp_import_std_bmi', tgt) + bmi_dep = dependencies.InternalDependency( + version='0.0', + incdirs=[], + compile_args=compiler_to_use.get_import_std_compile_args(self.environment), + # compile_args=[], + link_args=[], + libraries=[], + whole_libraries=[], + sources=[tgt], + extra_files=[], + ext_deps=[], + variables=[], + d_module_versions=[], + d_import_dirs=[], + objects=[]) + return bmi_dep def __getnewargs_ex__(self) -> T.Tuple[T.Tuple[object], T.Dict[str, object]]: raise MesonBugException('This class is unpicklable') @@ -3411,6 +3459,7 @@ def build_target(self, node: mparser.BaseNode, args: T.Tuple[str, SourcesVarargs if targetclass not in {build.Executable, build.SharedLibrary, build.SharedModule, build.StaticLibrary, build.Jar}: mlog.debug('Unknown target type:', str(targetclass)) raise RuntimeError('Unreachable code') + self.__process_language_args(kwargs) if targetclass is build.StaticLibrary: for lang in compilers.all_languages - {'java'}: @@ -3492,6 +3541,10 @@ def build_target(self, node: mparser.BaseNode, args: T.Tuple[str, SourcesVarargs target = targetclass(name, self.subdir, self.subproject, for_machine, srcs, struct, objs, self.environment, self.compilers[for_machine], kwargs) + if target.uses_cpp(): + if self.coredata.optstore.get_value('cpp_import_std') and self.subproject == '': + target.add_deps([self._cpp_import_std_bmi_dep]) + if objs and target.uses_rust(): FeatureNew.single_use('objects in Rust targets', '1.8.0', self.subproject) diff --git a/mesonbuild/options.py b/mesonbuild/options.py index b9f17cf72381..b1474dd5020d 100644 --- a/mesonbuild/options.py +++ b/mesonbuild/options.py @@ -581,6 +581,12 @@ def choices_are_different(a: _U, b: _U) -> bool: return False +class UseImportStd(UserBooleanOption): + def __init__(self, lang): + self.lang = lang.lower() + opt_name =f'{self.lang}_import_std' + super().__init__(opt_name, 'Whether to use import std; module in your targets', False) + class UserStdOption(UserComboOption): ''' UserOption specific to c_std and cpp_std options. User can set a list of diff --git a/mesonbuild/scripts/depscan.py b/mesonbuild/scripts/depscan.py index 6bd5cde9aac0..f69e187587a6 100644 --- a/mesonbuild/scripts/depscan.py +++ b/mesonbuild/scripts/depscan.py @@ -11,6 +11,7 @@ import pickle import re import typing as T +import subprocess as sp if T.TYPE_CHECKING: from typing_extensions import Literal, TypedDict, NotRequired @@ -201,8 +202,37 @@ def scan(self) -> int: return 0 + +class CppDependenciesScanner: + pass + +class ClangDependencyScanner(CppDependenciesScanner): + def __init__(self, compilation_db_file, json_output_file, dd_output_file=None): + self.compilation_db_file = compilation_db_file + self.output_file = output_file + self.dd_output_file = dd_output_file + + def scan(self): + try: + r = sp.run(["/usr/local/Cellar/llvm/20.1.1/bin/clang-scan-deps", + "-format=p1689", + "-compilation-database", self.compilation_db_file], + capture_output=True, + check=True) + print(r.stdout) + #json.loads(r.stdout) + return 0 + except sp.SubprocessError: + return 1 + except sp.TimeoutExpired: + return 2 + + def run(args: T.List[str]) -> int: - assert len(args) == 2, 'got wrong number of arguments!' - outfile, pickle_file = args - scanner = DependencyScanner(pickle_file, outfile) - return scanner.scan() + assert len(args) > 2, 'At least and arguments' + comp_db, json_output, dd_output = args + ClangDependencyScanner(compilation_db_file, output_file) + # assert len(args) == 2, 'got wrong number of arguments!' + # outfile, pickle_file = args + # scanner = DependencyScanner(pickle_file, outfile) + # return scanner.scan() diff --git a/mesonbuild/scripts/depscanaccumulate.py b/mesonbuild/scripts/depscanaccumulate.py new file mode 100644 index 000000000000..e2d3a3ba9087 --- /dev/null +++ b/mesonbuild/scripts/depscanaccumulate.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python3 +from collections import defaultdict +from dataclasses import dataclass +import json +import os +import subprocess as sp +import sys +import typing as T + +ModuleName: T.TypeAlias = str +ObjectFile: T.TypeAlias = str + + +@dataclass(frozen=True) +class ModuleProviderInfo: + logical_name: ModuleName + source_path: str + is_interface: bool = False + + +class CppDependenciesScanner: + pass + +class DynDepRule: + def __init__(self, out: str, imp_outs: T.Optional[T.List[str]], imp_ins: T.List[str]): + self.output = [f'build {out}'] + if imp_outs: + imp_out_str = " ".join(imp_outs) + self.output.append(f" | {imp_out_str}") + self.output.append(": dyndep") + if imp_ins: + imp_ins_str = " ".join(imp_ins) + self.output.append(" | " + imp_ins_str) + self.output_str = "".join(self.output) + "\n" + + def __str__(self): + return self.output_str + + +class ClangDependencyScanner(CppDependenciesScanner): + def __init__(self, compilation_db_file, json_output_file, dd_output_file=None): + self.compilation_db_file = compilation_db_file + self.json_output_file = json_output_file + self.dd_output_file = dd_output_file + + def scan(self) -> T.Tuple[T.Mapping[ObjectFile, ModuleName], + T.Mapping[ObjectFile, ModuleProviderInfo]]: + try: + r = sp.run(["/usr/local/Cellar/llvm/20.1.1/bin/clang-scan-deps", + "-format=p1689", + "-compilation-database", self.compilation_db_file], + capture_output=True) + if r.returncode != 0: + print(r.stderr) + raise sp.SubprocessError("Failed to run command") + process_output = r.stdout + with open(self.json_output_file, 'wb') as f: + f.write(process_output) + dependencies_info = json.loads(r.stdout) + all_deps_per_objfile = self.generate_dependencies(dependencies_info["rules"]) + self.generate_dd_file(all_deps_per_objfile) + except sp.SubprocessError: + return 1 + except sp.TimeoutExpired: + return 2 + + def generate_dd_file(self, deps_per_object_file): + with open('deps.dd', "w") as f: + f.write('ninja_dyndep_version = 1\n') + for obj, reqprov in deps_per_object_file.items(): + requires, provides = reqprov + dd = DynDepRule(obj, [p.logical_name + ".pcm" for p in provides], + [r + '.pcm' for r in requires]) + f.write(str(dd)) + + def generate_dependencies(self, rules: T.List): + all_entries: T.Mapping[ObjectFile, T.Tuple[T.Set(ModuleName), T.Set(ModuleProviderInfo)]] = defaultdict(lambda: (set(), set())) + for r in rules: + obj_processed = r["primary-output"] + # Add empty entries so that dyndep rule is generated for every file with a potential dyndep rule + # or ninja will complain + all_entries[obj_processed] = (set(), set()) + for req in r.get("requires", []): + all_entries[obj_processed][0].add(req["logical-name"]) + for prov in r.get("provides", []): + all_entries[obj_processed][1].add(ModuleProviderInfo( + logical_name=prov["logical-name"], + source_path=prov["source-path"], + is_interface=prov.get('is-interface', False))) + return all_entries + +def run(args: T.List[str]) -> int: + assert len(args) >= 2, 'At least and arguments' + comp_db_path, json_output_path, dd_output = args + scanner = ClangDependencyScanner(comp_db_path, json_output_path) + return scanner.scan() + +if __name__ == '__main__': + run(sys.argv[1:]) diff --git a/pyproject.toml b/pyproject.toml index 8fe2f47af9a1..892b84c778df 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,28 @@ +[project] +name = "python-forms-app" +version = "0.1.0" +description = "" +authors = [ + {name = "German Diago Gomez",email = "germandiago@gmail.com"} +] +readme = "README.md" +requires-python = ">=3.9" + +[tool.poetry] +packages = [{include = "meson", from = "mesonbuild"}] +package-mode = false + +[tool.poetry.group.dev.dependencies] +mypy = "^1.17.0" +python-lsp-server = "^1.13.0" +pyright = "^1.1.403" +flake8 = "^7.3.0" +pytest = "^8.4.1" +requests = "^2.32.4" +types-sqlalchemy = "^1.4.53.38" +types-jinja2 = "^2.11.9" +types-flask = "^1.1.6" + [build-system] -requires = ["setuptools>=42", "wheel"] -build-backend = "setuptools.build_meta" +requires = ["poetry-core>=2.0.0,<3.0.0"] +build-backend = "poetry.core.masonry.api" diff --git a/test cases/cpp/1 import std/import_std.cpp b/test cases/cpp/1 import std/import_std.cpp new file mode 100644 index 000000000000..193e9f54a0ce --- /dev/null +++ b/test cases/cpp/1 import std/import_std.cpp @@ -0,0 +1,12 @@ +import std; + +constexpr char const * const PROJECT_NAME = "import std"; + +int main(int argc, char **argv) { + if (argc != 1) { + std::cout << argv[0] << " takes no arguments.\n"; + return 1; + } + std::cout << "This is project " << PROJECT_NAME << ".\n"; + return 0; +} diff --git a/test cases/cpp/1 import std/meson.build b/test cases/cpp/1 import std/meson.build new file mode 100644 index 000000000000..f85064f61667 --- /dev/null +++ b/test cases/cpp/1 import std/meson.build @@ -0,0 +1,14 @@ +project( + 'import std', + 'cpp', + version : '0.1', + meson_version : '>= 1.3.0', + default_options : ['warning_level=3', 'cpp_std=c++23', + 'cpp_import_std=true'], +) + + +exe = executable( + 'import std', + 'import_std.cpp' +) diff --git a/test cases/cpp/1 import std/meson/native/clang19_modules.ini b/test cases/cpp/1 import std/meson/native/clang19_modules.ini new file mode 100644 index 000000000000..9b7fed419106 --- /dev/null +++ b/test cases/cpp/1 import std/meson/native/clang19_modules.ini @@ -0,0 +1,10 @@ +[binaries] +c = '/usr/local/Cellar/llvm/20.1.1/bin/clang' +cpp = '/usr/local/Cellar/llvm/20.1.1/bin/clang++' +objc = '/usr/local/Cellar/llvm/20.1.1/bin/clang' +objcpp = '/usr/local/Cellar/llvm/20.1.1/bin/clang++' + +[built-in options] +cpp_link_args='-L/usr/local/opt/llvm/lib/c++ -L/usr/local/opt/llvm/lib/unwind -lunwind -L/usr/local/opt/llvm/lib' +cpp_args = '-I/usr/local/opt/llvm/include' +c_args = '-I/usr/local/opt/llvm/include' From 09b1d0aad6061df442e248b37f028f2bf89ade9e Mon Sep 17 00:00:00 2001 From: German Diago Gomez Date: Mon, 8 Sep 2025 10:02:49 +0700 Subject: [PATCH 2/2] Add module partitions and implementation modules. This commit adds support for implementation partitions and modifies file conventions for a modularized target. The conventions are: - your primary interface unit for your module should always be called 'module.cppm'. It should export module . - your interface partitions can have any name supported by a module. For example: MyThings.cppm. The interface partition should declare 'export module :MyThings'. - Your *importable* interface implementation units should end with 'Impl.cppm'. For example 'MyThingsImpl.cppm'. This should 'module :MyThings' (without export). - Your *non-importable* implementation can have any name *with an extension .cpp*, not a .cppm, since implementation units are not importable. It is highly recommended, though, that if you have a single implementation file, it is called moduleImpl.cpp. You can see the project attached in Github draft PR: https://github.com/mesonbuild/meson/pull/14989 as an example that uses interface partitions, primary interface units, an implementation module (but not an implementation module partition, though it should work), multiple targets and import std. It can run a couple of tests when fully compiled and it resolves dependency order via scanner with Clang. --- mesonbuild/backend/ninjabackend.py | 40 +++++++++++++++++++++---- mesonbuild/scripts/depscanaccumulate.py | 7 +++-- 2 files changed, 40 insertions(+), 7 deletions(-) diff --git a/mesonbuild/backend/ninjabackend.py b/mesonbuild/backend/ninjabackend.py index 332727139e11..2c3452006b4d 100644 --- a/mesonbuild/backend/ninjabackend.py +++ b/mesonbuild/backend/ninjabackend.py @@ -26,6 +26,7 @@ from .. import compilers from ..arglist import CompilerArgs from ..compilers import Compiler +from ..compilers.cpp import CPPCompiler from ..linkers import ArLikeLinker, RSPFileSyntax from ..mesonlib import ( File, LibType, MachineChoice, MesonBugException, MesonException, OrderedSet, PerMachine, @@ -1134,12 +1135,12 @@ def should_use_dyndeps_for_target(self, target: 'build.BuildTarget') -> bool: if cpp.get_id() == 'clang': clang_version_ok = mesonlib.version_compare(cpp.version, '>=17') if not clang_version_ok: - raise MesonException('Tried to compile a library that uses modules. Modules feature is available in clang startint at version 17.') + raise MesonException('Tried to compile a library that uses modules. Modules feature is available in clang starting at version 17.') return (target_has_primary_module_interface or global_scan_enabled) and clang_version_ok and EXPERIMENTAL_CPP_MODULES_FEATURE != 'disabled' if cpp.get_id() == 'gcc': gcc_version_ok = mesonlib.version_compare(cpp.version, '>=14') if not gcc_version_ok: - raise MesonException('Tried to compile a library that uses modules. Modules feature is available in gcc startint at version 14.') + raise MesonException('Tried to compile a library that uses modules. Modules feature is available in gcc starting at version 14.') return (target_has_primary_module_interface or global_scan_enabled) and clang_version_ok and EXPERIMENTAL_CPP_MODULES_FEATURE != 'disabled' if cpp.get_id() != 'msvc': @@ -1191,8 +1192,6 @@ def generate_global_dependency_scan_target(self) -> None: elem.add_dep(self._all_scan_sources) self.add_build(elem) - - # def generate_dependency_scan_target(self, target: build.BuildTarget, # compiled_sources: T.List[str], # source2object: T.Dict[str, str], @@ -3166,6 +3165,35 @@ def generate_common_compile_args_per_src_type(self, target: build.BuildTarget) - src_type_to_args[src_type_str] = commands.to_native() return src_type_to_args + def _get_cpp_module_output_name(self, src_basename: str, + compiler: CPPCompiler, + target: build.BuildTarget): + is_module_file = src_basename.endswith('.cppm') + if not is_module_file: + # The compiler will not use this output bc it is not a module + return 'dummy' + + # Split the filename into root and extension, and take the root part + src_without_extension = os.path.splitext(src_basename)[0] + + # The primary interface unit should have the name 'module.cpp' in all cases. It should + # export as the module name the name of your target. For example, 'hello.world' target + # 'export module hello.world' + if src_without_extension == 'module': + return f"{target.name}{compiler.get_cpp20_module_bmi_extension()}" + # This is an implementation partition. by convention, it is the name that + # will be used as the name of the module. + # For example, given GreetImpl.cppm, then it should declare 'module hello.world:Greet' + # internally. + elif (src_without_extension.endswith('Impl')): + private_partition_name = src_without_extension.split('Impl')[0] + return f"{target.name}-{private_partition_name}{compiler.get_cpp20_module_bmi_extension()}" + # This is an interface partition. Same convention as the Impl with the difference that + # it does not end in 'Impl'. So for a given file 'MySalutation.cppm', the module would do + # 'export hello.world:MySalutation' + else: + return f"{target.name}-{src_without_extension}{compiler.get_cpp20_module_bmi_extension()}" + def generate_single_compile(self, target: build.BuildTarget, src, is_generated: bool = False, header_deps=None, order_deps: T.Optional[T.List[FileOrString]] = None, @@ -3316,8 +3344,10 @@ def quote_make_target(targetName: str) -> str: element.add_item('CUDA_ESCAPED_TARGET', quote_make_target(rel_obj)) if self.should_use_dyndeps_for_target(target) and compiler.get_language() == 'cpp' and compiler.get_id() == 'clang': + src_with_extension = os.path.basename(src.fname) + mod_output_name = self._get_cpp_module_output_name(src_with_extension, compiler, target) commands.extend(['--start-no-unused-arguments', - f'-fmodule-output={target.name}{compiler.get_cpp20_module_bmi_extension()}', + f'-fmodule-output={mod_output_name}', f'-fprebuilt-module-path={self.environment.get_build_dir()}', '--end-no-unused-arguments']) element.add_item('ARGS', commands) diff --git a/mesonbuild/scripts/depscanaccumulate.py b/mesonbuild/scripts/depscanaccumulate.py index e2d3a3ba9087..ccbfcecd58c8 100644 --- a/mesonbuild/scripts/depscanaccumulate.py +++ b/mesonbuild/scripts/depscanaccumulate.py @@ -21,15 +21,18 @@ class ModuleProviderInfo: class CppDependenciesScanner: pass +def normalize_filename(fname): + return fname.replace(':', '-') + class DynDepRule: def __init__(self, out: str, imp_outs: T.Optional[T.List[str]], imp_ins: T.List[str]): self.output = [f'build {out}'] if imp_outs: - imp_out_str = " ".join(imp_outs) + imp_out_str = " ".join([normalize_filename(o) for o in imp_outs]) self.output.append(f" | {imp_out_str}") self.output.append(": dyndep") if imp_ins: - imp_ins_str = " ".join(imp_ins) + imp_ins_str = " ".join([normalize_filename(inf) for inf in imp_ins]) self.output.append(" | " + imp_ins_str) self.output_str = "".join(self.output) + "\n"