Skip to content

Commit

Permalink
Reduce repeated template parsing to speed up builds
Browse files Browse the repository at this point in the history
* Separate writing arguments files from the builds of each generator or type support target.
* Write all arguments files first, before generating files.
* Generate all files for each IDL template at once, avoiding repeated template parsing.
  • Loading branch information
DanMesh committed Sep 27, 2023
1 parent 3491f4f commit 5f3eb65
Show file tree
Hide file tree
Showing 36 changed files with 730 additions and 411 deletions.
8 changes: 8 additions & 0 deletions rosidl_cmake/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,9 @@ find_package(ament_cmake_python REQUIRED)

ament_python_install_package(${PROJECT_NAME})

set(rosidl_cmake_generate_interfaces_BIN "${CMAKE_CURRENT_SOURCE_DIR}/bin/rosidl_cmake_generate_interfaces")
normalize_path(rosidl_cmake_generate_interfaces_BIN "${rosidl_cmake_generate_interfaces_BIN}")

if(BUILD_TESTING)
find_package(ament_lint_auto REQUIRED)
ament_lint_auto_find_test_dependencies()
Expand All @@ -16,6 +19,11 @@ ament_package(
CONFIG_EXTRAS "rosidl_cmake-extras.cmake"
)

install(
PROGRAMS bin/rosidl_cmake_generate_interfaces
DESTINATION lib/rosidl_cmake
)

install(
DIRECTORY cmake
DESTINATION share/${PROJECT_NAME}
Expand Down
40 changes: 40 additions & 0 deletions rosidl_cmake/bin/rosidl_cmake_generate_interfaces
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
#!/usr/bin/env python3

import argparse
import os
import sys

try:
from rosidl_pycommon import generate_files_from_arguments_files
except ImportError:
# modifying sys.path and importing the Python package with the same
# name as this script does not work on Windows
rosidl_pycommon_root = os.path.dirname(os.path.dirname(__file__))
rosidl_pycommon_module = os.path.join(
rosidl_pycommon_root, 'rosidl_pycommon', '__init__.py')
if not os.path.exists(rosidl_pycommon_module):
raise
from importlib.machinery import SourceFileLoader

loader = SourceFileLoader('rosidl_pycommon', rosidl_pycommon_module)
rosidl_pycommon = loader.load_module()
generate_files_from_arguments_files = rosidl_pycommon.generate_files_from_arguments_files


def main(argv=sys.argv[1:]):
parser = argparse.ArgumentParser(
description='Generate the ROS interfaces.',
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument(
'--generator-arguments-files',
required=True,
help='The location of the files containing the generator arguments')
args = parser.parse_args(argv)

print(args.generator_arguments_files)

generate_files_from_arguments_files(args.generator_arguments_files)


if __name__ == '__main__':
sys.exit(main())
20 changes: 19 additions & 1 deletion rosidl_cmake/cmake/rosidl_generate_interfaces.cmake
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# Copyright 2014-2018 Open Source Robotics Foundation, Inc.
# Copyright 2014-2023 Open Source Robotics Foundation, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
Expand Down Expand Up @@ -277,6 +277,24 @@ macro(rosidl_generate_interfaces target)
list(APPEND rosidl_generate_interfaces_ABS_IDL_FILES "${_abs_idl_file}")
endforeach()

# Create the type descriptions for use when writing the generator arguments files
ament_execute_extensions("rosidl_create_type_descriptions_extensions")

# Write the generator argument files for all registered languages and type supports
set(rosidl_generator_arguments_files)
ament_execute_extensions("rosidl_write_generator_arguments_extensions")

find_package(Python3 REQUIRED COMPONENTS Interpreter)

# Generate the interface source files for all registered languages and type supports
set(rosidl_cmake_generate_interfaces_BIN "${rosidl_cmake_DIR}/../../../lib/rosidl_cmake/rosidl_cmake_generate_interfaces")
execute_process(
COMMAND ${Python3_EXECUTABLE} ${rosidl_cmake_generate_interfaces_BIN}
--generator-arguments-files "${rosidl_generator_arguments_files}"
ECHO_OUTPUT_VARIABLE
)

# Build the interfaces from the generated files
ament_execute_extensions("rosidl_generate_idl_interfaces")

# check for extensions registered with the previous extension point
Expand Down
105 changes: 105 additions & 0 deletions rosidl_cmake/cmake/rosidl_write_additional_context.cmake
Original file line number Diff line number Diff line change
@@ -0,0 +1,105 @@
# Copyright 2023 Open Source Robotics Foundation, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

#
# Generate a JSON / YAML file containing additional context data for expanding
# IDL templates
#
#
# @public
#
function(rosidl_write_additional_context output_file)

set(OPTIONAL_ONE_VALUE_KEYWORDS
"DISABLE_DESCRIPTION_CODEGEN")
set(OPTIONAL_MULTI_VALUE_KEYWORDS
"TYPE_SUPPORTS")

cmake_parse_arguments(
ARG
""
"${OPTIONAL_ONE_VALUE_KEYWORDS}"
"${OPTIONAL_MULTI_VALUE_KEYWORDS}"
${ARGN})
if(ARG_UNPARSED_ARGUMENTS)
message(FATAL_ERROR "rosidl_write_additional_context() called with unused "
"arguments: ${ARG_UNPARSED_ARGUMENTS}")
endif()

# create folder
get_filename_component(output_path "${output_file}" PATH)
file(MAKE_DIRECTORY "${output_path}")

# open object
file(WRITE "${output_file}"
"{")

set(first_element TRUE)

# write string values
foreach(one_value_argument ${OPTIONAL_ONE_VALUE_KEYWORDS})
if(DEFINED ARG_${one_value_argument})
# write conditional comma and mandatory newline
if(NOT first_element)
file(APPEND "${output_file}" ",")
else()
set(first_element FALSE)
endif()
file(APPEND "${output_file}" "\n")

string(TOLOWER "${one_value_argument}" key)
string(REPLACE "\\" "\\\\" value "${ARG_${one_value_argument}}")
file(APPEND "${output_file}"
" \"${key}\": \"${value}\"")
endif()
endforeach()

# write array values
foreach(multi_value_argument ${OPTIONAL_MULTI_VALUE_KEYWORDS})
if(ARG_${multi_value_argument})
# write conditional comma and mandatory newline and indentation
if(NOT first_element)
file(APPEND "${output_file}" ",")
else()
set(first_element FALSE)
endif()
file(APPEND "${output_file}" "\n")

# write key, open array
string(TOLOWER "${multi_value_argument}" key)
file(APPEND "${output_file}"
" \"${key}\": [\n")

# write array values, last without trailing colon
list(GET ARG_${multi_value_argument} -1 last_value)
list(REMOVE_AT ARG_${multi_value_argument} -1)
foreach(value ${ARG_${multi_value_argument}})
string(REPLACE "\\" "\\\\" value "${value}")
file(APPEND "${output_file}"
" \"${value}\",\n")
endforeach()
string(REPLACE "\\" "\\\\" last_value "${last_value}")
file(APPEND "${output_file}"
" \"${last_value}\"\n")

# close array
file(APPEND "${output_file}"
" ]")
endif()
endforeach()

# close object
file(APPEND "${output_file}"
"\n}\n")
endfunction()
8 changes: 5 additions & 3 deletions rosidl_cmake/cmake/rosidl_write_generator_arguments.cmake
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# Copyright 2015 Open Source Robotics Foundation, Inc.
# Copyright 2023 Open Source Robotics Foundation, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
Expand All @@ -23,12 +23,14 @@ function(rosidl_write_generator_arguments output_file)
"PACKAGE_NAME")
set(OPTIONAL_ONE_VALUE_KEYWORDS
"OUTPUT_DIR"
"TEMPLATE_DIR")
"TEMPLATE_DIR"
"ADDITIONAL_CONTEXT_FILE")

set(REQUIRED_MULTI_VALUE_KEYWORDS # only require one of them
"IDL_TUPLES"
"NON_IDL_TUPLES"
"ROS_INTERFACE_FILES")
"ROS_INTERFACE_FILES"
"GENERATOR_FILES")
set(OPTIONAL_MULTI_VALUE_KEYWORDS
"ROS_INTERFACE_DEPENDENCIES" # since the dependencies can be empty
"TARGET_DEPENDENCIES"
Expand Down
1 change: 1 addition & 0 deletions rosidl_cmake/rosidl_cmake-extras.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ find_package(rosidl_adapter) # not required, being used when available
include("${rosidl_cmake_DIR}/rosidl_generate_interfaces.cmake")
include("${rosidl_cmake_DIR}/rosidl_get_typesupport_target.cmake")
include("${rosidl_cmake_DIR}/rosidl_target_interfaces.cmake")
include("${rosidl_cmake_DIR}/rosidl_write_additional_context.cmake")
include("${rosidl_cmake_DIR}/rosidl_write_generator_arguments.cmake")
include("${rosidl_cmake_DIR}/string_camel_case_to_lower_case_underscore.cmake")

Expand Down
4 changes: 0 additions & 4 deletions rosidl_generator_c/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -34,10 +34,6 @@ ament_package(
CONFIG_EXTRAS "rosidl_generator_c-extras.cmake.in"
)

install(
PROGRAMS bin/rosidl_generator_c
DESTINATION lib/rosidl_generator_c
)
install(
DIRECTORY cmake resource
DESTINATION share/${PROJECT_NAME}
Expand Down
42 changes: 0 additions & 42 deletions rosidl_generator_c/bin/rosidl_generator_c

This file was deleted.

12 changes: 7 additions & 5 deletions rosidl_generator_c/cmake/register_c.cmake
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# Copyright 2015 Open Source Robotics Foundation, Inc.
# Copyright 2023 Open Source Robotics Foundation, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
Expand All @@ -12,17 +12,19 @@
# See the License for the specific language governing permissions and
# limitations under the License.

macro(rosidl_generator_c_extras BIN GENERATOR_FILES TEMPLATE_DIR)
macro(rosidl_generator_c_extras GENERATOR_FILES TEMPLATE_DIR)
find_package(ament_cmake_core QUIET REQUIRED)
find_package(rosidl_generator_type_description QUIET REQUIRED)

ament_register_extension(
"rosidl_write_generator_arguments_extensions"
"rosidl_generator_c"
"rosidl_generator_c_write_arguments.cmake")
ament_register_extension(
"rosidl_generate_idl_interfaces"
"rosidl_generator_c"
"rosidl_generator_c_generate_interfaces.cmake")

normalize_path(BIN "${BIN}")
set(rosidl_generator_c_BIN "${BIN}")

normalize_path(GENERATOR_FILES "${GENERATOR_FILES}")
set(rosidl_generator_c_GENERATOR_FILES "${GENERATOR_FILES}")

Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# Copyright 2015-2018 Open Source Robotics Foundation, Inc.
# Copyright 2015-2023 Open Source Robotics Foundation, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
Expand Down Expand Up @@ -50,8 +50,9 @@ foreach(_pkg_name ${rosidl_generate_interfaces_DEPENDENCY_PACKAGE_NAMES})
endforeach()
endforeach()

set(generator_arguments_file "${CMAKE_CURRENT_BINARY_DIR}/rosidl_generator_c__arguments.json")

set(target_dependencies
"${rosidl_generator_c_BIN}"
${rosidl_generator_c_GENERATOR_FILES}
"${rosidl_generator_c_TEMPLATE_DIR}/action__type_support.h.em"
"${rosidl_generator_c_TEMPLATE_DIR}/action__type_support.c.em"
Expand All @@ -71,45 +72,14 @@ set(target_dependencies
"${rosidl_generator_c_TEMPLATE_DIR}/srv__type_support.c.em"
"${rosidl_generator_c_TEMPLATE_DIR}/srv__type_support.h.em"
${rosidl_generate_interfaces_ABS_IDL_FILES}
${generator_arguments_file}
${_dependency_files})
foreach(dep ${target_dependencies})
if(NOT EXISTS "${dep}")
message(FATAL_ERROR "Target dependency '${dep}' does not exist")
endif()
endforeach()

get_target_property(_target_sources ${rosidl_generate_interfaces_TARGET} SOURCES)
set(generator_arguments_file "${CMAKE_CURRENT_BINARY_DIR}/rosidl_generator_c__arguments.json")
rosidl_write_generator_arguments(
"${generator_arguments_file}"
PACKAGE_NAME "${PROJECT_NAME}"
IDL_TUPLES "${rosidl_generate_interfaces_c_IDL_TUPLES}"
ROS_INTERFACE_DEPENDENCIES "${_dependencies}"
OUTPUT_DIR "${_output_path}"
TEMPLATE_DIR "${rosidl_generator_c_TEMPLATE_DIR}"
TARGET_DEPENDENCIES ${target_dependencies}
TYPE_DESCRIPTION_TUPLES "${${rosidl_generate_interfaces_TARGET}__DESCRIPTION_TUPLES}"
ROS_INTERFACE_FILES "${_target_sources}"
)

find_package(Python3 REQUIRED COMPONENTS Interpreter)

set(disable_description_codegen_arg)
if(ROSIDL_GENERATOR_C_DISABLE_TYPE_DESCRIPTION_CODEGEN)
set(disable_description_codegen_arg "--disable-description-codegen")
endif()

add_custom_command(
OUTPUT ${_generated_headers} ${_generated_sources}
COMMAND Python3::Interpreter
ARGS ${rosidl_generator_c_BIN}
--generator-arguments-file "${generator_arguments_file}"
${disable_description_codegen_arg}
DEPENDS ${target_dependencies}
COMMENT "Generating C code for ROS interfaces"
VERBATIM
)

# generate header to switch between export and import for a specific package
set(_visibility_control_file
"${_output_path}/msg/rosidl_generator_c__visibility_control.h")
Expand Down
Loading

0 comments on commit 5f3eb65

Please sign in to comment.