ModelConfig Enhancement (#1516)

* Enhance ModelConfig to be more powerful * Empty-Commit * Incremented patch version * Test Python support * Fix compile error * Update library * Misc * Update Python version * Update directory * Should work. Hopefully * Enable Python in test install * Turn on FOEDAG_WITH_PYTHON * Why msys2 still failed * Deal with symbol array [] if contrains is read from file * Another try * Misc * Misc * Add debug * Suspect the issue, use hardcode method to confirm * Hardcode for testing * Use action * Revert * Mistake * DLL for MSCV only * Should work now * Should work now * Confuse * Simplify --------- Co-authored-by: chungshien-chai <[email protected]> Co-authored-by: alaindargelas <[email protected]>
os-fpga · Mar 27, 2024 · 02f26ae · 02f26ae
1 parent cf30c38
commit 02f26ae
Show file tree

Hide file tree

Showing 35 changed files with 2,341 additions and 393 deletions.
diff --git a/.github/workflows/install_centos_dependencies_build.sh b/.github/workflows/install_centos_dependencies_build.sh
@@ -31,6 +31,7 @@ yum install -y libusbx-devel libusb-devel
 yum install -y pkgconfig
 yum install -y perl-IPC-Cmd
 yum install -y alsa-lib mesa-dri-drivers openssl openssl-devel sudo
+yum install -y python3-devel
 ln -s $PWD/cmake-3.24.4-linux-x86_64/bin/ctest /usr/bin/ctest
 
 # downloads the Qt6 artifact from a specific URL

diff --git a/.github/workflows/install_ubuntu_dependencies_build.sh b/.github/workflows/install_ubuntu_dependencies_build.sh
@@ -15,7 +15,8 @@ sudo apt install -y \
   yosys \
   automake \
   libusb-1.0-0-dev \
-  pkg-config
+  pkg-config \
+  python3-dev
 
 wget -qO- https://packages.lunarg.com/lunarg-signing-key-pub.asc | sudo tee /etc/apt/trusted.gpg.d/lunarg.asc
 sudo wget -qO /etc/apt/sources.list.d/lunarg-vulkan-jammy.list http://packages.lunarg.com/vulkan/lunarg-vulkan-jammy.list

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -253,7 +253,8 @@ jobs:
           mingw-w64-x86_64-tcl
           mingw-w64-x86_64-zlib
           mingw-w64-x86_64-libusb
-    
+          python-devel
+
     - name: Configure shell environment variables
       run: |
         export CWD=`pwd`

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -133,15 +133,16 @@ endif()
 set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/lib)
 set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/bin)
 set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/bin)
+set(FOEDAG_WITH_PYTHON ON)
 
 # Python
 if (FOEDAG_WITH_PYTHON)
-find_package(Python3 3.3 REQUIRED COMPONENTS Interpreter Development)
-# find_package(SWIG 3.0 REQUIRED)
-message(STATUS "Python3_LIBRARIES = ${Python3_LIBRARIES}")
-message(STATUS "Python3_EXECUTABLE = ${Python3_EXECUTABLE}")
-message(STATUS "Python3_INCLUDE_DIRS = ${Python3_INCLUDE_DIRS}")
-message(STATUS "Python3_RUNTIME_LIBRARY_DIRS = ${Python3_RUNTIME_LIBRARY_DIRS}")
+  set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/cmake/")
+  find_package(CustomPython3 REQUIRED)
+  message(STATUS "Python3_LIBRARIES (main) = ${Python3_LIBRARIES}")
+  message(STATUS "Python3_EXECUTABLE (main) = ${Python3_EXECUTABLE}")
+  message(STATUS "Python3_INCLUDE_DIRS (main) = ${Python3_INCLUDE_DIRS}")
+  message(STATUS "Python3_RUNTIME_LIBRARY_DIRS (main) = ${Python3_RUNTIME_LIBRARY_DIRS}")
 endif()
 
 if(NOT NO_TCMALLOC)

diff --git a/cmake/FindCustomPython3.cmake b/cmake/FindCustomPython3.cmake
@@ -0,0 +1,52 @@
+# - Try to find the freetype library
+# Once done this defines
+#
+#  Python3_LIBRARIES - fullpath of the python3 library
+#  Python3_INCLUDE_DIRS - include folder of python3
+#  Python3_RUNTIME_LIBRARY_DIRS - bin folder of python3
+#  Python3_EXECUTABLE - exe of python3 
+#  Python3_FOUND - set when all four are found
+
+#  Work for msys2 only
+
+# Copyright (c) 2024 Chai, Chung Shien
+#
+# Redistribution and use is allowed according to the terms of the BSD license.
+# For details see the accompanying COPYING-CMAKE-SCRIPTS file.
+message("FindCustomPython3 MINGW: ${MINGW}")
+message("FindCustomPython3 MSVC: ${MSVC}")
+message("FindCustomPython3 WIN32: ${WIN32}")
+set(PYTHON3_FIND_PATH "C:/msys64/mingw64")
+if ((MINGW OR (NOT MSVC AND WIN32)) AND EXISTS ${PYTHON3_FIND_PATH})
+  file(GLOB potential_python3_libs "${PYTHON3_FIND_PATH}/lib/libpython3.*.dll.a")
+  if (potential_python3_libs)
+    list (GET potential_python3_libs 0 Python3_LIBRARIES)
+    if (Python3_LIBRARIES AND EXISTS ${Python3_LIBRARIES})
+      get_filename_component(python3_filename ${Python3_LIBRARIES} NAME)
+      string(REGEX REPLACE "libpython" "python" python3_filename ${python3_filename})
+      string(REGEX REPLACE ".dll.a" "" python3_filename ${python3_filename})
+      set(Python3_INCLUDE_DIRS ${PYTHON3_FIND_PATH}/include/${python3_filename})
+      set(Python3_RUNTIME_LIBRARY_DIRS ${PYTHON3_FIND_PATH}/bin)
+      set(Python3_EXECUTABLE ${Python3_RUNTIME_LIBRARY_DIRS}/${python3_filename}.exe)
+      if (Python3_INCLUDE_DIRS AND EXISTS ${Python3_INCLUDE_DIRS})
+        if (Python3_RUNTIME_LIBRARY_DIRS AND EXISTS ${Python3_RUNTIME_LIBRARY_DIRS})
+          if (Python3_EXECUTABLE AND EXISTS ${Python3_EXECUTABLE})
+            set(Python3_FOUND TRUE)
+          else()
+            message(SEND_ERROR "${Python3_EXECUTABLE} does not exist")
+          endif()
+        else()
+          message(SEND_ERROR "${Python3_RUNTIME_LIBRARY_DIRS} does not exist")
+        endif()
+      else()
+        message(SEND_ERROR "${Python3_INCLUDE_DIRS} does not exist")
+      endif()
+    else()
+      message(SEND_ERROR "Fail to look for libpython3.*.dll.a in ${PYTHON3_FIND_PATH}/lib")
+    endif()
+  else()
+    message(SEND_ERROR "Fail to look for libpython3.*.dll.a in ${PYTHON3_FIND_PATH}/lib")
+  endif()
+else()
+  find_package(Python3 3.3 REQUIRED COMPONENTS Interpreter Development)
+endif()
diff --git a/src/Compiler/Constraints.cpp b/src/Compiler/Constraints.cpp
@@ -128,6 +128,41 @@ static std::vector<std::string> constraint_procs = {
     "set_pvt", "set_pvt_min_max", "default_operating_conditions", "cell_regexp",
     "cell_regexp_hsc", "port_regexp", "port_regexp_hsc"};
 
+static std::string get_rid_array_name(const std::string& name) {
+  std::string final_name = name;
+  uint32_t tracking = 0;
+  size_t index = final_name.size() - 1;
+  for (auto c = final_name.rbegin(); c < final_name.rend(); c++, index--) {
+    if (tracking == 0) {
+      if (*c == '%') {
+        tracking++;
+      } else {
+        break;
+      }
+    } else if (tracking == 1) {
+      if (*c >= '0' && *c <= '9') {
+        tracking++;
+      } else {
+        break;
+      }
+    } else if (tracking == 2) {
+      if (*c >= '0' && *c <= '9') {
+        continue;
+      } else if (*c == '@') {
+        tracking++;
+        break;
+      } else {
+        break;
+      }
+    }
+  }
+  if (tracking == 3 && index > 0) {
+    final_name[index] = '[';
+    final_name[final_name.size() - 1] = ']';
+  }
+  return final_name;
+}
+
 void Constraints::registerCommands(TclInterpreter* interp) {
   // SDC constraints
   // https://github.com/The-OpenROAD-Project/OpenSTA/blob/master/tcl/Sdc.tcl
@@ -613,6 +648,11 @@ void Constraints::registerCommands(TclInterpreter* interp) {
       return TCL_ERROR;
     }
     Constraints* constraints = (Constraints*)clientData;
+#if 0
+    for (int i = 0; i < argc; i++) {
+      constraints->GetCompiler()->Message(std::string{"CONSTRAINT set_property DEBUG: "} + std::string{argv[i]});
+    }
+#endif
     if (!verifyTimingLimits(argc, argv)) {
       Tcl_AppendResult(interp, TimingLimitErrorMessage, nullptr);
       return TCL_ERROR;
@@ -648,7 +688,7 @@ void Constraints::registerCommands(TclInterpreter* interp) {
     }
     for (int i = 3; i < argc; i++) {
       // No duplication
-      std::string obj = std::string(argv[i]);
+      std::string obj = get_rid_array_name(std::string(argv[i]));
       if (CFG_find_string_in_vector(objects, obj) < 0) {
         objects.push_back(obj);
       }

diff --git a/src/Configuration/CFGCommon/CFGCommon.cpp b/src/Configuration/CFGCommon/CFGCommon.cpp
@@ -13,7 +13,12 @@ GNU General Public License for more details.
 You should have received a copy of the GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
+
+// clang-format off
+#include <Python.h>
+
 #include "CFGCommon.h"
+// clang-format on
 
 #ifdef _WIN32
 #include <Windows.h>
@@ -699,4 +704,55 @@ bool CFG_compare_two_binary_files(const std::string& filepath1,
   CFG_read_binary_file(filepath2, data2);
   return (data1.size() == data2.size()) &&
          (data1.size() == 0 || memcmp(&data1[0], &data2[0], data1.size()) == 0);
-}
+}
+
+static void CFG_Python_get_string(
+    PyObject* dict, const std::string& key,
+    std::map<std::string, std::string>& str_maps) {
+  PyObject* value = PyDict_GetItemString(dict, key.c_str());
+  if (value != nullptr) {
+    CFG_ASSERT_MSG(PyUnicode_Check(value),
+                   "Python Dict key=%s is expected to be string, but it is not",
+                   key.c_str());
+    str_maps[key] = std::string(PyUnicode_AsUTF8(value));
+    // Py_DECREF(value);
+  }
+}
+
+static void CFG_Python_get_u32(PyObject* dict, const std::string& key,
+                               std::map<std::string, uint32_t>& int_maps) {
+  PyObject* value = PyDict_GetItemString(dict, key.c_str());
+  if (value != nullptr) {
+    CFG_ASSERT_MSG(PyLong_Check(value),
+                   "Python Dict key=%s is expected to be long, but it is not",
+                   key.c_str());
+    int_maps[key] = (uint32_t)(PyLong_AsLong(value));
+    // Py_DECREF(value);
+  }
+}
+
+void CFG_Python(std::vector<std::string> commands,
+                std::vector<std::string> strs, std::vector<std::string> ints,
+                std::map<std::string, std::string>& str_maps,
+                std::map<std::string, uint32_t>& int_maps) {
+  Py_Initialize();
+  PyObject* dict = PyDict_New();
+  PyObject* o = nullptr;
+  for (auto& command : commands) {
+#if 0
+    printf("Python Command: %s\n", command.c_str());
+#endif
+    o = PyRun_String(command.c_str(), Py_single_input, dict, dict);
+  }
+  for (auto key : strs) {
+    CFG_Python_get_string(dict, key, str_maps);
+  }
+  for (auto key : ints) {
+    CFG_Python_get_u32(dict, key, int_maps);
+  }
+  if (o != nullptr) {
+    Py_DECREF(o);
+  }
+  Py_DECREF(dict);
+  Py_Finalize();
+}
diff --git a/src/Configuration/CFGCommon/CFGCommon.h b/src/Configuration/CFGCommon/CFGCommon.h
@@ -6,6 +6,7 @@
 #include <cstring>
 #include <filesystem>
 #include <functional>
+#include <map>
 #include <memory>
 #include <regex>
 #include <string>
@@ -193,6 +194,11 @@ bool CFG_compare_two_text_files(const std::string& filepath1,
 bool CFG_compare_two_binary_files(const std::string& filepath1,
                                   const std::string& filepath2);
 
+void CFG_Python(std::vector<std::string> commands,
+                std::vector<std::string> strs, std::vector<std::string> ints,
+                std::map<std::string, std::string>& str_maps,
+                std::map<std::string, uint32_t>& int_maps);
+
 #define CFG_POST_MSG(...) \
   { CFG_post_msg(CFG_print(__VA_ARGS__)); }
 

diff --git a/src/Configuration/CFGCommon/CMakeLists.txt b/src/Configuration/CFGCommon/CMakeLists.txt
@@ -28,6 +28,13 @@ set(CMAKE_INCLUDE_CURRENT_DIR ON)
 set(CMAKE_CXX_STANDARD 17)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 
+set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/../../../cmake/")
+find_package(CustomPython3 REQUIRED)
+message(STATUS "Python3_LIBRARIES (CFG) = ${Python3_LIBRARIES}")
+message(STATUS "Python3_INCLUDE_DIRS (CFG) = ${Python3_INCLUDE_DIRS}")
+message(STATUS "Python3_RUNTIME_LIBRARY_DIRS (CFG) = ${Python3_RUNTIME_LIBRARY_DIRS}")
+include_directories(${Python3_INCLUDE_DIRS})
+
 if (MSVC)
 else()
   set(CMAKE_CXX_FLAGS_DEBUG
@@ -67,6 +74,7 @@ add_library(
   CFGArg.cpp
   ${CMAKE_CURRENT_BINARY_DIR}/CFGArg_auto.h
 )
+target_link_libraries(${subsystem} ${Python3_LIBRARIES})
 
 ###################
 #

diff --git a/src/Configuration/ModelConfig/ModelConfig.cpp b/src/Configuration/ModelConfig/ModelConfig.cpp
@@ -221,24 +221,26 @@ class ModelConfig_DEVICE {
   }
   void set_attr(const std::string& instance, const std::string& name,
                 const std::string& value, const std::string& reason) {
-    /*
-    printf("set_attr: %s: %s -> %s\n", instance.c_str(), name.c_str(),
-           value.c_str());
-    */
+#if 0
+    printf("DEBUG: Design Set Attr 1: %s : %s -> %s\n", instance.c_str(),
+           name.c_str(), value.c_str());
+#endif
     ModelConfig_BITFIELD* bitfield = get_bitfield(instance, name);
     CFG_ASSERT_MSG(bitfield != nullptr,
                    "Could not find bitfield '%s' for block instance '%s'",
                    name.c_str(), instance.c_str());
-    uint32_t v = 0;
-    if (!is_number(value, v)) {
-      CFG_ASSERT(bitfield->m_type != nullptr);
-      CFG_ASSERT(bitfield->m_type.get() != nullptr);
-      v = bitfield->m_type.get()->get_enum_value(value);
+    if (value != "__DONT__") {
+      uint32_t v = 0;
+      if (!is_number(value, v)) {
+        CFG_ASSERT(bitfield->m_type != nullptr);
+        CFG_ASSERT(bitfield->m_type.get() != nullptr);
+        v = bitfield->m_type.get()->get_enum_value(value);
+      }
+      CFG_ASSERT(bitfield->m_size == 32 ||
+                 (v < ((uint32_t)(1) << bitfield->m_size)));
+      bitfield->m_value = v;
+      bitfield->reasons.push_back(reason);
     }
-    CFG_ASSERT(bitfield->m_size == 32 ||
-               (v < ((uint32_t)(1) << bitfield->m_size)));
-    bitfield->m_value = v;
-    bitfield->reasons.push_back(reason);
   }
   void set_attr(const std::map<std::string, std::string>& options,
                 std::string reason = "") {
@@ -304,7 +306,8 @@ class ModelConfig_DEVICE {
         reason = CFG_print("%s [from %s]", reason.c_str(), instance.c_str());
       }
 #if 0
-      printf("DEBUG: Design Set Attr: %s : %s -> %s\n", final_instance.c_str(), key_str.c_str(), value_str.c_str());
+      printf("DEBUG: Design Set Attr 0: %s : %s -> %s\n",
+             final_instance.c_str(), key_str.c_str(), value_str.c_str());
 #endif
       set_attr({{"instance", final_instance},
                 {"name", key_str},
@@ -338,7 +341,8 @@ class ModelConfig_DEVICE {
   }
   void set_design(const std::string& filepath) {
     std::ifstream file(filepath.c_str());
-    CFG_ASSERT(file.is_open() && file.good());
+    CFG_ASSERT_MSG(file.is_open() && file.good(), "Cannot open design file %s",
+                   filepath.c_str());
     nlohmann::json api = nlohmann::json::parse(file);
     file.close();
     // Must start with a dict/map

diff --git a/src/Configuration/ModelConfig/ModelConfig.h b/src/Configuration/ModelConfig/ModelConfig.h
@@ -30,6 +30,8 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 struct CFGCommon_ARG;
 
+enum ARG_PROPERTY { IS_NONE_ARG, IS_ARG, IS_ARG_WITH_DEFAULT };
+
 namespace FOEDAG {
 
 void model_config_entry(CFGCommon_ARG* cmdarg);
@@ -56,21 +58,30 @@ class ModelConfig_IO {
   static void set_config_attribute(nlohmann::json& config_attributes,
                                    const std::string& module,
                                    nlohmann::json inputs,
-                                   nlohmann::json mapping);
+                                   nlohmann::json mapping,
+                                   std::map<std::string, std::string>& args,
+                                   nlohmann::json define);
   static void set_config_attribute(nlohmann::json& config_attributes,
                                    nlohmann::json inputs, nlohmann::json rules,
                                    nlohmann::json results,
-                                   nlohmann::json neg_results);
+                                   nlohmann::json neg_results,
+                                   std::map<std::string, std::string>& args,
+                                   nlohmann::json define);
   static void set_config_attribute(nlohmann::json& config_attributes,
                                    nlohmann::json& results,
-                                   std::map<std::string, std::string>& args);
+                                   std::map<std::string, std::string>& args,
+                                   nlohmann::json define);
   static void set_config_attribute(nlohmann::json& config_attributes,
                                    std::map<std::string, std::string>& args,
                                    nlohmann::json object, std::string key,
                                    nlohmann::json value);
   static bool config_attribute_rule_match(
       nlohmann::json inputs, const std::string& input, nlohmann::json options,
       std::map<std::string, std::string>& args);
+  static void define_args(nlohmann::json define,
+                          std::map<std::string, std::string>& args);
+  static ARG_PROPERTY get_arg_info(std::string str, std::string& name,
+                                   std::string& value);
 };
 
 }  // namespace FOEDAG