Add a script that converts FLINT headers to julia types

.gitignore

@@ -1,3 +1,5 @@
 /docs/build/
 Manifest.toml
 .DS_Store
+
+FLINT-3.2.0-rc1/

Project.toml

@@ -7,16 +7,18 @@ AbstractAlgebra = "c3fe647b-3220-5bb0-a1ea-a7954cac585d"
 FLINT_jll = "e134572f-a0d5-539d-bddf-3cad8db41a82"
 Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb"
 LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
+MacroTools = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09"
 Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
 RandomExtensions = "fb686558-2515-59ef-acaa-46db3789a887"
 SHA = "ea8e919c-243c-51af-8825-aaa63cd721ce"
 TOML = "fa267f1f-6049-4f14-aa54-33bafae1ed76"
 
 [compat]
 AbstractAlgebra = "0.44.2"
-FLINT_jll = "^300.100.100"
+FLINT_jll = "~300.100.301" # should be "~300.200.0"
 Libdl = "1.6"
 LinearAlgebra = "1.6"
+MacroTools = "0.5.15"
 Random = "1.6"
 RandomExtensions = "0.4.2"
 SHA = "~1.6, ~1.7, 0.7"

etc/FlintCTypes_template.jl

@@ -0,0 +1,58 @@
+###############################################################################
+#
+#   Flint C types
+#
+###############################################################################
+
+module FlintC
+
+using ..Nemo: @flintstruct, FlintStruct
+
+#
+# FLINT configuration and global definitions
+#
+
+const ulong = UInt
+const slong = Int
+
+WORD(n) = Int(n)
+UWORD(n) = UInt(n)
+
+const FLINT_BITS = UInt == UInt64 ? 64 : 32
+
+
+@include_c_header("flint.h")
+
+@include_c_header("limb_types.h")
+
+@include_c_header("nmod_types.h")
+
+@include_c_header("fmpz_types.h")
+
+@include_c_header("fmpq_types.h")
+
+@include_c_header("fmpz_mod_types.h")
+
+@include_c_header("fq_nmod_types.h")
+
+@include_c_header("fq_zech_types.h")
+
+@include_c_header("fq_types.h")
+
+@include_c_header("gr_types.h")
+
+@include_c_header("fq_default.h")
+
+@include_c_header("fq_default_mat.h")
+
+@include_c_header("fq_default_poly.h")
+
+@include_c_header("fq_default_poly_factor.h")
+
+@include_c_header("padic_types.h")
+
+@include_c_header("n_poly_types.h")
+
+@include_c_header("mpoly_types.h")
+
+end # module FlintC

etc/generate_FLINT_structs.jl

@@ -0,0 +1,189 @@
+VERSION >= v"1.9" || error("This script requires Julia 1.7 or later")
+
+function expand_templates(input, flintdir)
+  matches = eachmatch(r"^@include_c_header\(\"([A-Za-z_.]+)\"\)$"m, input)
+  substitutions = Pair{String,String}[]
+  for m in matches
+    filename = m.captures[1]
+    @info "Including header \"$filename\""
+    template_key = "@include_c_header(\"$filename\")"
+
+    # Read the file
+    content = open(joinpath(flintdir, filename)) do headerfile
+      read(headerfile, String)
+    end
+    # Convert to julia
+    translated_content = c2julia(content)
+    # Build the substitution value
+    template_val =
+      "###############################################################################\n" *
+      "# begin $filename\n\n" *
+      strip(translated_content) *
+      "\n\n# end $filename\n" *
+      "###############################################################################\n"
+    push!(substitutions, template_key => template_val)
+  end
+  return replace(input, substitutions...)
+end
+
+const regex_typedef_struct_fields_name = r"^typedef struct\s*\{([^{}]+)\}\s*([A-Za-z0-9_]+);"m
+const regex_typedef_struct_fields_refname = r"^typedef struct\s*\{([^{}]+)\}\s*([A-Za-z0-9_]+)\[1\];"m
+const regex_struct_structname_fields = r"^struct *([A-Za-z0-9_]+)\s*\{([^{}]+)\}\s*;"m
+const regex_struct_structname = r"^struct *([A-Za-z0-9_]+);"m
+const regex_typedef_struct_structname_fields_name = r"^typedef struct *([A-Za-z0-9_]+)\s*\{([^{}]+)\}\s*([A-Za-z0-9_]+);"m
+
+const regex_typedef_enum_values_name = r"^typedef enum\s*\{([^{}]+)\}\s*([A-Za-z0-9_]+);"m
+const regex_enum_enumname_values = r"^enum *([A-Za-z0-9_]+)\s*\{([^{}]+)\}\s*;"m
+const regex_typedef_enum_enumname_values_name = r"^typedef enum *([A-Za-z0-9_]+)\s*\{([^{}]+)\}\s*([A-Za-z0-9_]+);"m
+
+const regex_typedef_union_values_name = r"^typedef union\s*\{([^{}]+)\}\s*([A-Za-z0-9_]+);"m
+const regex_union_unionname_values = r"^union *([A-Za-z0-9_]+)\s*\{([^{}]+)\}\s*;"m
+const regex_typedef_union_unionname_values_name = r"^typedef union *([A-Za-z0-9_]+)\s*\{([^{}]+)\}\s*([A-Za-z0-9_]+);"m
+
+function convert_struct(str::AbstractString)
+  substitutions = Pair{Regex,Union{SubstitutionString,Function}}[
+    regex_typedef_struct_fields_name => s"@flintstruct struct \2\1end",                                        # whole typedef struct construct
+    regex_struct_structname_fields => s"@flintstruct struct struct_\1\2end",                                   # whole struct construct
+    regex_struct_structname => s"@flintstruct struct struct_\1 end",                                           # whole struct construct without fields
+    regex_typedef_struct_fields_refname => s"@flintstruct struct struct_\2\1end\nconst \2 = Tuple{struct_\2}", # whole typedef struct singleton array construct
+    regex_typedef_struct_structname_fields_name => s"@flintstruct struct \3\2end\nconst struct_\1 = \3",       # whole typedef struct construct with two names
+    r"^ +([A-Za-z0-9_]+) +([A-Za-z0-9_]+);"m => s"  \2::\1",                                      # simple fields (one to five declared on one line)
+    r"^ +([A-Za-z0-9_]+) +([A-Za-z0-9_]+) *, *([A-Za-z0-9_]+);"m => s"  \2::\1\n  \3::\1",
+    r"^ +([A-Za-z0-9_]+) +([A-Za-z0-9_]+) *, *([A-Za-z0-9_]+) *, *([A-Za-z0-9_]+);"m => s"  \2::\1\n  \3::\1\n  \4::\1",
+    r"^ +([A-Za-z0-9_]+) +([A-Za-z0-9_]+) *, *([A-Za-z0-9_]+) *, *([A-Za-z0-9_]+) *, *([A-Za-z0-9_]+);"m => s"  \2::\1\n  \3::\1\n  \4::\1\n  \5::\1",
+    r"^ +([A-Za-z0-9_]+) +([A-Za-z0-9_]+) *, *([A-Za-z0-9_]+) *, *([A-Za-z0-9_]+) *, *([A-Za-z0-9_]+) *, *([A-Za-z0-9_]+);"m => s"  \2::\1\n  \3::\1\n  \4::\1\n  \5::\1\n  \6::\1",
+    r"^ +([A-Za-z0-9_]+) *\* *([A-Za-z0-9_]+);"m => s"  \2::Ptr{\1}",                             # pointer field (one to five declared on one line)
+    r"^ +([A-Za-z0-9_]+) *\* *([A-Za-z0-9_]+) *, *\* *([A-Za-z0-9_]+);"m => s"  \2::Ptr{\1}\n  \3::Ptr{\1}",
+    r"^ +([A-Za-z0-9_]+) *\* *([A-Za-z0-9_]+) *, *\* *([A-Za-z0-9_]+) *, *\* *([A-Za-z0-9_]+);"m => s"  \2::Ptr{\1}\n  \3::Ptr{\1}\n  \4::Ptr{\1}",
+    r"^ +([A-Za-z0-9_]+) *\* *([A-Za-z0-9_]+) *, *\* *([A-Za-z0-9_]+) *, *\* *([A-Za-z0-9_]+) *, *\* *([A-Za-z0-9_]+);"m => s"  \2::Ptr{\1}\n  \3::Ptr{\1}\n  \4::Ptr{\1}\n  \5::Ptr{\1}",
+    r"^ +([A-Za-z0-9_]+) *\* *([A-Za-z0-9_]+) *, *\* *([A-Za-z0-9_]+) *, *\* *([A-Za-z0-9_]+) *, *\* *([A-Za-z0-9_]+) *, *\* *([A-Za-z0-9_]+);"m => s"  \2::Ptr{\1}\n  \3::Ptr{\1}\n  \4::Ptr{\1}\n  \5::Ptr{\1}\n  \6::Ptr{\1}",
+    r"^ +([A-Za-z0-9_]+) *\* *\* *([A-Za-z0-9_]+);"m => s"  \2::Ptr{Ptr{\1}}",                    # double pointer field
+    r"^ +([A-Za-z0-9_]+) +([A-Za-z0-9_]+)\[([A-Za-z0-9_]+)\];"m => s"  \2::NTuple{\3, \1}",       # fixed len array field
+    r"^ +[A-Za-z0-9_]+ *\( *\* *([A-Za-z0-9_]+) *\) *\([A-Za-z0-9_, *]+\);"m => s"  \1::Ptr{Cvoid}",  # function pointer field
+    r"^ +struct *([A-Za-z0-9_]+) +([A-Za-z0-9_]+);"m => s"  \2::struct_\1",                       # struct field (without typedef)
+    r"^ +enum *([A-Za-z0-9_]+) +([A-Za-z0-9_]+);"m => s"  \2::enum_\1",                           # enum field (without typedef)
+  ]
+  for substitution in substitutions
+    str = replace(str, substitution)
+  end
+  return str
+end
+
+function convert_enum(str::AbstractString)
+  substitutions = Pair{Regex,Union{SubstitutionString,Function}}[
+    regex_typedef_enum_values_name => s"@enum \2 begin\1end",                               # whole typedef enum construct
+    regex_enum_enumname_values => s"@enum enum_\1 begin\2end",                              # whole enum construct
+    regex_typedef_enum_enumname_values_name => s"@enum \3 begin\2end\nconst enum_\1 = \3",  # whole typedef enum construct with two names
+    r"^ +([A-Za-z0-9_]+),?"m => s"  \1",                                                    # simple enum values
+  ]
+  for substitution in substitutions
+    str = replace(str, substitution)
+  end
+  return str
+end
+
+function convert_union(str::AbstractString)
+  substitutions = Pair{Regex,Union{SubstitutionString,Function}}[
+    regex_typedef_union_values_name => s"struct \2\n  uniondata::NTuple{maximum(sizeof, (\1  )), UInt8}\nend",                                 # whole typedef union construct
+    regex_union_unionname_values => s"struct union_\1\n  uniondata::NTuple{maximum(sizeof, (\2  )), UInt8}\nend",                             # whole union construct
+    regex_typedef_union_unionname_values_name => s"struct \3\n  uniondata::NTuple{maximum(sizeof, (\2  )), UInt8}\nend\nconst union_\1 = \3", # whole typedef union construct with two names
+    r"^ +([a-z_]+) +([A-Za-z0-9_]+);"m => s"    \1,",
+  ]
+  for substitution in substitutions
+    str = replace(str, substitution)
+  end
+  return str
+end
+
+function c2julia(str::String)
+  preprocessing = Pair{Regex,String}[
+    r"^//(.*)$"m => "",                     # remove line comment
+    r"/\*(.*?)\*/"s => "",                  # remove block comment
+    r" const " => " ",                      # remove all `const`
+    r"\bvoid\b" => "Cvoid",                 # replace `void` C type
+    r"\bunsigned int\b" => "Cuint",         # replace `unsigned int` C type
+    r"\bint\b" => "Cint",                    # replace `int` C type
+    r"\bunsigned char\b" => "Cuchar",       # replace `unsigned char` C type
+    r"\bchar\b" => "Cchar",                 # replace `char` C type
+    r"\bdouble\b" => "Cdouble",             # replace `double` C type
+  ]
+  for substitution in preprocessing
+    str = replace(str, substitution)
+  end
+  substitutions = Pair{Regex,Union{SubstitutionString,Function}}[
+    regex_typedef_struct_fields_name => convert_struct,                       # whole typedef struct construct
+    regex_typedef_struct_fields_refname => convert_struct,                    # whole typedef struct singleton array construct
+    regex_struct_structname_fields => convert_struct,                         # whole struct construct
+    regex_struct_structname => convert_struct,                                # whole struct construct without fields
+    regex_typedef_struct_structname_fields_name => convert_struct,            # whole typedef struct construct with two names
+    regex_typedef_enum_values_name => convert_enum,                           # whole typedef enum construct
+    regex_enum_enumname_values => convert_enum,                               # whole enum construct
+    regex_typedef_enum_enumname_values_name => convert_enum,                  # whole typedef enum construct with two names
+    regex_typedef_union_values_name => convert_union,                         # whole typedef union construct
+    regex_union_unionname_values => convert_union,                            # whole union construct
+    regex_typedef_union_unionname_values_name => convert_union,               # whole typedef union construct with two names
+    r"^typedef +([A-Za-z_]+) +([A-Za-z_]+);"m => s"const \2 = \1",            # simple typedef
+    r"^typedef +([A-Za-z_]+) *\* *([A-Za-z_]+);"m => s"const \2 = Ptr{\1}",   # pointer typedef
+    r"^typedef +([A-Za-z_]+) +([A-Za-z_]+)\[1\];"m => s"const \2 = Tuple{\1}",  # singleton array typedef
+    r"^typedef +struct +([A-Za-z_]+) +([A-Za-z_]+);"m => s"const \2 = struct_\1", # struct typedef
+    r"^typedef +enum +([A-Za-z_]+) +([A-Za-z_]+);"m => s"const \2 = enum_\1",   # enum typedef
+    r"^typedef +union +([A-Za-z_]+) +([A-Za-z_]+);"m => s"const \2 = union_\1", # union typedef
+    r"^typedef +[A-Za-z0-9_]+ *\( *\* *([A-Za-z0-9_]+) *\) *\([A-Za-z0-9_, *]+\);"m => s"const \1 = Ptr{Cvoid}", # function pointer typedef
+    r"^#define +([A-Za-z_]+) +(\d+) *$"m => s"const \1 = \2",                 # defines of integer constants
+    r"^#define +([A-Za-z_]+) +\(([A-Za-z0-9+*() ]+)\) *$"m => s"const \1 = \2",   # defines of more complex constants
+  ]
+  combined_regex = Regex(join(map(re -> re.pattern, first.(substitutions)), "|"), "m")
+  output = join(
+    map(m -> replace(m.match, substitutions...), eachmatch(combined_regex, str)), "\n\n"
+  )
+  output = replace(output, r"\h*\n" => "\n") # remove trailing whitespace
+  return output
+end
+
+file_header_notice(FLINT_jll_version) = """
+# Most of this file is generated from FLINT's header files.
+# Do not edit manually, only the corresponding `etc/*_template.jl` file should be edited.
+
+# This file was generated using FLINT_jll v$(FLINT_jll_version).
+
+"""
+
+################################################################################
+#
+# Main script
+#
+################################################################################
+
+function expand_template_file(
+  infile::String, outfile::String, flintpath::String, file_header::String
+)
+  @info "Expanding file" infile outfile
+  open(outfile, "w") do io
+    write(io, file_header)
+    write(
+      io, expand_templates(read(infile, String), joinpath(flintpath, "include", "flint"))
+    )
+  end
+end
+
+function main()
+  flintpath = FLINT_jll.find_artifact_dir()
+  flintversion = pkgversion(FLINT_jll)
+  @info "Found FLINT" flintpath flintversion
+
+  infile = joinpath(nemopath, "etc", "FlintCTypes_template.jl")
+  outfile = joinpath(nemopath, "src", "flint", "FlintCTypes.jl")
+  expand_template_file(infile, outfile, flintpath, file_header_notice(flintversion))
+end
+
+const nemopath = dirname(dirname(@__FILE__))
+
+@info "Setting up environment"
+using Pkg
+ENV["JULIA_PKG_PRECOMPILE_AUTO"]=0
+Pkg.activate(; temp=true)
+Pkg.develop(PackageSpec(; path=nemopath))
+Pkg.add("FLINT_jll") # version is fixed by Nemo.jl in the line above
+using FLINT_jll
+
+main()

src/Nemo.jl

@@ -1,3 +1,5 @@
+# Note to self: Remember to remove the Overrides.toml once FLINT 3.2.0 is released.
+
 @doc raw"""
   Nemo is a computer algebra package for the Julia programming language, maintained by William Hart, Tommy Hofmann, Claus Fieker and Fredrik Johansson with additional code by Oleksandr Motsak and other contributors.
 

src/flint/FlintCHelpers.jl

@@ -0,0 +1,26 @@
+import MacroTools
+
+abstract type FlintStruct end
+
+function Base.getproperty(val::FlintStruct, name::Symbol)
+  if fieldtype(typeof(val), name) <: Tuple{T} where T # unwrap singleton tuples
+    return only(getfield(val, name))
+  else
+    getfield(val, name)
+  end
+end
+
+macro flintstruct(structdef::Expr)
+  structprops = MacroTools.splitstructdef(structdef)
+  @assert !structprops[:mutable]
+  @assert isempty(structprops[:params])
+  @assert structprops[:supertype] == :Any
+
+  @show structprops
+
+  structprops[:supertype] = :(FlintStruct)
+
+  return esc(quote
+    $(MacroTools.combinestructdef(structprops))
+  end)
+end