Merge pull request #3 from pocketlinux32/staging-llvm

Merging LLVM Test Branch

.vscode/settings.json

@@ -0,0 +1,6 @@
+{
+    "files.watcherExclude": {
+		"**/build/**": true,
+		"**/tarballs/**": true
+	}
+}

compile

@@ -2,8 +2,6 @@
 # SPDX-License-Identifier: GPL-2.0-or-later
 set -e
 
-# TODO: refactor code in modules to use _exec() and _compile_pkg() when possible and useful
-# TODO: maybe make a _timed_exec() and/or _timed_run() function to potentialy remove some additional redundant lines of code
 # TODO: document the usage and params of every function
 # TODO: document every modules needed dependencies in comments
 # TODO: finish llvm support
@@ -16,6 +14,10 @@ pldir="$(dirname $(realpath $0))"
 plfiles="$pldir/pl-files"
 threads=$(nproc)
 
+source "$plfiles/compile-modules/defaults.sh"
+source "$plfiles/compile-modules/pkg/get_names.sh"
+source "$plfiles/compile-modules/runtime_calc.sh"
+
 if [ $threads -gt 4 ]; then
 	if [ $threads -lt 8 ]; then
 		threads=4
@@ -24,10 +26,6 @@ if [ $threads -gt 4 ]; then
 	fi
 fi
 
-source "$plfiles/compile-modules/defaults.sh"
-source "$plfiles/compile-modules/pkg/get_names.sh"
-source "$plfiles/compile-modules/runtime_calc.sh"
-
 _print_help(){ # TODO: Make sure this is up to date
 	printf "Usage: $0 {--target-system|--threads} [--init|--clean|--hard-clean|--build] [value]\n\n"
 	echo "--init					Initializes the build system by downloading necessary components and unpacking them"
@@ -74,7 +72,19 @@ _exec(){
 	set -e
 }
 
-_compile_pkg(){
+_compiler_check(){
+	if [ "$LLVM" != "" ]; then
+		cross_cc="$toolchain_bin/clang"
+		cross_cflags="--gcc-toolchain='' --target=$compile_target --sysroot=$sysroot --ld-path=$toolchain_bin/ld.lld --rtlib=compiler-rt"
+		kbuild_flags="HOSTCC='cc' HOSTLD='ld' LLVM='$toolchain_bin' "
+
+		printf "WARNING: LLVM PortaLinux is experimental, and currently only tested and supported on i486-musl. extra-pkgs are currently not supported on LLVM.\n\n"
+	else
+		cross_cc="$toolchain_bin/$compile_target-gcc"
+	fi
+}
+
+_compile_ac_pkg(){
 	if [ ! -r "$1" ]; then
 		cd "$2"
 		if [ ! -r "$2/Makefile" ]; then
@@ -85,16 +95,42 @@ _compile_pkg(){
 			if [ "$dir" = "" ]; then
 				dir=".."
 			fi
-			_exec "$3" "$dir/configure $4"
+			PATH="$toolchain_bin:$PATH" _exec "$3" "$dir/configure $4"
 		fi
 
 		while [ $# -gt 4 ]; do
-			_exec "$5" "make -j$threads $6"
+			PATH="$toolchain_bin:$PATH" _exec "$5" "make -j$threads $6"
 			shift 2
 		done
 	fi
 }
 
+#_compile_cmake_pkg 1)fileToCheckIfExists 2)mainProjectDir 3)projectToConfig 4)buildSubDir(optional)
+#                   5)installPrefix 6)cmakeArgs 7)projectName 8)noCleanBuildDir(optional) 9)noSilentBuild(optional)
+_compile_cmake_pkg(){
+	if [ ! -r "$1" ]; then
+		cd "$2"
+		if [ "$8" = "" ]; then
+			rm -rf "build/$4"
+		fi
+
+		# arg parser
+		_args=""
+		for i in $6; do
+			_args="$_args -D$i"
+		done
+
+		# configure the project
+		_exec "Configuring $7"			"cmake -S './$3' -B 'build/$4' -GNinja -DCMAKE_BUILD_TYPE=MinSizeRel -DCMAKE_INSTALL_PREFIX='$5' $_args"
+		if [ "$9" = "" ]; then
+			_exec "Compiling $7"		"cmake --build 'build/$4' -j$threads"
+		else
+			_exec "Compiling $7...\n"	"cmake --build 'build/$4' -j$threads" no-silent
+		fi
+		_exec "Installing $7"			"cmake --install 'build/$4' --strip"
+	fi
+}
+
 _generate_stuff(){
 	case $1 in
 		"musl")
@@ -131,6 +167,32 @@ _generate_stuff(){
 	esac
 }
 
+_compile_musl(){
+	if [ "$3" = "rootfs" ]; then
+		install_dir="$output_rootfs"
+	else
+		install_dir=""
+	fi
+
+	if [ ! -r "$install_dir/lib/libc.so" ]; then
+		cd "$libc_dir"
+		libcc="$sysroot/lib/linux/libclang_rt.builtins-$linux_arch.a"
+		ar="$toolchain_bin/llvm-ar"
+		ranlib="$toolchain_bin/llvm-ranlib"
+
+		if [ "$LLVM" = "" ]; then
+			libcc="$toolchain_prefix/lib/gcc/$compile_target/$(_generate_stuff pkg_ver gcc)/libgcc.a"
+			ar="$toolchain_bin/$compile_target-ar"
+			ranlib="$toolchain_bin/$compile_target-ranlib"
+		fi
+
+		_exec "Configuring musl libc" "ARCH=$arch LIBCC='$libcc' CC='$cross_cc $cross_cflags' ./configure --prefix='$1' --disable-multilib --host=$compile_target $2"
+		_exec "Compiling musl libc" "make -j$threads AR='$ar' RANLIB='$ranlib'"
+		_exec "Installing musl libc" "make AR='$ar' RANLIB='$ranlib' DESTDIR='$install_dir' install"
+	fi
+}
+
+
 _setup_gcc(){
 	if [ ! -r "$gcc_dir/mpfr" ]; then
 		ln "$gmp_dir" "$gcc_dir/gmp" -s 2>/dev/null || true
@@ -139,23 +201,24 @@ _setup_gcc(){
 	fi
 
 	if [ "$1" = "rfs" ]; then
-		ln -s "$toolchain_prefix/bin/$compile_target-gcc" "$toolchain_prefix/bin/cc" 2>/dev/null || true
-		ln -s "$toolchain_prefix/bin/$compile_target-g++" "$toolchain_prefix/bin/c++" 2>/dev/null || true
+		ln -s "$toolchain_bin/$compile_target-gcc" "$toolchain_bin/cc" 2>/dev/null || true
+		ln -s "$toolchain_bin/$compile_target-g++" "$toolchain_bin/c++" 2>/dev/null || true
 	fi
 }
 
 setup_toolchain(){
-	if [ "$LLVM" = "1" ]; then
-		source "$plfiles/compile-modules/llvm.sh"  
+	if [ "$LLVM" != "" ]; then
+		source "$plfiles/compile-modules/llvm.sh"
 	else
 		source "$plfiles/compile-modules/gcc.sh"
 	fi
 
 	compile_toolchain
 }
 
-echo "PortaLinux Build System v0.09"
+echo "PortaLinux Build System v0.10"
 printf "(c) 2022 pocketlinux32 & raisinware, Under GPLv2+\n\n"
+_compiler_check
 
 if [ $# -eq 0 ]; then
 	echo "Error: No command given. Run $0 --help for a list of supported commands"
@@ -180,6 +243,8 @@ while [ $# -gt 0 ]; do
 			;;
 		"--toolchain-prefix")
 			toolchain_prefix="$2"
+			sysroot="$toolchain_prefix/$compile_target"
+			toolchain_bin="$toolchain_prefix/bin"
 			shift
 			;;
 		"--test")

pl-files/compile-modules/bootloader.sh

@@ -3,7 +3,7 @@
 compile_bootloader(){
 	_get_pkg_names
 
-	_compile_pkg "$toolchain_prefix/sbin/$compile_target-grub-install" "$grub_dir" \
+	_compile_ac_pkg "$toolchain_prefix/sbin/$compile_target-grub-install" "$grub_dir" \
 				"Configuring GRUB" "--prefix=$toolchain_prefix --target=$compile_target --program-prefix=$compile_target- --with-platform=$grub_platform" \
 				"Compiling GRUB" "" \
 				"Installing GRUB Tools" "install"

pl-files/compile-modules/defaults.sh

@@ -9,12 +9,7 @@ toolchain_prefix="$(echo ~/cross)"
 logfile="$pldir/log.txt"
 
 ## Default URLS
-kernel_url="https://cdn.kernel.org/pub/linux/kernel/v5.x/linux-5.15.77.tar.xz"
-binutils_url="http://ftp.gnu.org/gnu/binutils/binutils-2.37.tar.gz"
-gcc_url="http://ftp.gnu.org/gnu/gcc/gcc-10.3.0/gcc-10.3.0.tar.gz" # update to 10.4? ver num is hardcoded in some places
-gmp_url="http://ftp.gnu.org/gnu/gmp/gmp-6.2.1.tar.xz"
-mpc_url="http://ftp.gnu.org/gnu/mpc/mpc-1.2.1.tar.gz"
-mpfr_url="http://ftp.gnu.org/gnu/mpfr/mpfr-4.1.0.tar.gz"
+kernel_url="https://cdn.kernel.org/pub/linux/kernel/v5.x/linux-5.15.79.tar.xz"
 glibc_url="http://ftp.gnu.org/gnu/glibc/glibc-2.31.tar.gz"
 musl_url="https://musl.libc.org/releases/musl-1.2.3.tar.gz"
 busybox_url="http://busybox.net/downloads/busybox-1.34.1.tar.bz2"
@@ -25,10 +20,19 @@ python_url="https://www.python.org/ftp/python/3.10.8/Python-3.10.8.tar.xz"
 nano_url="http://ftp.gnu.org/gnu/nano/nano-6.2.tar.gz"
 ncurses_url="http://ftp.gnu.org/gnu/ncurses/ncurses-6.2.tar.gz"
 grub_url="https://ftp.gnu.org/gnu/grub/grub-2.06.tar.xz"
-xserver_url="https://www.x.org/releases/X11R7.7/src/xserver/xorg-server-1.12.2.tar.bz2"
+xserver_url="https://www.x.org/releases/individual/xserver/xorg-server-1.20.14.tar.xz"
 bison_url="http://ftp.gnu.org/gnu/bison/bison-3.7.6.tar.gz"
+# llvm toolchain url
+llvm_url="https://github.com/llvm/llvm-project/archive/refs/tags/llvmorg-14.0.6.tar.gz"
+# gcc toolchain urls
+binutils_url="http://ftp.gnu.org/gnu/binutils/binutils-2.37.tar.gz"
+gcc_url="http://ftp.gnu.org/gnu/gcc/gcc-10.3.0/gcc-10.3.0.tar.gz" # update to 10.4? ver num is hardcoded in some places
+gmp_url="http://ftp.gnu.org/gnu/gmp/gmp-6.2.1.tar.xz"
+mpc_url="http://ftp.gnu.org/gnu/mpc/mpc-1.2.1.tar.gz"
+mpfr_url="http://ftp.gnu.org/gnu/mpfr/mpfr-4.1.0.tar.gz"
 
 ## Default Target
+llvm_targets="AArch64;ARM;Mips;PowerPC;RISCV;Sparc;SystemZ;X86"
 compile_target="i486-pocket-linux-musl"
 linux_arch="i386"
 specific_arch="i486"
@@ -38,6 +42,8 @@ libdir="lib"
 grub_platform="efi"
 abi=""
 with_aoc=""
+sysroot="$toolchain_prefix/$compile_target"
+toolchain_bin="$toolchain_prefix/bin"
 
 ## Default Configs
 kdefconfig="defconfig"

pl-files/compile-modules/gcc.sh

@@ -6,7 +6,7 @@ _setup_glibc(){
 	touch "$output_rootfs/opt/include/gnu/stubs-32.h"
 	touch "$output_rootfs/opt/include/gnu/stubs-64.h"
 	if [ "$1" = "tc" ]; then
-		$compile_target-gcc -nostdlib -nostartfiles -shared -x c /dev/null -o "$toolchain_prefix/$compile_target/lib/libc.so"
+		$compile_target-gcc -nostdlib -nostartfiles -shared -x c /dev/null -o "$sysroot/lib/libc.so"
 	fi
 }
 
@@ -18,7 +18,7 @@ compile_toolchain(){
 	_get_pkg_names $dist
 
 	# binutils
-	_compile_pkg "$toolchain_prefix/bin/$compile_target-as" "$binutils_dir" \
+	_compile_ac_pkg "$toolchain_bin/$compile_target-as" "$binutils_dir" \
 				"Configuring Binutils" "--prefix=$toolchain_prefix --target=$gnu_flags" \
 				"Compiling Binutils" "" \
 				"Installing Binutils" "install-strip"
@@ -28,35 +28,35 @@ compile_toolchain(){
 	if [ "$dist" = "musl" ]; then
 		extra_flags="--disable-libsanitizer --enable-initfini-array"
 	fi
-	_compile_pkg "$toolchain_prefix/bin/$compile_target-gcc" "$gcc_dir" \
+	_compile_ac_pkg "$toolchain_bin/$compile_target-gcc" "$gcc_dir" \
 				"Configuring GCC" "--prefix=$toolchain_prefix --target=$gnu_flags --enable-languages=c,c++,$extra_gcc_langs --disable-libstdcxx-debug --disable-bootstrap $extra_flags $extra_gcc_flags" \
 				"Compiling GCC C/C++ compilers" "all-gcc" \
 				"Installing GCC C/C++ compilers" "install-strip-gcc"
 
 	# linux headers
-	if [ ! -r "$toolchain_prefix/$compile_target/include/linux" ]; then
+	if [ ! -r "$sysroot/include/linux" ]; then
 		cd "$linux_dir"
-		_exec "Installing Linux headers" "make ARCH=$linux_arch INSTALL_HDR_PATH=$toolchain_prefix/$compile_target headers_install"
+		_exec "Installing Linux headers" "make ARCH=$linux_arch INSTALL_HDR_PATH=$sysroot headers_install"
 	fi
 
 	# libc headers + start files (glibc-only)
-	if [ ! -r "$toolchain_prefix/$compile_target/include/stdio.h" ]; then
+	if [ ! -r "$sysroot/include/stdio.h" ]; then
 		cd "$libc_dir"
 		if [ "$dist" = "gnu" ]; then
 			mkdir -p "build" && cd "build"
 			if [ ! -r "$libc_dir/build/Makefile" ]; then
-				_exec "Configuring glibc" "../configure --prefix=$toolchain_prefix/$compile_target --host=$gnu_flags --with-headers=$toolchain_prefix/$compile_target/include libc_cv_forced_unwind=yes"
+				_exec "Configuring glibc" "../configure --prefix=$sysroot --host=$gnu_flags --with-headers=$sysroot/include libc_cv_forced_unwind=yes"
 			fi
 			_exec "Compiling glibc start files" "make -j$threads csu/subdir_lib CFLAGS_FOR_TARGET='-s -O2' CXXFLAGS_FOR_TARGET='-s -O2'"
-			_exec "Installing glibc start files" "install csu/crti.o csu/crtn.o csu/crt1.o '$toolchain_prefix/$compile_target/lib'"
+			_exec "Installing glibc start files" "install csu/crti.o csu/crtn.o csu/crt1.o '$sysroot/lib'"
 			_exec "Installing glibc headers" "make install-bootstrap-headers=yes install-headers"
 		else
-			_exec "Installing musl headers" "make ARCH=$arch prefix=$toolchain_prefix/$compile_target install-headers"
+			_exec "Installing musl headers" "make ARCH=$arch prefix=$sysroot install-headers"
 		fi
 	fi
 
 	# libgcc (libgcc-static for musl)
-	if [ ! -r "$toolchain_prefix/lib/gcc/$compile_target/10.3.0/libgcc.a" ]; then
+	if [ ! -r "$toolchain_prefix/lib/gcc/$compile_target/$(_generate_stuff pkg_ver gcc)/libgcc.a" ]; then
 		cd "$gcc_dir/build"
 		name="libgcc"
 		printf "Preparing to compile libgcc..."
@@ -69,47 +69,43 @@ compile_toolchain(){
 		echo "Done."
 		_exec "Compiling $name" "make -j$threads $extra_flags all-target-libgcc"
 		_exec "Installing libgcc" "make install-strip-target-libgcc"
-		rm -rf "$toolchain_prefix/$compile_target/lib/libc.so"
+		rm -rf "$sysroot/lib/libc.so"
 	fi
 
 	# libc
-	if [ ! -r "$toolchain_prefix/$compile_target/lib/libc.so" ]; then
-		cd "$libc_dir"
-		if [ "$dist" = "gnu" ]; then
-			cd "build"
-		else
-			_exec "Configuring libc" "ARCH=$arch CC=$compile_target-gcc CROSS_COMPILE=$compile_target- LIBCC=$toolchain_prefix/lib/gcc/$compile_target/$(_generate_stuff pkg_ver gcc)/libgcc.a ./configure --prefix=$toolchain_prefix/$compile_target --host=$common_flags"
-		fi
-
+	if [ ! -r "$sysroot/lib/libc.so" ] && [ "$dist" = "gnu" ]; then
+		cd "$libc_dir/build"
 		_exec "Compiling libc" "make -j$threads AR=$compile_target-ar RANLIB=$compile_target-ranlib"
 		_exec "Installing libc" "make AR=$compile_target-ar RANLIB=$compile_target-ranlib install"
+	elif [ ! -r "$sysroot/lib/libc.so" ]; then
+		_compile_musl "$sysroot"
 	fi
 
 	# libgcc-shared (musl-only)
-	if [ ! -r "$toolchain_prefix/$compile_target/$libdir/libgcc_s.so" ]; then
+	if [ ! -r "$sysroot/$libdir/libgcc_s.so" ]; then
 		cd "$gcc_dir/build"
 		_exec "Cleaning libgcc" "make -C $compile_target/libgcc distclean"
 		_exec "Compiling libgcc-shared" "make enable_shared=yes -j$threads all-target-libgcc"
 		_exec "Installing libgcc" "make install-strip-target-libgcc"
 	fi
 
 	# libstdc++
-	_compile_pkg "$toolchain_prefix/$compile_target/$libdir/libstdc++.so" "$gcc_dir" "" "" \
+	_compile_ac_pkg "$sysroot/$libdir/libstdc++.so" "$gcc_dir" "" "" \
 				"Compiling libstdc++" "" \
 				"Installing libstdc++" "install-strip-target-libstdc++-v3"
 
 	# ncurses
-	_compile_pkg "$toolchain_prefix/$compile_target/lib/libncurses.so" "$ncurses_dir" \
-				"Configuring Ncurses" "--prefix=$toolchain_prefix/$compile_target --host=$compile_target --with-cxx-shared --with-shared --enable-overwrite --with-termlib" \
+	_compile_ac_pkg "$sysroot/lib/libncurses.so" "$ncurses_dir" \
+				"Configuring Ncurses" "--prefix=$sysroot --host=$compile_target --with-cxx-shared --with-shared --enable-overwrite --with-termlib" \
 				"Compiling Ncurses" "" \
-				"Installing Ncurses" "install INSTALL_PROG='/usr/bin/env install --strip-program=$compile_target-strip -c -s'"
+				"Installing Ncurses" "install INSTALL_PROG='/usr/bin/env install --strip-program=$toolchain_bin/$compile_target-strip -c -s'"
 
 	# ncursesw
 	if [ -r "$ncurses_dir/build" ]; then
 		_exec "Cleaning Ncurses" "rm -rf $ncurses_dir/build"
 	fi
-	_compile_pkg "$toolchain_prefix/$compile_target/lib/libncursesw.so" "$ncurses_dir" \
-				"Configuring NcursesW" "--prefix=$toolchain_prefix/$compile_target --host=$compile_target --with-cxx-shared --with-shared --enable-overwrite --with-termlib --enable-widec" \
+	_compile_ac_pkg "$sysroot/lib/libncursesw.so" "$ncurses_dir" \
+				"Configuring NcursesW" "--prefix=$sysroot --host=$compile_target --with-cxx-shared --with-shared --enable-overwrite --with-termlib --enable-widec" \
 				"Compiling NcursesW" "" \
-				"Installing NcursesW" "install INSTALL_PROG='/usr/bin/env install --strip-program=$compile_target-strip -c -s'"
+				"Installing NcursesW" "install INSTALL_PROG='/usr/bin/env install --strip-program=$toolchain_bin/$compile_target-strip -c -s'"
 }

pl-files/compile-modules/kernel.sh

@@ -9,11 +9,11 @@ compile_kernel(){
 		exit 4
 	fi
 
-	script -qeac "make CROSS_COMPILE=$compile_target- ARCH=$linux_arch -j$threads" "$logfile"
+	script -qeac "PATH="$toolchain_bin:$PATH" make $kbuild_flags CROSS_COMPILE=$compile_target- ARCH=$linux_arch -j$threads" "$logfile"
 
 	modules_support=$(grep -w "CONFIG_MODULES" .config)
 	if [ "$modules_support" != "" ] && [ "$(echo $modules_support | grep '#')" = "" ]; then
-		script -qeac "make CROSS_COMPILE=$compile_target- ARCH=$linux_arch INSTALL_MOD_PATH=$output_rootfs modules_install" "$logfile"
+		script -qeac "PATH="$toolchain_bin:$PATH" make $kbuild_flags CROSS_COMPILE=$compile_target- ARCH=$linux_arch INSTALL_MOD_PATH=$output_rootfs modules_install" "$logfile"
 	fi
 	cp arch/$linux_arch/boot/dts/*.dtb "$output" 2>/dev/null || true
 	cp arch/$linux_arch/boot/*Image "$output"
@@ -28,9 +28,9 @@ configure_kernel(){
 	fi
 
 	if [ ! -f .config ]; then
-		make CROSS_COMPILE=$compile_target- ARCH=$linux_arch $kdefconfig
+		PATH="$toolchain_bin:$PATH" make $kbuild_flags CROSS_COMPILE=$compile_target- ARCH=$linux_arch $kdefconfig
 	fi
 
-	make ARCH=$linux_arch menuconfig
+	PATH="$toolchain_bin:$PATH" make $kbuild_flags ARCH=$linux_arch menuconfig
 	exit 0
 }