pkg/tlsf: Early initialization of memory pool.

pkg/tlsf/contrib/Makefile.include

@@ -0,0 +1 @@
+UNDEF += $(BINDIR)/tlsf-malloc/tlsf-malloc.o

pkg/tlsf/contrib/native.c

@@ -28,6 +28,19 @@
 #include "tlsf-malloc.h"
 #include "tlsf-malloc-internal.h"
 
+/*
+ * On native the linker dos not define the Heap area.
+ * It is handled by the C library (it requests memory from the OS ??)
+ * TODO: make this configurable
+ */
+static char _sheap[TLSF_NATIVE_HEAPSIZE];
+
+/* Define the initialization function */
+void init_tlsf_malloc(void)
+{
+    tlsf_add_global_pool(_sheap, ROUND_DOWN4(TLSF_NATIVE_HEAPSIZE));
+}
+
 /* TODO: Add defines for other compilers */
 #if defined(__GNUC__) && !defined(__clang__)    /* Clang supports __GNUC__ but
                                                  * not the alloc_size()

pkg/tlsf/contrib/newlib.c

@@ -21,15 +21,58 @@
  *
  */
 
+#define _DEFAULT_SOURCE
 #include <string.h>
 #include <reent.h>
 #include <errno.h>
+#include <unistd.h>
 
 #include "irq.h"
+#include "panic.h"
 #include "tlsf.h"
 #include "tlsf-malloc.h"
 #include "tlsf-malloc-internal.h"
 
+/*
+ * The linker script for embedded target defines _sheap and _eheap:
+ *
+ *  / * heap section * /
+ *  . = ALIGN(4);
+ *  _sheap = . ;
+ *  _eheap = ORIGIN(ram) + LENGTH(ram);
+ *
+ * For some reason, at program startup there is something at _sbreak. so
+ * initializing the tlsf heap there leads to failure. In fact, sbrk(0)
+ * points quite some bytes after _sheap.
+ * The solution here is to request new memory to sbrk, use only that and
+ * request all memory up to _eheap.
+ * Keep in mind that this may mean less memory available for growing the
+ * stack! The way to solve it is to configure a smaller _eheap.
+ */
+extern char _eheap[];  /* FIXME: what to do with platforms without this symbol? */
+
+/* Define the initialization function */
+void init_tlsf_malloc(void)
+{
+    /* Use sbrk(0) instead if _sheap since they may not be equal. */
+    size_t request_size = ROUND_DOWN4(_eheap - (char*)sbrk(0));
+    void *mem_start = sbrk(request_size);
+
+    if (mem_start == (void *)(-1)) {
+        /* FIXME: This message does not show. The UART is probably not yet
+         * initialized.
+         */
+        core_panic(PANIC_GENERAL_ERROR, "Could not enlarge heap");
+    }
+
+    /* Why do we use sbrk instead of _sheap and _eheap?
+     * Because of the (unlikely) possibility that some other libc procedure is
+     * using sbrk, which - if we were to bypass it by directly using _{s,e}heap
+     * would result in weird and hard to debug memory errors.
+     */
+    tlsf_add_global_pool(mem_start, request_size);
+}
+
 
 /* TODO: Add defines for other compilers */
 #if defined(__GNUC__) && !defined(__clang__)    /* Clang supports __GNUC__ but

pkg/tlsf/contrib/tlsf-malloc-internal.h

@@ -25,8 +25,24 @@
 extern "C" {
 #endif
 
+#define ROUND_DOWN4(x) (((x)/4)*4) /* Is this necessary??? */
+
+#ifndef TLSF_NATIVE_HEAPSIZE
+/**
+ * Fixed heap size to use in native.
+ *
+ * In native there is virtually unlimited memory and no predefined heap area
+ * in the linker script, so one needs to define it manually.
+ * The default here is 8MiB which should be plenty for RIOT.
+ * Note that this has no effect on other platforms.
+ */
+#define TLSF_NATIVE_HEAPSIZE        (0x800000)
+#endif
+
 extern tlsf_t tlsf_malloc_gheap;
 
+void init_tlsf_malloc(void);
+
 #ifdef __cplusplus
 }
 #endif

pkg/tlsf/contrib/tlsf-malloc.c

@@ -15,6 +15,16 @@
  * @brief   TLSF-based global memory allocator.
  * @author  Juan I Carrano
  *
+ * # About initialization
+ *
+ * Some system (standard library) routines trigger the allocation of buffers
+ * which results in segmentation faults/ hard faults if using tlsf and the
+ * heap is not yet allocated. This function can possibly be used super early
+ * in the boot process, so the TLSF initialization must run earlier still.
+ *
+ * On native we define a static array and on embedded platforms we use the
+ * _sheap and _eheap linker symbols and sbrk.
+ *
  */
 
 #include <stdio.h>
@@ -56,6 +66,14 @@ void tlsf_size_walker(void* ptr, size_t size, int used, void* user)
     }
 }
 
+/* The C library runs the functions in this array before it initializes itself.
+ * This section constitutes a cross-file array.
+ * Note that in order for this to work this object file has to be added to
+ * UNDEF (see ../Makefile.include) otherwise this symbol gets discarded.
+ */
+void (* const init_tlsf_malloc_p) (void) __attribute__((section (".preinit_array"))) =
+    init_tlsf_malloc;
+
 /**
  * @}
  */

tests/pkg_tlsf_malloc/Makefile

@@ -0,0 +1,13 @@
+DEVELHELP ?= 1
+
+include ../Makefile.tests_common
+
+# The bug is not fixed yet, so blacklist this test.
+TEST_ON_CI_BLACKLIST += all
+TEST_ON_CI_WHITELIST += native samr21-xpro
+
+USEMODULE += tlsf-malloc
+
+CFLAGS += -DDEBUG_ASSERT_VERBOSE -DTLSF_NATIVE_HEAPSIZE=0x4000
+
+include $(RIOTBASE)/Makefile.include

tests/pkg_tlsf_malloc/main.c

@@ -0,0 +1,77 @@
+/*
+ * Copyright (C) 2019 Freie Universität Berlin
+ *
+ * This file is subject to the terms and conditions of the GNU Lesser
+ * General Public License v2.1. See the file LICENSE in the top level
+ * directory for more details.
+ */
+
+/**
+ * @ingroup     examples
+ * @{
+ *
+ * @file
+ * @brief       TLSF initialization tests.
+ *
+ * @author      Juan I Carrano <[email protected]>
+ *
+ * Verify that TLSF is correctly and automatically initialized when used as
+ * the default global allocator. Failure to do so will result in segfaults /
+ * hardfaults, even if the application does not use malloc (this is because
+ * the C library does buffer allocations internally).
+ *
+ * This is currently failing as the initialization is not being done.
+ *
+ * @}
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <assert.h>
+
+#include "tlsf.h"
+#include "tlsf-malloc.h"
+
+extern char _eheap[];
+
+#define BLOCKSIZE 42
+
+static void walker(void* ptr, size_t size, int used, void* _accumulator)
+{
+	size_t *accumulator = _accumulator;
+    *accumulator += size;
+	printf("\t%p %s size: %u\n", ptr, used ? "used" : "free", (unsigned int)size);
+}
+
+int main(void)
+{
+    size_t accumulator = 0;
+    void *some_block;
+    size_t requested_size = 0;
+
+    printf("Using TLSF-malloc, global pool at %p\n", (void*)_tlsf_get_global_control());
+
+    #ifndef BOARD_NATIVE
+        requested_size = _eheap - (char*)_tlsf_get_global_control();
+    #else
+        requested_size = TLSF_NATIVE_HEAPSIZE;
+    #endif
+
+    /* increase the chances of a crash */
+    some_block = calloc(1, BLOCKSIZE);
+
+    assert(some_block != NULL);
+
+    puts("TLSF heap report:");
+
+    tlsf_walk_pool(tlsf_get_pool(_tlsf_get_global_control()), walker, &accumulator);
+
+    printf("According to TLSF heap has %u bytes\n", accumulator);
+    printf("The size should be %u bytes.\n", requested_size);
+
+    printf("Overhead: %d bytes\n", requested_size-accumulator);
+
+    free(some_block);
+
+    return 0;
+}

tests/pkg_tlsf_malloc/tests/01-run.py

@@ -0,0 +1,26 @@
+#!/usr/bin/env python3
+
+# Copyright (C) 2019 Freie Universität Berlin
+#
+# This file is subject to the terms and conditions of the GNU Lesser
+# General Public License v2.1. See the file LICENSE in the top level
+# directory for more details.
+
+# Tell the lua interpreter running in riot to load some modules and print
+# the value of a variable inside that module.
+
+import os
+import sys
+
+
+def test(child):
+    # this is the worst test ever.
+    child.expect('Overhead: (-?[0-9]+) bytes')
+
+    assert(int(child.match[1]) >= 0)
+
+
+if __name__ == "__main__":
+    sys.path.append(os.path.join(os.environ['RIOTTOOLS'], 'testrunner'))
+    from testrunner import run
+    sys.exit(run(test))