From 530072491a2180eb6e702b2e25a541c75257bd4c Mon Sep 17 00:00:00 2001 From: Michal Moskal Date: Sun, 20 Jan 2019 17:58:10 +0000 Subject: [PATCH 1/4] Add Fiber->user_data and list_fiber() This is port of relevant codal changes in support of GC in the PXT runtime. Also, limit the number of fibers in pool to 3. --- inc/core/MicroBitFiber.h | 13 +++ source/core/MicroBitFiber.cpp | 144 +++++++++++++++++++++++++--------- 2 files changed, 122 insertions(+), 35 deletions(-) diff --git a/inc/core/MicroBitFiber.h b/inc/core/MicroBitFiber.h index 2e042609..f01519aa 100644 --- a/inc/core/MicroBitFiber.h +++ b/inc/core/MicroBitFiber.h @@ -89,6 +89,9 @@ struct Fiber uint32_t flags; // Information about this fiber. Fiber **queue; // The queue this fiber is stored on. Fiber *next, *prev; // Position of this Fiber on the run queue. +#if CONFIG_ENABLED(MICROBIT_FIBER_USER_DATA) + void *user_data; +#endif }; extern Fiber *currentFiber; @@ -364,6 +367,16 @@ inline int inInterruptContext() return (((int)__get_IPSR()) & 0x003F) > 0; } +/** + * Return all current fibers. + * + * @param dest If non-null, it points to an array of pointers to fibers to store results in. + * + * @return the number of fibers (potentially) stored + */ +int list_fibers(Fiber **dest); + + /** * Assembler Context switch routing. * Defined in CortexContextSwitch.s. diff --git a/source/core/MicroBitFiber.cpp b/source/core/MicroBitFiber.cpp index 484e621b..7c47f0d3 100644 --- a/source/core/MicroBitFiber.cpp +++ b/source/core/MicroBitFiber.cpp @@ -65,6 +65,43 @@ static EventModel *messageBus = NULL; // Array of components which are iterated during idle thread execution. static MicroBitComponent* idleThreadComponents[MICROBIT_IDLE_COMPONENTS]; +static void get_fibers_from(Fiber ***dest, int *sum, Fiber *queue) +{ + if (queue && queue->prev) target_panic(30); + while (queue) { + if (*dest) + *(*dest)++ = queue; + (*sum)++; + queue = queue->next; + } +} + +/** + * Return all current fibers. + * + * @param dest If non-null, it points to an array of pointers to fibers to store results in. + * + * @return the number of fibers (potentially) stored + */ +int list_fibers(Fiber **dest) +{ + int sum = 0; + + // interrupts might move fibers between queues, but should not create new ones + __disable_irq(); + get_fibers_from(&dest, &sum, runQueue); + get_fibers_from(&dest, &sum, sleepQueue); + get_fibers_from(&dest, &sum, waitQueue); + __enable_irq(); + + // idleFiber is used to start event handlers using invoke(), + // so it may in fact have the user_data set if in FOB context + if (dest) + *dest++ = idleFiber; + sum++; + return sum; +} + /** * Utility function to add the currenty running fiber to the given queue. * @@ -171,6 +208,10 @@ Fiber *getFiberContext() f->flags = 0; f->tcb.stack_base = CORTEX_M0_STACK_BASE; + #if CONFIG_ENABLED(MICROBIT_FIBER_USER_DATA) + f->user_data = 0; + #endif + return f; } @@ -314,6 +355,29 @@ void scheduler_event(MicroBitEvent evt) messageBus->ignore(evt.source, evt.value, scheduler_event); } +static Fiber* handle_fob() +{ + Fiber *f = currentFiber; + + // This is a blocking call, so if we're in a fork on block context, + // it's time to spawn a new fiber... + if (f->flags & MICROBIT_FIBER_FLAG_FOB) + { + // Allocate a TCB from the new fiber. This will come from the tread pool if availiable, + // else a new one will be allocated on the heap. + forkedFiber = getFiberContext(); + // If we're out of memory, there's nothing we can do. + // keep running in the context of the current thread as a best effort. + if (forkedFiber != NULL) { +#if CONFIG_ENABLED(MICROBIT_FIBER_USER_DATA) + forkedFiber->user_data = f->user_data; + f->user_data = NULL; +#endif + f = forkedFiber; + } + } + return f; +} /** * Blocks the calling thread for the given period of time. @@ -327,8 +391,6 @@ void scheduler_event(MicroBitEvent evt) */ void fiber_sleep(unsigned long t) { - Fiber *f = currentFiber; - // If the scheduler is not running, then simply perform a spin wait and exit. if (!fiber_scheduler_running()) { @@ -336,19 +398,7 @@ void fiber_sleep(unsigned long t) return; } - // Sleep is a blocking call, so if we're in a fork on block context, - // it's time to spawn a new fiber... - if (currentFiber->flags & MICROBIT_FIBER_FLAG_FOB) - { - // Allocate a new fiber. This will come from the fiber pool if availiable, - // else a new one will be allocated on the heap. - forkedFiber = getFiberContext(); - - // If we're out of memory, there's nothing we can do. - // keep running in the context of the current thread as a best effort. - if (forkedFiber != NULL) - f = forkedFiber; - } + Fiber *f = handle_fob(); // Calculate and store the time we want to wake up. f->context = system_timer_current_time() + t; @@ -412,28 +462,17 @@ int fiber_wait_for_event(uint16_t id, uint16_t value) */ int fiber_wake_on_event(uint16_t id, uint16_t value) { - Fiber *f = currentFiber; - if (messageBus == NULL || !fiber_scheduler_running()) return MICROBIT_NOT_SUPPORTED; - // Sleep is a blocking call, so if we're in a fork on block context, - // it's time to spawn a new fiber... - if (currentFiber->flags & MICROBIT_FIBER_FLAG_FOB) - { - // Allocate a TCB from the new fiber. This will come from the tread pool if availiable, - // else a new one will be allocated on the heap. - forkedFiber = getFiberContext(); + Fiber *f = handle_fob(); - // If we're out of memory, there's nothing we can do. - // keep running in the context of the current thread as a best effort. - if (forkedFiber != NULL) - { - f = forkedFiber; - dequeue_fiber(f); - queue_fiber(f, &runQueue); - schedule(); - } + // in case we created a new fiber, make sure to initialize its context + // in case schedule() isn't called immedietly afterwards + if (f != currentFiber) { + dequeue_fiber(f); + queue_fiber(f, &runQueue); + schedule(); } // Encode the event data in the context field. It's handy having a 32 bit core. :-) @@ -453,6 +492,12 @@ int fiber_wake_on_event(uint16_t id, uint16_t value) return MICROBIT_OK; } +#if CONFIG_ENABLED(MICROBIT_FIBER_USER_DATA) +#define HAS_THREAD_USER_DATA (currentFiber->user_data != NULL) +#else +#define HAS_THREAD_USER_DATA false +#endif + /** * Executes the given function asynchronously if necessary. * @@ -475,7 +520,7 @@ int invoke(void (*entry_fn)(void)) if (!fiber_scheduler_running()) return MICROBIT_NOT_SUPPORTED; - if (currentFiber->flags & MICROBIT_FIBER_FLAG_FOB) + if (currentFiber->flags & (MICROBIT_FIBER_FLAG_FOB | MICROBIT_FIBER_FLAG_PARENT | MICROBIT_FIBER_FLAG_CHILD) || HAS_THREAD_USER_DATA) { // If we attempt a fork on block whilst already in fork n block context, // simply launch a fiber to deal with the request and we're done. @@ -504,6 +549,9 @@ int invoke(void (*entry_fn)(void)) // spawn a thread to deal with it. currentFiber->flags |= MICROBIT_FIBER_FLAG_FOB; entry_fn(); + #if CONFIG_ENABLED(MICROBIT_FIBER_USER_DATA) + f->user_data = 0; + #endif currentFiber->flags &= ~MICROBIT_FIBER_FLAG_FOB; // If this is is an exiting fiber that for spawned to handle a blocking call, recycle it. @@ -538,7 +586,7 @@ int invoke(void (*entry_fn)(void *), void *param) if (!fiber_scheduler_running()) return MICROBIT_NOT_SUPPORTED; - if (currentFiber->flags & (MICROBIT_FIBER_FLAG_FOB | MICROBIT_FIBER_FLAG_PARENT | MICROBIT_FIBER_FLAG_CHILD)) + if (currentFiber->flags & (MICROBIT_FIBER_FLAG_FOB | MICROBIT_FIBER_FLAG_PARENT | MICROBIT_FIBER_FLAG_CHILD) || HAS_THREAD_USER_DATA) { // If we attempt a fork on block whilst already in a fork on block context, // simply launch a fiber to deal with the request and we're done. @@ -567,6 +615,9 @@ int invoke(void (*entry_fn)(void *), void *param) // spawn a thread to deal with it. currentFiber->flags |= MICROBIT_FIBER_FLAG_FOB; entry_fn(param); + #if CONFIG_ENABLED(MICROBIT_FIBER_USER_DATA) + f->user_data = 0; + #endif currentFiber->flags &= ~MICROBIT_FIBER_FLAG_FOB; // If this is is an exiting fiber that for spawned to handle a blocking call, recycle it. @@ -710,6 +761,21 @@ void release_fiber(void) // Remove ourselves form the runqueue. dequeue_fiber(currentFiber); + // limit the number of fibers in the pool + int numFree = 0; + for (Fiber *p = fiberPool; p; p = p->next) { + if (!p->next && numFree > 3) { + p->prev->next = NULL; + free(p->tcb); + free((void *)p->stack_bottom); + memset(p, 0, sizeof(*p)); + free(p); + break; + } + numFree++; + } + + // Add ourselves to the list of free fibers queue_fiber(currentFiber, &fiberPool); @@ -742,6 +808,12 @@ void verify_stack_size(Fiber *f) // If we're too small, increase our buffer size. if (bufferSize < stackDepth) { + // We are only here, when the current stack is the stack of fiber [f]. + // Make sure the contents of [currentFiber] variable reflects that, otherwise + // an external memory allocator might get confused when scanning fiber stacks. + Fiber *prevCurrFiber = currentFiber; + currentFiber = f; + // To ease heap churn, we choose the next largest multple of 32 bytes. bufferSize = (stackDepth + 32) & 0xffffffe0; @@ -754,6 +826,8 @@ void verify_stack_size(Fiber *f) // Recalculate where the top of the stack is and we're done. f->stack_top = f->stack_bottom + bufferSize; + + currentFiber = prevCurrFiber; } } From 62754373005fffe6de6fbd6d51fa5b9f14b553b3 Mon Sep 17 00:00:00 2001 From: Michal Moskal Date: Sun, 20 Jan 2019 17:59:10 +0000 Subject: [PATCH 2/4] Allow overriding of the heap allocator and getting heap sizes Also port from codal-core in support of PXT GC. Also, fix compiler optimization bug in calloc(). --- inc/core/MicroBitHeapAllocator.h | 34 +++++++++++++++++++++++++++ source/core/MicroBitHeapAllocator.cpp | 29 +++++++++++++++++++---- 2 files changed, 58 insertions(+), 5 deletions(-) diff --git a/inc/core/MicroBitHeapAllocator.h b/inc/core/MicroBitHeapAllocator.h index 10aa5c94..43886f77 100644 --- a/inc/core/MicroBitHeapAllocator.h +++ b/inc/core/MicroBitHeapAllocator.h @@ -85,4 +85,38 @@ struct HeapDefinition int microbit_create_heap(uint32_t start, uint32_t end); void microbit_heap_print(); + +/** + * Returns the size of a given heap. + * + * @param heap_index index between 0 and MICROBIT_MAXIMUM_HEAPS-1 + * + * @return the size of heap in bytes, or zero if no such heap exists. + */ +uint32_t microbit_heap_size(uint8_t heap_index); + +/** + * Attempt to allocate a given amount of memory from any of our configured heap areas. + * + * @param size The amount of memory, in bytes, to allocate. + * + * @return A pointer to the allocated memory, or NULL if insufficient memory is available. + */ +extern "C" void* microbit_alloc(size_t size); + +/** + * Release a given area of memory from the heap. + * + * @param mem The memory area to release. + */ +extern "C" void microbit_free(void *mem); + +/** + * Copy existing contents of ptr to a new memory block of given size. + * + * @param ptr The existing memory block (can be NULL) + * @param size The size of new block (can be smaller or larger than the old one) + */ +extern "C" void* microbit_realloc(void* ptr, size_t size); + #endif diff --git a/source/core/MicroBitHeapAllocator.cpp b/source/core/MicroBitHeapAllocator.cpp index c42b3daa..278dfab1 100644 --- a/source/core/MicroBitHeapAllocator.cpp +++ b/source/core/MicroBitHeapAllocator.cpp @@ -173,6 +173,14 @@ int microbit_create_heap(uint32_t start, uint32_t end) return MICROBIT_OK; } +uint32_t device_heap_size(uint8_t heap_index) +{ + if (heap_index >= heap_count) + return 0; + HeapDefinition *h = &heap[heap_index]; + return (uint8_t*)h->heap_end - (uint8_t*)h->heap_start; +} + /** * Attempt to allocate a given amount of memory from a given heap area. * @@ -271,11 +279,14 @@ void *microbit_malloc(size_t size, HeapDefinition &heap) * * @return A pointer to the allocated memory, or NULL if insufficient memory is available. */ -void *malloc(size_t size) +void *microbit_alloc(size_t size) { static uint8_t initialised = 0; void *p; + if (size == 0) + return NULL; + if (!initialised) { heap_count = 0; @@ -319,7 +330,7 @@ void *malloc(size_t size) * * @param mem The memory area to release. */ -void free(void *mem) +void microbit_free(void *mem) { uint32_t *memory = (uint32_t *)mem; uint32_t *cb = memory-1; @@ -355,13 +366,17 @@ void* calloc (size_t num, size_t size) { void *mem = malloc(num*size); - if (mem) - memclr(mem, num*size); + if (mem) { + // without this write, GCC will happily optimize malloc() above into calloc() + // and remove the memset + ((uint32_t*)mem)[0] = 1; + memset(mem, 0, num*size); + } return mem; } -void* realloc (void* ptr, size_t size) +void* microbit_realloc (void* ptr, size_t size) { void *mem = malloc(size); @@ -380,6 +395,10 @@ void* realloc (void* ptr, size_t size) return mem; } +void *malloc(size_t sz) __attribute__ ((weak, alias ("microbit_alloc"))); +void free(void *mem) __attribute__ ((weak, alias ("microbit_free"))); +void* realloc (void* ptr, size_t size) __attribute__ ((weak, alias ("microbit_realloc"))); + // make sure the libc allocator is not pulled in void *_malloc_r(struct _reent *, size_t len) { From 517f2ea0acf07ebad1b6d936999470b885019da1 Mon Sep 17 00:00:00 2001 From: Michal Moskal Date: Sun, 20 Jan 2019 18:03:47 +0000 Subject: [PATCH 3/4] Compilation fixes --- source/core/MicroBitFiber.cpp | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/source/core/MicroBitFiber.cpp b/source/core/MicroBitFiber.cpp index 7c47f0d3..b3630a97 100644 --- a/source/core/MicroBitFiber.cpp +++ b/source/core/MicroBitFiber.cpp @@ -34,6 +34,8 @@ DEALINGS IN THE SOFTWARE. #include "MicroBitConfig.h" #include "MicroBitFiber.h" #include "MicroBitSystemTimer.h" +#include "ErrorNo.h" +#include "MicroBitDevice.h" /* * Statically allocated values used to create and destroy Fibers. @@ -67,7 +69,8 @@ static MicroBitComponent* idleThreadComponents[MICROBIT_IDLE_COMPONENTS]; static void get_fibers_from(Fiber ***dest, int *sum, Fiber *queue) { - if (queue && queue->prev) target_panic(30); + if (queue && queue->prev) + microbit_panic(MICROBIT_HEAP_ERROR); while (queue) { if (*dest) *(*dest)++ = queue; @@ -232,7 +235,7 @@ void scheduler_init(EventModel &_messageBus) // Store a reference to the messageBus provided. // This parameter will be NULL if we're being run without a message bus. - messageBus = &_messageBus; + messageBus = &_messageBus; // Create a new fiber context currentFiber = getFiberContext(); @@ -438,7 +441,7 @@ int fiber_wait_for_event(uint16_t id, uint16_t value) if(ret == MICROBIT_OK) schedule(); - return ret; + return ret; } /** @@ -766,7 +769,6 @@ void release_fiber(void) for (Fiber *p = fiberPool; p; p = p->next) { if (!p->next && numFree > 3) { p->prev->next = NULL; - free(p->tcb); free((void *)p->stack_bottom); memset(p, 0, sizeof(*p)); free(p); From 6e9b1d5890c4df9c19432282a9c5db92660f1249 Mon Sep 17 00:00:00 2001 From: Michal Moskal Date: Sun, 20 Jan 2019 18:29:29 +0000 Subject: [PATCH 4/4] Add fiber_user_data to yotta mappings; fix some errors --- inc/core/MicroBitConfig.h | 5 +++++ inc/platform/yotta_cfg_mappings.h | 4 ++++ source/core/MicroBitFiber.cpp | 4 ++-- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/inc/core/MicroBitConfig.h b/inc/core/MicroBitConfig.h index 9f5e3855..ba2ff7e9 100644 --- a/inc/core/MicroBitConfig.h +++ b/inc/core/MicroBitConfig.h @@ -149,6 +149,11 @@ extern uint32_t __etext; #define SYSTEM_TICK_PERIOD_MS 6 #endif +// Enable used_data field in Fiber structure (for thread-local data) +#ifndef MICROBIT_FIBER_USER_DATA +#define MICROBIT_FIBER_USER_DATA 0 +#endif + // // Message Bus: // Default behaviour for event handlers, if not specified in the listen() call diff --git a/inc/platform/yotta_cfg_mappings.h b/inc/platform/yotta_cfg_mappings.h index 08ef57fc..fbf565b5 100644 --- a/inc/platform/yotta_cfg_mappings.h +++ b/inc/platform/yotta_cfg_mappings.h @@ -15,6 +15,10 @@ #define MICROBIT_NESTED_HEAP_SIZE YOTTA_CFG_MICROBIT_DAL_NESTED_HEAP_PROPORTION #endif +#ifdef YOTTA_CFG_MICROBIT_DAL_FIBER_USER_DATA + #define MICROBIT_FIBER_USER_DATA YOTTA_CFG_MICROBIT_DAL_FIBER_USER_DATA +#endif + #ifdef YOTTA_CFG_MICROBIT_DAL_REUSE_SD #define MICROBIT_HEAP_REUSE_SD YOTTA_CFG_MICROBIT_DAL_REUSE_SD #endif diff --git a/source/core/MicroBitFiber.cpp b/source/core/MicroBitFiber.cpp index b3630a97..5e3068c2 100644 --- a/source/core/MicroBitFiber.cpp +++ b/source/core/MicroBitFiber.cpp @@ -553,7 +553,7 @@ int invoke(void (*entry_fn)(void)) currentFiber->flags |= MICROBIT_FIBER_FLAG_FOB; entry_fn(); #if CONFIG_ENABLED(MICROBIT_FIBER_USER_DATA) - f->user_data = 0; + currentFiber->user_data = 0; #endif currentFiber->flags &= ~MICROBIT_FIBER_FLAG_FOB; @@ -619,7 +619,7 @@ int invoke(void (*entry_fn)(void *), void *param) currentFiber->flags |= MICROBIT_FIBER_FLAG_FOB; entry_fn(param); #if CONFIG_ENABLED(MICROBIT_FIBER_USER_DATA) - f->user_data = 0; + currentFiber->user_data = 0; #endif currentFiber->flags &= ~MICROBIT_FIBER_FLAG_FOB;