diff --git a/arch/sim/src/sim/sim_heap.c b/arch/sim/src/sim/sim_heap.c index 17cb5e198d..79c5e4bd81 100644 --- a/arch/sim/src/sim/sim_heap.c +++ b/arch/sim/src/sim/sim_heap.c @@ -25,6 +25,7 @@ #include #include +#include #include #include #include @@ -51,6 +52,11 @@ struct mm_delaynode_s struct mm_heap_s { struct mm_delaynode_s *mm_delaylist[CONFIG_SMP_NCPUS]; + +#if CONFIG_MM_FREE_DELAYCOUNT_MAX > 0 + size_t mm_delaycount[CONFIG_SMP_NCPUS]; +#endif + atomic_int aordblks; atomic_int uordblks; atomic_int usmblks; @@ -60,6 +66,12 @@ struct mm_heap_s #endif }; +/**************************************************************************** + * Private Function Prototypes + ****************************************************************************/ + +static void mm_delayfree(struct mm_heap_s *heap, void *mem, bool delay); + /**************************************************************************** * Private Functions ****************************************************************************/ @@ -77,12 +89,17 @@ static void mm_add_delaylist(struct mm_heap_s *heap, void *mem) tmp->flink = heap->mm_delaylist[up_cpu_index()]; heap->mm_delaylist[up_cpu_index()] = tmp; +#if CONFIG_MM_FREE_DELAYCOUNT_MAX > 0 + heap->mm_delaycount[up_cpu_index()]++; +#endif + up_irq_restore(flags); #endif } -static void mm_free_delaylist(struct mm_heap_s *heap) +static bool mm_free_delaylist(struct mm_heap_s *heap, bool force) { + bool ret = false; #if defined(CONFIG_BUILD_FLAT) || defined(__KERNEL__) struct mm_delaynode_s *tmp; irqstate_t flags; @@ -92,12 +109,26 @@ static void mm_free_delaylist(struct mm_heap_s *heap) flags = up_irq_save(); tmp = heap->mm_delaylist[up_cpu_index()]; + +#if CONFIG_MM_FREE_DELAYCOUNT_MAX > 0 + if (tmp == NULL || + (!force && + heap->mm_delaycount[up_cpu_index()] < CONFIG_MM_FREE_DELAYCOUNT_MAX)) + { + up_irq_restore(flags); + return false; + } + + heap->mm_delaycount[up_cpu_index()] = 0; +#endif heap->mm_delaylist[up_cpu_index()] = NULL; up_irq_restore(flags); /* Test if the delayed is empty */ + ret = tmp != NULL; + while (tmp) { void *address; @@ -111,9 +142,51 @@ static void mm_free_delaylist(struct mm_heap_s *heap) * 'while' condition above. */ - mm_free(heap, address); + mm_delayfree(heap, address, false); } + #endif + return ret; +} + +/**************************************************************************** + * Name: mm_delayfree + * + * Description: + * Delay free memory if `delay` is true, otherwise free it immediately. + * + ****************************************************************************/ + +static void mm_delayfree(struct mm_heap_s *heap, void *mem, bool delay) +{ +#if defined(CONFIG_BUILD_FLAT) || defined(__KERNEL__) + /* Check current environment */ + + if (up_interrupt_context()) + { + /* We are in ISR, add to the delay list */ + + mm_add_delaylist(heap, mem); + } + else +#endif + + if (nxsched_gettid() < 0 || delay) + { + /* nxsched_gettid() return -ESRCH, means we are in situations + * during context switching(See nxsched_gettid's comment). + * Then add to the delay list. + */ + + mm_add_delaylist(heap, mem); + } + else + { + int size = host_mallocsize(mem); + atomic_fetch_sub(&heap->aordblks, 1); + atomic_fetch_sub(&heap->uordblks, size); + host_free(mem); + } } /**************************************************************************** @@ -208,34 +281,16 @@ void *mm_malloc(struct mm_heap_s *heap, size_t size) void mm_free(struct mm_heap_s *heap, void *mem) { -#if defined(CONFIG_BUILD_FLAT) || defined(__KERNEL__) - /* Check current environment */ + minfo("Freeing %p\n", mem); - if (up_interrupt_context()) + /* Protect against attempts to free a NULL reference */ + + if (mem == NULL) { - /* We are in ISR, add to the delay list */ - - mm_add_delaylist(heap, mem); + return; } - else -#endif - if (nxsched_gettid() < 0) - { - /* nxsched_gettid() return -ESRCH, means we are in situations - * during context switching(See nxsched_gettid's comment). - * Then add to the delay list. - */ - - mm_add_delaylist(heap, mem); - } - else - { - int size = host_mallocsize(mem); - atomic_fetch_sub(&heap->aordblks, 1); - atomic_fetch_sub(&heap->uordblks, size); - host_free(mem); - } + mm_delayfree(heap, mem, CONFIG_MM_FREE_DELAYCOUNT_MAX > 0); } /**************************************************************************** @@ -269,7 +324,7 @@ void *mm_realloc(struct mm_heap_s *heap, void *oldmem, int usmblks; int newsize; - mm_free_delaylist(heap); + mm_free_delaylist(heap, false); if (size == 0) { @@ -295,6 +350,13 @@ void *mm_realloc(struct mm_heap_s *heap, void *oldmem, } while (atomic_compare_exchange_weak(&heap->usmblks, &usmblks, uordblks)); +#if CONFIG_MM_FREE_DELAYCOUNT_MAX > 0 + if (mem == NULL && mm_free_delaylist(heap, true)) + { + return mm_realloc(heap, oldmem, size); + } +#endif + return mem; } @@ -358,7 +420,7 @@ void *mm_memalign(struct mm_heap_s *heap, size_t alignment, size_t size) int uordblks; int usmblks; - mm_free_delaylist(heap); + mm_free_delaylist(heap, false); mem = host_memalign(alignment, size); if (mem == NULL) @@ -381,6 +443,13 @@ void *mm_memalign(struct mm_heap_s *heap, size_t alignment, size_t size) } while (atomic_compare_exchange_weak(&heap->usmblks, &usmblks, uordblks)); +#if CONFIG_MM_FREE_DELAYCOUNT_MAX > 0 + if (mem == NULL && mm_free_delaylist(heap, true)) + { + return mm_memalign(heap, alignment, size); + } +#endif + return mem; } diff --git a/mm/Kconfig b/mm/Kconfig index 2c3d32aa1d..72ce1a8533 100644 --- a/mm/Kconfig +++ b/mm/Kconfig @@ -358,4 +358,12 @@ config MM_PANIC_ON_FAILURE default n depends on DEBUG_MM +config MM_FREE_DELAYCOUNT_MAX + int "Maximum memory nodes can be delayed to free" + default 0 + ---help--- + Set to 0 to disable the delayed free mechanism. Otherwise, + the value decides the maximum number of memory nodes that + will be delayed to free. + source "mm/iob/Kconfig" diff --git a/mm/mm_heap/mm.h b/mm/mm_heap/mm.h index 28cf0bae0d..e12303b97c 100644 --- a/mm/mm_heap/mm.h +++ b/mm/mm_heap/mm.h @@ -260,6 +260,10 @@ struct mm_heap_s FAR struct mm_delaynode_s *mm_delaylist[CONFIG_SMP_NCPUS]; +#if CONFIG_MM_FREE_DELAYCOUNT_MAX > 0 + size_t mm_delaycount[CONFIG_SMP_NCPUS]; +#endif + /* The is a multiple mempool of the heap */ #if CONFIG_MM_HEAP_MEMPOOL_THRESHOLD != 0 @@ -304,4 +308,8 @@ int mm_size2ndx(size_t size); void mm_foreach(FAR struct mm_heap_s *heap, mm_node_handler_t handler, FAR void *arg); +/* Functions contained in mm_free.c *****************************************/ + +void mm_delayfree(FAR struct mm_heap_s *heap, FAR void *mem, bool delay); + #endif /* __MM_MM_HEAP_MM_H */ diff --git a/mm/mm_heap/mm_free.c b/mm/mm_heap/mm_free.c index 881b025e02..d29f901e84 100644 --- a/mm/mm_heap/mm_free.c +++ b/mm/mm_heap/mm_free.c @@ -50,6 +50,10 @@ static void add_delaylist(FAR struct mm_heap_s *heap, FAR void *mem) tmp->flink = heap->mm_delaylist[up_cpu_index()]; heap->mm_delaylist[up_cpu_index()] = tmp; +#if CONFIG_MM_FREE_DELAYCOUNT_MAX > 0 + heap->mm_delaycount[up_cpu_index()]++; +#endif + up_irq_restore(flags); #endif } @@ -59,15 +63,14 @@ static void add_delaylist(FAR struct mm_heap_s *heap, FAR void *mem) ****************************************************************************/ /**************************************************************************** - * Name: mm_free + * Name: mm_delayfree * * Description: - * Returns a chunk of memory to the list of free nodes, merging with - * adjacent free chunks if possible. + * Delay free memory if `delay` is true, otherwise free it immediately. * ****************************************************************************/ -void mm_free(FAR struct mm_heap_s *heap, FAR void *mem) +void mm_delayfree(FAR struct mm_heap_s *heap, FAR void *mem, bool delay) { FAR struct mm_freenode_s *node; FAR struct mm_freenode_s *prev; @@ -75,24 +78,6 @@ void mm_free(FAR struct mm_heap_s *heap, FAR void *mem) size_t nodesize; size_t prevsize; - minfo("Freeing %p\n", mem); - - /* Protect against attempts to free a NULL reference */ - - if (!mem) - { - return; - } - - DEBUGASSERT(mm_heapmember(heap, mem)); - -#if CONFIG_MM_HEAP_MEMPOOL_THRESHOLD != 0 - if (mempool_multiple_free(heap->mm_mpool, mem) >= 0) - { - return; - } -#endif - if (mm_lock(heap) < 0) { /* Meet -ESRCH return, which means we are in situations @@ -110,6 +95,13 @@ void mm_free(FAR struct mm_heap_s *heap, FAR void *mem) kasan_poison(mem, mm_malloc_size(heap, mem)); + if (delay) + { + mm_unlock(heap); + add_delaylist(heap, mem); + return; + } + /* Map the memory chunk into a free node */ node = (FAR struct mm_freenode_s *)((FAR char *)mem - MM_SIZEOF_ALLOCNODE); @@ -202,3 +194,35 @@ void mm_free(FAR struct mm_heap_s *heap, FAR void *mem) mm_addfreechunk(heap, node); mm_unlock(heap); } + +/**************************************************************************** + * Name: mm_free + * + * Description: + * Returns a chunk of memory to the list of free nodes, merging with + * adjacent free chunks if possible. + * + ****************************************************************************/ + +void mm_free(FAR struct mm_heap_s *heap, FAR void *mem) +{ + minfo("Freeing %p\n", mem); + + /* Protect against attempts to free a NULL reference */ + + if (mem == NULL) + { + return; + } + + DEBUGASSERT(mm_heapmember(heap, mem)); + +#if CONFIG_MM_HEAP_MEMPOOL_THRESHOLD != 0 + if (mempool_multiple_free(heap->mm_mpool, mem) >= 0) + { + return; + } +#endif + + mm_delayfree(heap, mem, CONFIG_MM_FREE_DELAYCOUNT_MAX > 0); +} diff --git a/mm/mm_heap/mm_malloc.c b/mm/mm_heap/mm_malloc.c index 46f9648fea..30e307f577 100644 --- a/mm/mm_heap/mm_malloc.c +++ b/mm/mm_heap/mm_malloc.c @@ -39,8 +39,23 @@ * Private Functions ****************************************************************************/ -static void free_delaylist(FAR struct mm_heap_s *heap) +/**************************************************************************** + * Name: free_delaylist + * + * Description: + * Free the memory in delay list either added because of mm_lock failed or + * added because of CONFIG_MM_FREE_DELAYCOUNT_MAX. + * Set force to true to free all the memory in delay list immediately, set + * to false will only free delaylist when time is up if + * CONFIG_MM_FREE_DELAYCOUNT_MAX is enabled. + * + * Return true if there is memory freed. + * + ****************************************************************************/ + +static bool free_delaylist(FAR struct mm_heap_s *heap, bool force) { + bool ret = false; #if defined(CONFIG_BUILD_FLAT) || defined(__KERNEL__) FAR struct mm_delaynode_s *tmp; irqstate_t flags; @@ -50,12 +65,27 @@ static void free_delaylist(FAR struct mm_heap_s *heap) flags = up_irq_save(); tmp = heap->mm_delaylist[up_cpu_index()]; + +#if CONFIG_MM_FREE_DELAYCOUNT_MAX > 0 + if (tmp == NULL || + (!force && + heap->mm_delaycount[up_cpu_index()] < CONFIG_MM_FREE_DELAYCOUNT_MAX)) + { + up_irq_restore(flags); + return false; + } + + heap->mm_delaycount[up_cpu_index()] = 0; +#endif + heap->mm_delaylist[up_cpu_index()] = NULL; up_irq_restore(flags); /* Test if the delayed is empty */ + ret = tmp != NULL; + while (tmp) { FAR void *address; @@ -69,9 +99,11 @@ static void free_delaylist(FAR struct mm_heap_s *heap) * 'while' condition above. */ - mm_free(heap, address); + mm_delayfree(heap, address, false); } + #endif + return ret; } #if CONFIG_MM_BACKTRACE >= 0 @@ -126,7 +158,7 @@ FAR void *mm_malloc(FAR struct mm_heap_s *heap, size_t size) /* Free the delay list first */ - free_delaylist(heap); + free_delaylist(heap, false); #if CONFIG_MM_HEAP_MEMPOOL_THRESHOLD != 0 ret = mempool_multiple_alloc(heap->mm_mpool, size); @@ -279,6 +311,16 @@ FAR void *mm_malloc(FAR struct mm_heap_s *heap, size_t size) minfo("Allocated %p, size %zu\n", ret, alignsize); #endif } + +#if CONFIG_MM_FREE_DELAYCOUNT_MAX > 0 + /* Try again after free delay list */ + + else if (free_delaylist(heap, true)) + { + return mm_malloc(heap, size); + } +#endif + #ifdef CONFIG_DEBUG_MM else if (MM_INTERNAL_HEAP(heap)) { diff --git a/mm/tlsf/mm_tlsf.c b/mm/tlsf/mm_tlsf.c index 8162976147..ad7801f670 100644 --- a/mm/tlsf/mm_tlsf.c +++ b/mm/tlsf/mm_tlsf.c @@ -103,10 +103,14 @@ struct mm_heap_s FAR struct mempool_multiple_s *mm_mpool; #endif - /* Free delay list, for some situation can't do free immdiately */ + /* Free delay list, for some situation can't do free immediately */ struct mm_delaynode_s *mm_delaylist[CONFIG_SMP_NCPUS]; +#if CONFIG_MM_FREE_DELAYCOUNT_MAX > 0 + size_t mm_delaycount[CONFIG_SMP_NCPUS]; +#endif + #if defined(CONFIG_FS_PROCFS) && !defined(CONFIG_FS_PROCFS_EXCLUDE_MEMINFO) struct procfs_meminfo_entry_s mm_procfs; #endif @@ -129,6 +133,12 @@ struct mm_mallinfo_handler_s FAR struct mallinfo_task *info; }; +/**************************************************************************** + * Private Function Prototypes + ****************************************************************************/ + +static void mm_delayfree(struct mm_heap_s *heap, void *mem, bool delay); + /**************************************************************************** * Private Functions ****************************************************************************/ @@ -182,6 +192,10 @@ static void add_delaylist(FAR struct mm_heap_s *heap, FAR void *mem) tmp->flink = heap->mm_delaylist[up_cpu_index()]; heap->mm_delaylist[up_cpu_index()] = tmp; +#if CONFIG_MM_FREE_DELAYCOUNT_MAX > 0 + heap->mm_delaycount[up_cpu_index()]++; +#endif + up_irq_restore(flags); #endif } @@ -190,8 +204,9 @@ static void add_delaylist(FAR struct mm_heap_s *heap, FAR void *mem) * Name: free_delaylist ****************************************************************************/ -static void free_delaylist(FAR struct mm_heap_s *heap) +static bool free_delaylist(FAR struct mm_heap_s *heap, bool force) { + bool ret = false; #if defined(CONFIG_BUILD_FLAT) || defined(__KERNEL__) FAR struct mm_delaynode_s *tmp; irqstate_t flags; @@ -201,12 +216,27 @@ static void free_delaylist(FAR struct mm_heap_s *heap) flags = up_irq_save(); tmp = heap->mm_delaylist[up_cpu_index()]; + +#if CONFIG_MM_FREE_DELAYCOUNT_MAX > 0 + if (tmp == NULL || + (!force && + heap->mm_delaycount[up_cpu_index()] < CONFIG_MM_FREE_DELAYCOUNT_MAX)) + { + up_irq_restore(flags); + return false; + } + + heap->mm_delaycount[up_cpu_index()] = 0; +#endif + heap->mm_delaylist[up_cpu_index()] = NULL; up_irq_restore(flags); /* Test if the delayed is empty */ + ret = tmp != NULL; + while (tmp) { FAR void *address; @@ -220,9 +250,11 @@ static void free_delaylist(FAR struct mm_heap_s *heap) * 'while' condition above. */ - mm_free(heap, address); + mm_delayfree(heap, address, false); } + #endif + return ret; } #if CONFIG_MM_HEAP_MEMPOOL_THRESHOLD != 0 && CONFIG_MM_BACKTRACE >= 0 @@ -457,6 +489,50 @@ static void memdump_handler(FAR void *ptr, size_t size, int used, } } +/**************************************************************************** + * Name: mm_delayfree + * + * Description: + * Delay free memory if `delay` is true, otherwise free it immediately. + * + ****************************************************************************/ + +static void mm_delayfree(FAR struct mm_heap_s *heap, FAR void *mem, + bool delay) +{ + if (mm_lock(heap) == 0) + { +#ifdef CONFIG_MM_FILL_ALLOCATIONS + memset(mem, 0x55, mm_malloc_size(heap, mem)); +#endif + + kasan_poison(mem, mm_malloc_size(heap, mem)); + + /* Update heap statistics */ + + heap->mm_curused -= mm_malloc_size(heap, mem); + + /* Pass, return to the tlsf pool */ + + if (delay) + { + add_delaylist(heap, mem); + } + else + { + tlsf_free(heap->mm_tlsf, mem); + } + + mm_unlock(heap); + } + else + { + /* Add to the delay list(see the comment in mm_lock) */ + + add_delaylist(heap, mem); + } +} + /**************************************************************************** * Public Functions ****************************************************************************/ @@ -678,18 +754,17 @@ void mm_extend(FAR struct mm_heap_s *heap, FAR void *mem, size_t size, void mm_free(FAR struct mm_heap_s *heap, FAR void *mem) { - int ret; - - UNUSED(ret); minfo("Freeing %p\n", mem); /* Protect against attempts to free a NULL reference */ - if (!mem) + if (mem == NULL) { return; } + DEBUGASSERT(mm_heapmember(heap, mem)); + #if CONFIG_MM_HEAP_MEMPOOL_THRESHOLD != 0 if (mempool_multiple_free(heap->mm_mpool, mem) >= 0) { @@ -697,29 +772,7 @@ void mm_free(FAR struct mm_heap_s *heap, FAR void *mem) } #endif - if (mm_lock(heap) == 0) - { -#ifdef CONFIG_MM_FILL_ALLOCATIONS - memset(mem, 0x55, mm_malloc_size(heap, mem)); -#endif - - kasan_poison(mem, mm_malloc_size(heap, mem)); - - /* Update heap statistics */ - - heap->mm_curused -= mm_malloc_size(heap, mem); - - /* Pass, return to the tlsf pool */ - - tlsf_free(heap->mm_tlsf, mem); - mm_unlock(heap); - } - else - { - /* Add to the delay list(see the comment in mm_lock) */ - - add_delaylist(heap, mem); - } + mm_delayfree(heap, mem, CONFIG_MM_FREE_DELAYCOUNT_MAX > 0); } /**************************************************************************** @@ -1061,7 +1114,7 @@ FAR void *mm_malloc(FAR struct mm_heap_s *heap, size_t size) /* Free the delay list first */ - free_delaylist(heap); + free_delaylist(heap, false); /* Allocate from the tlsf pool */ @@ -1095,6 +1148,15 @@ FAR void *mm_malloc(FAR struct mm_heap_s *heap, size_t size) #endif } +#if CONFIG_MM_FREE_DELAYCOUNT_MAX > 0 + /* Try again after free delay list */ + + else if (free_delaylist(heap, true)) + { + return mm_malloc(heap, size); + } +#endif + return ret; } @@ -1126,7 +1188,7 @@ FAR void *mm_memalign(FAR struct mm_heap_s *heap, size_t alignment, /* Free the delay list first */ - free_delaylist(heap); + free_delaylist(heap, false); /* Allocate from the tlsf pool */ @@ -1156,6 +1218,15 @@ FAR void *mm_memalign(FAR struct mm_heap_s *heap, size_t alignment, kasan_unpoison(ret, mm_malloc_size(heap, ret)); } +#if CONFIG_MM_FREE_DELAYCOUNT_MAX > 0 + /* Try again after free delay list */ + + else if (free_delaylist(heap, true)) + { + return mm_memalign(heap, alignment, size); + } +#endif + return ret; } @@ -1237,7 +1308,7 @@ FAR void *mm_realloc(FAR struct mm_heap_s *heap, FAR void *oldmem, #else /* Free the delay list first */ - free_delaylist(heap); + free_delaylist(heap, false); /* Allocate from the tlsf pool */ @@ -1268,8 +1339,16 @@ FAR void *mm_realloc(FAR struct mm_heap_s *heap, FAR void *oldmem, } #endif +#if CONFIG_MM_FREE_DELAYCOUNT_MAX > 0 + /* Try again after free delay list */ + + if (newmem == NULL && free_delaylist(heap, true)) + { + return mm_realloc(heap, oldmem, size); + } #endif +#endif return newmem; }