//     Copyright 2026, Kay Hayen, mailto:kay.hayen@gmail.com find license text at end of file

// For interacting with the Python GC unexposed details

// This file is included from another C file, help IDEs to still parse it on
// its own.
#ifdef __IDE_ONLY__
#include "nuitka/prelude.h"
#endif

// spell-checker: ignore PYMEM_DOMAIN,Nuitka_gc_decref,gcstate,uncollectable,sisnsn,QSBR,wrasgc
// spell-checker: ignore Nuitka_visit_decref,finalizers,wrcb,wrlist,_PyObject_GET_WEAKREFS_LISTPTR
// spell-checker: ignore objmalloc,qbsr,stoptheworld

void *(*python_obj_malloc)(void *ctx, size_t size) = NULL;
void *(*python_mem_malloc)(void *ctx, size_t size) = NULL;
void *(*python_mem_calloc)(void *ctx, size_t nelem, size_t elsize) = NULL;
#ifndef Py_GIL_DISABLED
void *(*python_mem_realloc)(void *ctx, void *ptr, size_t new_size) = NULL;
#else
void (*python_mem_free)(void *ctx, void *ptr) = NULL;
#endif

#if defined(Py_DEBUG)
void *python_obj_ctx = NULL;
void *python_mem_ctx = NULL;
#endif

void initNuitkaAllocators(void) {
    PyMemAllocatorEx allocators;

    PyMem_GetAllocator(PYMEM_DOMAIN_OBJ, &allocators);

#if defined(Py_DEBUG)
    python_obj_ctx = allocators.ctx;
#endif

    python_obj_malloc = allocators.malloc;

    PyMem_GetAllocator(PYMEM_DOMAIN_MEM, &allocators);

#if defined(Py_DEBUG)
    python_mem_ctx = allocators.ctx;
#endif

    python_mem_malloc = allocators.malloc;
    python_mem_calloc = allocators.calloc;
#ifndef Py_GIL_DISABLED
    python_mem_realloc = allocators.realloc;
#else
    python_mem_free = allocators.free;
#endif
}

#if defined(Py_GIL_DISABLED) && defined(Py_REF_DEBUG) && PYTHON_VERSION >= 0x3d0 && PYTHON_VERSION < 0x3e0
void _Py_AddRefTotal(PyThreadState *tstate, Py_ssize_t n) { tstate->interp->object_state.reftotal += n; }

void _Py_IncRefTotal(PyThreadState *tstate) { _Py_AddRefTotal(tstate, 1); }

void _Py_DecRefTotal(PyThreadState *tstate) { _Py_AddRefTotal(tstate, -1); }
#endif

#if PYTHON_VERSION >= 0x3b0

typedef struct _gc_runtime_state GCState;

#if PYTHON_VERSION < 0x3d0

#define AS_GC(o) ((PyGC_Head *)(((char *)(o)) - sizeof(PyGC_Head)))
#define FROM_GC(g) ((PyObject *)(((char *)(g)) + sizeof(PyGC_Head)))

static inline bool Nuitka_gc_is_collecting(PyGC_Head *g) { return (g->_gc_prev & _PyGC_PREV_MASK_COLLECTING) != 0; }

static inline void Nuitka_gc_clear_collecting(PyGC_Head *g) { g->_gc_prev &= ~_PyGC_PREV_MASK_COLLECTING; }

static inline Py_ssize_t Nuitka_gc_get_refs(PyGC_Head *g) { return (Py_ssize_t)(g->_gc_prev >> _PyGC_PREV_SHIFT); }

static inline void Nuitka_gc_set_refs(PyGC_Head *g, Py_ssize_t refs) {
    g->_gc_prev = (g->_gc_prev & ~_PyGC_PREV_MASK) | ((uintptr_t)(refs) << _PyGC_PREV_SHIFT);
}

static inline void Nuitka_gc_reset_refs(PyGC_Head *g, Py_ssize_t refs) {
    g->_gc_prev = (g->_gc_prev & _PyGC_PREV_MASK_FINALIZED) | _PyGC_PREV_MASK_COLLECTING |
                  ((uintptr_t)(refs) << _PyGC_PREV_SHIFT);
}

static inline void Nuitka_gc_decref(PyGC_Head *g) { g->_gc_prev -= 1 << _PyGC_PREV_SHIFT; }

#define GEN_HEAD(gcstate, n) (&(gcstate)->generations[n].head)
#define GC_NEXT _PyGCHead_NEXT
#define GC_PREV _PyGCHead_PREV

static void Nuitka_invoke_gc_callback(PyThreadState *tstate, const char *phase, int generation, Py_ssize_t collected,
                                      Py_ssize_t uncollectable) {
    assert(!HAS_ERROR_OCCURRED(tstate));

    GCState *gcstate = &tstate->interp->gc;
    if (gcstate->callbacks == NULL) {
        return;
    }

    PyObject *info = NULL;

    if (PyList_GET_SIZE(gcstate->callbacks) != 0) {
        info =
            Py_BuildValue("{sisnsn}", "generation", generation, "collected", collected, "uncollectable", uncollectable);

        if (info == NULL) {
            PyErr_WriteUnraisable(NULL);
            return;
        }
    }

    for (Py_ssize_t i = 0; i < PyList_GET_SIZE(gcstate->callbacks); i++) {
        PyObject *r, *cb = PyList_GET_ITEM(gcstate->callbacks, i);
        Py_INCREF(cb);

        r = PyObject_CallFunction(cb, "sO", phase, info);

        if (r == NULL) {
            PyErr_WriteUnraisable(cb);
        } else {
            Py_DECREF(r);
        }

        Py_DECREF(cb);
    }

    Py_XDECREF(info);

    assert(!HAS_ERROR_OCCURRED(tstate));
}

static inline bool Nuitka_gc_list_is_empty(PyGC_Head *list) { return (list->_gc_next == (uintptr_t)list); }

static inline void Nuitka_gc_list_init(PyGC_Head *list) {
    list->_gc_prev = (uintptr_t)list;
    list->_gc_next = (uintptr_t)list;
}

static void Nuitka_gc_list_merge(PyGC_Head *from, PyGC_Head *to) {
    if (Nuitka_gc_list_is_empty(from) == false) {
        PyGC_Head *to_tail = GC_PREV(to);
        PyGC_Head *from_head = GC_NEXT(from);
        PyGC_Head *from_tail = GC_PREV(from);

        assert(from_head != from);
        assert(from_tail != from);

        _PyGCHead_SET_NEXT(to_tail, from_head);
        _PyGCHead_SET_PREV(from_head, to_tail);

        _PyGCHead_SET_NEXT(from_tail, to);
        _PyGCHead_SET_PREV(to, from_tail);
    }

    Nuitka_gc_list_init(from);
}

static void Nuitka_update_refs(PyGC_Head *containers) {
    for (PyGC_Head *gc = GC_NEXT(containers); gc != containers; gc = GC_NEXT(gc)) {
        Nuitka_gc_reset_refs(gc, Py_REFCNT(FROM_GC(gc)));
    }
}

static int Nuitka_visit_decref(PyObject *op, void *parent) {
    if (_PyObject_IS_GC(op)) {
        PyGC_Head *gc = AS_GC(op);

        if (Nuitka_gc_is_collecting(gc)) {
            Nuitka_gc_decref(gc);
        }
    }

    return 0;
}

static void Nuitka_subtract_refs(PyGC_Head *containers) {
    for (PyGC_Head *gc = GC_NEXT(containers); gc != containers; gc = GC_NEXT(gc)) {
        PyObject *op = FROM_GC(gc);
        traverseproc traverse = Py_TYPE(op)->tp_traverse;
        (void)traverse(op, (visitproc)Nuitka_visit_decref, op);
    }
}

#define NEXT_MASK_UNREACHABLE 1

static inline void Nuitka_gc_list_append(PyGC_Head *node, PyGC_Head *list) {
    PyGC_Head *last = (PyGC_Head *)list->_gc_prev;

    _PyGCHead_SET_PREV(node, last);
    _PyGCHead_SET_NEXT(last, node);

    _PyGCHead_SET_NEXT(node, list);
    list->_gc_prev = (uintptr_t)node;
}

static inline void Nuitka_gc_list_remove(PyGC_Head *node) {
    PyGC_Head *prev = GC_PREV(node);
    PyGC_Head *next = GC_NEXT(node);

    _PyGCHead_SET_NEXT(prev, next);
    _PyGCHead_SET_PREV(next, prev);

    node->_gc_next = 0;
}

static int Nuitka_visit_reachable(PyObject *op, PyGC_Head *reachable) {
    if (!_PyObject_IS_GC(op)) {
        return 0;
    }

    PyGC_Head *gc = AS_GC(op);
    const Py_ssize_t gc_refs = Nuitka_gc_get_refs(gc);

    if (Nuitka_gc_is_collecting(gc) == false) {
        return 0;
    }

    assert(gc->_gc_next != 0);

    if (gc->_gc_next & NEXT_MASK_UNREACHABLE) {
        PyGC_Head *prev = GC_PREV(gc);
        PyGC_Head *next = (PyGC_Head *)(gc->_gc_next & ~NEXT_MASK_UNREACHABLE);
        prev->_gc_next = gc->_gc_next;
        _PyGCHead_SET_PREV(next, prev);

        Nuitka_gc_list_append(gc, reachable);
        Nuitka_gc_set_refs(gc, 1);
    } else if (gc_refs == 0) {
        Nuitka_gc_set_refs(gc, 1);
    }

    return 0;
}

static void Nuitka_move_unreachable(PyGC_Head *young, PyGC_Head *unreachable) {
    PyGC_Head *prev = young;
    PyGC_Head *gc = GC_NEXT(young);

    while (gc != young) {
        if (Nuitka_gc_get_refs(gc)) {
            PyObject *op = FROM_GC(gc);
            traverseproc traverse = Py_TYPE(op)->tp_traverse;

            (void)traverse(op, (visitproc)Nuitka_visit_reachable, (void *)young);
            _PyGCHead_SET_PREV(gc, prev);

            Nuitka_gc_clear_collecting(gc);
            prev = gc;
        } else {
            prev->_gc_next = gc->_gc_next;

            PyGC_Head *last = GC_PREV(unreachable);
            last->_gc_next = (NEXT_MASK_UNREACHABLE | (uintptr_t)gc);
            _PyGCHead_SET_PREV(gc, last);
            gc->_gc_next = (NEXT_MASK_UNREACHABLE | (uintptr_t)unreachable);
            unreachable->_gc_prev = (uintptr_t)gc;
        }
        gc = (PyGC_Head *)prev->_gc_next;
    }

    young->_gc_prev = (uintptr_t)prev;
    unreachable->_gc_next &= ~NEXT_MASK_UNREACHABLE;
}

static inline void Nuitka_deduce_unreachable(PyGC_Head *base, PyGC_Head *unreachable) {
    Nuitka_update_refs(base);
    Nuitka_subtract_refs(base);

    Nuitka_gc_list_init(unreachable);
    Nuitka_move_unreachable(base, unreachable);
}

static void Nuitka_untrack_tuples(PyGC_Head *head) {
    PyGC_Head *gc = GC_NEXT(head);

    while (gc != head) {
        PyObject *op = FROM_GC(gc);
        PyGC_Head *next = GC_NEXT(gc);

        if (PyTuple_CheckExact(op)) {
            // TODO: API function
            _PyTuple_MaybeUntrack(op);
        }

        gc = next;
    }
}

static Py_ssize_t Nuitka_gc_list_size(PyGC_Head *list) {
    Py_ssize_t n = 0;

    for (PyGC_Head *gc = GC_NEXT(list); gc != list; gc = GC_NEXT(gc)) {
        n += 1;
    }

    return n;
}

static void Nuitka_untrack_dicts(PyGC_Head *head) {
    PyGC_Head *next, *gc = GC_NEXT(head);
    while (gc != head) {
        PyObject *op = FROM_GC(gc);
        next = GC_NEXT(gc);
        if (PyDict_CheckExact(op)) {
            // TODO: API function
            _PyDict_MaybeUntrack(op);
        }
        gc = next;
    }
}

static bool Nuitka_has_legacy_finalizer(PyObject *op) { return Py_TYPE(op)->tp_del != NULL; }

static void Nuitka_gc_list_move(PyGC_Head *node, PyGC_Head *list) {
    PyGC_Head *from_prev = GC_PREV(node);
    PyGC_Head *from_next = GC_NEXT(node);
    _PyGCHead_SET_NEXT(from_prev, from_next);
    _PyGCHead_SET_PREV(from_next, from_prev);

    PyGC_Head *to_prev = (PyGC_Head *)list->_gc_prev;
    _PyGCHead_SET_PREV(node, to_prev);
    _PyGCHead_SET_NEXT(to_prev, node);
    list->_gc_prev = (uintptr_t)node;
    _PyGCHead_SET_NEXT(node, list);
}

static void _Nuitka_move_legacy_finalizers(PyGC_Head *unreachable, PyGC_Head *finalizers) {
    PyGC_Head *next;
    for (PyGC_Head *gc = GC_NEXT(unreachable); gc != unreachable; gc = next) {
        PyObject *op = FROM_GC(gc);

        _PyObject_ASSERT(op, gc->_gc_next & NEXT_MASK_UNREACHABLE);
        gc->_gc_next &= ~NEXT_MASK_UNREACHABLE;
        next = (PyGC_Head *)gc->_gc_next;

        if (Nuitka_has_legacy_finalizer(op)) {
            Nuitka_gc_clear_collecting(gc);
            Nuitka_gc_list_move(gc, finalizers);
        }
    }
}

static int _Nuitka_visit_move(PyObject *op, PyGC_Head *to_list) {
    if (_PyObject_IS_GC(op)) {
        PyGC_Head *gc = AS_GC(op);
        if (Nuitka_gc_is_collecting(gc)) {
            Nuitka_gc_list_move(gc, to_list);
            Nuitka_gc_clear_collecting(gc);
        }
    }

    // Required for traverseproc
    return 0;
}

static void Nuitka_move_legacy_finalizer_reachable(PyGC_Head *finalizers) {
    for (PyGC_Head *gc = GC_NEXT(finalizers); gc != finalizers; gc = GC_NEXT(gc)) {
        traverseproc traverse = Py_TYPE(FROM_GC(gc))->tp_traverse;
        (void)traverse(FROM_GC(gc), (visitproc)_Nuitka_visit_move, (void *)finalizers);
    }
}

static void Nuitka_finalize_garbage(PyThreadState *tstate, PyGC_Head *collectable) {
    destructor finalize;
    PyGC_Head seen;

    Nuitka_gc_list_init(&seen);

    while (Nuitka_gc_list_is_empty(collectable) == false) {
        PyGC_Head *gc = GC_NEXT(collectable);
        PyObject *object = FROM_GC(gc);
        Nuitka_gc_list_move(gc, &seen);

        if (!_PyGCHead_FINALIZED(gc) && (finalize = Py_TYPE(object)->tp_finalize) != NULL) {
            _PyGCHead_SET_FINALIZED(gc);
            Py_INCREF(object);
            finalize(object);
            assert(!HAS_ERROR_OCCURRED(tstate));
            Py_DECREF(object);
        }
    }
    Nuitka_gc_list_merge(&seen, collectable);
}

static int Nuitka_handle_weakrefs(PyThreadState *tstate, PyGC_Head *unreachable, PyGC_Head *old) {
    PyGC_Head *gc;
    PyObject *object;
    PyWeakReference *wr;
    PyGC_Head wrcb_to_call;
    PyGC_Head *next;
    int num_freed = 0;

    Nuitka_gc_list_init(&wrcb_to_call);

    for (gc = GC_NEXT(unreachable); gc != unreachable; gc = next) {
        PyWeakReference **wrlist;

        object = FROM_GC(gc);
        next = GC_NEXT(gc);

        if (PyWeakref_Check(object)) {
            _PyWeakref_ClearRef((PyWeakReference *)object);
        }

        if (!_PyType_SUPPORTS_WEAKREFS(Py_TYPE(object)))
            continue;

        wrlist = (PyWeakReference **)_PyObject_GET_WEAKREFS_LISTPTR(object);

        for (wr = *wrlist; wr != NULL; wr = *wrlist) {
            PyGC_Head *wrasgc;

            _PyWeakref_ClearRef(wr);

            if (wr->wr_callback == NULL) {
                continue;
            }

            if (Nuitka_gc_is_collecting(AS_GC(wr))) {
                continue;
            }

            Py_INCREF(wr);

            wrasgc = AS_GC(wr);
            Nuitka_gc_list_move(wrasgc, &wrcb_to_call);
        }
    }

    while (Nuitka_gc_list_is_empty(&wrcb_to_call) == false) {
        PyObject *temp;
        PyObject *callback;

        gc = (PyGC_Head *)wrcb_to_call._gc_next;
        object = FROM_GC(gc);
        wr = (PyWeakReference *)object;
        callback = wr->wr_callback;

        temp = CALL_FUNCTION_WITH_SINGLE_ARG(tstate, callback, (PyObject *)wr);
        if (temp == NULL)
            PyErr_WriteUnraisable(callback);
        else
            Py_DECREF(temp);

        Py_DECREF(object);
        if (wrcb_to_call._gc_next == (uintptr_t)gc) {
            Nuitka_gc_list_move(gc, old);
        } else {
            ++num_freed;
        }
    }

    return num_freed;
}

static inline void Nuitka_gc_list_clear_collecting(PyGC_Head *collectable) {
    for (PyGC_Head *gc = GC_NEXT(collectable); gc != collectable; gc = GC_NEXT(gc)) {
        Nuitka_gc_clear_collecting(gc);
    }
}

static inline void Nuitka_clear_unreachable_mask(PyGC_Head *unreachable) {
    PyGC_Head *next;

    for (PyGC_Head *gc = GC_NEXT(unreachable); gc != unreachable; gc = next) {
        gc->_gc_next &= ~NEXT_MASK_UNREACHABLE;
        next = (PyGC_Head *)gc->_gc_next;
    }
}

static inline void handle_resurrected_objects(PyGC_Head *unreachable, PyGC_Head *still_unreachable,
                                              PyGC_Head *old_generation) {
    Nuitka_gc_list_clear_collecting(unreachable);

    PyGC_Head *resurrected = unreachable;
    Nuitka_deduce_unreachable(resurrected, still_unreachable);
    Nuitka_clear_unreachable_mask(still_unreachable);

    Nuitka_gc_list_merge(resurrected, old_generation);
}

static void Nuitka_delete_garbage(PyThreadState *tstate, GCState *gcstate, PyGC_Head *collectable, PyGC_Head *old) {
    assert(!HAS_ERROR_OCCURRED(tstate));

    while (Nuitka_gc_list_is_empty(collectable) == false) {
        PyGC_Head *gc = GC_NEXT(collectable);
        PyObject *object = FROM_GC(gc);

        _PyObject_ASSERT_WITH_MSG(object, Py_REFCNT(object) > 0, "refcount is too small");

        {
            inquiry clear;
            if ((clear = Py_TYPE(object)->tp_clear) != NULL) {
                Py_INCREF(object);
                (void)clear(object);
                if (HAS_ERROR_OCCURRED(tstate)) {
                    _PyErr_WriteUnraisableMsg("in tp_clear of", (PyObject *)Py_TYPE(object));
                }
                Py_DECREF(object);
            }
        }
        if (GC_NEXT(collectable) == gc) {
            Nuitka_gc_clear_collecting(gc);
            Nuitka_gc_list_move(gc, old);
        }
    }
}

static void Nuitka_handle_legacy_finalizers(PyThreadState *tstate, GCState *gcstate, PyGC_Head *finalizers,
                                            PyGC_Head *old) {
    assert(!HAS_ERROR_OCCURRED(tstate));

    for (PyGC_Head *gc = GC_NEXT(finalizers); gc != finalizers; gc = GC_NEXT(gc)) {
        PyObject *op = FROM_GC(gc);

        if (Nuitka_has_legacy_finalizer(op)) {
            if (PyList_Append(gcstate->garbage, op) < 0) {
                CLEAR_ERROR_OCCURRED(tstate);
                break;
            }
        }
    }

    Nuitka_gc_list_merge(finalizers, old);
}

static Py_ssize_t Nuitka_gc_collect_main(PyThreadState *tstate, int generation, Py_ssize_t *n_collected,
                                         Py_ssize_t *n_uncollectable) {
    int i;
    Py_ssize_t m = 0;
    Py_ssize_t n = 0;
    PyGC_Head *young;
    PyGC_Head *old;
    PyGC_Head unreachable;
    PyGC_Head finalizers;
    PyGC_Head *gc;

    GCState *gcstate = &tstate->interp->gc;

    assert(gcstate->garbage != NULL);
    assert(!HAS_ERROR_OCCURRED(tstate));

    if (generation + 1 < NUM_GENERATIONS) {
        gcstate->generations[generation + 1].count += 1;
    }

    for (i = 0; i <= generation; i++) {
        gcstate->generations[i].count = 0;
    }

    for (i = 0; i < generation; i++) {
        Nuitka_gc_list_merge(GEN_HEAD(gcstate, i), GEN_HEAD(gcstate, generation));
    }

    young = GEN_HEAD(gcstate, generation);
    if (generation < NUM_GENERATIONS - 1) {
        old = GEN_HEAD(gcstate, generation + 1);
    } else {
        old = young;
    }

    Nuitka_deduce_unreachable(young, &unreachable);

    Nuitka_untrack_tuples(young);

    if (young != old) {
        if (generation == NUM_GENERATIONS - 2) {
            gcstate->long_lived_pending += Nuitka_gc_list_size(young);
        }

        Nuitka_gc_list_merge(young, old);
    } else {
        Nuitka_untrack_dicts(young);

        gcstate->long_lived_pending = 0;
        gcstate->long_lived_total = Nuitka_gc_list_size(young);
    }

    Nuitka_gc_list_init(&finalizers);

    _Nuitka_move_legacy_finalizers(&unreachable, &finalizers);

    Nuitka_move_legacy_finalizer_reachable(&finalizers);

    m += Nuitka_handle_weakrefs(tstate, &unreachable, old);

    Nuitka_finalize_garbage(tstate, &unreachable);

    PyGC_Head final_unreachable;
    handle_resurrected_objects(&unreachable, &final_unreachable, old);

    m += Nuitka_gc_list_size(&final_unreachable);
    Nuitka_delete_garbage(tstate, gcstate, &final_unreachable, old);

    for (gc = GC_NEXT(&finalizers); gc != &finalizers; gc = GC_NEXT(gc)) {
        n++;
    }

    Nuitka_handle_legacy_finalizers(tstate, gcstate, &finalizers, old);

    if (HAS_ERROR_OCCURRED(tstate)) {
        _PyErr_WriteUnraisableMsg("in garbage collection", NULL);
    }

    if (n_collected) {
        *n_collected = m;
    }
    if (n_uncollectable) {
        *n_uncollectable = n;
    }

    struct gc_generation_stats *stats = &gcstate->generation_stats[generation];
    stats->collections++;
    stats->collected += m;
    stats->uncollectable += n;

    assert(!HAS_ERROR_OCCURRED(tstate));
    return n + m;
}

static Py_ssize_t Nuitka_gc_collect_with_callback(PyThreadState *tstate, int generation) {
    assert(!HAS_ERROR_OCCURRED(tstate));

    Nuitka_invoke_gc_callback(tstate, "start", generation, 0, 0);
    Py_ssize_t collected, uncollectable;
    Py_ssize_t result = Nuitka_gc_collect_main(tstate, generation, &collected, &uncollectable);
    Nuitka_invoke_gc_callback(tstate, "stop", generation, collected, uncollectable);

    assert(!HAS_ERROR_OCCURRED(tstate));
    return result;
}

static Py_ssize_t Nuitka_gc_collect_generations(PyThreadState *tstate) {
    GCState *gcstate = &tstate->interp->gc;

    Py_ssize_t n = 0;

    for (int i = NUM_GENERATIONS - 1; i >= 0; i--) {
        if (gcstate->generations[i].count > gcstate->generations[i].threshold) {
            if (i == NUM_GENERATIONS - 1 && gcstate->long_lived_pending < gcstate->long_lived_total / 4)
                continue;

            n = Nuitka_gc_collect_with_callback(tstate, i);
            break;
        }
    }

    return n;
}

#elif PYTHON_VERSION < 0x3e0

static void Nuitka_Py_ScheduleGC(PyThreadState *tstate) {
    if (!_Py_eval_breaker_bit_is_set(tstate, _PY_GC_SCHEDULED_BIT)) {
        _Py_set_eval_breaker_bit(tstate, _PY_GC_SCHEDULED_BIT);
    }
}

#endif

// This is called during object creation and might trigger garbage collection
void Nuitka_PyObject_GC_Link(PyObject *op) {
#if PYTHON_VERSION < 0x3d0
    PyGC_Head *g = AS_GC(op);

    PyThreadState *tstate = _PyThreadState_GET();
    GCState *gcstate = &tstate->interp->gc;

    g->_gc_next = 0;
    g->_gc_prev = 0;
    gcstate->generations[0].count++;

    if (gcstate->generations[0].count > gcstate->generations[0].threshold && gcstate->enabled &&
        gcstate->generations[0].threshold && !gcstate->collecting && !HAS_ERROR_OCCURRED(tstate)) {

        gcstate->collecting = 1;
        Nuitka_gc_collect_generations(tstate);
        gcstate->collecting = 0;
    }
#elif PYTHON_VERSION < 0x3e0
    // TODO: This is not the no-GIL implementation
    PyGC_Head *gc = _Py_AS_GC(op);

    // gc must be correctly aligned
    _PyObject_ASSERT(op, ((uintptr_t)gc & (sizeof(uintptr_t) - 1)) == 0);

    // TODO: Have this passed.
    PyThreadState *tstate = _PyThreadState_GET();
    GCState *gcstate = &tstate->interp->gc;

    gc->_gc_next = 0;
    gc->_gc_prev = 0;

    gcstate->generations[0].count++;

    if (gcstate->generations[0].count > gcstate->generations[0].threshold && gcstate->enabled &&
        gcstate->generations[0].threshold && !_Py_atomic_load_int_relaxed(&gcstate->collecting) &&
        !_PyErr_Occurred(tstate)) {
        Nuitka_Py_ScheduleGC(tstate);
    }
#elif _NUITKA_MODULE_MODE && PYTHON_VERSION >= 0x3e0 && PYTHON_VERSION < 0x3f0 && !defined(Py_GIL_DISABLED)
    // TODO: This is not the no-GIL implementation
    PyGC_Head *gc = _Py_AS_GC(op);

    // gc must be correctly aligned
    _PyObject_ASSERT(op, ((uintptr_t)gc & (sizeof(uintptr_t) - 1)) == 0);

    // TODO: Have this passed.
    PyThreadState *tstate = _PyThreadState_GET();
    GCState *gcstate = &tstate->interp->gc;

    gc->_gc_next = 0;
    gc->_gc_prev = 0;

    if (Nuitka_GC_UsesGeneration0List()) {
        Nuitka_GCStateGenerational *gcstate_generational = Nuitka_GC_GetGenerationalState(gcstate);

        gcstate_generational->generations[0].count++;

        if (gcstate_generational->generations[0].count > gcstate_generational->generations[0].threshold &&
            gcstate_generational->enabled && gcstate_generational->generations[0].threshold &&
            !_Py_atomic_load_int_relaxed(&gcstate_generational->collecting) && !_PyErr_Occurred(tstate)) {
            Nuitka_Py_ScheduleGC(tstate);
        }
    } else if (!Nuitka_GC_TrackOwnsAccounting()) {
        Nuitka_GCStateIncremental *gcstate_incremental = Nuitka_GC_GetIncrementalState(gcstate);

        gcstate_incremental->young.count++;
        gcstate_incremental->heap_size++;

        if (gcstate_incremental->young.count > gcstate_incremental->young.threshold && gcstate_incremental->enabled &&
            gcstate_incremental->young.threshold && !_Py_atomic_load_int_relaxed(&gcstate_incremental->collecting) &&
            !_PyErr_Occurred(tstate)) {
            Nuitka_Py_ScheduleGC(tstate);
        }
    }
#elif PYTHON_VERSION < 0x3e5
    // TODO: This is not the no-GIL implementation
    PyGC_Head *gc = _Py_AS_GC(op);

    // gc must be correctly aligned
    _PyObject_ASSERT(op, ((uintptr_t)gc & (sizeof(uintptr_t) - 1)) == 0);

    // TODO: Have this passed.
    PyThreadState *tstate = _PyThreadState_GET();
    GCState *gcstate = &tstate->interp->gc;

    gc->_gc_next = 0;
    gc->_gc_prev = 0;

    if (!Nuitka_GC_TrackOwnsAccounting()) {
        gcstate->young.count++;
        gcstate->heap_size++;

        if (gcstate->young.count > gcstate->young.threshold && gcstate->enabled && gcstate->young.threshold &&
            !_Py_atomic_load_int_relaxed(&gcstate->collecting) && !_PyErr_Occurred(tstate)) {
            Nuitka_Py_ScheduleGC(tstate);
        }
    }
#else
    // TODO: This is not the no-GIL implementation
    PyGC_Head *gc = _Py_AS_GC(op);

    // gc must be correctly aligned
    _PyObject_ASSERT(op, ((uintptr_t)gc & (sizeof(uintptr_t) - 1)) == 0);

    // TODO: Have this passed.
    PyThreadState *tstate = _PyThreadState_GET();
    GCState *gcstate = &tstate->interp->gc;

    gc->_gc_next = 0;
    gc->_gc_prev = 0;

    gcstate->generations[0].count++;

    if (gcstate->generations[0].count > gcstate->generations[0].threshold && gcstate->enabled &&
        gcstate->generations[0].threshold && !_Py_atomic_load_int_relaxed(&gcstate->collecting) &&
        !_PyErr_Occurred(tstate)) {
        Nuitka_Py_ScheduleGC(tstate);
    }
#endif
}

#endif

#ifdef Py_GIL_DISABLED

// Needs to align with CPython "objmalloc.c"
#define WORK_ITEMS_PER_CHUNK 254
struct _mem_work_item {
    uintptr_t ptr; // lowest bit tagged 1 for objects freed with PyObject_Free
    uint64_t qsbr_goal;
};

struct _mem_work_chunk {
    struct llist_node node;

    Py_ssize_t rd_idx; // index of next item to read
    Py_ssize_t wr_idx; // index of next item to write
    struct _mem_work_item array[WORK_ITEMS_PER_CHUNK];
};

// Aligns with CPython "qsbr.c"
#define QSBR_DEFERRED_LIMIT 10

static uint64_t Nuitka_qsbr_shared_next(struct _qsbr_shared *shared) {
    return _Py_qsbr_shared_current(shared) + QSBR_INCR;
}

static uint64_t Nuitka_qsbr_advance(struct _qsbr_shared *shared) {
    return _Py_atomic_add_uint64(&shared->wr_seq, QSBR_INCR) + QSBR_INCR;
}

static uint64_t Nuitka_qsbr_deferred_advance(struct _qsbr_thread_state *qsbr) {
    if (++qsbr->deferred_count < QSBR_DEFERRED_LIMIT) {
        return Nuitka_qsbr_shared_next(qsbr->shared);
    }

    qsbr->deferred_count = 0;
    qsbr->should_process = true;
    return Nuitka_qsbr_advance(qsbr->shared);
}

static uint64_t _Nuitka_qsbr_poll_scan(struct _qsbr_shared *shared) {
    _Py_atomic_fence_seq_cst();

    uint64_t min_seq = _Py_atomic_load_uint64(&shared->wr_seq);
    struct _qsbr_pad *array = shared->array;
    for (Py_ssize_t i = 0, size = shared->size; i != size; i++) {
        struct _qsbr_thread_state *qsbr = &array[i].qsbr;

        uint64_t seq = _Py_atomic_load_uint64(&qsbr->seq);
        if (seq != QSBR_OFFLINE && QSBR_LT(seq, min_seq)) {
            min_seq = seq;
        }
    }

    uint64_t rd_seq = _Py_atomic_load_uint64(&shared->rd_seq);
    if (QSBR_LT(rd_seq, min_seq)) {
        (void)_Py_atomic_compare_exchange_uint64(&shared->rd_seq, &rd_seq, min_seq);
        rd_seq = min_seq;
    }

    return rd_seq;
}

static bool _Nuitka_qsbr_poll(struct _qsbr_thread_state *qsbr, uint64_t goal) {
    assert(_Py_atomic_load_int_relaxed(&_PyThreadState_GET()->state) == _Py_THREAD_ATTACHED);

    if (_Py_qbsr_goal_reached(qsbr, goal)) {
        return true;
    }

    uint64_t rd_seq = _Nuitka_qsbr_poll_scan(qsbr->shared);
    return QSBR_LEQ(goal, rd_seq);
}

static void _NuitkaMem_free_work_item(uintptr_t ptr) {
    if (ptr & 0x01) {
        PyObject_Free((char *)(ptr - 1));
    } else {
        NuitkaMem_Free((void *)ptr);
    }
}

static struct _mem_work_chunk *work_queue_first(struct llist_node *head) {
    return llist_data(head->next, struct _mem_work_chunk, node);
}

static void _Nuitka_process_queue(struct llist_node *head, struct _qsbr_thread_state *qsbr, bool keep_empty) {
    while (!llist_empty(head)) {
        struct _mem_work_chunk *buf = work_queue_first(head);

        while (buf->rd_idx < buf->wr_idx) {
            struct _mem_work_item *item = &buf->array[buf->rd_idx];
            if (!_Nuitka_qsbr_poll(qsbr, item->qsbr_goal)) {
                return;
            }

            _NuitkaMem_free_work_item(item->ptr);
            buf->rd_idx++;
        }

        assert(buf->rd_idx == buf->wr_idx);

        if (keep_empty && buf->node.next == head) {
            // Keeping the last buffer as a free-list entry.
            buf->rd_idx = buf->wr_idx = 0;
            return;
        }

        llist_remove(&buf->node);
        NuitkaMem_Free(buf);
    }
}

struct mutex_entry {
    // The time after which the unlocking thread should hand off lock ownership
    // directly to the waiting thread. Written by the waiting thread.
    PyTime_t time_to_be_fair;

    // Set to 1 if the lock was handed off. Written by the unlocking thread.
    int handed_off;
};

static const PyTime_t TIME_TO_BE_FAIR_NS = 1000 * 1000;
static const int MAX_SPIN_COUNT = 40;

static void _Nuitka_yield(void) {
#ifdef _WIN32
    SwitchToThread();
#else
    sched_yield();
#endif
}

PyLockStatus _NuitkaMutex_LockTimed(PyMutex *m) {
    uint8_t v = _Py_atomic_load_uint8_relaxed(&m->_bits);
    if ((v & _Py_LOCKED) == 0) {
        if (_Py_atomic_compare_exchange_uint8(&m->_bits, &v, v | _Py_LOCKED)) {
            return PY_LOCK_ACQUIRED;
        }
    } else {
        return PY_LOCK_FAILURE;
    }

    Py_ssize_t spin_count = 0;

    for (;;) {
        if ((v & _Py_LOCKED) == 0) {
            if (_Py_atomic_compare_exchange_uint8(&m->_bits, &v, v | _Py_LOCKED)) {
                return PY_LOCK_ACQUIRED;
            }
            continue;
        }

        if (!(v & _Py_HAS_PARKED) && spin_count < MAX_SPIN_COUNT) {
            // Spin for a bit.
            _Nuitka_yield();

            spin_count++;
            continue;
        }

        return PY_LOCK_FAILURE;
    }
}

static void _Nuitka_process_interp_queue(struct _Py_mem_interp_free_queue *queue, struct _qsbr_thread_state *qsbr) {
    if (!_Py_atomic_load_int_relaxed(&queue->has_work)) {
        return;
    }

    // Try to acquire the lock, but don't block if it's already held.
    if (_NuitkaMutex_LockTimed(&queue->mutex) == PY_LOCK_ACQUIRED) {
        _Nuitka_process_queue(&queue->head, qsbr, false);

        int more_work = !llist_empty(&queue->head);
        _Py_atomic_store_int_relaxed(&queue->has_work, more_work);

        PyMutex_Unlock(&queue->mutex);
    }
}

void _NuitkaMem_ProcessDelayed(PyThreadState *tstate) {
    PyInterpreterState *interp = tstate->interp;
    _PyThreadStateImpl *tstate_impl = (_PyThreadStateImpl *)tstate;

    // Release thread-local queue
    _Nuitka_process_queue(&tstate_impl->mem_free_queue, tstate_impl->qsbr, true);

    // Release interpreter queue
    _Nuitka_process_interp_queue(Nuitka_PyInterpreterState_GetMemFreeQueue(interp), tstate_impl->qsbr);
}

static void _NuitkaMem_FreeDelayed2(uintptr_t ptr) {
    // Free immediately if possible.
    if (_PyRuntime.stoptheworld.world_stopped) {
        _NuitkaMem_free_work_item(ptr);
        return;
    }

    // Allocate an entry for later processing.
    _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)_PyThreadState_GET();
    struct llist_node *head = &tstate->mem_free_queue;

    struct _mem_work_chunk *buf = NULL;
    if (!llist_empty(head)) {
        // Try to reuse the last buffer
        buf = llist_data(head->prev, struct _mem_work_chunk, node);
        if (buf->wr_idx == WORK_ITEMS_PER_CHUNK) {
            // already full
            buf = NULL;
        }
    }

    if (buf == NULL) {
        buf = NuitkaMem_Calloc(1, sizeof(*buf));
    }

    assert(buf != NULL && buf->wr_idx < WORK_ITEMS_PER_CHUNK);
    uint64_t seq = Nuitka_qsbr_deferred_advance(tstate->qsbr);
    buf->array[buf->wr_idx].ptr = ptr;
    buf->array[buf->wr_idx].qsbr_goal = seq;
    buf->wr_idx++;

    if (buf->wr_idx == WORK_ITEMS_PER_CHUNK || _Py_qsbr_should_process(tstate->qsbr)) {
        tstate->qsbr->should_process = false;
        _NuitkaMem_ProcessDelayed((PyThreadState *)tstate);
    }
}

void NuitkaMem_FreeDelayed(void *ptr) {
    assert(!((uintptr_t)ptr & 0x01));
    _NuitkaMem_FreeDelayed2((uintptr_t)ptr);
}

#endif

//     Part of "Nuitka", an optimizing Python compiler that is compatible and
//     integrates with CPython, but also works on its own.
//
//     Licensed under the GNU Affero General Public License, Version 3 (the "License");
//     you may not use this file except in compliance with the License.
//     You may obtain a copy of the License at
//
//        https://www.gnu.org/licenses/agpl-3.0.txt
//
//     See also: "Nuitka Runtime Library Exception, Version 1.0" in file
//     "LICENSE-RUNTIME.txt" for additional permissions granted under Section 7.
//
//     Unless required by applicable law or agreed to in writing, software
//     distributed under the License is distributed on an "AS IS" BASIS,
//     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//     See the License for the specific language governing permissions and
//     limitations under the License.
