nybble
/
rosella
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity
rosella/gc.c

1585 lines
39 KiB
C
Raw Blame History

#define _POSIX_C_SOURCE 199309L
#include <assert.h>
#include <ctype.h>
#include <inttypes.h>
#include <setjmp.h>
#include <stdbool.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
#include "gc.h"
#if _CLOCK_MONOTONIC
# define TIMING_CLOCK CLOCK_MONOTONIC
#else
# define TIMING_CLOCK CLOCK_REALTIME
#endif
gc_stats_t gc_stats;
/* Helper macros to reduce duplication */
#define VECTOR_BYTES(nelem) (sizeof(vector_t) + (sizeof(value_t) * (nelem)))
#define BYTESTR_BYTES(size) (sizeof(byte_string_t) + (size))
#define STRUCT_BYTES(nslots) (sizeof(struct_t) + (sizeof(value_t) * (nslots)))
/* Alignment must ensure each object has enough room to hold a forwarding object */
#define GC_ALIGNMENT ((size_t)(sizeof(object_t)))
/* Smaller block sizes allow for more precise GC, but require more memory for the bitmap. */
/* The block bitmap will be allocated at startup assuming a max. heap size of 2GB. */
/* The default of 128 KB/block should require a 2 KB bitmap. */
#define GC_DIRTY_BLOCK_SIZE (128UL << 10)
typedef struct seen_value
{
value_t value;
struct seen_value *prev;
} seen_value_t;
/****************************************************************************/
static value_t gc_weak_box_list;
static value_t gc_will_list;
static value_t gc_will_active_list;
static gc_root_t gc_root_list = {
.value = UNDEFINED,
.prev = &gc_root_list,
.next = &gc_root_list
};
void register_gc_root(gc_root_t *root, value_t v)
{
root->value = v;
root->next = &gc_root_list;
gc_root_list.prev->next = root;
root->prev = gc_root_list.prev;
gc_root_list.prev = root;
}
void unregister_gc_root(gc_root_t *root)
{
assert(root && root->prev && root->next); /* Uninitialized */
assert((root->prev != root) && (root->next != root)); /* Already removed */
/* Cut the given root out of the list */
root->prev->next = root->next;
root->next->prev = root->prev;
/* Remove dead references to root list; protects against double-removal */
root->prev = root->next = root;
}
/****************************************************************************/
static value_t make_hash_value(void)
{
static fixnum_t hash_seed = 0x67f76bc8;
hash_seed = (33 * hash_seed) ^ (fixnum_t)clock();
return fixnum_value(hash_seed);
}
bool get_boolean(value_t v)
{
release_assert(is_boolean(v));
return (v != FALSE_VALUE);
}
fixnum_t get_fixnum(value_t v)
{
release_assert(is_fixnum(v));
return _get_fixnum(v);
}
object_t *get_object(value_t v)
{
release_assert(is_object(v));
return _get_object(v);
}
/* No one outside this module should care... */
static inline bool is_broken_heart(value_t v)
{
return is_object(v) && (_get_object(v)->tag == BROKEN_HEART);
}
value_t cons(value_t car, value_t cdr)
{
gc_root_t car_root, cdr_root;
pair_t *p;
register_gc_root(&car_root, car);
register_gc_root(&cdr_root, cdr);
p = (pair_t*)gc_alloc(sizeof(pair_t));
p->car = car_root.value;
p->cdr = cdr_root.value;
unregister_gc_root(&car_root);
unregister_gc_root(&cdr_root);
return pair_value(p);
}
pair_t *get_pair(value_t v)
{
release_assert(is_pair(v));
return _get_pair(v);
}
value_t make_box(value_t initial_value)
{
gc_root_t iv_root;
box_t *box;
register_gc_root(&iv_root, initial_value);
box = (box_t*)gc_alloc(sizeof(box_t));
box->tag = TYPE_TAG_BOX;
box->value = iv_root.value;
unregister_gc_root(&iv_root);
return object_value(box);
}
box_t *get_box(value_t v)
{
release_assert(is_box(v));
return _get_box(v);
}
value_t make_vector(size_t nelem, value_t initial_value)
{
gc_root_t iv_root;
vector_t *vec;
register_gc_root(&iv_root, initial_value);
vec = (vector_t*)gc_alloc(VECTOR_BYTES(nelem));
vec->tag = TYPE_TAG_VECTOR;
vec->size = nelem;
vec->hash = make_hash_value();
for (int i = 0; i < nelem; ++i)
vec->elements[i] = iv_root.value;
unregister_gc_root(&iv_root);
return object_value(vec);
}
vector_t *get_vector(value_t v)
{
release_assert(is_vector(v));
return _get_vector(v);
}
value_t make_byte_string(size_t size, int default_value)
{
const size_t nbytes = BYTESTR_BYTES(size);
byte_string_t *str;
str = (byte_string_t*)gc_alloc(nbytes);
str->tag = TYPE_TAG_BYTESTR;
str->size = size;
memset(str->bytes, default_value, size);
return object_value(str);
}
byte_string_t *get_byte_string(value_t v)
{
release_assert(is_byte_string(v));
return _get_byte_string(v);
}
value_t string_to_value(const char *s)
{
size_t len = strlen(s);
value_t v = make_byte_string(len, '\0');
memcpy(_get_byte_string(v)->bytes, s, len);
return v;
}
char *value_to_string(value_t v)
{
byte_string_t *str = get_byte_string(v);
char *s = (char*)malloc(str->size + 1);
memcpy(s, str->bytes, str->size);
s[str->size] = '\0';
return s;
}
int byte_strcmp(value_t s1, value_t s2)
{
byte_string_t *str1 = get_byte_string(s1);
byte_string_t *str2 = get_byte_string(s2);
if (str1->size < str2->size)
return -1;
else if (str1->size > str2->size)
return 1;
else
return memcmp(str1->bytes, str2->bytes, str1->size);
}
value_t make_struct(value_t type, size_t nslots)
{
gc_root_t type_root;
struct_t *s;
register_gc_root(&type_root, type);
s = (struct_t*)gc_alloc(STRUCT_BYTES(nslots));
s->tag = TYPE_TAG_STRUCT;
s->type = type_root.value;
s->nslots = nslots;
s->hash = make_hash_value();
for (int i = 0; i < nslots; ++i)
s->slots[i] = UNDEFINED;
unregister_gc_root(&type_root);
return object_value(s);
}
struct_t *get_struct(value_t v)
{
release_assert(is_struct(v));
return _get_struct(v);
}
value_t make_weak_box(value_t initial_value)
{
gc_root_t iv_root;
weak_box_t *box;
register_gc_root(&iv_root, initial_value);
box = (weak_box_t*)gc_alloc(sizeof(weak_box_t));
box->tag = TYPE_TAG_WEAK_BOX;
box->value = iv_root.value;
box->next = gc_weak_box_list;
gc_weak_box_list = object_value(box);
unregister_gc_root(&iv_root);
return object_value(box);
}
weak_box_t *get_weak_box(value_t v)
{
release_assert(is_weak_box(v));
return _get_weak_box(v);
}
void register_finalizer(value_t value, value_t finalizer)
{
/* Non-objects are never GC'd, so their finalizers will never be invoked. */
if (is_object(value))
{
gc_root_t value_root, finalizer_root;
will_t *w;
register_gc_root(&value_root, value);
register_gc_root(&finalizer_root, finalizer);
w = (will_t*)gc_alloc(sizeof(will_t));
w->tag = TYPE_TAG_WILL;
w->value = value_root.value;
w->finalizer = finalizer_root.value;
w->next = gc_will_list;
gc_will_list = object_value(w);
unregister_gc_root(&value_root);
unregister_gc_root(&finalizer_root);
}
}
/* The 'will' accessors are private to the GC, unlike other similar routines. */
static inline will_t *_get_will(value_t v)
{
return (will_t*)_get_object(v);
}
static will_t *get_will(value_t v)
{
release_assert(is_will(v));
return _get_will(v);
}
value_t make_float(native_float_t value)
{
float_object_t *obj;
obj = (float_object_t*)gc_alloc(sizeof(float_object_t));
obj->tag = TYPE_TAG_FLOAT;
obj->value = value;
return object_value(obj);
}
native_float_t get_float(value_t v)
{
if (is_fixnum(v))
{
return (native_float_t)_get_fixnum(v);
}
else
{
release_assert(is_float(v));
return _get_float(v);
}
}
value_t make_builtin_fn(builtin_fn_t *fn)
{
builtin_fn_object_t *obj;
obj = (builtin_fn_object_t*)gc_alloc(sizeof(builtin_fn_object_t));
obj->tag = TYPE_TAG_BUILTIN;
obj->fn = fn;
return object_value(obj);
}
builtin_fn_t *get_builtin_fn(value_t v)
{
release_assert(is_builtin_fn(v));
return _get_builtin_fn(v);
}
static size_t dbj2_hash(uint8_t *bytes, size_t size)
{
size_t hash = 5381;
for (size_t i = 0; i < size; ++i)
{
hash = (33 * hash) ^ (size_t)bytes[i];
}
return hash;
}
static value_t _get_hash_value(value_t v, seen_value_t *seen)
{
seen_value_t new_seen = { v, seen };
if (is_object(v) && !(is_float(v) || is_byte_string(v) || is_builtin_fn(v)))
{
for (seen_value_t *sv = seen; sv; sv = sv->prev)
{
if (v == sv->value)
{
return 0;
}
}
}
if (!is_object(v))
{
/* Non-objects compare by value */
return fixnum_value(v ^ (v >> 2));
}
else if (is_float(v))
{
double d = _get_float(v);
return fixnum_value(dbj2_hash((uint8_t*)&d, sizeof d));
}
else if (is_builtin_fn(v))
{
return fixnum_value((uintptr_t)_get_builtin_fn(v) >> 2);
}
else if (is_pair(v))
{
/* Lists and trees compare by value, not reference. */
value_t seed = fixnum_value(0);
seed = combine_hash_values(seed, _get_hash_value(_CAR(v), &new_seen));
seed = combine_hash_values(seed, _get_hash_value(_CDR(v), &new_seen));
return seed;
}
else if (is_box(v) || is_weak_box(v))
{
/* Boxes compare by value, not by reference. */
return _get_hash_value(_get_box(v)->value, &new_seen);
}
else if (is_byte_string(v))
{
byte_string_t *str = _get_byte_string(v);
return fixnum_value(dbj2_hash(str->bytes, str->size));
}
else if (is_vector(v))
{
/* Vectors compare by reference. */
return _get_vector(v)->hash;
}
else if (is_struct(v))
{
/* Structures compare by reference. */
return _get_struct(v)->hash;
}
else /* unknown object type */
{
return _get_hash_value(_get_object(v)->tag, &new_seen);
}
}
value_t get_hash_value(value_t v)
{
return _get_hash_value(v, NULL);
}
value_t combine_hash_values(value_t f1, value_t f2)
{
fixnum_t h1 = _get_fixnum(f1);
fixnum_t h2 = _get_fixnum(f2);
return fixnum_value(h1 ^ (h2 + 0x9e3779b9 + (h1 << 6) + (h1 >> 2)));
}
/*************************** Common Collector Code **************************/
static bool gc_enabled;
static bool gc_in_gen0_collection;
static bool gc_in_gen1_collection;
/* Also used from Gen-0 code to track new Gen-1 objects */
static char *gc_gen1_ranges[2];
static size_t gc_gen1_min_size;
static size_t gc_gen1_max_size;
static size_t gc_gen1_soft_limit;
static int gc_gen1_current_range;
static char *gc_gen1_free_ptr;
static char *gc_gen1_range_end;
static size_t gc_gen1_max_blocks;
static uint32_t *gc_gen1_dirty_bits;
static char **gc_gen1_block_starts;
/* A convenient shorthand */
#define gc_gen1_other_range() (1-gc_gen1_current_range)
static inline size_t gc_align(size_t nbytes) __attribute__ ((const));
static void transfer_roots(void);
static size_t transfer_children(object_t *obj);
static void process_weak_boxes(void);
static void process_wills(void);
static void update_weak_box_list(void);
static void gc_gen0_init(size_t gen0_size);
static void gc_gen1_init(size_t min_size, size_t max_size);
static int gc_gen1_range_of(void *object) __attribute__ ((const));
static size_t gc_gen1_block_of(void *obj) __attribute__ ((const));
static inline size_t gc_gen1_free_space(void);
static void *gc_alloc_gen1(size_t nbytes);
static void collect_gen1_garbage(size_t min_free);
static void gc_gen1_clear_dirty_bits(void);
static inline size_t gc_gen1_free_space(void)
{
return gc_gen1_range_end - gc_gen1_free_ptr;
}
static inline size_t gc_align(size_t nbytes)
{
return ((nbytes + GC_ALIGNMENT - 1) & ~(GC_ALIGNMENT - 1));
}
void gc_init(size_t gen0_size, size_t gen1_min_size, size_t gen1_max_size)
{
gc_gen0_init(gen0_size);
gc_gen1_init(gen1_min_size, gen1_max_size);
gc_weak_box_list = NIL;
gc_will_list = NIL;
gc_will_active_list = NIL;
clear_gc_stats();
gc_enabled = true;
}
void clear_gc_stats(void)
{
int i;
for (i = 0; i < 2; ++i)
{
gc_stats.gen[i].passes = 0;
gc_stats.gen[i].total_ns = 0;
gc_stats.gen[i].max_ns = 0;
gc_stats.gen[i].total_freed = 0;
}
gc_stats.gen1_high_water = 0;
}
#ifndef NDEBUG
static void gc_poison_region(void *start, size_t size, value_t tag)
{
size_t count = size / GC_ALIGNMENT;
object_t *obj = (object_t*)start;
while (count--)
*obj++ = (object_t){ .tag = tag, .forward = tag };
}
#endif
/****************************** Gen-0 Collector *****************************/
/* These private variables are exported ONLY for use by is_gen0_object(). */
char *gc_gen0_range;
char *gc_gen0_range_end;
static size_t gc_gen0_size;
static char *gc_gen0_free_ptr;
/* Used to signal that Gen-0 pass has been obviated by Gen-1 collection. */
static jmp_buf gc_gen0_end_ctx;
static inline size_t gc_gen0_free_space(void)
{
return gc_gen0_range_end - gc_gen0_free_ptr;
}
static void gc_gen0_init(size_t gen0_size)
{
assert(gen0_size >= GC_ALIGNMENT);
gc_gen0_size = gen0_size;
gc_gen0_range = (char*)malloc(gc_gen0_size);
release_assert(gc_gen0_range);
gc_gen0_free_ptr = gc_gen0_range;
gc_gen0_range_end = gc_gen0_range + gc_gen0_size;
}
static void collect_gen0_garbage(void)
{
if (gc_enabled)
{
#ifndef NDEBUG
size_t initial_gen1_free_space;
#endif
#ifndef NO_STATS
size_t initial_free_space;
#ifndef NO_TIMING_STATS
struct timespec start_time;
clock_gettime(TIMING_CLOCK, &start_time);
#endif
initial_free_space = gc_gen0_free_space() + gc_gen1_free_space();
#endif
debug(("Performing Gen-0 garbage collection pass...\n"));
assert(!gc_in_gen0_collection);
assert(!gc_in_gen1_collection);
#ifndef NDEBUG
initial_gen1_free_space = gc_gen1_free_space();
#endif
/* If we trigger a Gen-1 collection at any point then we are done. */
/* Full collection will pull in any current Gen-0 objects. */
if (setjmp(gc_gen0_end_ctx) == 0)
{
char *object_ptr = gc_gen1_free_ptr;
const size_t used_bytes = gc_gen1_free_ptr - gc_gen1_ranges[gc_gen1_current_range];
const int current_blocks = (used_bytes + GC_DIRTY_BLOCK_SIZE - 1) / GC_DIRTY_BLOCK_SIZE;
const int current_block_groups = (current_blocks + 31) / 32;
int group;
gc_in_gen0_collection = true;
/* 1. Transfer Gen-0 roots (ignore Gen-1). */
transfer_roots();
/* 2. Locate and transfer Gen-0 references from dirty Gen-1 blocks. */
for (group = 0; group < current_block_groups; ++group)
{
uint32_t bits = gc_gen1_dirty_bits[group];
if (bits)
{
int block;
for (block = group * 32; bits; bits >>= 1, ++block)
{
if (bits & 1)
{
/* Find first object in the block */
char *block_obj = gc_gen1_block_starts[block];
char *block_end = gc_gen1_ranges[gc_gen1_current_range]
+ ((block+1) * GC_DIRTY_BLOCK_SIZE);
assert(block_obj && (gc_gen1_block_of(block_obj) == block));
/* For each object in block: transfer children */
do {
block_obj += gc_align(transfer_children((object_t*)block_obj));
assert(gc_gen1_range_of(block_obj) == gc_gen1_current_range);
} while (block_obj < block_end);
}
}
}
}
gc_gen1_clear_dirty_bits();
/* Transfer Gen-0 children of objects newly moved to Gen-1 */
while (object_ptr < gc_gen1_free_ptr)
{
object_ptr += gc_align(transfer_children((object_t*)object_ptr));
}
/* These have to be examined after normal reachability has been determined */
process_weak_boxes();
process_wills();
/*
* Keep transferring until no more objects in the new range refer to the old one.
* This is so that values which are otherwise unreachable, but have finalizers which
* may be able to reach them, are not collected prematurely. process_wills() transfers
* the value of any will newly placed on the active list. Note that these values may
* be finalized in any order, and that any weak references have already been cleared.
*/
while (object_ptr < gc_gen1_free_ptr)
{
object_ptr += gc_align(transfer_children((object_t*)object_ptr));
}
update_weak_box_list();
#ifndef NDEBUG
/* Clear old range, to make it easier to detect bugs. */
gc_poison_region(gc_gen0_range, gc_gen0_size, GC_GEN0_POISON);
#endif
/* 4. Reset Gen-0 range to 'empty' state. */
gc_gen0_free_ptr = gc_gen0_range;
//debug(("Finished Gen-0 collection; added %d bytes to Gen-1 heap.\n", initial_gen1_free_space - gc_gen1_free_space()));
#ifndef NO_STATS
#ifndef NO_TIMING_STATS
{
struct timespec end_time;
nsec_t nsec;
clock_gettime(TIMING_CLOCK, &end_time);
nsec = (end_time.tv_sec - start_time.tv_sec) * 1000000000LL;
nsec += (end_time.tv_nsec - start_time.tv_nsec);
gc_stats.gen[0].total_ns += nsec;
if (nsec > gc_stats.gen[0].max_ns)
gc_stats.gen[0].max_ns = nsec;
}
#endif
gc_stats.gen[0].total_freed -= initial_free_space;
gc_stats.gen[0].total_freed += gc_gen0_free_space();
gc_stats.gen[0].total_freed += gc_gen1_free_space();
++gc_stats.gen[0].passes;
#endif
}
gc_in_gen0_collection = false;
}
}
void *gc_alloc(size_t nbytes)
{
nbytes = gc_align(nbytes);
assert(!gc_in_gen0_collection);
assert(!gc_in_gen1_collection);
if (nbytes >= gc_gen0_size)
{
//debug(("Allocating directly from Gen-0...\n"));
return gc_alloc_gen1(nbytes);
}
else
{
if (nbytes > gc_gen0_free_space())
{
if (gc_enabled)
collect_gen0_garbage();
else
return gc_alloc_gen1(nbytes);
}
assert(nbytes <= gc_gen0_free_space());
{
void *const p = gc_gen0_free_ptr;
gc_gen0_free_ptr += nbytes;
return p;
}
}
}
/****************************** Gen-1 Collector *****************************/
static int gc_gen1_range_of(void *object)
{
const char *const ptr = (char*)object;
if ((ptr >= gc_gen1_ranges[0]) && (ptr < (gc_gen1_ranges[0] + gc_gen1_max_size)))
return 0;
else if ((ptr >= gc_gen1_ranges[1]) && (ptr < (gc_gen1_ranges[1] + gc_gen1_max_size)))
return 1;
else
return -1;
}
static inline bool gc_object_has_moved(value_t v)
{
return is_broken_heart(v) &&
(gc_gen1_range_of(_get_object(_get_object(v)->forward)) == gc_gen1_current_range);
}
/* Only useful AFTER all reachable objects have been processed. */
static inline bool gc_object_left_behind(value_t v)
{
/* Must provide a reference to the original location, not the new one (if moved). */
assert(!is_object(v) ||
is_gen0_object(v) ||
(gc_gen1_range_of(_get_object(v)) != gc_gen1_current_range));
return is_object(v) &&
(gc_in_gen1_collection || is_gen0_object(v)) &&
(!is_broken_heart(v) || (gc_gen1_range_of(_get_object(_get_object(v)->forward)) !=
gc_gen1_current_range));
}
static void gc_gen1_init(size_t min_size, size_t max_size)
{
release_assert(gc_gen0_size > 0);
release_assert(max_size >= gc_gen0_size);
release_assert(min_size <= (max_size - gc_gen0_size));
gc_gen1_ranges[0] = (char*)malloc(max_size);
gc_gen1_ranges[1] = (char*)malloc(max_size);
release_assert(gc_gen1_ranges[0] && gc_gen1_ranges[1]);
gc_gen1_current_range = 0;
gc_gen1_free_ptr = gc_gen1_ranges[gc_gen1_current_range];
gc_gen1_min_size = min_size;
gc_gen1_max_size = max_size;
gc_gen1_soft_limit = min_size + gc_gen0_size;
gc_gen1_range_end = gc_gen1_free_ptr + gc_gen1_soft_limit;
gc_gen1_max_blocks = ((size_t)2 << 30) / GC_DIRTY_BLOCK_SIZE;
gc_gen1_dirty_bits = (uint32_t*)calloc((gc_gen1_max_blocks + 31) / 32, sizeof(uint32_t));
release_assert(gc_gen1_dirty_bits);
gc_gen1_block_starts = (char**)calloc(gc_gen1_max_blocks, sizeof(char*));
release_assert(gc_gen1_block_starts);
}
static void gc_gen1_clear_dirty_bits(void)
{
const size_t block_groups = (gc_gen1_max_blocks + 31) / 32;
memset(gc_gen1_dirty_bits, 0, sizeof(uint32_t) * block_groups);
}
static void *gc_alloc_gen1(size_t nbytes)
{
size_t min_free;
min_free = nbytes = gc_align(nbytes);
/* Ensure there is always enough room for a full collection. */
if (!gc_in_gen1_collection)
min_free += gc_gen0_size;
if (gc_gen1_free_space() < min_free)
collect_gen1_garbage(min_free);
assert(nbytes <= gc_gen1_free_space());
{
void *const p = gc_gen1_free_ptr;
const size_t block = gc_gen1_block_of(p);
if (!gc_gen1_block_starts[block])
gc_gen1_block_starts[block] = (char*)p;
gc_gen1_free_ptr += nbytes;
return p;
}
}
static void transfer_object(value_t *value)
{
/* During Gen-0 collection pass, leave Gen-1 objects alone. Always ignore non-objects. */
if (is_object(*value) && (gc_in_gen1_collection || is_gen0_object(*value)))
{
object_t *obj = _get_object(*value);
size_t nbytes;
void *newobj;
assert(is_gen0_object(*value) ||
(gc_gen1_range_of(obj) == gc_gen1_other_range()));
if (obj->tag == BROKEN_HEART)
{
if (gc_gen1_range_of(_get_object(obj->forward)) != gc_gen1_current_range)
{
/* Gen-0 object was transferred into old range; needs to move to current range */
transfer_object(&obj->forward);
}
assert(gc_gen1_range_of(_get_object(obj->forward)) == gc_gen1_current_range);
/* Object has already been moved; just update the reference */
*value = obj->forward;
return;
}
switch (obj->tag)
{
case TYPE_TAG_BOX:
nbytes = sizeof(box_t);
break;
case TYPE_TAG_VECTOR:
nbytes = VECTOR_BYTES(((const vector_t*)obj)->size);
break;
case TYPE_TAG_BYTESTR:
nbytes = BYTESTR_BYTES(((const byte_string_t*)obj)->size);
break;
case TYPE_TAG_STRUCT:
nbytes = STRUCT_BYTES(((const struct_t*)obj)->nslots);
break;
case TYPE_TAG_WEAK_BOX:
nbytes = sizeof(weak_box_t);
break;
case TYPE_TAG_WILL:
nbytes = sizeof(will_t);
break;
case TYPE_TAG_FLOAT:
nbytes = sizeof(float_object_t);
break;
case TYPE_TAG_BUILTIN:
nbytes = sizeof(builtin_fn_object_t);
break;
default: /* pair */
nbytes = sizeof(pair_t);
break;
}
newobj = gc_alloc_gen1(nbytes);
memcpy(newobj, obj, nbytes);
/* Keep the original tag bits (pair or object) */
obj->tag = BROKEN_HEART;
*value = obj->forward = (object_value(newobj) & ~2) | (*value & 2);
}
}
static size_t transfer_vector(vector_t *vec)
{
assert(vec->tag == TYPE_TAG_VECTOR);
/* Should be fixnum, but just in case... */
transfer_object(&vec->hash);
for (size_t i = 0; i < vec->size; ++i)
transfer_object(&vec->elements[i]);
return VECTOR_BYTES(vec->size);
}
static size_t transfer_struct(struct_t *s)
{
assert(s->tag == TYPE_TAG_STRUCT);
transfer_object(&s->type);
/* Should be fixnum, but just in case... */
transfer_object(&s->hash);
for (size_t i = 0; i < s->nslots; ++i)
transfer_object(&s->slots[i]);
return STRUCT_BYTES(s->nslots);
}
static size_t transfer_box(box_t *b)
{
transfer_object(&b->value);
return sizeof(box_t);
}
static size_t transfer_pair(pair_t *p)
{
transfer_object(&p->car);
transfer_object(&p->cdr);
return sizeof(pair_t);
}
static size_t transfer_will(will_t *w)
{
assert(w->tag == TYPE_TAG_WILL);
transfer_object(&w->finalizer);
/* Weak boxes are discarded when there are no other references,
* but wills need to remain until their finalizers are invoked. */
transfer_object(&w->next);
return sizeof(will_t);
}
static size_t transfer_children(object_t *obj)
{
switch (obj->tag)
{
case TYPE_TAG_BOX:
return transfer_box((box_t*)obj);
case TYPE_TAG_VECTOR:
return transfer_vector((vector_t*)obj);
case TYPE_TAG_BYTESTR:
return BYTESTR_BYTES(((const byte_string_t*)obj)->size);
case TYPE_TAG_STRUCT:
return transfer_struct((struct_t*)obj);
case TYPE_TAG_WEAK_BOX:
return sizeof(weak_box_t);
case TYPE_TAG_WILL:
return transfer_will((will_t*)obj);
case TYPE_TAG_FLOAT:
return sizeof(float_object_t);
case TYPE_TAG_BUILTIN:
return sizeof(builtin_fn_object_t);
default: /* pair */
return transfer_pair((pair_t*)obj);
}
}
static void swap_gen1_gc_ranges(void)
{
gc_gen1_current_range = gc_gen1_other_range();
gc_gen1_free_ptr = gc_gen1_ranges[gc_gen1_current_range];
gc_gen1_range_end = gc_gen1_free_ptr + gc_gen1_soft_limit;
}
static void transfer_roots(void)
{
/* Transfer registered GC roots */
for (gc_root_t *root = gc_root_list.next; root != &gc_root_list; root = root->next)
transfer_object(&root->value);
/* Ensure pending will list is transferred */
transfer_object(&gc_will_list);
/* The values associated with active wills are also roots */
for (value_t will = gc_will_active_list; !is_nil(will); will = _get_will(will)->next)
transfer_object(&get_will(will)->value);
/* Ensure active list itself is transferred */
transfer_object(&gc_will_active_list);
}
static void process_weak_boxes(void)
{
value_t wb = gc_weak_box_list;
while (!is_nil(wb))
{
weak_box_t *box;
if (gc_object_has_moved(wb))
{
/* Box has been moved; get a pointer to the new location, but don't update list yet. */
value_t fw = _get_object(wb)->forward;
assert(is_weak_box(fw));
box = _get_weak_box(fw);
}
else
{
/* Box hasn't been moved. Could be Gen-0 pass, or may live on as the value of a will. */
assert(is_weak_box(wb));
box = _get_weak_box(wb);
}
if (gc_object_left_behind(box->value))
{
/* The value in the box is an unreachable object; change to #f. */
/*
* NOTE: An object is considered unreachable via weak box when it could be finalized, even
* though it will be kept alive until any finalizers are removed from the 'active' list.
* The finalizer(s) may restore the object to a reachable state, in which case it will not
* be collected--but the weak reference will remain broken.
*
* Restoring references to otherwise GC'able objects is not recommended.
*
* The only known alternative would have been to invoke the finalizer while other objects
* may still be able to access the object (and create new references) via the weak box.
*/
box->value = FALSE_VALUE;
}
else if (gc_object_has_moved(box->value))
{
/* The value in the box is reachable; update w/ new location. */
box->value = _get_object(box->value)->forward;
}
/* Move on to this box's 'next' pointer */
wb = box->next;
}
}
/* Precondition: The wills themselves, and their finalizers,
* have already been transferred (recursively). */
static void process_wills(void)
{
/*
* Was value transferred (broken heart), and thus reachable?
* Yes ==> update value.
* No ==> transfer value and move will to active list.
*/
value_t *will = &gc_will_list;
while (!is_nil(*will))
{
will_t *w = get_will(*will);
if (gc_object_left_behind(w->value))
{
/*
* The will is associated with an unreachable object; activate it.
*/
/* First, ensure that the value remains reachable for the finalizer. */
transfer_object(&w->value);
/* Remove the will from the 'pending' list. */
*will = w->next;
/* Insert the will into the 'active' list. */
w->next = gc_will_active_list;
gc_will_active_list = object_value(w);
}
else
{
/* The value associated with the will is still reachable; update w/ new location. */
if (gc_object_has_moved(w->value))
{
w->value = _get_object(w->value)->forward;
}
/* Move on to this will's 'next' pointer */
will = &w->next;
}
}
}
static void update_weak_box_list(void)
{
value_t *wb = &gc_weak_box_list;
while (!is_nil(*wb))
{
if (gc_object_left_behind(*wb))
{
/* Box is no longer reachable; remove it from the list by updating 'next' pointer. */
assert(is_weak_box(*wb));
*wb = _get_weak_box(*wb)->next;
}
else
{
/* The box itself is reachable; need to update 'next' pointer to new location */
if (gc_object_has_moved(*wb))
{
*wb = _get_object(*wb)->forward;
}
/* Move on to next box's 'next' pointer */
assert(is_weak_box(*wb));
wb = &_get_weak_box(*wb)->next;
}
}
}
#define GC_DEFLATE_SIZE (64*1024)
static void update_soft_limit(size_t min_free)
{
size_t bytes_used = gc_gen1_free_ptr - gc_gen1_ranges[gc_gen1_current_range];
size_t min_limit = bytes_used + min_free;
size_t new_limit = 2 * min_limit;
if (gc_gen1_soft_limit > GC_DEFLATE_SIZE)
{
size_t deflate_limit = gc_gen1_soft_limit - GC_DEFLATE_SIZE;
if (new_limit < deflate_limit)
new_limit = deflate_limit;
}
if (new_limit < gc_gen1_min_size)
new_limit = gc_gen1_min_size;
else if (new_limit > gc_gen1_max_size)
new_limit = gc_gen1_max_size;
gc_gen1_soft_limit = new_limit;
/* Update end of range to reflect new limit */
gc_gen1_range_end = gc_gen1_ranges[gc_gen1_current_range] + gc_gen1_soft_limit;
#ifndef NO_STATS
if (gc_gen1_soft_limit > gc_stats.gen1_high_water)
{
gc_stats.gen1_high_water = gc_gen1_soft_limit;
}
#endif
}
static size_t gc_gen1_block_of(void *obj)
{
const intptr_t offset = (uintptr_t)obj
- (uintptr_t)gc_gen1_ranges[gc_gen1_current_range];
return (offset & (((uintptr_t)2 << 30) - 1)) / GC_DIRTY_BLOCK_SIZE;
}
void _gc_mark_updated_gen1_object(value_t v)
{
assert(is_object(v));
{
const size_t block = gc_gen1_block_of(_get_object(v));
assert(is_object(v) && !is_gen0_object(v) && gc_gen1_block_starts[block]);
gc_gen1_dirty_bits[block / 32] |= (1UL << (block % 32));
}
}
static void _out_of_memory(void) __attribute__ ((noreturn));
static void _out_of_memory(void)
{
out_of_memory();
abort();
}
static void collect_gen1_garbage(size_t min_free)
{
bool was_in_gen0_collection = gc_in_gen0_collection;
bool collected_garbage = false;
/* If Gen-1 free space falls below used portion of Gen-0, chaos may ensue. */
assert(gc_gen1_free_space() >= (gc_gen0_free_ptr - gc_gen0_range));
min_free += gc_gen0_size;
if (gc_enabled)
{
char *object_ptr;
#ifndef NO_STATS
size_t initial_free_space = gc_gen0_free_space() + gc_gen1_free_space();
#ifndef NO_TIMING_STATS
struct timespec start_time;
clock_gettime(TIMING_CLOCK, &start_time);
#endif
#endif
debug(("Performing Gen-1 garbage collection pass...\n"));
gc_enabled = false;
gc_in_gen0_collection = false;
gc_in_gen1_collection = true;
gc_gen1_clear_dirty_bits();
swap_gen1_gc_ranges();
/* Record the start of each Gen-1 block as objects are moved to the new range. */
memset(gc_gen1_block_starts, 0, gc_gen1_max_blocks * sizeof(char*));
/* New "current" range is initially empty, old one is full */
object_ptr = gc_gen1_free_ptr;
/* Prime the pump */
transfer_roots();
/* Keep transferring until no more objects in the new range refer to the old one,
* other than pending wills and weak boxes. */
while (object_ptr < gc_gen1_free_ptr)
{
object_ptr += gc_align(transfer_children((object_t*)object_ptr));
assert(gc_gen1_range_of(object_ptr) == gc_gen1_current_range);
}
/* These have to be examined after normal reachability has been determined */
process_weak_boxes();
process_wills();
/* Keep transferring until no more objects in the new range refer to the old one.
* This is so that values which are otherwise unreachable, but have finalizers which
* may be able to reach them, are not collected prematurely. process_wills() transfers
* the value of any will newly placed on the active list. Note that these values may
* be finalized in any order, and that any weak references have already been cleared. */
while (object_ptr < gc_gen1_free_ptr)
{
object_ptr += gc_align(transfer_children((object_t*)object_ptr));
assert(gc_gen1_range_of(object_ptr) == gc_gen1_current_range);
}
update_weak_box_list();
#ifndef NDEBUG
/* Clear old range, to make it easier to detect bugs. */
gc_poison_region(gc_gen1_ranges[gc_gen1_other_range()], gc_gen1_soft_limit, GC_GEN1_POISON);
gc_poison_region(gc_gen0_range, gc_gen0_size, GC_GEN0_POISON);
#endif
/*
* Gen-0 should be empty at this point; all active objects
* have been moved to the Gen-1 memory region.
*/
gc_gen0_free_ptr = gc_gen0_range;
collected_garbage = true;
//debug(("Finished Gen-1 collection; active set is %d bytes.\n", gc_gen1_free_ptr - gc_gen1_ranges[gc_gen1_current_range]));
gc_in_gen1_collection = false;
gc_enabled = true;
#ifndef NO_STATS
#ifndef NO_TIMING_STATS
{
struct timespec end_time;
nsec_t nsec;
clock_gettime(TIMING_CLOCK, &end_time);
nsec = (end_time.tv_sec - start_time.tv_sec) * 1000000000LL;
nsec += (end_time.tv_nsec - start_time.tv_nsec);
gc_stats.gen[1].total_ns += nsec;
if (nsec > gc_stats.gen[1].max_ns)
gc_stats.gen[1].max_ns = nsec;
}
#endif
gc_stats.gen[1].total_freed -= initial_free_space;
gc_stats.gen[1].total_freed += gc_gen0_free_space();
gc_stats.gen[1].total_freed += gc_gen1_free_space();
++gc_stats.gen[1].passes;
#endif
}
update_soft_limit(min_free);
if (gc_gen1_free_space() < min_free)
{
size_t bytes_used = gc_gen1_free_ptr - gc_gen1_ranges[gc_gen1_current_range];
size_t need_bytes = bytes_used + min_free + gc_gen0_size;
/* If GC is disabled then we can't move anything, so reallocating is impossible. */
if (!gc_enabled)
_out_of_memory();
/*
* Try to get more memory from the C runtime.
*/
debug(("Ran out of free memory; will try to allocate more...\n"));
do {
release_assert(gc_gen1_max_size < (SIZE_MAX/2));
gc_gen1_max_size *= 2;
} while (gc_gen1_max_size < need_bytes);
/* Reallocate the unused space. */
{
char *unused_range = gc_gen1_ranges[gc_gen1_other_range()];
gc_gen1_ranges[gc_gen1_other_range()] = (char*)malloc(gc_gen1_max_size);
free(unused_range);
}
/* See if reallocation succeeded. */
if (!gc_gen1_ranges[gc_gen1_other_range()])
_out_of_memory();
/* Move everything into the newly enlarged space.
* Will update the soft-limit. */
collect_gen1_garbage(0);
/* Reallocate the other space, now unused. */
free(gc_gen1_ranges[gc_gen1_other_range()]);
gc_gen1_ranges[gc_gen1_other_range()] = (char*)malloc(gc_gen1_max_size);
/* Ensure second reallocation succeeded. */
if (!gc_gen1_ranges[gc_gen1_other_range()])
_out_of_memory();
}
/* If Gen-1 was invoked within Gen-0, skip the rest: Gen-0 is empty, we're done. */
if (was_in_gen0_collection && collected_garbage)
longjmp(gc_gen0_end_ctx, 1);
}
void collect_garbage(size_t min_free)
{
bool was_enabled = set_gc_enabled(true);
collect_gen1_garbage(min_free);
set_gc_enabled(was_enabled);
}
bool set_gc_enabled(bool enable)
{
bool was_enabled = gc_enabled;
gc_enabled = enable;
return was_enabled;
}
bool are_finalizers_pending(void)
{
return !is_nil(gc_will_active_list);
}
/* Finalizer can be registered as #f, but value must be an object.
* Returning with value == #f means there are no more finalizers. */
void get_next_finalizer(value_t *value, value_t *finalizer)
{
assert(value && finalizer);
if (is_nil(gc_will_active_list))
{
*value = *finalizer = FALSE_VALUE;
}
else
{
will_t *w = get_will(gc_will_active_list);
*value = w->value;
*finalizer = w->finalizer;
/* Remove this finalizer from the list -- up to caller to keep values reachable. */
gc_will_active_list = w->next;
}
}
void _release_assert(bool expr, const char *str, const char *file, int line)
{
if (!expr)
{
fprintf(stderr, "ERROR: Invalid state detected in %s, line %d.\n"
"Assertion failed: %s\n",
file, line, str);
abort();
}
}
static void _fprint_value(FILE *f, value_t v, seen_value_t *seen)
{
seen_value_t new_seen = { v, seen };
int depth = 0;
if (is_object(v) && !(is_float(v) || is_byte_string(v) || is_builtin_fn(v)))
{
for (seen_value_t *sv = seen; sv; sv = sv->prev)
{
if (v == sv->value)
{
fputs("#<cycle>", f);
return;
}
if (++depth >= 3)
{
fputs("...", f);
return;
}
}
}
if (v == NIL)
{
fputs("nil", f);
}
else if (v == FALSE_VALUE)
{
fputs("#f", f);
}
else if (v == TRUE_VALUE)
{
fputs("#t", f);
}
else if (v == UNDEFINED)
{
fputs("#<undefined>", f);
}
else if (is_fixnum(v))
{
fprintf(f, "%d", (int)get_fixnum(v));
}
else if (is_box(v))
{
fputs("#&", f);
_fprint_value(f, _get_box(v)->value, &new_seen);
}
else if (is_pair(v))
{
fputc('(', f);
_fprint_value(f, _get_pair(v)->car, &new_seen);
v = _get_pair(v)->cdr;
while (is_pair(v))
{
fputc(' ', f);
_fprint_value(f, _get_pair(v)->car, &new_seen);
v = _get_pair(v)->cdr;
}
if (v != NIL)
{
fputs(" . ", f);
_fprint_value(f, v, &new_seen);
}
fputc(')', f);
}
else if (is_vector(v))
{
fputs("#(", f);
for (size_t i = 0; i < _get_vector(v)->size; ++i)
{
if (i != 0) fputc(' ', f);
_fprint_value(f, _get_vector(v)->elements[i], &new_seen);
}
fputc(')', f);
}
else if (is_byte_string(v))
{
byte_string_t *str = _get_byte_string(v);
size_t written = 0;
fputc('"', f);
for (size_t i = 0; i < str->size; ++i)
{
int ch = str->bytes[i];
if (isprint(ch) && (ch != '\\') && (ch != '\"'))
{
fputc(str->bytes[i], f);
++written;
}
else
{
fprintf(f, "\\x%.2X", (int)str->bytes[i]);
written += 4;
}
if (written >= 20)
{
fputs("...", f);
break;
}
}
fputc('"', f);
}
else if (is_struct(v))
{
struct_t *meta = get_struct(_get_struct(v)->type);
byte_string_t *str = get_byte_string(meta->slots[0]);
fputs("#S(", f);
fwrite(str->bytes, str->size, 1, f);
for (size_t i = 0; i < _get_struct(v)->nslots; ++i)
{
fputc(' ', f);
_fprint_value(f, _get_struct(v)->slots[i], &new_seen);
}
fputc(')', f);
}
else if (is_weak_box(v))
{
fputs("#W&", f);
_fprint_value(f, _get_weak_box(v)->value, &new_seen);
}
else if (is_float(v))
{
fprintf(f, "%f", (double)_get_float(v));
}
else if (is_builtin_fn(v))
{
fputs("#<builtin>", f);
}
else
{
fputs("#<unknown>", f);
}
}
void fprint_value(FILE *f, value_t v)
{
_fprint_value(f, v, NULL);
}
static double ns2sec(nsec_t ns) __attribute__ ((const));
static double ns2sec(nsec_t ns)
{
return ns / 1.0e9;
}
void fprint_gc_stats(FILE *f)
{
if (gc_stats.gen[0].passes || gc_stats.gen[1].passes)
{
const llsize_t total_freed = gc_stats.gen[0].total_freed + gc_stats.gen[1].total_freed;
const nsec_t total_ns = gc_stats.gen[0].total_ns + gc_stats.gen[1].total_ns;
fprintf(f, "GC: %lld bytes freed by %d GCs in %0.6f sec => %0.3f MB/sec.\n",
total_freed,
gc_stats.gen[0].passes + gc_stats.gen[1].passes,
ns2sec(total_ns),
(total_freed / ns2sec(total_ns)) / (1024*1024));
fprintf(f, "GC: %d Gen-0 passes; avg. time was %0.6f sec, max %0.6f.\n",
gc_stats.gen[0].passes,
ns2sec(gc_stats.gen[0].total_ns) / gc_stats.gen[0].passes,
ns2sec(gc_stats.gen[0].max_ns));
fprintf(f, "GC: %d Gen-1 passes; avg. time was %0.6f sec, max %0.6f.\n",
gc_stats.gen[1].passes,
ns2sec(gc_stats.gen[1].total_ns) / gc_stats.gen[1].passes,
ns2sec(gc_stats.gen[1].max_ns));
fprintf(f, "GC: The Gen-1 soft-limit peaked at %d bytes out of %d allocated.\n",
gc_stats.gen1_high_water, gc_gen1_max_size);
}
else
{
fputs("GC: No garbage collection was performed.\n", f);
}
}
/* vim:set sw=2 expandtab: */
Reference in New Issue View Git Blame Copy Permalink