#include "jemalloc/internal/jemalloc_preamble.h"

#include "jemalloc/internal/jemalloc_internal_includes.h"

#include "jemalloc/internal/assert.h"

#include "jemalloc/internal/ckh.h"

#include "jemalloc/internal/hash.h"

#include "jemalloc/internal/malloc_io.h"

#include "jemalloc/internal/prof_data.h"

/*

* This file defines and manages the core profiling data structures.

*

* Conceptually, profiling data can be imagined as a table with three columns:

* thread, stack trace, and current allocation size. (When prof_accum is on,

* there's one additional column which is the cumulative allocation size.)

*

* Implementation wise, each thread maintains a hash recording the stack trace

* to allocation size correspondences, which are basically the individual rows

* in the table. In addition, two global "indices" are built to make data

* aggregation efficient (for dumping): bt2gctx and tdatas, which are basically

* the "grouped by stack trace" and "grouped by thread" views of the same table,

* respectively. Note that the allocation size is only aggregated to the two

* indices at dumping time, so as to optimize for performance.

*/

/******************************************************************************/

malloc_mutex_t bt2gctx_mtx;

malloc_mutex_t tdatas_mtx;

malloc_mutex_t prof_dump_mtx;

/*

* Table of mutexes that are shared among gctx's. These are leaf locks, so

* there is no problem with using them for more than one gctx at the same time.

* The primary motivation for this sharing though is that gctx's are ephemeral,

* and destroying mutexes causes complications for systems that allocate when

* creating/destroying mutexes.

*/

malloc_mutex_t *gctx_locks;

static atomic_u_t cum_gctxs; /* Atomic counter. */

/*

* Table of mutexes that are shared among tdata's. No operations require

* holding multiple tdata locks, so there is no problem with using them for more

* than one tdata at the same time, even though a gctx lock may be acquired

* while holding a tdata lock.

*/

malloc_mutex_t *tdata_locks;

/*

* Global hash of (prof_bt_t *)-->(prof_gctx_t *). This is the master data

* structure that knows about all backtraces currently captured.

*/

static ckh_t bt2gctx;

/*

* Tree of all extant prof_tdata_t structures, regardless of state,

* {attached,detached,expired}.

*/

static prof_tdata_tree_t tdatas;

size_t prof_unbiased_sz[PROF_SC_NSIZES];

size_t prof_shifted_unbiased_cnt[PROF_SC_NSIZES];

/******************************************************************************/

/* Red-black trees. */

static int

prof_tctx_comp(const prof_tctx_t *a, const prof_tctx_t *b) {

uint64_t a_thr_uid = a->thr_uid;

uint64_t b_thr_uid = b->thr_uid;

int ret = (a_thr_uid > b_thr_uid) - (a_thr_uid < b_thr_uid);

if (ret == 0) {

uint64_t a_thr_discrim = a->thr_discrim;

uint64_t b_thr_discrim = b->thr_discrim;

ret = (a_thr_discrim > b_thr_discrim)

- (a_thr_discrim < b_thr_discrim);

if (ret == 0) {

uint64_t a_tctx_uid = a->tctx_uid;

uint64_t b_tctx_uid = b->tctx_uid;

ret = (a_tctx_uid > b_tctx_uid)

- (a_tctx_uid < b_tctx_uid);

}

}

return ret;

}

/* NOLINTBEGIN(performance-no-int-to-ptr) */

rb_gen(static UNUSED, tctx_tree_, prof_tctx_tree_t, prof_tctx_t, tctx_link,

prof_tctx_comp)

/* NOLINTEND(performance-no-int-to-ptr) */

static int prof_gctx_comp(const prof_gctx_t *a, const prof_gctx_t *b) {

unsigned a_len = a->bt.len;

unsigned b_len = b->bt.len;

unsigned comp_len = (a_len < b_len) ? a_len : b_len;

int ret = memcmp(a->bt.vec, b->bt.vec, comp_len * sizeof(void *));

if (ret == 0) {

ret = (a_len > b_len) - (a_len < b_len);

}

return ret;

}

/* NOLINTBEGIN(performance-no-int-to-ptr) */

rb_gen(static UNUSED, gctx_tree_, prof_gctx_tree_t, prof_gctx_t, dump_link,

prof_gctx_comp)

/* NOLINTEND(performance-no-int-to-ptr) */

static int prof_tdata_comp(const prof_tdata_t *a, const prof_tdata_t *b) {

int ret;

uint64_t a_uid = a->thr_uid;

uint64_t b_uid = b->thr_uid;

ret = ((a_uid > b_uid) - (a_uid < b_uid));

if (ret == 0) {

uint64_t a_discrim = a->thr_discrim;

uint64_t b_discrim = b->thr_discrim;

ret = ((a_discrim > b_discrim) - (a_discrim < b_discrim));

}

return ret;

}

/* NOLINTBEGIN(performance-no-int-to-ptr) */

rb_gen(static UNUSED, tdata_tree_, prof_tdata_tree_t, prof_tdata_t, tdata_link,

prof_tdata_comp)

/* NOLINTEND(performance-no-int-to-ptr) */

/******************************************************************************/

static malloc_mutex_t *prof_gctx_mutex_choose(void) {

unsigned ngctxs = atomic_fetch_add_u(&cum_gctxs, 1, ATOMIC_RELAXED);

return &gctx_locks[(ngctxs - 1) % PROF_NCTX_LOCKS];

}

static malloc_mutex_t *

prof_tdata_mutex_choose(uint64_t thr_uid) {

return &tdata_locks[thr_uid % PROF_NTDATA_LOCKS];

}

bool

prof_data_init(tsd_t *tsd) {

tdata_tree_new(&tdatas);

return ckh_new(

tsd, &bt2gctx, PROF_CKH_MINITEMS, prof_bt_hash, prof_bt_keycomp);

}

static void

prof_enter(tsd_t *tsd, prof_tdata_t *tdata) {

cassert(config_prof);

assert(tdata == prof_tdata_get(tsd, false));

if (tdata != NULL) {

assert(!tdata->enq);

tdata->enq = true;

}

malloc_mutex_lock(tsd_tsdn(tsd), &bt2gctx_mtx);

}

static void

prof_leave(tsd_t *tsd, prof_tdata_t *tdata) {

cassert(config_prof);

assert(tdata == prof_tdata_get(tsd, false));

malloc_mutex_unlock(tsd_tsdn(tsd), &bt2gctx_mtx);

if (tdata != NULL) {

bool idump, gdump;

assert(tdata->enq);

tdata->enq = false;

idump = tdata->enq_idump;

tdata->enq_idump = false;

gdump = tdata->enq_gdump;

tdata->enq_gdump = false;

if (idump) {

prof_idump(tsd_tsdn(tsd));

}

if (gdump) {

prof_gdump(tsd_tsdn(tsd));

}

}

}

static prof_gctx_t *

prof_gctx_create(tsdn_t *tsdn, prof_bt_t *bt) {

/*

* Create a single allocation that has space for vec of length bt->len.

*/

size_t size = offsetof(prof_gctx_t, vec) + (bt->len * sizeof(void *));

prof_gctx_t *gctx = (prof_gctx_t *)iallocztm(tsdn, size,

sz_size2index(size), false, NULL, true,

arena_get(TSDN_NULL, 0, true), true);

if (gctx == NULL) {

return NULL;

}

gctx->lock = prof_gctx_mutex_choose();

/*

* Set nlimbo to 1, in order to avoid a race condition with

* prof_tctx_destroy()/prof_gctx_try_destroy().

*/

gctx->nlimbo = 1;

tctx_tree_new(&gctx->tctxs);

/* Duplicate bt. */

memcpy(gctx->vec, bt->vec, bt->len * sizeof(void *));

gctx->bt.vec = gctx->vec;

gctx->bt.len = bt->len;

return gctx;

}

static void

prof_gctx_try_destroy(tsd_t *tsd, prof_tdata_t *tdata_self, prof_gctx_t *gctx) {

cassert(config_prof);

/*

* Check that gctx is still unused by any thread cache before destroying

* it. prof_lookup() increments gctx->nlimbo in order to avoid a race

* condition with this function, as does prof_tctx_destroy() in order to

* avoid a race between the main body of prof_tctx_destroy() and entry

* into this function.

*/

prof_enter(tsd, tdata_self);

malloc_mutex_lock(tsd_tsdn(tsd), gctx->lock);

assert(gctx->nlimbo != 0);

if (tctx_tree_empty(&gctx->tctxs) && gctx->nlimbo == 1) {

/* Remove gctx from bt2gctx. */

if (ckh_remove(tsd, &bt2gctx, &gctx->bt, NULL, NULL)) {

not_reached();

}

prof_leave(tsd, tdata_self);

/* Destroy gctx. */

malloc_mutex_unlock(tsd_tsdn(tsd), gctx->lock);

idalloctm(tsd_tsdn(tsd), gctx, NULL, NULL, true, true);

} else {

/*

* Compensate for increment in prof_tctx_destroy() or

* prof_lookup().

*/

gctx->nlimbo--;

malloc_mutex_unlock(tsd_tsdn(tsd), gctx->lock);

prof_leave(tsd, tdata_self);

}

}

static bool

prof_gctx_should_destroy(prof_gctx_t *gctx) {

if (opt_prof_accum) {

return false;

}

if (!tctx_tree_empty(&gctx->tctxs)) {

return false;

}

if (gctx->nlimbo != 0) {

return false;

}

return true;

}

static bool

prof_lookup_global(tsd_t *tsd, prof_bt_t *bt, prof_tdata_t *tdata,

void **p_btkey, prof_gctx_t **p_gctx, bool *p_new_gctx) {

union {

prof_gctx_t *p;

void *v;

} gctx, tgctx;

union {

prof_bt_t *p;

void *v;

} btkey;

bool new_gctx;

prof_enter(tsd, tdata);

if (ckh_search(&bt2gctx, bt, &btkey.v, &gctx.v)) {

/* bt has never been seen before. Insert it. */

prof_leave(tsd, tdata);

tgctx.p = prof_gctx_create(tsd_tsdn(tsd), bt);

if (tgctx.v == NULL) {

return true;

}

prof_enter(tsd, tdata);

if (ckh_search(&bt2gctx, bt, &btkey.v, &gctx.v)) {

gctx.p = tgctx.p;

btkey.p = &gctx.p->bt;

if (ckh_insert(tsd, &bt2gctx, btkey.v, gctx.v)) {

/* OOM. */

prof_leave(tsd, tdata);

idalloctm(tsd_tsdn(tsd), gctx.v, NULL, NULL,

true, true);

return true;

}

new_gctx = true;

} else {

new_gctx = false;

}

} else {

tgctx.v = NULL;

new_gctx = false;

}

if (!new_gctx) {

/*

* Increment nlimbo, in order to avoid a race condition with

* prof_tctx_destroy()/prof_gctx_try_destroy().

*/

malloc_mutex_lock(tsd_tsdn(tsd), gctx.p->lock);

gctx.p->nlimbo++;

malloc_mutex_unlock(tsd_tsdn(tsd), gctx.p->lock);

new_gctx = false;

if (tgctx.v != NULL) {

/* Lost race to insert. */

idalloctm(

tsd_tsdn(tsd), tgctx.v, NULL, NULL, true, true);

}

}

prof_leave(tsd, tdata);

*p_btkey = btkey.v;

*p_gctx = gctx.p;

*p_new_gctx = new_gctx;

return false;

}

prof_tctx_t *

prof_lookup(tsd_t *tsd, prof_bt_t *bt) {

union {

prof_tctx_t *p;

void *v;

} ret;

prof_tdata_t *tdata;

bool not_found;

cassert(config_prof);

tdata = prof_tdata_get(tsd, false);

assert(tdata != NULL);

malloc_mutex_lock(tsd_tsdn(tsd), tdata->lock);

not_found = ckh_search(&tdata->bt2tctx, bt, NULL, &ret.v);

if (!not_found) { /* Note double negative! */

ret.p->prepared = true;

}

malloc_mutex_unlock(tsd_tsdn(tsd), tdata->lock);

if (not_found) {

void *btkey;

prof_gctx_t *gctx;

bool new_gctx, error;

/*

* This thread's cache lacks bt. Look for it in the global

* cache.

*/

if (prof_lookup_global(

tsd, bt, tdata, &btkey, &gctx, &new_gctx)) {

return NULL;

}

/* Link a prof_tctx_t into gctx for this thread. */

ret.v = iallocztm(tsd_tsdn(tsd), sizeof(prof_tctx_t),

sz_size2index(sizeof(prof_tctx_t)), false, NULL, true,

arena_ichoose(tsd, NULL), true);

if (ret.p == NULL) {

if (new_gctx) {

prof_gctx_try_destroy(tsd, tdata, gctx);

}

return NULL;

}

ret.p->tdata = tdata;

ret.p->thr_uid = tdata->thr_uid;

ret.p->thr_discrim = tdata->thr_discrim;

ret.p->recent_count = 0;

memset(&ret.p->cnts, 0, sizeof(prof_cnt_t));

ret.p->gctx = gctx;

ret.p->tctx_uid = tdata->tctx_uid_next++;

ret.p->prepared = true;

ret.p->state = prof_tctx_state_initializing;

malloc_mutex_lock(tsd_tsdn(tsd), tdata->lock);

error = ckh_insert(tsd, &tdata->bt2tctx, btkey, ret.v);

malloc_mutex_unlock(tsd_tsdn(tsd), tdata->lock);

if (error) {

if (new_gctx) {

prof_gctx_try_destroy(tsd, tdata, gctx);

}

idalloctm(tsd_tsdn(tsd), ret.v, NULL, NULL, true, true);

return NULL;

}

malloc_mutex_lock(tsd_tsdn(tsd), gctx->lock);

ret.p->state = prof_tctx_state_nominal;

tctx_tree_insert(&gctx->tctxs, ret.p);

gctx->nlimbo--;

malloc_mutex_unlock(tsd_tsdn(tsd), gctx->lock);

}

return ret.p;

}

/* Used in unit tests. */

static prof_tdata_t *

prof_tdata_count_iter(

prof_tdata_tree_t *tdatas_ptr, prof_tdata_t *tdata, void *arg) {

size_t *tdata_count = (size_t *)arg;

(*tdata_count)++;

return NULL;

}

/* Used in unit tests. */

size_t

prof_tdata_count(void) {

size_t tdata_count = 0;

tsdn_t *tsdn;

tsdn = tsdn_fetch();

malloc_mutex_lock(tsdn, &tdatas_mtx);

tdata_tree_iter(

&tdatas, NULL, prof_tdata_count_iter, (void *)&tdata_count);

malloc_mutex_unlock(tsdn, &tdatas_mtx);

return tdata_count;

}

/* Used in unit tests. */

size_t

prof_bt_count(void) {

size_t bt_count;

tsd_t *tsd;

prof_tdata_t *tdata;

tsd = tsd_fetch();

tdata = prof_tdata_get(tsd, false);

if (tdata == NULL) {

return 0;

}

malloc_mutex_lock(tsd_tsdn(tsd), &bt2gctx_mtx);

bt_count = ckh_count(&bt2gctx);

malloc_mutex_unlock(tsd_tsdn(tsd), &bt2gctx_mtx);

return bt_count;

}

static void

prof_thread_name_write_tdata(prof_tdata_t *tdata, const char *thread_name) {

strncpy(tdata->thread_name, thread_name, PROF_THREAD_NAME_MAX_LEN);

tdata->thread_name[PROF_THREAD_NAME_MAX_LEN - 1] = '\0';

}

int

prof_thread_name_set_impl(tsd_t *tsd, const char *thread_name) {

assert(tsd_reentrancy_level_get(tsd) == 0);

assert(thread_name != NULL);

for (unsigned i = 0; thread_name[i] != '\0'; i++) {

char c = thread_name[i];

if (!isgraph(c) && !isblank(c)) {

return EINVAL;

}

}

prof_tdata_t *tdata = prof_tdata_get(tsd, true);

if (tdata == NULL) {

return ENOMEM;

}

prof_thread_name_write_tdata(tdata, thread_name);

return 0;

}

JEMALLOC_FORMAT_PRINTF(3, 4)

static void

prof_dump_printf(

write_cb_t *prof_dump_write, void *cbopaque, const char *format, ...) {

va_list ap;

char buf[PROF_PRINTF_BUFSIZE];

va_start(ap, format);

malloc_vsnprintf(buf, sizeof(buf), format, ap);

va_end(ap);

prof_dump_write(cbopaque, buf);

}

/*

* Casting a double to a uint64_t may not necessarily be in range; this can be

* UB. I don't think this is practically possible with the cur counters, but

* plausibly could be with the accum counters.

*/

#ifdef JEMALLOC_PROF

static uint64_t

prof_double_uint64_cast(double d) {

/*

* Note: UINT64_MAX + 1 is exactly representable as a double on all

* reasonable platforms (certainly those we'll support). Writing this

* as !(a < b) instead of (a >= b) means that we're NaN-safe.

*/

double rounded = round(d);

if (!(rounded < (double)UINT64_MAX)) {

return UINT64_MAX;

}

return (uint64_t)rounded;

}

#endif

void

prof_unbias_map_init(void) {

/* See the comment in prof_sample_new_event_wait */

#ifdef JEMALLOC_PROF

for (szind_t i = 0; i < SC_NSIZES; i++) {

/*

* With large size classes disabled, the unbiased calculation

* here is not as accurate as it was because usize now changes

* in a finer grain while the unbiased_sz is still calculated

* using the old way.

*/

double sz = (double)sz_index2size_unsafe(i);

double rate = (double)(ZU(1) << lg_prof_sample);

double div_val = 1.0 - exp(-sz / rate);

double unbiased_sz = sz / div_val;

/*

* The "true" right value for the unbiased count is

* 1.0/(1 - exp(-sz/rate)). The problem is, we keep the counts

* as integers (for a variety of reasons -- rounding errors

* could trigger asserts, and not all libcs can properly handle

* floating point arithmetic during malloc calls inside libc).

* Rounding to an integer, though, can lead to rounding errors

* of over 30% for sizes close to the sampling rate. So

* instead, we multiply by a constant, dividing the maximum

* possible roundoff error by that constant. To avoid overflow

* in summing up size_t values, the largest safe constant we can

* pick is the size of the smallest allocation.

*/

double cnt_shift = (double)(ZU(1) << SC_LG_TINY_MIN);

double shifted_unbiased_cnt = cnt_shift / div_val;

prof_unbiased_sz[i] = (size_t)round(unbiased_sz);

prof_shifted_unbiased_cnt[i] = (size_t)round(

shifted_unbiased_cnt);

}

#else

unreachable();

#endif

}

/*

* The unbiasing story is long. The jeprof unbiasing logic was copied from

* pprof. Both shared an issue: they unbiased using the average size of the

* allocations at a particular stack trace. This can work out OK if allocations

* are mostly of the same size given some stack, but not otherwise. We now

* internally track what the unbiased results ought to be. We can't just report

* them as they are though; they'll still go through the jeprof unbiasing

* process. Instead, we figure out what values we can feed *into* jeprof's

* unbiasing mechanism that will lead to getting the right values out.

*

* It'll unbias count and aggregate size as:

*

* c_out = c_in * 1/(1-exp(-s_in/c_in/R)

* s_out = s_in * 1/(1-exp(-s_in/c_in/R)

*

* We want to solve for the values of c_in and s_in that will

* give the c_out and s_out that we've computed internally.

*

* Let's do a change of variables (both to make the math easier and to make it

* easier to write):

* x = s_in / c_in

* y = s_in

* k = 1/R.

*

* Then

* c_out = y/x * 1/(1-exp(-k*x))

* s_out = y * 1/(1-exp(-k*x))

*

* The first equation gives:

* y = x * c_out * (1-exp(-k*x))

* The second gives:

* y = s_out * (1-exp(-k*x))

* So we have

* x = s_out / c_out.

* And all the other values fall out from that.

*

* This is all a fair bit of work. The thing we get out of it is that we don't

* break backwards compatibility with jeprof (and the various tools that have

* copied its unbiasing logic). Eventually, we anticipate a v3 heap profile

* dump format based on JSON, at which point I think much of this logic can get

* cleaned up (since we'll be taking a compatibility break there anyways).

*/

static void

prof_do_unbias(uint64_t c_out_shifted_i, uint64_t s_out_i, uint64_t *r_c_in,

uint64_t *r_s_in) {

#ifdef JEMALLOC_PROF

if (c_out_shifted_i == 0 || s_out_i == 0) {

*r_c_in = 0;

*r_s_in = 0;

return;

}

/*

* See the note in prof_unbias_map_init() to see why we take c_out in a

* shifted form.

*/

double c_out = (double)c_out_shifted_i

/ (double)(ZU(1) << SC_LG_TINY_MIN);

double s_out = (double)s_out_i;

double R = (double)(ZU(1) << lg_prof_sample);

double x = s_out / c_out;

double y = s_out * (1.0 - exp(-x / R));

double c_in = y / x;

double s_in = y;

*r_c_in = prof_double_uint64_cast(c_in);

*r_s_in = prof_double_uint64_cast(s_in);

#else

unreachable();

#endif

}

static void

prof_dump_print_cnts(

write_cb_t *prof_dump_write, void *cbopaque, const prof_cnt_t *cnts) {

uint64_t curobjs;

uint64_t curbytes;

uint64_t accumobjs;

uint64_t accumbytes;

if (opt_prof_unbias) {

prof_do_unbias(cnts->curobjs_shifted_unbiased,

cnts->curbytes_unbiased, &curobjs, &curbytes);

prof_do_unbias(cnts->accumobjs_shifted_unbiased,

cnts->accumbytes_unbiased, &accumobjs, &accumbytes);

} else {

curobjs = cnts->curobjs;

curbytes = cnts->curbytes;

accumobjs = cnts->accumobjs;

accumbytes = cnts->accumbytes;

}

prof_dump_printf(prof_dump_write, cbopaque,

"%" FMTu64 ": %" FMTu64 " [%" FMTu64 ": %" FMTu64 "]", curobjs,

curbytes, accumobjs, accumbytes);

}

static void

prof_tctx_merge_tdata(tsdn_t *tsdn, prof_tctx_t *tctx, prof_tdata_t *tdata) {

malloc_mutex_assert_owner(tsdn, tctx->tdata->lock);

malloc_mutex_lock(tsdn, tctx->gctx->lock);

switch (tctx->state) {

case prof_tctx_state_initializing:

malloc_mutex_unlock(tsdn, tctx->gctx->lock);

return;

case prof_tctx_state_nominal:

tctx->state = prof_tctx_state_dumping;

malloc_mutex_unlock(tsdn, tctx->gctx->lock);

memcpy(&tctx->dump_cnts, &tctx->cnts, sizeof(prof_cnt_t));

tdata->cnt_summed.curobjs += tctx->dump_cnts.curobjs;

tdata->cnt_summed.curobjs_shifted_unbiased +=

tctx->dump_cnts.curobjs_shifted_unbiased;

tdata->cnt_summed.curbytes += tctx->dump_cnts.curbytes;

tdata->cnt_summed.curbytes_unbiased +=

tctx->dump_cnts.curbytes_unbiased;

if (opt_prof_accum) {

tdata->cnt_summed.accumobjs +=

tctx->dump_cnts.accumobjs;

tdata->cnt_summed.accumobjs_shifted_unbiased +=

tctx->dump_cnts.accumobjs_shifted_unbiased;

tdata->cnt_summed.accumbytes +=

tctx->dump_cnts.accumbytes;

tdata->cnt_summed.accumbytes_unbiased +=

tctx->dump_cnts.accumbytes_unbiased;

}

break;

case prof_tctx_state_dumping:

case prof_tctx_state_purgatory:

not_reached();

}

}

static void

prof_tctx_merge_gctx(tsdn_t *tsdn, prof_tctx_t *tctx, prof_gctx_t *gctx) {

malloc_mutex_assert_owner(tsdn, gctx->lock);

gctx->cnt_summed.curobjs += tctx->dump_cnts.curobjs;

gctx->cnt_summed.curobjs_shifted_unbiased +=

tctx->dump_cnts.curobjs_shifted_unbiased;

gctx->cnt_summed.curbytes += tctx->dump_cnts.curbytes;

gctx->cnt_summed.curbytes_unbiased += tctx->dump_cnts.curbytes_unbiased;

if (opt_prof_accum) {

gctx->cnt_summed.accumobjs += tctx->dump_cnts.accumobjs;

gctx->cnt_summed.accumobjs_shifted_unbiased +=

tctx->dump_cnts.accumobjs_shifted_unbiased;

gctx->cnt_summed.accumbytes += tctx->dump_cnts.accumbytes;

gctx->cnt_summed.accumbytes_unbiased +=

tctx->dump_cnts.accumbytes_unbiased;

}

}

static prof_tctx_t *

prof_tctx_merge_iter(prof_tctx_tree_t *tctxs, prof_tctx_t *tctx, void *arg) {

tsdn_t *tsdn = (tsdn_t *)arg;

malloc_mutex_assert_owner(tsdn, tctx->gctx->lock);

switch (tctx->state) {

case prof_tctx_state_nominal:

/* New since dumping started; ignore. */

break;

case prof_tctx_state_dumping:

case prof_tctx_state_purgatory:

prof_tctx_merge_gctx(tsdn, tctx, tctx->gctx);

break;

case prof_tctx_state_initializing:

default:

not_reached();

}

return NULL;

}

typedef struct prof_dump_iter_arg_s prof_dump_iter_arg_t;

struct prof_dump_iter_arg_s {

tsdn_t *tsdn;

write_cb_t *prof_dump_write;

void *cbopaque;

};

static prof_tctx_t *

prof_tctx_dump_iter(prof_tctx_tree_t *tctxs, prof_tctx_t *tctx, void *opaque) {

prof_dump_iter_arg_t *arg = (prof_dump_iter_arg_t *)opaque;

malloc_mutex_assert_owner(arg->tsdn, tctx->gctx->lock);

switch (tctx->state) {

case prof_tctx_state_initializing:

case prof_tctx_state_nominal:

/* Not captured by this dump. */

break;

case prof_tctx_state_dumping:

case prof_tctx_state_purgatory:

prof_dump_printf(arg->prof_dump_write, arg->cbopaque,

" t%" FMTu64 ": ", tctx->thr_uid);

prof_dump_print_cnts(

arg->prof_dump_write, arg->cbopaque, &tctx->dump_cnts);

arg->prof_dump_write(arg->cbopaque, "\n");

break;

default:

not_reached();

}

return NULL;

}

static prof_tctx_t *

prof_tctx_finish_iter(prof_tctx_tree_t *tctxs, prof_tctx_t *tctx, void *arg) {

tsdn_t *tsdn = (tsdn_t *)arg;

prof_tctx_t *ret;

malloc_mutex_assert_owner(tsdn, tctx->gctx->lock);

switch (tctx->state) {

case prof_tctx_state_nominal:

/* New since dumping started; ignore. */

break;

case prof_tctx_state_dumping:

tctx->state = prof_tctx_state_nominal;

break;

case prof_tctx_state_purgatory:

ret = tctx;

goto label_return;

case prof_tctx_state_initializing:

default:

not_reached();

}

ret = NULL;

label_return:

return ret;

}

static void

prof_dump_gctx_prep(tsdn_t *tsdn, prof_gctx_t *gctx, prof_gctx_tree_t *gctxs) {

cassert(config_prof);

malloc_mutex_lock(tsdn, gctx->lock);

/*

* Increment nlimbo so that gctx won't go away before dump.

* Additionally, link gctx into the dump list so that it is included in

* prof_dump()'s second pass.

*/

gctx->nlimbo++;

gctx_tree_insert(gctxs, gctx);

memset(&gctx->cnt_summed, 0, sizeof(prof_cnt_t));

malloc_mutex_unlock(tsdn, gctx->lock);

}

typedef struct prof_gctx_merge_iter_arg_s prof_gctx_merge_iter_arg_t;

struct prof_gctx_merge_iter_arg_s {

tsdn_t *tsdn;

size_t *leak_ngctx;

};

static prof_gctx_t *

prof_gctx_merge_iter(prof_gctx_tree_t *gctxs, prof_gctx_t *gctx, void *opaque) {

prof_gctx_merge_iter_arg_t *arg = (prof_gctx_merge_iter_arg_t *)opaque;

malloc_mutex_lock(arg->tsdn, gctx->lock);

tctx_tree_iter(

&gctx->tctxs, NULL, prof_tctx_merge_iter, (void *)arg->tsdn);

if (gctx->cnt_summed.curobjs != 0) {

(*arg->leak_ngctx)++;

}

malloc_mutex_unlock(arg->tsdn, gctx->lock);

return NULL;

}

static void

prof_gctx_finish(tsd_t *tsd, prof_gctx_tree_t *gctxs) {

prof_tdata_t *tdata = prof_tdata_get(tsd, false);

prof_gctx_t *gctx;

/*

* Standard tree iteration won't work here, because as soon as we

* decrement gctx->nlimbo and unlock gctx, another thread can

* concurrently destroy it, which will corrupt the tree. Therefore,

* tear down the tree one node at a time during iteration.

*/

while ((gctx = gctx_tree_first(gctxs)) != NULL) {

gctx_tree_remove(gctxs, gctx);

malloc_mutex_lock(tsd_tsdn(tsd), gctx->lock);

{

prof_tctx_t *next;

next = NULL;

do {

prof_tctx_t *to_destroy = tctx_tree_iter(

&gctx->tctxs, next, prof_tctx_finish_iter,

(void *)tsd_tsdn(tsd));

if (to_destroy != NULL) {

next = tctx_tree_next(

&gctx->tctxs, to_destroy);

tctx_tree_remove(

&gctx->tctxs, to_destroy);

idalloctm(tsd_tsdn(tsd), to_destroy,

NULL, NULL, true, true);

} else {

next = NULL;

}

} while (next != NULL);

}

gctx->nlimbo--;

if (prof_gctx_should_destroy(gctx)) {

gctx->nlimbo++;

malloc_mutex_unlock(tsd_tsdn(tsd), gctx->lock);

prof_gctx_try_destroy(tsd, tdata, gctx);

} else {

malloc_mutex_unlock(tsd_tsdn(tsd), gctx->lock);

}

}

}

typedef struct prof_tdata_merge_iter_arg_s prof_tdata_merge_iter_arg_t;

struct prof_tdata_merge_iter_arg_s {

tsdn_t *tsdn;

prof_cnt_t *cnt_all;

};

static prof_tdata_t *

prof_tdata_merge_iter(

prof_tdata_tree_t *tdatas_ptr, prof_tdata_t *tdata, void *opaque) {

prof_tdata_merge_iter_arg_t *arg = (prof_tdata_merge_iter_arg_t *)

opaque;

malloc_mutex_lock(arg->tsdn, tdata->lock);

if (!tdata->expired) {

size_t tabind;

union {

prof_tctx_t *p;

void *v;

} tctx;

tdata->dumping = true;

memset(&tdata->cnt_summed, 0, sizeof(prof_cnt_t));

for (tabind = 0;

!ckh_iter(&tdata->bt2tctx, &tabind, NULL, &tctx.v);) {

prof_tctx_merge_tdata(arg->tsdn, tctx.p, tdata);

}

arg->cnt_all->curobjs += tdata->cnt_summed.curobjs;

arg->cnt_all->curobjs_shifted_unbiased +=

tdata->cnt_summed.curobjs_shifted_unbiased;

arg->cnt_all->curbytes += tdata->cnt_summed.curbytes;

arg->cnt_all->curbytes_unbiased +=

tdata->cnt_summed.curbytes_unbiased;

if (opt_prof_accum) {

arg->cnt_all->accumobjs += tdata->cnt_summed.accumobjs;

arg->cnt_all->accumobjs_shifted_unbiased +=

tdata->cnt_summed.accumobjs_shifted_unbiased;

arg->cnt_all->accumbytes +=

tdata->cnt_summed.accumbytes;

arg->cnt_all->accumbytes_unbiased +=

tdata->cnt_summed.accumbytes_unbiased;

}

} else {

tdata->dumping = false;

}

malloc_mutex_unlock(arg->tsdn, tdata->lock);

return NULL;

}

static prof_tdata_t *

prof_tdata_dump_iter(

prof_tdata_tree_t *tdatas_ptr, prof_tdata_t *tdata, void *opaque) {

if (!tdata->dumping) {

return NULL;

}

prof_dump_iter_arg_t *arg = (prof_dump_iter_arg_t *)opaque;

prof_dump_printf(arg->prof_dump_write, arg->cbopaque,

" t%" FMTu64 ": ", tdata->thr_uid);

prof_dump_print_cnts(

arg->prof_dump_write, arg->cbopaque, &tdata->cnt_summed);

if (!prof_thread_name_empty(tdata)) {

arg->prof_dump_write(arg->cbopaque, " ");

arg->prof_dump_write(arg->cbopaque, tdata->thread_name);

}

arg->prof_dump_write(arg->cbopaque, "\n");

return NULL;

}

static void

prof_dump_header(prof_dump_iter_arg_t *arg, const prof_cnt_t *cnt_all) {

prof_dump_printf(arg->prof_dump_write, arg->cbopaque,

"heap_v2/%" FMTu64 "\n t*: ", ((uint64_t)1U << lg_prof_sample));

prof_dump_print_cnts(arg->prof_dump_write, arg->cbopaque, cnt_all);

arg->prof_dump_write(arg->cbopaque, "\n");

malloc_mutex_lock(arg->tsdn, &tdatas_mtx);

tdata_tree_iter(&tdatas, NULL, prof_tdata_dump_iter, arg);

malloc_mutex_unlock(arg->tsdn, &tdatas_mtx);

}

static void

prof_dump_gctx(prof_dump_iter_arg_t *arg, prof_gctx_t *gctx,

const prof_bt_t *bt, prof_gctx_tree_t *gctxs) {

cassert(config_prof);

malloc_mutex_assert_owner(arg->tsdn, gctx->lock);

/* Avoid dumping such gctx's that have no useful data. */

if ((!opt_prof_accum && gctx->cnt_summed.curobjs == 0)

|| (opt_prof_accum && gctx->cnt_summed.accumobjs == 0)) {

assert(gctx->cnt_summed.curobjs == 0);

assert(gctx->cnt_summed.curbytes == 0);

/*

* These asserts would not be correct -- see the comment on races

* in prof.c

* assert(gctx->cnt_summed.curobjs_unbiased == 0);

* assert(gctx->cnt_summed.curbytes_unbiased == 0);

*/

assert(gctx->cnt_summed.accumobjs == 0);

assert(gctx->cnt_summed.accumobjs_shifted_unbiased == 0);

assert(gctx->cnt_summed.accumbytes == 0);

assert(gctx->cnt_summed.accumbytes_unbiased == 0);

return;

}

arg->prof_dump_write(arg->cbopaque, "@");

for (unsigned i = 0; i < bt->len; i++) {

prof_dump_printf(arg->prof_dump_write, arg->cbopaque,

" %#" FMTxPTR, (uintptr_t)bt->vec[i]);

}

arg->prof_dump_write(arg->cbopaque, "\n t*: ");

prof_dump_print_cnts(

arg->prof_dump_write, arg->cbopaque, &gctx->cnt_summed);

arg->prof_dump_write(arg->cbopaque, "\n");

tctx_tree_iter(&gctx->tctxs, NULL, prof_tctx_dump_iter, arg);

}

/*

* See prof_sample_new_event_wait() comment for why the body of this function

* is conditionally compiled.

*/

static void

prof_leakcheck(const prof_cnt_t *cnt_all, size_t leak_ngctx) {

#ifdef JEMALLOC_PROF

/*

* Scaling is equivalent AdjustSamples() in jeprof, but the result may

* differ slightly from what jeprof reports, because here we scale the

* summary values, whereas jeprof scales each context individually and

* reports the sums of the scaled values.

*/

if (cnt_all->curbytes != 0) {

double sample_period = (double)((uint64_t)1 << lg_prof_sample);