1147997Srwatson/*- 2147997Srwatson * Copyright (c) 2005 Robert N. M. Watson 3147997Srwatson * All rights reserved. 4147997Srwatson * 5147997Srwatson * Redistribution and use in source and binary forms, with or without 6147997Srwatson * modification, are permitted provided that the following conditions 7147997Srwatson * are met: 8147997Srwatson * 1. Redistributions of source code must retain the above copyright 9147997Srwatson * notice, this list of conditions and the following disclaimer. 10147997Srwatson * 2. Redistributions in binary form must reproduce the above copyright 11147997Srwatson * notice, this list of conditions and the following disclaimer in the 12147997Srwatson * documentation and/or other materials provided with the distribution. 13147997Srwatson * 14147997Srwatson * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15147997Srwatson * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16147997Srwatson * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17147997Srwatson * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18147997Srwatson * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19147997Srwatson * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20147997Srwatson * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21147997Srwatson * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22147997Srwatson * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23147997Srwatson * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24147997Srwatson * SUCH DAMAGE. 25147997Srwatson * 26147997Srwatson * $FreeBSD$ 27147997Srwatson */ 28147997Srwatson 29169838Srwatson#include <sys/cdefs.h> 30147997Srwatson#include <sys/param.h> 31147997Srwatson#include <sys/malloc.h> 32147997Srwatson#include <sys/sysctl.h> 33147997Srwatson 34147997Srwatson#include <err.h> 35147997Srwatson#include <errno.h> 36148789Srwatson#include <kvm.h> 37148789Srwatson#include <nlist.h> 38147997Srwatson#include <stdio.h> 39147997Srwatson#include <stdlib.h> 40147997Srwatson#include <string.h> 41147997Srwatson 42147997Srwatson#include "memstat.h" 43147997Srwatson#include "memstat_internal.h" 44147997Srwatson 45148789Srwatsonstatic struct nlist namelist[] = { 46148789Srwatson#define X_KMEMSTATISTICS 0 47148789Srwatson { .n_name = "_kmemstatistics" }, 48148789Srwatson#define X_MP_MAXCPUS 1 49148789Srwatson { .n_name = "_mp_maxcpus" }, 50148789Srwatson { .n_name = "" }, 51148789Srwatson}; 52148789Srwatson 53147997Srwatson/* 54147997Srwatson * Extract malloc(9) statistics from the running kernel, and store all memory 55147997Srwatson * type information in the passed list. For each type, check the list for an 56147997Srwatson * existing entry with the right name/allocator -- if present, update that 57147997Srwatson * entry. Otherwise, add a new entry. On error, the entire list will be 58147997Srwatson * cleared, as entries will be in an inconsistent state. 59147997Srwatson * 60147997Srwatson * To reduce the level of work for a list that starts empty, we keep around a 61147997Srwatson * hint as to whether it was empty when we began, so we can avoid searching 62147997Srwatson * the list for entries to update. Updates are O(n^2) due to searching for 63147997Srwatson * each entry before adding it. 64147997Srwatson */ 65147997Srwatsonint 66147997Srwatsonmemstat_sysctl_malloc(struct memory_type_list *list, int flags) 67147997Srwatson{ 68147997Srwatson struct malloc_type_stream_header *mtshp; 69147997Srwatson struct malloc_type_header *mthp; 70147997Srwatson struct malloc_type_stats *mtsp; 71147997Srwatson struct memory_type *mtp; 72148357Srwatson int count, hint_dontsearch, i, j, maxcpus; 73147997Srwatson char *buffer, *p; 74147997Srwatson size_t size; 75147997Srwatson 76148357Srwatson hint_dontsearch = LIST_EMPTY(&list->mtl_list); 77147997Srwatson 78147997Srwatson /* 79147997Srwatson * Query the number of CPUs, number of malloc types so that we can 80147997Srwatson * guess an initial buffer size. We loop until we succeed or really 81147997Srwatson * fail. Note that the value of maxcpus we query using sysctl is not 82147997Srwatson * the version we use when processing the real data -- that is read 83147997Srwatson * from the header. 84147997Srwatson */ 85147997Srwatsonretry: 86147997Srwatson size = sizeof(maxcpus); 87147997Srwatson if (sysctlbyname("kern.smp.maxcpus", &maxcpus, &size, NULL, 0) < 0) { 88148357Srwatson if (errno == EACCES || errno == EPERM) 89148357Srwatson list->mtl_error = MEMSTAT_ERROR_PERMISSION; 90148357Srwatson else 91148357Srwatson list->mtl_error = MEMSTAT_ERROR_DATAERROR; 92147997Srwatson return (-1); 93147997Srwatson } 94147997Srwatson if (size != sizeof(maxcpus)) { 95148357Srwatson list->mtl_error = MEMSTAT_ERROR_DATAERROR; 96147997Srwatson return (-1); 97147997Srwatson } 98147997Srwatson 99147997Srwatson size = sizeof(count); 100147997Srwatson if (sysctlbyname("kern.malloc_count", &count, &size, NULL, 0) < 0) { 101148357Srwatson if (errno == EACCES || errno == EPERM) 102148357Srwatson list->mtl_error = MEMSTAT_ERROR_PERMISSION; 103148357Srwatson else 104148357Srwatson list->mtl_error = MEMSTAT_ERROR_VERSION; 105147997Srwatson return (-1); 106147997Srwatson } 107147997Srwatson if (size != sizeof(count)) { 108148357Srwatson list->mtl_error = MEMSTAT_ERROR_DATAERROR; 109147997Srwatson return (-1); 110147997Srwatson } 111147997Srwatson 112147997Srwatson size = sizeof(*mthp) + count * (sizeof(*mthp) + sizeof(*mtsp) * 113147997Srwatson maxcpus); 114147997Srwatson 115147997Srwatson buffer = malloc(size); 116147997Srwatson if (buffer == NULL) { 117148357Srwatson list->mtl_error = MEMSTAT_ERROR_NOMEMORY; 118147997Srwatson return (-1); 119147997Srwatson } 120147997Srwatson 121147997Srwatson if (sysctlbyname("kern.malloc_stats", buffer, &size, NULL, 0) < 0) { 122147997Srwatson /* 123147997Srwatson * XXXRW: ENOMEM is an ambiguous return, we should bound the 124147997Srwatson * number of loops, perhaps. 125147997Srwatson */ 126147997Srwatson if (errno == ENOMEM) { 127147997Srwatson free(buffer); 128147997Srwatson goto retry; 129147997Srwatson } 130148357Srwatson if (errno == EACCES || errno == EPERM) 131148357Srwatson list->mtl_error = MEMSTAT_ERROR_PERMISSION; 132148357Srwatson else 133148357Srwatson list->mtl_error = MEMSTAT_ERROR_VERSION; 134147997Srwatson free(buffer); 135147997Srwatson return (-1); 136147997Srwatson } 137147997Srwatson 138147997Srwatson if (size == 0) { 139147997Srwatson free(buffer); 140147997Srwatson return (0); 141147997Srwatson } 142147997Srwatson 143147997Srwatson if (size < sizeof(*mtshp)) { 144148357Srwatson list->mtl_error = MEMSTAT_ERROR_VERSION; 145147997Srwatson free(buffer); 146147997Srwatson return (-1); 147147997Srwatson } 148147997Srwatson p = buffer; 149147997Srwatson mtshp = (struct malloc_type_stream_header *)p; 150147997Srwatson p += sizeof(*mtshp); 151147997Srwatson 152147997Srwatson if (mtshp->mtsh_version != MALLOC_TYPE_STREAM_VERSION) { 153148357Srwatson list->mtl_error = MEMSTAT_ERROR_VERSION; 154147997Srwatson free(buffer); 155147997Srwatson return (-1); 156147997Srwatson } 157147997Srwatson 158147997Srwatson /* 159147997Srwatson * For the remainder of this function, we are quite trusting about 160147997Srwatson * the layout of structures and sizes, since we've determined we have 161147997Srwatson * a matching version and acceptable CPU count. 162147997Srwatson */ 163147997Srwatson maxcpus = mtshp->mtsh_maxcpus; 164147997Srwatson count = mtshp->mtsh_count; 165147997Srwatson for (i = 0; i < count; i++) { 166147997Srwatson mthp = (struct malloc_type_header *)p; 167147997Srwatson p += sizeof(*mthp); 168147997Srwatson 169147997Srwatson if (hint_dontsearch == 0) { 170147997Srwatson mtp = memstat_mtl_find(list, ALLOCATOR_MALLOC, 171147997Srwatson mthp->mth_name); 172147997Srwatson } else 173147997Srwatson mtp = NULL; 174147997Srwatson if (mtp == NULL) 175148354Srwatson mtp = _memstat_mt_allocate(list, ALLOCATOR_MALLOC, 176224569Spluknet mthp->mth_name, maxcpus); 177147997Srwatson if (mtp == NULL) { 178148619Srwatson _memstat_mtl_empty(list); 179147997Srwatson free(buffer); 180148357Srwatson list->mtl_error = MEMSTAT_ERROR_NOMEMORY; 181147997Srwatson return (-1); 182147997Srwatson } 183147997Srwatson 184147997Srwatson /* 185147997Srwatson * Reset the statistics on a current node. 186147997Srwatson */ 187224569Spluknet _memstat_mt_reset_stats(mtp, maxcpus); 188147997Srwatson 189147997Srwatson for (j = 0; j < maxcpus; j++) { 190147997Srwatson mtsp = (struct malloc_type_stats *)p; 191147997Srwatson p += sizeof(*mtsp); 192147997Srwatson 193147997Srwatson /* 194147997Srwatson * Sumarize raw statistics across CPUs into coalesced 195147997Srwatson * statistics. 196147997Srwatson */ 197147997Srwatson mtp->mt_memalloced += mtsp->mts_memalloced; 198147997Srwatson mtp->mt_memfreed += mtsp->mts_memfreed; 199147997Srwatson mtp->mt_numallocs += mtsp->mts_numallocs; 200147997Srwatson mtp->mt_numfrees += mtsp->mts_numfrees; 201147997Srwatson mtp->mt_sizemask |= mtsp->mts_size; 202147997Srwatson 203147997Srwatson /* 204147997Srwatson * Copies of per-CPU statistics. 205147997Srwatson */ 206147997Srwatson mtp->mt_percpu_alloc[j].mtp_memalloced = 207147997Srwatson mtsp->mts_memalloced; 208147997Srwatson mtp->mt_percpu_alloc[j].mtp_memfreed = 209147997Srwatson mtsp->mts_memfreed; 210147997Srwatson mtp->mt_percpu_alloc[j].mtp_numallocs = 211147997Srwatson mtsp->mts_numallocs; 212147997Srwatson mtp->mt_percpu_alloc[j].mtp_numfrees = 213147997Srwatson mtsp->mts_numfrees; 214147997Srwatson mtp->mt_percpu_alloc[j].mtp_sizemask = 215147997Srwatson mtsp->mts_size; 216147997Srwatson } 217147997Srwatson 218147997Srwatson /* 219147997Srwatson * Derived cross-CPU statistics. 220147997Srwatson */ 221147997Srwatson mtp->mt_bytes = mtp->mt_memalloced - mtp->mt_memfreed; 222147997Srwatson mtp->mt_count = mtp->mt_numallocs - mtp->mt_numfrees; 223147997Srwatson } 224147997Srwatson 225147997Srwatson free(buffer); 226147997Srwatson 227147997Srwatson return (0); 228147997Srwatson} 229148789Srwatson 230148789Srwatsonstatic int 231148789Srwatsonkread(kvm_t *kvm, void *kvm_pointer, void *address, size_t size, 232148789Srwatson size_t offset) 233148789Srwatson{ 234148789Srwatson ssize_t ret; 235148789Srwatson 236148789Srwatson ret = kvm_read(kvm, (unsigned long)kvm_pointer + offset, address, 237148789Srwatson size); 238148789Srwatson if (ret < 0) 239148789Srwatson return (MEMSTAT_ERROR_KVM); 240148789Srwatson if ((size_t)ret != size) 241148789Srwatson return (MEMSTAT_ERROR_KVM_SHORTREAD); 242148789Srwatson return (0); 243148789Srwatson} 244148789Srwatson 245148789Srwatsonstatic int 246169838Srwatsonkread_string(kvm_t *kvm, const void *kvm_pointer, char *buffer, int buflen) 247148789Srwatson{ 248148789Srwatson ssize_t ret; 249148789Srwatson int i; 250148789Srwatson 251148789Srwatson for (i = 0; i < buflen; i++) { 252169838Srwatson ret = kvm_read(kvm, __DECONST(unsigned long, kvm_pointer) + 253169838Srwatson i, &(buffer[i]), sizeof(char)); 254148789Srwatson if (ret < 0) 255148789Srwatson return (MEMSTAT_ERROR_KVM); 256148789Srwatson if ((size_t)ret != sizeof(char)) 257148789Srwatson return (MEMSTAT_ERROR_KVM_SHORTREAD); 258148789Srwatson if (buffer[i] == '\0') 259148789Srwatson return (0); 260148789Srwatson } 261148789Srwatson /* Truncate. */ 262148789Srwatson buffer[i-1] = '\0'; 263148789Srwatson return (0); 264148789Srwatson} 265148789Srwatson 266148789Srwatsonstatic int 267148789Srwatsonkread_symbol(kvm_t *kvm, int index, void *address, size_t size, 268148789Srwatson size_t offset) 269148789Srwatson{ 270148789Srwatson ssize_t ret; 271148789Srwatson 272148789Srwatson ret = kvm_read(kvm, namelist[index].n_value + offset, address, size); 273148789Srwatson if (ret < 0) 274148789Srwatson return (MEMSTAT_ERROR_KVM); 275148789Srwatson if ((size_t)ret != size) 276148789Srwatson return (MEMSTAT_ERROR_KVM_SHORTREAD); 277148789Srwatson return (0); 278148789Srwatson} 279148789Srwatson 280148789Srwatsonint 281148789Srwatsonmemstat_kvm_malloc(struct memory_type_list *list, void *kvm_handle) 282148789Srwatson{ 283148789Srwatson struct memory_type *mtp; 284148789Srwatson void *kmemstatistics; 285148789Srwatson int hint_dontsearch, j, mp_maxcpus, ret; 286148789Srwatson char name[MEMTYPE_MAXNAME]; 287224569Spluknet struct malloc_type_stats *mts, *mtsp; 288192148Sjhb struct malloc_type_internal *mtip; 289148789Srwatson struct malloc_type type, *typep; 290148789Srwatson kvm_t *kvm; 291148789Srwatson 292148789Srwatson kvm = (kvm_t *)kvm_handle; 293148789Srwatson 294148789Srwatson hint_dontsearch = LIST_EMPTY(&list->mtl_list); 295148789Srwatson 296148789Srwatson if (kvm_nlist(kvm, namelist) != 0) { 297148789Srwatson list->mtl_error = MEMSTAT_ERROR_KVM; 298148789Srwatson return (-1); 299148789Srwatson } 300148789Srwatson 301148789Srwatson if (namelist[X_KMEMSTATISTICS].n_type == 0 || 302148789Srwatson namelist[X_KMEMSTATISTICS].n_value == 0) { 303148789Srwatson list->mtl_error = MEMSTAT_ERROR_KVM_NOSYMBOL; 304148789Srwatson return (-1); 305148789Srwatson } 306148789Srwatson 307148789Srwatson ret = kread_symbol(kvm, X_MP_MAXCPUS, &mp_maxcpus, 308148789Srwatson sizeof(mp_maxcpus), 0); 309148789Srwatson if (ret != 0) { 310148789Srwatson list->mtl_error = ret; 311148789Srwatson return (-1); 312148789Srwatson } 313148789Srwatson 314148789Srwatson ret = kread_symbol(kvm, X_KMEMSTATISTICS, &kmemstatistics, 315148789Srwatson sizeof(kmemstatistics), 0); 316148789Srwatson if (ret != 0) { 317148789Srwatson list->mtl_error = ret; 318148789Srwatson return (-1); 319148789Srwatson } 320148789Srwatson 321224569Spluknet mts = malloc(sizeof(struct malloc_type_stats) * mp_maxcpus); 322224569Spluknet if (mts == NULL) { 323224569Spluknet list->mtl_error = MEMSTAT_ERROR_NOMEMORY; 324224569Spluknet return (-1); 325224569Spluknet } 326224569Spluknet 327148789Srwatson for (typep = kmemstatistics; typep != NULL; typep = type.ks_next) { 328148789Srwatson ret = kread(kvm, typep, &type, sizeof(type), 0); 329148789Srwatson if (ret != 0) { 330148789Srwatson _memstat_mtl_empty(list); 331224569Spluknet free(mts); 332148789Srwatson list->mtl_error = ret; 333148789Srwatson return (-1); 334148789Srwatson } 335148789Srwatson ret = kread_string(kvm, (void *)type.ks_shortdesc, name, 336148789Srwatson MEMTYPE_MAXNAME); 337148789Srwatson if (ret != 0) { 338148789Srwatson _memstat_mtl_empty(list); 339224569Spluknet free(mts); 340148789Srwatson list->mtl_error = ret; 341148789Srwatson return (-1); 342148789Srwatson } 343148789Srwatson 344148789Srwatson /* 345192148Sjhb * Since our compile-time value for MAXCPU may differ from the 346192148Sjhb * kernel's, we populate our own array. 347148789Srwatson */ 348192148Sjhb mtip = type.ks_handle; 349192148Sjhb ret = kread(kvm, mtip->mti_stats, mts, mp_maxcpus * 350148789Srwatson sizeof(struct malloc_type_stats), 0); 351148789Srwatson if (ret != 0) { 352148789Srwatson _memstat_mtl_empty(list); 353224569Spluknet free(mts); 354148789Srwatson list->mtl_error = ret; 355148789Srwatson return (-1); 356148789Srwatson } 357148789Srwatson 358148789Srwatson if (hint_dontsearch == 0) { 359148789Srwatson mtp = memstat_mtl_find(list, ALLOCATOR_MALLOC, name); 360148789Srwatson } else 361148789Srwatson mtp = NULL; 362148789Srwatson if (mtp == NULL) 363148789Srwatson mtp = _memstat_mt_allocate(list, ALLOCATOR_MALLOC, 364224569Spluknet name, mp_maxcpus); 365148789Srwatson if (mtp == NULL) { 366148789Srwatson _memstat_mtl_empty(list); 367224569Spluknet free(mts); 368148789Srwatson list->mtl_error = MEMSTAT_ERROR_NOMEMORY; 369148789Srwatson return (-1); 370148789Srwatson } 371148789Srwatson 372148789Srwatson /* 373148789Srwatson * This logic is replicated from kern_malloc.c, and should 374148789Srwatson * be kept in sync. 375148789Srwatson */ 376224569Spluknet _memstat_mt_reset_stats(mtp, mp_maxcpus); 377148789Srwatson for (j = 0; j < mp_maxcpus; j++) { 378148789Srwatson mtsp = &mts[j]; 379148789Srwatson mtp->mt_memalloced += mtsp->mts_memalloced; 380148789Srwatson mtp->mt_memfreed += mtsp->mts_memfreed; 381148789Srwatson mtp->mt_numallocs += mtsp->mts_numallocs; 382148789Srwatson mtp->mt_numfrees += mtsp->mts_numfrees; 383148789Srwatson mtp->mt_sizemask |= mtsp->mts_size; 384148789Srwatson 385148789Srwatson mtp->mt_percpu_alloc[j].mtp_memalloced = 386148789Srwatson mtsp->mts_memalloced; 387148789Srwatson mtp->mt_percpu_alloc[j].mtp_memfreed = 388148789Srwatson mtsp->mts_memfreed; 389148789Srwatson mtp->mt_percpu_alloc[j].mtp_numallocs = 390148789Srwatson mtsp->mts_numallocs; 391148789Srwatson mtp->mt_percpu_alloc[j].mtp_numfrees = 392148789Srwatson mtsp->mts_numfrees; 393148789Srwatson mtp->mt_percpu_alloc[j].mtp_sizemask = 394148789Srwatson mtsp->mts_size; 395148789Srwatson } 396148789Srwatson 397148789Srwatson mtp->mt_bytes = mtp->mt_memalloced - mtp->mt_memfreed; 398148789Srwatson mtp->mt_count = mtp->mt_numallocs - mtp->mt_numfrees; 399148789Srwatson } 400148789Srwatson 401148789Srwatson return (0); 402148789Srwatson} 403