40#include <mach/mach_time.h>
46#include <sys/kdebug.h>
47#include <sys/sysctl.h>
125#define KPC_CLASS_FIXED (0)
126#define KPC_CLASS_CONFIGURABLE (1)
127#define KPC_CLASS_POWER (2)
128#define KPC_CLASS_RAWPMU (3)
131#define KPC_CLASS_FIXED_MASK (1u << KPC_CLASS_FIXED)
132#define KPC_CLASS_CONFIGURABLE_MASK (1u << KPC_CLASS_CONFIGURABLE)
133#define KPC_CLASS_POWER_MASK (1u << KPC_CLASS_POWER)
134#define KPC_CLASS_RAWPMU_MASK (1u << KPC_CLASS_RAWPMU)
137#define KPC_PMU_ERROR (0)
138#define KPC_PMU_INTEL_V3 (1)
139#define KPC_PMU_ARM_APPLE (2)
140#define KPC_PMU_INTEL_V2 (3)
141#define KPC_PMU_ARM_V2 (4)
148#define KPC_MAX_COUNTERS 32
152#define KPERF_SAMPLER_TH_INFO (1U << 0)
153#define KPERF_SAMPLER_TH_SNAPSHOT (1U << 1)
154#define KPERF_SAMPLER_KSTACK (1U << 2)
155#define KPERF_SAMPLER_USTACK (1U << 3)
156#define KPERF_SAMPLER_PMC_THREAD (1U << 4)
157#define KPERF_SAMPLER_PMC_CPU (1U << 5)
158#define KPERF_SAMPLER_PMC_CONFIG (1U << 6)
159#define KPERF_SAMPLER_MEMINFO (1U << 7)
160#define KPERF_SAMPLER_TH_SCHEDULING (1U << 8)
161#define KPERF_SAMPLER_TH_DISPATCH (1U << 9)
162#define KPERF_SAMPLER_TK_SNAPSHOT (1U << 10)
163#define KPERF_SAMPLER_SYS_MEM (1U << 11)
164#define KPERF_SAMPLER_TH_INSCYC (1U << 12)
165#define KPERF_SAMPLER_TK_INFO (1U << 13)
168#define KPERF_ACTION_MAX (32)
171#define KPERF_TIMER_MAX (8)
361 if (!enabled)
return -1;
363 return sysctlbyname(
"kperf.lightweight_pet", enabled, &size, NULL, 0);
368 return sysctlbyname(
"kperf.lightweight_pet", NULL, NULL, &enabled, 4);
378#define KPEP_ARCH_I386 0
379#define KPEP_ARCH_X86_64 1
380#define KPEP_ARCH_ARM 2
381#define KPEP_ARCH_ARM64 3
467 "current system unknown",
468 "database path invalid",
469 "database not found",
470 "database architecture unsupported",
471 "database version unsupported",
474 "conflicting events",
475 "all counters must be forced",
484 return "unknown error";
616#define lib_nelems(x) (sizeof(x) / sizeof((x)[0]))
617#define lib_symbol_def(name) \
618 { #name, (void **)&name }
683#define lib_path_kperf "/System/Library/PrivateFrameworks/kperf.framework/kperf"
684#define lib_path_kperfdata \
685 "/System/Library/PrivateFrameworks/kperfdata.framework/kperfdata"
703 *symbol->
impl = NULL;
707 *symbol->
impl = NULL;
712#define return_err() \
716 lib_has_err = true; \
726 "Failed to load kperf.framework, message: %s.", dlerror());
732 "Failed to load kperfdata.framework, message: %s.", dlerror());
740 if (!*symbol->
impl) {
742 "Failed to load kperf function: %s.", symbol->
name);
749 if (!*symbol->
impl) {
751 "Failed to load kperfdata function: %s.", symbol->
name);
772#if defined(__arm64__)
791#if defined(__LP64__) || defined(__arm64__)
798#define KDBG_CLASSTYPE 0x10000
799#define KDBG_SUBCLSTYPE 0x20000
800#define KDBG_RANGETYPE 0x40000
801#define KDBG_TYPENONE 0x80000
802#define KDBG_CKTYPES 0xF0000
805#define KDBG_VALCHECK 0x00200000U
835 int mib[3] = {CTL_KERN, KERN_KDEBUG, KERN_KDREMOVE};
836 return sysctl(mib, 3, NULL, NULL, NULL, 0);
842 int mib[3] = {CTL_KERN, KERN_KDEBUG, KERN_KDSETUP};
843 return sysctl(mib, 3, NULL, NULL, NULL, 0);
848 int mib[3] = {CTL_KERN, KERN_KDEBUG, KERN_KDSETREG};
850 return sysctl(mib, 3, kdr, &size, NULL, 0);
857 int mib[4] = {CTL_KERN, KERN_KDEBUG, KERN_KDSETBUF, nbufs};
858 return sysctl(mib, 4, NULL, NULL, NULL, 0);
865 int mib[4] = {CTL_KERN, KERN_KDEBUG, KERN_KDENABLE, enable};
866 return sysctl(mib, 4, NULL, 0, NULL, 0);
872 if (!info)
return -1;
873 int mib[3] = {CTL_KERN, KERN_KDEBUG, KERN_KDGETBUF};
875 return sysctl(mib, 3, info, &needed, NULL, 0);
884 if (count) *count = 0;
885 if (!buf || !len)
return -1;
890 int mib[3] = {CTL_KERN, KERN_KDEBUG, KERN_KDREADTR};
891 int ret = sysctl(mib, 3, buf, &len, NULL, 0);
892 if (ret != 0)
return ret;
902 if (timeout_ms == 0)
return -1;
903 int mib[3] = {CTL_KERN, KERN_KDEBUG, KERN_KDBUFWAIT};
904 usize val = timeout_ms;
905 int ret = sysctl(mib, 3, NULL, &val, NULL, 0);
906 if (suc) *suc = !!val;
914#define EVENT_NAME_MAX 8
926 "CPU_CLK_UNHALTED.THREAD",
927 "CPU_CLK_UNHALTED.CORE",
931 "FIXED_INSTRUCTIONS",
937 "BR_INST_RETIRED.ALL_BRANCHES",
942 "BRANCH_MISPRED_NONSPEC",
944 "BR_MISP_RETIRED.ALL_BRANCHES",
945 "BR_INST_RETIRED.MISPRED",
951 const char *name = alias->
names[j];
968 static bool init =
false;
969 static bool worked =
false;
979 return (worked =
false);
985 printf(
"Permission denied, xnu/kpc requires root privileges.\n");
986 return (worked =
false);
992 printf(
"Error: cannot load pmc database: %d.\n", ret);
993 return (worked =
false);
1002 printf(
"Failed to create kpep config: %d (%s).\n", ret,
1004 return (worked =
false);
1007 printf(
"Failed to force counters: %d (%s).\n", ret,
1009 return (worked =
false);
1018 printf(
"Cannot find event: %s.\n", alias->
alias);
1019 return (worked =
false);
1027 printf(
"Failed to add event: %d (%s).\n", ret,
1029 return (worked =
false);
1035 usize reg_count = 0;
1037 printf(
"Failed get kpc classes: %d (%s).\n", ret,
1039 return (worked =
false);
1042 printf(
"Failed get kpc count: %d (%s).\n", ret,
1044 return (worked =
false);
1048 return (worked =
false);
1051 printf(
"Failed get kpc registers: %d (%s).\n", ret,
1053 return (worked =
false);
1058 printf(
"Failed force all ctrs: %d.\n", ret);
1059 return (worked =
false);
1063 printf(
"Failed set kpc config: %d.\n", ret);
1064 return (worked =
false);
1070 printf(
"Failed set counting: %d.\n", ret);
1071 return (worked =
false);
1074 printf(
"Failed set thread counting: %d.\n", ret);
1075 return (worked =
false);
1078 return (worked =
true);
1082 static bool warned =
false;
1087 printf(
"Failed get thread counters before: %d.\n", ret);
static int kdebug_setreg(kd_regtype *kdr)
Set debug filter.
static void * lib_handle_kperf
static int kdebug_get_bufinfo(kbufinfo_t *info)
static int(* kperf_reset)(void)
static int(* kperf_action_filter_set_by_task)(u32 actionid, i32 port)
static int(* kperf_timer_pet_get)(u32 *timerid)
static int kdebug_reinit(void)
#define lib_path_kperfdata
static int kdebug_wait(usize timeout_ms, bool *suc)
static int(* kperf_action_samplers_set)(u32 actionid, u32 sample)
static kpep_event * get_event(kpep_db *db, const event_alias *alias)
static u32(* kpc_get_counting)(void)
static bool lib_init(void)
static int(* kpep_db_create)(const char *name, kpep_db **db_ptr)
static const char * kpep_config_error_names[KPEP_CONFIG_ERROR_MAX]
Error description for kpep_config_error_code.
static int(* kpep_config_kpc_map)(kpep_config *cfg, usize *buf, usize buf_size)
static int(* kpc_set_thread_counting)(u32 classes)
kpep_config_error_code
Error code for kpep_config_xxx() and kpep_db_xxx() functions.
@ KPEP_CONFIG_ERROR_ERRNO
@ KPEP_CONFIG_ERROR_BUFFER_TOO_SMALL
@ KPEP_CONFIG_ERROR_DB_NOT_FOUND
@ KPEP_CONFIG_ERROR_DB_VERSION_UNSUPPORTED
@ KPEP_CONFIG_ERROR_EVENT_NOT_FOUND
@ KPEP_CONFIG_ERROR_INVALID_ARGUMENT
@ KPEP_CONFIG_ERROR_EVENT_UNAVAILABLE
@ KPEP_CONFIG_ERROR_DB_PATH_INVALID
@ KPEP_CONFIG_ERROR_DB_CORRUPT
@ KPEP_CONFIG_ERROR_OUT_OF_MEMORY
@ KPEP_CONFIG_ERROR_CUR_SYSTEM_UNKNOWN
@ KPEP_CONFIG_ERROR_COUNTERS_NOT_FORCED
@ KPEP_CONFIG_ERROR_CONFLICTING_EVENTS
@ KPEP_CONFIG_ERROR_DB_ARCH_UNSUPPORTED
static const lib_symbol lib_symbols_kperfdata[]
static void(* kpep_db_free)(kpep_db *db)
Free the kpep database.
#define lib_symbol_def(name)
static int kdebug_trace_enable(bool enable)
static int(* kperf_action_samplers_get)(u32 actionid, u32 *sample)
static int(* kpep_event_name)(kpep_event *ev, const char **name_ptr)
static int(* kpep_config_add_event)(kpep_config *cfg, kpep_event **ev_ptr, u32 flag, u32 *err)
static int(* kpep_config_events_count)(kpep_config *cfg, usize *count_ptr)
static int(* kperf_sample_set)(u32 enabled)
static int(* kpc_set_counting)(u32 classes)
static const event_alias profile_events[]
Event names from /usr/share/kpep/<name>.plist.
static int kperf_lightweight_pet_get(u32 *enabled)
Get lightweight PET mode (not in kperf.framework).
static u64(* kperf_ticks_to_ns)(u64 ticks)
CPU ticks to nanoseconds.
static int(* kperf_timer_period_set)(u32 actionid, u64 tick)
static int kperf_lightweight_pet_set(u32 enabled)
Set lightweight PET mode (not in kperf.framework).
static int(* kpep_config_kpc_classes)(kpep_config *cfg, u32 *classes_ptr)
static int(* kperf_timer_action_set)(u32 actionid, u32 timerid)
static int(* kpep_db_events)(kpep_db *db, kpep_event **buf, usize buf_size)
static int(* kpep_db_aliases_count)(kpep_db *db, usize *count)
static int(* kpc_cpu_string)(char *buf, usize buf_size)
static u32(* kpc_pmu_version)(void)
static void * lib_handle_kperfdata
static u32(* kpc_get_config_count)(u32 classes)
static int(* kpep_config_kpc)(kpep_config *cfg, kpc_config_t *buf, usize buf_size)
static int(* kpc_get_thread_counters)(u32 tid, u32 buf_count, u64 *buf)
static int(* kperf_timer_pet_set)(u32 timerid)
u64 counters_1[KPC_MAX_COUNTERS]
static int kdebug_trace_read(void *buf, usize len, usize *count)
static void(* kpep_config_free)(kpep_config *cfg)
Free the config.
static int(* kpc_force_all_ctrs_set)(int val)
static int(* kperf_action_filter_set_by_pid)(u32 actionid, i32 pid)
static u64(* kperf_tick_frequency)(void)
CPU ticks frequency (mach_absolute_time).
static int(* kpep_db_aliases)(kpep_db *db, const char **buf, usize buf_size)
static const lib_symbol lib_symbols_kperf[]
static int(* kperf_timer_count_get)(u32 *count)
static int(* kpep_event_description)(kpep_event *ev, const char **str_ptr)
kpc_config_t regs[KPC_MAX_COUNTERS]
static void lib_deinit(void)
static int(* kpep_config_create)(kpep_db *db, kpep_config **cfg_ptr)
static int kdebug_trace_setbuf(int nbufs)
static int kdebug_reset(void)
u64 counters_0[KPC_MAX_COUNTERS]
static int(* kpep_db_event)(kpep_db *db, const char *name, kpep_event **ev_ptr)
static int(* kpep_config_kpc_count)(kpep_config *cfg, usize *count_ptr)
static char lib_err_msg[256]
static int(* kpep_db_events_count)(kpep_db *db, usize *count)
static int(* kpep_db_counters_count)(kpep_db *db, u8 classes, usize *count)
static u32(* kpc_get_thread_counting)(void)
bool setup_performance_counters()
static int(* kpep_event_alias)(kpep_event *ev, const char **alias_ptr)
static int(* kperf_action_count_get)(u32 *count)
static int(* kperf_timer_period_get)(u32 actionid, u64 *tick)
static u64(* kperf_ns_to_ticks)(u64 ns)
Nanoseconds to CPU ticks.
static int(* kperf_sample_get)(u32 *enabled)
static const char * kpep_config_error_desc(int code)
Error description.
static u32(* kpc_get_counter_count)(u32 classes)
static int(* kpep_config_events)(kpep_config *cfg, kpep_event **buf, usize buf_size)
performance_counters get_counters()
static int(* kpc_get_config)(u32 classes, kpc_config_t *config)
static int(* kpep_config_remove_event)(kpep_config *cfg, usize idx)
static int(* kperf_timer_action_get)(u32 actionid, u32 *timerid)
#define KPC_CLASS_CONFIGURABLE_MASK
static int(* kperf_timer_count_set)(u32 count)
static int(* kpep_db_name)(kpep_db *db, const char **name)
static int(* kpc_force_all_ctrs_get)(int *val_out)
static int(* kpep_config_force_counters)(kpep_config *cfg)
usize counter_map[KPC_MAX_COUNTERS]
static int(* kpc_set_config)(u32 classes, kpc_config_t *config)
static int(* kperf_action_count_set)(u32 count)
performance_counters operator-(const performance_counters &a, const performance_counters &b)
static int(* kpc_get_cpu_counters)(bool all_cpus, u32 classes, int *curcpu, u64 *buf)
const char * names[EVENT_NAME_MAX]
name for print
KPEP config (size: 80/44 bytes on 64/32 bit OS).
usize counter_count
kpep_config_events_count()
usize * ev_idx
(sizeof(usize *) * counter_count), init -1
usize * ev_map
(sizeof(usize *) * counter_count), init 0
u32 classes
See class mask constants above.
u32 * flags
(sizeof(u32 *) * counter_count), init 0
kpep_event ** ev_arr
(sizeof(kpep_event *) * counter_count), init NULL
u64 * kpc_periods
(sizeof(u64 *) * counter_count), init 0
KPEP database (size: 144/80 bytes on 64/32 bit OS).
kpep_event ** fixed_event_arr
u32 archtecture
see KPEP CPU archtecture constants above.
void * alias_map
All aliases (CFDict<CFSTR(event_name), kpep_event *>).
void * event_map
All events (CFDict<CFSTR(event_name), kpep_event *>).
usize fixed_counter_count
usize power_counter_count
usize config_counter_count
const char * cpu_id
Plist name, such as "cpu_7_8_10b282dc".
kpep_event * event_arr
Event struct buffer (sizeof(kpep_event) * events_count).
const char * name
Database name, such as "haswell".
const char * marketing_name
Marketing name, such as "Intel Haswell".
usize event_count
All events count.
void * plist_data
Plist data (CFDataRef), currently NULL.
KPEP event (size: 48/28 bytes on 64/32 bit OS).
const char * fallback
Fallback event name for fixed counter.
const char * description
Description for this event.
const char * name
Unique name of a event, such as "INST_RETIRED.ANY".
const char * errata
Errata, currently NULL.
const char * alias
Alias name, such as "Instructions", "Cycles".