From fd0b256f7c8c71b2485543b211d6a96bf9d098b8 Mon Sep 17 00:00:00 2001 From: Alexander Boettcher Date: Wed, 9 Aug 2017 16:28:44 +0200 Subject: [PATCH] sel4: support cpu utilization via TRACE service using benchmark infrastructure of the seL4 kernel Issue #2451 --- repos/base-sel4/lib/mk/syscall-sel4.inc | 3 +- repos/base-sel4/patches/autoconf_32.config | 4 +- repos/base-sel4/patches/autoconf_64.config | 4 +- repos/base-sel4/patches/benchmark.patch | 49 +++++++++++++++++ repos/base-sel4/patches/wand_quad.config | 6 +++ repos/base-sel4/ports/sel4.hash | 2 +- .../src/core/include/platform_thread.h | 2 +- .../base-sel4/src/core/include/thread_sel4.h | 33 ++++++++---- repos/base-sel4/src/core/platform.cc | 53 +++++++++++++++++++ repos/base-sel4/src/core/platform_thread.cc | 20 +++++++ repos/base-sel4/src/core/spec/arm/thread.cc | 9 +++- .../base-sel4/src/core/spec/x86_32/thread.cc | 7 ++- .../base-sel4/src/core/spec/x86_64/thread.cc | 7 ++- repos/base-sel4/src/core/thread_start.cc | 8 +++ 14 files changed, 187 insertions(+), 20 deletions(-) create mode 100644 repos/base-sel4/patches/benchmark.patch diff --git a/repos/base-sel4/lib/mk/syscall-sel4.inc b/repos/base-sel4/lib/mk/syscall-sel4.inc index 37a32d1373..c7313d46b1 100644 --- a/repos/base-sel4/lib/mk/syscall-sel4.inc +++ b/repos/base-sel4/lib/mk/syscall-sel4.inc @@ -35,7 +35,8 @@ ARCH_INCLUDES += objecttype.h types.h constants.h functions.h deprecated.h \ INCLUDES := objecttype.h types.h bootinfo.h bootinfo_types.h errors.h \ constants.h messages.h sel4.h macros.h simple_types.h types_gen.h \ syscall.h invocation.h shared_types_gen.h debug_assert.h \ - shared_types.h sel4.h deprecated.h autoconf.h syscalls.h faults.h + shared_types.h sel4.h deprecated.h autoconf.h syscalls.h faults.h \ + benchmark_utilisation_types.h PLAT_API_INCLUDES := constants.h diff --git a/repos/base-sel4/patches/autoconf_32.config b/repos/base-sel4/patches/autoconf_32.config index a94218e598..479f11bf40 100644 --- a/repos/base-sel4/patches/autoconf_32.config +++ b/repos/base-sel4/patches/autoconf_32.config @@ -56,7 +56,7 @@ #define CONFIG_HAVE_LIB_UTILS 1 #define CONFIG_USER_OPTIMISATION_O2 1 #define CONFIG_LIB_CPIO 1 -@@ -92,9 +93,6 @@ +@@ -92,9 +92,8 @@ #define CONFIG_KERNEL_EXTRA_CPPFLAGS "" #define CONFIG_LIBSEL4DEBUG_ALLOC_BUFFER_ENTRIES 128 #define CONFIG_CACHE_LN_SZ 64 @@ -66,3 +66,5 @@ #define CONFIG_BUILDSYS_USE_CCACHE 1 -#define CONFIG_MAX_NUM_NODES 1 -#define CONFIG_KERNEL_STACK_BITS 12 ++#define CONFIG_ENABLE_BENCHMARKS 1 ++#define CONFIG_BENCHMARK_TRACK_UTILISATION 1 diff --git a/repos/base-sel4/patches/autoconf_64.config b/repos/base-sel4/patches/autoconf_64.config index 0bddc79306..75133c48a7 100644 --- a/repos/base-sel4/patches/autoconf_64.config +++ b/repos/base-sel4/patches/autoconf_64.config @@ -50,7 +50,7 @@ #define CONFIG_HAVE_LIB_PLATSUPPORT 1 #define CONFIG_NUM_DOMAINS 1 #define CONFIG_HAVE_LIB_UTILS 1 -@@ -93,8 +93,6 @@ +@@ -93,8 +93,8 @@ #define CONFIG_LIBSEL4DEBUG_ALLOC_BUFFER_ENTRIES 128 #define CONFIG_CACHE_LN_SZ 64 #define CONFIG_ARCH_X86_64 1 @@ -59,3 +59,5 @@ #define CONFIG_BUILDSYS_USE_CCACHE 1 -#define CONFIG_MAX_NUM_NODES 1 #define CONFIG_KERNEL_STACK_BITS 12 ++#define CONFIG_ENABLE_BENCHMARKS 1 ++#define CONFIG_BENCHMARK_TRACK_UTILISATION 1 diff --git a/repos/base-sel4/patches/benchmark.patch b/repos/base-sel4/patches/benchmark.patch new file mode 100644 index 0000000000..ac49286895 --- /dev/null +++ b/repos/base-sel4/patches/benchmark.patch @@ -0,0 +1,49 @@ +--- src/kernel/sel4/src/benchmark/benchmark_utilisation.c ++++ src/kernel/sel4/src/benchmark/benchmark_utilisation.c +@@ -36,6 +36,11 @@ + + tcb = TCB_PTR(cap_thread_cap_get_capTCBPtr(lu_ret.cap)); + buffer[BENCHMARK_TCB_UTILISATION] = tcb->benchmark.utilisation; /* Requested thread utilisation */ ++#if CONFIG_MAX_NUM_NODES > 1 ++ if (tcb->tcbAffinity < ksNumCPUs) ++ buffer[BENCHMARK_IDLE_UTILISATION] = NODE_STATE_ON_CORE(ksIdleThread, tcb->tcbAffinity)->benchmark.utilisation; /* Idle thread utilisation */ ++ else ++#endif + buffer[BENCHMARK_IDLE_UTILISATION] = NODE_STATE(ksIdleThread)->benchmark.utilisation; /* Idle thread utilisation */ + + #ifdef CONFIG_ARM_ENABLE_PMU_OVERFLOW_INTERRUPT +--- src/kernel/sel4/include/arch/arm/arch/benchmark_overflowHandler.h ++++ src/kernel/sel4/include/arch/arm/arch/benchmark_overflowHandler.h +@@ -21,6 +21,7 @@ + + extern bool_t benchmark_log_utilisation_enabled; + ++#ifdef CONFIG_ARM_ENABLE_PMU_OVERFLOW_INTERRUPT + static inline void handleOverflowIRQ(void) + { + if (likely(benchmark_log_utilisation_enabled)) { +@@ -31,6 +32,7 @@ + armv_handleOverflowIRQ(); + } + } ++#endif /* CONFIG_ARM_ENABLE_PMU_OVERFLOW_INTERRUPT */ + #endif /* CONFIG_BENCHMARK_TRACK_UTILISATION */ + #endif /* CONFIG_ENABLE_BENCHMARKS */ + #endif /* ARCH_BENCHMARK_OV_H */ +--- src/kernel/sel4/include/arch/arm/armv/armv7-a/armv/benchmark.h ++++ src/kernel/sel4/include/arch/arm/armv/armv7-a/armv/benchmark.h +@@ -14,10 +14,14 @@ + #ifdef CONFIG_ENABLE_BENCHMARKS + + #ifdef CONFIG_BENCHMARK_TRACK_UTILISATION ++#ifdef CONFIG_ARM_ENABLE_PMU_OVERFLOW_INTERRUPT + extern uint64_t ccnt_num_overflows; ++#endif /* CONFIG_ARM_ENABLE_PMU_OVERFLOW_INTERRUPT */ + static inline void benchmark_arch_utilisation_reset(void) + { ++#ifdef CONFIG_ARM_ENABLE_PMU_OVERFLOW_INTERRUPT + ccnt_num_overflows = 0; ++#endif /* CONFIG_ARM_ENABLE_PMU_OVERFLOW_INTERRUPT */ + } + #endif /* CONFIG_BENCHMARK_TRACK_UTILISATION */ + diff --git a/repos/base-sel4/patches/wand_quad.config b/repos/base-sel4/patches/wand_quad.config index ffede51129..caf6b87068 100644 --- a/repos/base-sel4/patches/wand_quad.config +++ b/repos/base-sel4/patches/wand_quad.config @@ -25,3 +25,9 @@ #define CONFIG_NUM_PRIORITIES 256 #define CONFIG_TESTPRINTER_REGEX ".*" #define CONFIG_APP_SEL4TEST 1 +@@ -93,3 +93,5 @@ + #define CONFIG_BUILDSYS_USE_CCACHE 1 + #define CONFIG_MAX_NUM_NODES 1 + #define CONFIG_KERNEL_STACK_BITS 12 ++#define CONFIG_ENABLE_BENCHMARKS 1 ++#define CONFIG_BENCHMARK_TRACK_UTILISATION 1 diff --git a/repos/base-sel4/ports/sel4.hash b/repos/base-sel4/ports/sel4.hash index afe68c2861..7ede8150a5 100644 --- a/repos/base-sel4/ports/sel4.hash +++ b/repos/base-sel4/ports/sel4.hash @@ -1 +1 @@ -5de2f37b4752f45b087a874f0e3f34dac92f5b5d +112234357bb0cd71d9401f183734fa5b784543e1 diff --git a/repos/base-sel4/src/core/include/platform_thread.h b/repos/base-sel4/src/core/include/platform_thread.h index 94ce828ec9..67ea43ae13 100644 --- a/repos/base-sel4/src/core/include/platform_thread.h +++ b/repos/base-sel4/src/core/include/platform_thread.h @@ -137,7 +137,7 @@ class Genode::Platform_thread : public List::Element /** * Return execution time consumed by the thread */ - unsigned long long execution_time() const { return 0; } + unsigned long long execution_time() const; /************************ diff --git a/repos/base-sel4/src/core/include/thread_sel4.h b/repos/base-sel4/src/core/include/thread_sel4.h index 972fffa33b..3497e11e26 100644 --- a/repos/base-sel4/src/core/include/thread_sel4.h +++ b/repos/base-sel4/src/core/include/thread_sel4.h @@ -39,6 +39,8 @@ namespace Genode { Thread_info() { } + inline void init_tcb(Platform &, Range_allocator &, + unsigned const prio, unsigned const cpu); inline void init(addr_t const utcb_virt_addr, unsigned const prio); inline void destruct(); @@ -49,8 +51,27 @@ namespace Genode { * start the seL4 thread */ void start_sel4_thread(Cap_sel tcb_sel, addr_t ip, addr_t sp, unsigned cpu); + void affinity_sel4_thread(Cap_sel const &tcb_sel, unsigned cpu); }; +void Genode::Thread_info::init_tcb(Platform &platform, + Range_allocator &phys_alloc, + unsigned const prio, unsigned const cpu) +{ + /* allocate TCB within core's CNode */ + addr_t const phys_addr = Untyped_memory::alloc_page(phys_alloc); + seL4_Untyped const service = Untyped_memory::untyped_sel(phys_addr).value(); + + tcb_sel = platform.core_sel_alloc().alloc(); + create(service, platform.core_cnode().sel(), tcb_sel); + + /* set scheduling priority */ + seL4_TCB_SetMCPriority(tcb_sel.value(), prio); + seL4_TCB_SetPriority(tcb_sel.value(), prio); + + /* place at cpu */ + affinity_sel4_thread(tcb_sel, cpu); +} void Genode::Thread_info::init(addr_t const utcb_virt_addr, unsigned const prio) { @@ -62,13 +83,7 @@ void Genode::Thread_info::init(addr_t const utcb_virt_addr, unsigned const prio) Untyped_memory::convert_to_page_frames(ipc_buffer_phys, 1); /* allocate TCB within core's CNode */ - { - addr_t const phys_addr = Untyped_memory::alloc_page(phys_alloc); - seL4_Untyped const service = Untyped_memory::untyped_sel(phys_addr).value(); - - tcb_sel = platform.core_sel_alloc().alloc(); - create(service, platform.core_cnode().sel(), tcb_sel); - } + init_tcb(platform, phys_alloc, prio, 0); /* allocate synchronous endpoint within core's CNode */ { @@ -97,10 +112,6 @@ void Genode::Thread_info::init(addr_t const utcb_virt_addr, unsigned const prio) ipc_buffer_sel.value()); ASSERT(ret == 0); } - - /* set scheduling priority */ - seL4_TCB_SetMCPriority(tcb_sel.value(), prio); - seL4_TCB_SetPriority(tcb_sel.value(), prio); } diff --git a/repos/base-sel4/src/core/platform.cc b/repos/base-sel4/src/core/platform.cc index 800d726c09..eccb183d92 100644 --- a/repos/base-sel4/src/core/platform.cc +++ b/repos/base-sel4/src/core/platform.cc @@ -15,6 +15,7 @@ #include #include #include +#include /* core includes */ #include @@ -22,11 +23,15 @@ #include #include #include +#include /* base-internal includes */ #include #include +/* seL4 includes */ +#include + using namespace Genode; static bool const verbose_boot_info = true; @@ -372,6 +377,9 @@ Platform::Platform() { platform_in_construction = this; + /* start benchmarking for CPU utilization in Genode TRACE service */ + seL4_BenchmarkResetLog(); + /* create notification object for Genode::Lock used by this first thread */ Cap_sel lock_sel (INITIAL_SEL_LOCK); Cap_sel core_sel = _core_sel_alloc.alloc(); @@ -412,6 +420,51 @@ Platform::Platform() _core_vm_space.unsynchronized_alloc_page_tables(virt_addr, virt_size); } + /* add idle thread trace subjects */ + for (unsigned cpu_id = 0; cpu_id < affinity_space().width(); cpu_id ++) { + + struct Idle_trace_source : Trace::Source::Info_accessor, Trace::Control, + Trace::Source, Genode::Thread_info + { + Affinity::Location const affinity; + + /** + * Trace::Source::Info_accessor interface + */ + Info trace_source_info() const override + { + Genode::String<8> name("idle", affinity.xpos()); + + Genode::Thread * me = Genode::Thread::myself(); + addr_t const ipc_buffer = reinterpret_cast(me->utcb()); + seL4_IPCBuffer * ipcbuffer = reinterpret_cast(ipc_buffer); + uint64_t const * buf = reinterpret_cast(ipcbuffer->msg); + + seL4_BenchmarkGetThreadUtilisation(tcb_sel.value()); + uint64_t execution_time = buf[BENCHMARK_IDLE_UTILISATION]; + + return { Session_label("kernel"), Trace::Thread_name(name), + Trace::Execution_time(execution_time), affinity }; + } + + Idle_trace_source(Platform &platform, Range_allocator &phys_alloc, + Affinity::Location affinity) + : + Trace::Source(*this, *this), affinity(affinity) + { + Thread_info::init_tcb(platform, phys_alloc, 0, affinity.xpos()); + } + }; + + Idle_trace_source *source = new (core_mem_alloc()) + Idle_trace_source(*this, *_core_mem_alloc.phys_alloc(), + Affinity::Location(cpu_id, 0, + affinity_space().width(), + affinity_space().height())); + + Trace::sources().insert(source); + } + /* I/O port allocator (only meaningful for x86) */ _io_port_alloc.add_range(0, 0x10000); diff --git a/repos/base-sel4/src/core/platform_thread.cc b/repos/base-sel4/src/core/platform_thread.cc index 5ab4ac0c46..ef4b795412 100644 --- a/repos/base-sel4/src/core/platform_thread.cc +++ b/repos/base-sel4/src/core/platform_thread.cc @@ -23,6 +23,9 @@ #include #include +/* seL4 includes */ +#include + using namespace Genode; @@ -255,4 +258,21 @@ Platform_thread::~Platform_thread() platform_specific()->core_sel_alloc().free(_pager_obj_sel); } +unsigned long long Platform_thread::execution_time() const +{ + Genode::Thread * me = Genode::Thread::myself(); + if (!me || !me->utcb()) { + Genode::error("don't know myself"); + return 0; + } + + seL4_IPCBuffer * ipcbuffer = reinterpret_cast(me->utcb()); + uint64_t const * values = reinterpret_cast(ipcbuffer->msg); + + /* kernel puts execution time on ipc buffer of calling thread */ + seL4_BenchmarkGetThreadUtilisation(_info.tcb_sel.value()); + + uint64_t const execution_time = values[BENCHMARK_TCB_UTILISATION]; + return execution_time; +} diff --git a/repos/base-sel4/src/core/spec/arm/thread.cc b/repos/base-sel4/src/core/spec/arm/thread.cc index e45bd37590..509bde6ed6 100644 --- a/repos/base-sel4/src/core/spec/arm/thread.cc +++ b/repos/base-sel4/src/core/spec/arm/thread.cc @@ -36,12 +36,17 @@ void Genode::start_sel4_thread(Cap_sel tcb_sel, addr_t ip, addr_t sp, num_regs, ®s); ASSERT(ret == 0); - if (cpu != 0) - error("could not set affinity of thread"); + affinity_sel4_thread(tcb_sel, cpu); seL4_TCB_Resume(tcb_sel.value()); } +void Genode::affinity_sel4_thread(Cap_sel const &tcb_sel, unsigned cpu) +{ + if (cpu != 0) + error("could not set affinity of thread"); +} + Genode::Thread_state Genode::Platform_thread::state() { seL4_TCB const thread = _info.tcb_sel.value(); diff --git a/repos/base-sel4/src/core/spec/x86_32/thread.cc b/repos/base-sel4/src/core/spec/x86_32/thread.cc index f0ce8ecbac..191f8a2b7a 100644 --- a/repos/base-sel4/src/core/spec/x86_32/thread.cc +++ b/repos/base-sel4/src/core/spec/x86_32/thread.cc @@ -37,11 +37,16 @@ void Genode::start_sel4_thread(Cap_sel tcb_sel, addr_t ip, addr_t sp, num_regs, ®s); ASSERT(ret == 0); - seL4_TCB_SetAffinity(tcb_sel.value(), cpu); + affinity_sel4_thread(tcb_sel, cpu); seL4_TCB_Resume(tcb_sel.value()); } +void Genode::affinity_sel4_thread(Cap_sel const &tcb_sel, unsigned cpu) +{ + seL4_TCB_SetAffinity(tcb_sel.value(), cpu); +} + Genode::Thread_state Genode::Platform_thread::state() { seL4_TCB const thread = _info.tcb_sel.value(); diff --git a/repos/base-sel4/src/core/spec/x86_64/thread.cc b/repos/base-sel4/src/core/spec/x86_64/thread.cc index 25af07cd82..dce39cc3f7 100644 --- a/repos/base-sel4/src/core/spec/x86_64/thread.cc +++ b/repos/base-sel4/src/core/spec/x86_64/thread.cc @@ -36,11 +36,16 @@ void Genode::start_sel4_thread(Cap_sel tcb_sel, addr_t ip, addr_t sp, num_regs, ®s); ASSERT(ret == 0); - seL4_TCB_SetAffinity(tcb_sel.value(), cpu); + affinity_sel4_thread(tcb_sel, cpu); seL4_TCB_Resume(tcb_sel.value()); } +void Genode::affinity_sel4_thread(Cap_sel const &tcb_sel, unsigned cpu) +{ + seL4_TCB_SetAffinity(tcb_sel.value(), cpu); +} + Genode::Thread_state Genode::Platform_thread::state() { seL4_TCB const thread = _info.tcb_sel.value(); diff --git a/repos/base-sel4/src/core/thread_start.cc b/repos/base-sel4/src/core/thread_start.cc index 15a5cd1d3d..b14e811206 100644 --- a/repos/base-sel4/src/core/thread_start.cc +++ b/repos/base-sel4/src/core/thread_start.cc @@ -109,3 +109,11 @@ void Thread::cancel_blocking() warning(__func__, " not implemented"); } + +Native_utcb *Thread::utcb() +{ + if (!_stack) + return nullptr; + + return &_stack->utcb(); +}