Makefile | 2 +- arch/sparc64/Kconfig | 2 + arch/sparc64/kernel/smp.c | 1 + arch/sparc64/kernel/systbls.S | 20 +- arch/x86/Kconfig | 2 + arch/x86/kernel/Makefile_32 | 3 + arch/x86/kernel/ft_event.c | 104 +++++ arch/x86/kernel/smp_32.c | 1 + arch/x86/kernel/syscall_table_32.S | 16 + fs/exec.c | 3 + fs/inode.c | 2 + include/asm-sparc64/feather_trace.h | 22 + include/asm-sparc64/unistd.h | 6 +- include/asm-x86/feather_trace.h | 104 +++++ include/asm-x86/unistd_32.h | 6 +- include/linux/completion.h | 2 +- include/linux/fs.h | 5 + include/linux/sched.h | 11 + include/linux/tick.h | 3 + include/linux/uaccess.h | 16 + include/litmus/edf_common.h | 25 + include/litmus/fdso.h | 69 +++ include/litmus/feather_buffer.h | 94 ++++ include/litmus/feather_trace.h | 37 ++ include/litmus/heap.h | 301 +++++++++++++ include/litmus/jobs.h | 9 + include/litmus/litmus.h | 205 +++++++++ include/litmus/norqlock.h | 26 ++ include/litmus/rt_domain.h | 127 ++++++ include/litmus/rt_param.h | 170 +++++++ include/litmus/sched_plugin.h | 144 ++++++ include/litmus/sched_trace.h | 31 ++ include/litmus/trace.h | 107 +++++ include/litmus/unistd.h | 20 + kernel/exit.c | 4 + kernel/fork.c | 8 + kernel/printk.c | 10 +- kernel/sched.c | 93 ++++- kernel/sched_fair.c | 2 +- kernel/sched_rt.c | 2 +- kernel/time/tick-sched.c | 37 ++- litmus/Kconfig | 78 ++++ litmus/Makefile | 14 + litmus/edf_common.c | 94 ++++ litmus/fdso.c | 282 ++++++++++++ litmus/fmlp.c | 262 +++++++++++ litmus/ft_event.c | 43 ++ litmus/jobs.c | 43 ++ litmus/litmus.c | 826 +++++++++++++++++++++++++++++++++++ litmus/norqlock.c | 56 +++ litmus/rt_domain.c | 138 ++++++ litmus/sched_cedf.c | 717 ++++++++++++++++++++++++++++++ litmus/sched_gsn_edf.c | 731 +++++++++++++++++++++++++++++++ litmus/sched_litmus.c | 228 ++++++++++ litmus/sched_pfair.c | 785 +++++++++++++++++++++++++++++++++ litmus/sched_plugin.c | 185 ++++++++ litmus/sched_psn_edf.c | 454 +++++++++++++++++++ litmus/sched_trace.c | 569 ++++++++++++++++++++++++ litmus/srp.c | 318 ++++++++++++++ litmus/sync.c | 86 ++++ litmus/trace.c | 335 ++++++++++++++ 61 files changed, 8075 insertions(+), 21 deletions(-) diff --git a/Makefile b/Makefile index 189d8ef..d9e4495 100644 --- a/Makefile +++ b/Makefile @@ -597,7 +597,7 @@ export mod_strip_cmd ifeq ($(KBUILD_EXTMOD),) -core-y += kernel/ mm/ fs/ ipc/ security/ crypto/ block/ +core-y += kernel/ mm/ fs/ ipc/ security/ crypto/ block/ litmus/ vmlinux-dirs := $(patsubst %/,%,$(filter %/, $(init-y) $(init-m) \ $(core-y) $(core-m) $(drivers-y) $(drivers-m) \ diff --git a/arch/sparc64/Kconfig b/arch/sparc64/Kconfig index 10b212a..8d90b5a 100644 --- a/arch/sparc64/Kconfig +++ b/arch/sparc64/Kconfig @@ -471,3 +471,5 @@ source "security/Kconfig" source "crypto/Kconfig" source "lib/Kconfig" + +source "litmus/Kconfig" diff --git a/arch/sparc64/kernel/smp.c b/arch/sparc64/kernel/smp.c index c399449..cd2bc7e 100644 --- a/arch/sparc64/kernel/smp.c +++ b/arch/sparc64/kernel/smp.c @@ -1033,6 +1033,7 @@ void smp_receive_signal(int cpu) void smp_receive_signal_client(int irq, struct pt_regs *regs) { clear_softint(1 << irq); + set_tsk_need_resched(current); } void smp_new_mmu_context_version_client(int irq, struct pt_regs *regs) diff --git a/arch/sparc64/kernel/systbls.S b/arch/sparc64/kernel/systbls.S index 06d1090..7fc7615 100644 --- a/arch/sparc64/kernel/systbls.S +++ b/arch/sparc64/kernel/systbls.S @@ -82,6 +82,13 @@ sys_call_table32: .word compat_sys_set_mempolicy, compat_sys_kexec_load, compat_sys_move_pages, sys_getcpu, compat_sys_epoll_pwait /*310*/ .word compat_sys_utimensat, compat_sys_signalfd, compat_sys_timerfd, sys_eventfd, compat_sys_fallocate +/*LITMUS, 315*/ + .word sys_set_rt_task_param, sys_get_rt_task_param, sys_complete_job, sys_register_np_flag, sys_exit_np +/*320*/ + .word sys_od_open, sys_od_close, sys_fmlp_down, sys_fmlp_up, sys_srp_down +/*325*/ .word sys_srp_up, sys_query_job_no, sys_wait_for_job_release, sys_wait_for_ts_release, sys_release_ts + + #endif /* CONFIG_COMPAT */ /* Now the 64-bit native Linux syscall table. */ @@ -154,6 +161,12 @@ sys_call_table: .word sys_set_mempolicy, sys_kexec_load, sys_move_pages, sys_getcpu, sys_epoll_pwait /*310*/ .word sys_utimensat, sys_signalfd, sys_timerfd, sys_eventfd, sys_fallocate +/*LITMUS, 315*/ + .word sys_set_rt_task_param, sys_get_rt_task_param, sys_complete_job, sys_register_np_flag, sys_exit_np +/*320*/ + .word sys_od_open, sys_od_close, sys_fmlp_down, sys_fmlp_up, sys_srp_down +/*325*/ .word sys_srp_up, sys_query_job_no, sys_wait_for_job_release, sys_wait_for_ts_release, sys_release_ts + #if defined(CONFIG_SUNOS_EMUL) || defined(CONFIG_SOLARIS_EMUL) || \ defined(CONFIG_SOLARIS_EMUL_MODULE) /* Now the 32-bit SunOS syscall table. */ @@ -271,6 +284,11 @@ sunos_sys_table: .word sunos_nosys, sunos_nosys, sunos_nosys .word sunos_nosys /*310*/ .word sunos_nosys, sunos_nosys, sunos_nosys - .word sunos_nosys, sunos_nosys + .word sunos_nosys, sunos_nosys, sunos_nosys + .word sunos_nosys, sunos_nosys, sunos_nosys + .word sunos_nosys +/*320*/ .word sunos_nosys, sunos_nosys, sunos_nosys + .word sunos_nosys, sunos_nosys, sunos_nosys + .word sunos_nosys, sunos_nosys, sunos_nosys #endif diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index 80b7ba4..f99330f 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -1620,3 +1620,5 @@ source "security/Kconfig" source "crypto/Kconfig" source "lib/Kconfig" + +source "litmus/Kconfig" diff --git a/arch/x86/kernel/Makefile_32 b/arch/x86/kernel/Makefile_32 index a7bc93c..5f87f32 100644 --- a/arch/x86/kernel/Makefile_32 +++ b/arch/x86/kernel/Makefile_32 @@ -49,6 +49,9 @@ obj-y += pcspeaker.o obj-$(CONFIG_SCx200) += scx200_32.o +obj-$(CONFIG_FEATHER_TRACE) += ft_event.o + + # vsyscall_32.o contains the vsyscall DSO images as __initdata. # We must build both images before we can assemble it. # Note: kbuild does not track this dependency due to usage of .incbin diff --git a/arch/x86/kernel/ft_event.c b/arch/x86/kernel/ft_event.c new file mode 100644 index 0000000..b1d80c5 --- /dev/null +++ b/arch/x86/kernel/ft_event.c @@ -0,0 +1,104 @@ +#include + +#include + +/* the feather trace management functions assume + * exclusive access to the event table + */ + + +#define BYTE_JUMP 0xeb +#define BYTE_JUMP_LEN 0x02 + +/* for each event, there is an entry in the event table */ +struct trace_event { + long id; + long count; + long start_addr; + long end_addr; +}; + +extern struct trace_event __start___event_table[]; +extern struct trace_event __stop___event_table[]; + +int ft_enable_event(unsigned long id) +{ + struct trace_event* te = __start___event_table; + int count = 0; + char* delta; + unsigned char* instr; + + while (te < __stop___event_table) { + if (te->id == id && ++te->count == 1) { + instr = (unsigned char*) te->start_addr; + /* make sure we don't clobber something wrong */ + if (*instr == BYTE_JUMP) { + delta = (((unsigned char*) te->start_addr) + 1); + *delta = 0; + } + } + if (te->id == id) + count++; + te++; + } + return count; +} + +int ft_disable_event(unsigned long id) +{ + struct trace_event* te = __start___event_table; + int count = 0; + char* delta; + unsigned char* instr; + + while (te < __stop___event_table) { + if (te->id == id && --te->count == 0) { + instr = (unsigned char*) te->start_addr; + if (*instr == BYTE_JUMP) { + delta = (((unsigned char*) te->start_addr) + 1); + *delta = te->end_addr - te->start_addr - + BYTE_JUMP_LEN; + } + } + if (te->id == id) + count++; + te++; + } + return count; +} + +int ft_disable_all_events(void) +{ + struct trace_event* te = __start___event_table; + int count = 0; + char* delta; + unsigned char* instr; + + while (te < __stop___event_table) { + if (te->count) { + instr = (unsigned char*) te->start_addr; + if (*instr == BYTE_JUMP) { + delta = (((unsigned char*) te->start_addr) + + 1); + *delta = te->end_addr - te->start_addr - + BYTE_JUMP_LEN; + te->count = 0; + count++; + } + } + te++; + } + return count; +} + +int ft_is_event_enabled(unsigned long id) +{ + struct trace_event* te = __start___event_table; + + while (te < __stop___event_table) { + if (te->id == id) + return te->count; + te++; + } + return 0; +} diff --git a/arch/x86/kernel/smp_32.c b/arch/x86/kernel/smp_32.c index fcaa026..1063dfc 100644 --- a/arch/x86/kernel/smp_32.c +++ b/arch/x86/kernel/smp_32.c @@ -641,6 +641,7 @@ static void native_smp_send_stop(void) fastcall void smp_reschedule_interrupt(struct pt_regs *regs) { ack_APIC_irq(); + set_tsk_need_resched(current); __get_cpu_var(irq_stat).irq_resched_count++; } diff --git a/arch/x86/kernel/syscall_table_32.S b/arch/x86/kernel/syscall_table_32.S index 8344c70..f6fdb0a 100644 --- a/arch/x86/kernel/syscall_table_32.S +++ b/arch/x86/kernel/syscall_table_32.S @@ -324,3 +324,19 @@ ENTRY(sys_call_table) .long sys_timerfd .long sys_eventfd .long sys_fallocate + /* LITMUS */ + .long sys_set_rt_task_param /* 325 */ + .long sys_get_rt_task_param + .long sys_complete_job + .long sys_register_np_flag + .long sys_exit_np + .long sys_od_open /* 330 */ + .long sys_od_close + .long sys_fmlp_down + .long sys_fmlp_up + .long sys_srp_down + .long sys_srp_up /* 335 */ + .long sys_query_job_no + .long sys_wait_for_job_release + .long sys_wait_for_ts_release + .long sys_release_ts /* 339 */ diff --git a/fs/exec.c b/fs/exec.c index 282240a..6f47786 100644 --- a/fs/exec.c +++ b/fs/exec.c @@ -56,6 +56,8 @@ #include #include +#include + #ifdef CONFIG_KMOD #include #endif @@ -1309,6 +1311,7 @@ int do_execve(char * filename, goto out_kfree; sched_exec(); + litmus_exec(); bprm->file = file; bprm->filename = filename; diff --git a/fs/inode.c b/fs/inode.c index ed35383..ef71ea0 100644 --- a/fs/inode.c +++ b/fs/inode.c @@ -220,6 +220,8 @@ void inode_init_once(struct inode *inode) INIT_LIST_HEAD(&inode->inotify_watches); mutex_init(&inode->inotify_mutex); #endif + INIT_LIST_HEAD(&inode->i_obj_list); + mutex_init(&inode->i_obj_mutex); } EXPORT_SYMBOL(inode_init_once); diff --git a/include/asm-sparc64/feather_trace.h b/include/asm-sparc64/feather_trace.h new file mode 100644 index 0000000..35ec70f --- /dev/null +++ b/include/asm-sparc64/feather_trace.h @@ -0,0 +1,22 @@ +#ifndef _ARCH_FEATHER_TRACE_H +#define _ARCH_FEATHER_TRACE_H + +#include +#include + +static inline int fetch_and_inc(int *val) +{ + return atomic_add_ret(1, (atomic_t*) val) - 1; +} + +static inline int fetch_and_dec(int *val) +{ + return atomic_sub_ret(1, (atomic_t*) val) + 1; +} + +static inline unsigned long long ft_timestamp(void) +{ + return get_cycles(); +} + +#endif diff --git a/include/asm-sparc64/unistd.h b/include/asm-sparc64/unistd.h index cb751b4..ebebde6 100644 --- a/include/asm-sparc64/unistd.h +++ b/include/asm-sparc64/unistd.h @@ -333,7 +333,11 @@ #define __NR_eventfd 313 #define __NR_fallocate 314 -#define NR_SYSCALLS 315 +#define __NR_LITMUS 315 + +#include "litmus/unistd.h" + +#define NR_SYSCALLS 315 + NR_litmus_syscalls #ifdef __KERNEL__ /* sysconf options, for SunOS compatibility */ diff --git a/include/asm-x86/feather_trace.h b/include/asm-x86/feather_trace.h new file mode 100644 index 0000000..253067e --- /dev/null +++ b/include/asm-x86/feather_trace.h @@ -0,0 +1,104 @@ +#ifndef _ARCH_FEATHER_TRACE_H +#define _ARCH_FEATHER_TRACE_H + +static inline int fetch_and_inc(int *val) +{ + int ret = 1; + __asm__ __volatile__("lock; xaddl %0, %1" : "+r" (ret), "+m" (*val) : : "memory" ); + return ret; +} + +static inline int fetch_and_dec(int *val) +{ + int ret = -1; + __asm__ __volatile__("lock; xaddl %0, %1" : "+r" (ret), "+m" (*val) : : "memory" ); + return ret; +} + +#define feather_callback __attribute__((regparm(0))) + +/* make the compiler reload any register that is not saved in + * a cdecl function call + */ +#define CLOBBER_LIST "memory", "cc", "eax", "ecx", "edx" + +#define ft_event(id, callback) \ + __asm__ __volatile__( \ + "1: jmp 2f \n\t" \ + " call " #callback " \n\t" \ + ".section __event_table, \"aw\" \n\t" \ + ".long " #id ", 0, 1b, 2f \n\t" \ + ".previous \n\t" \ + "2: \n\t" \ + : : : CLOBBER_LIST) + +#define ft_event0(id, callback) \ + __asm__ __volatile__( \ + "1: jmp 2f \n\t" \ + " subl $4, %%esp \n\t" \ + " movl $" #id ", (%%esp) \n\t" \ + " call " #callback " \n\t" \ + " addl $4, %%esp \n\t" \ + ".section __event_table, \"aw\" \n\t" \ + ".long " #id ", 0, 1b, 2f \n\t" \ + ".previous \n\t" \ + "2: \n\t" \ + : : : CLOBBER_LIST) + +#define ft_event1(id, callback, param) \ + __asm__ __volatile__( \ + "1: jmp 2f \n\t" \ + " subl $8, %%esp \n\t" \ + " movl %0, 4(%%esp) \n\t" \ + " movl $" #id ", (%%esp) \n\t" \ + " call " #callback " \n\t" \ + " addl $8, %%esp \n\t" \ + ".section __event_table, \"aw\" \n\t" \ + ".long " #id ", 0, 1b, 2f \n\t" \ + ".previous \n\t" \ + "2: \n\t" \ + : : "r" (param) : CLOBBER_LIST) + +#define ft_event2(id, callback, param, param2) \ + __asm__ __volatile__( \ + "1: jmp 2f \n\t" \ + " subl $12, %%esp \n\t" \ + " movl %1, 8(%%esp) \n\t" \ + " movl %0, 4(%%esp) \n\t" \ + " movl $" #id ", (%%esp) \n\t" \ + " call " #callback " \n\t" \ + " addl $12, %%esp \n\t" \ + ".section __event_table, \"aw\" \n\t" \ + ".long " #id ", 0, 1b, 2f \n\t" \ + ".previous \n\t" \ + "2: \n\t" \ + : : "r" (param), "r" (param2) : CLOBBER_LIST) + + +#define ft_event3(id, callback, p, p2, p3) \ + __asm__ __volatile__( \ + "1: jmp 2f \n\t" \ + " subl $16, %%esp \n\t" \ + " movl %1, 12(%%esp) \n\t" \ + " movl %1, 8(%%esp) \n\t" \ + " movl %0, 4(%%esp) \n\t" \ + " movl $" #id ", (%%esp) \n\t" \ + " call " #callback " \n\t" \ + " addl $16, %%esp \n\t" \ + ".section __event_table, \"aw\" \n\t" \ + ".long " #id ", 0, 1b, 2f \n\t" \ + ".previous \n\t" \ + "2: \n\t" \ + : : "r" (p), "r" (p2), "r" (p3) : CLOBBER_LIST) + + +static inline unsigned long long ft_timestamp(void) +{ + unsigned long long ret; + __asm__ __volatile__("rdtsc" : "=A" (ret)); + return ret; +} + +#define __ARCH_HAS_FEATHER_TRACE + +#endif diff --git a/include/asm-x86/unistd_32.h b/include/asm-x86/unistd_32.h index 9b15545..36fec84 100644 --- a/include/asm-x86/unistd_32.h +++ b/include/asm-x86/unistd_32.h @@ -331,9 +331,13 @@ #define __NR_eventfd 323 #define __NR_fallocate 324 +#define __NR_LITMUS 325 + +#include "litmus/unistd.h" + #ifdef __KERNEL__ -#define NR_syscalls 325 +#define NR_syscalls 324 + NR_litmus_syscalls #define __ARCH_WANT_IPC_PARSE_VERSION #define __ARCH_WANT_OLD_READDIR diff --git a/include/linux/completion.h b/include/linux/completion.h index 33d6aaf..5b55e97 100644 --- a/include/linux/completion.h +++ b/include/linux/completion.h @@ -51,7 +51,7 @@ extern unsigned long wait_for_completion_interruptible_timeout( extern void complete(struct completion *); extern void complete_all(struct completion *); - +extern void complete_n(struct completion *, int n); #define INIT_COMPLETION(x) ((x).done = 0) #endif diff --git a/include/linux/fs.h b/include/linux/fs.h index b3ec4a4..22f856c 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -588,6 +588,8 @@ static inline int mapping_writably_mapped(struct address_space *mapping) #define i_size_ordered_init(inode) do { } while (0) #endif +struct inode_obj_id_table; + struct inode { struct hlist_node i_hash; struct list_head i_list; @@ -653,6 +655,9 @@ struct inode { void *i_security; #endif void *i_private; /* fs or device private pointer */ + + struct list_head i_obj_list; + struct mutex i_obj_mutex; }; /* diff --git a/include/linux/sched.h b/include/linux/sched.h index cc14656..76e28f1 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -37,6 +37,7 @@ #define SCHED_BATCH 3 /* SCHED_ISO: reserved but not implemented yet */ #define SCHED_IDLE 5 +#define SCHED_LITMUS 6 #ifdef __KERNEL__ @@ -91,6 +92,8 @@ struct sched_param { #include +#include + struct exec_domain; struct futex_pi_state; struct bio; @@ -914,6 +917,8 @@ struct sched_entity { #endif }; +struct od_table_entry; + struct task_struct { volatile long state; /* -1 unrunnable, 0 runnable, >0 stopped */ void *stack; @@ -1178,6 +1183,12 @@ struct task_struct { int make_it_fail; #endif struct prop_local_single dirties; + + /* litmus parameters and state */ + struct rt_param rt_param; + + /* references to PI semaphores, etc. */ + struct od_table_entry* od_table; }; /* diff --git a/include/linux/tick.h b/include/linux/tick.h index f4a1395..7eae358 100644 --- a/include/linux/tick.h +++ b/include/linux/tick.h @@ -64,6 +64,9 @@ extern int tick_is_oneshot_available(void); extern struct tick_device *tick_get_device(int cpu); # ifdef CONFIG_HIGH_RES_TIMERS +#define LINUX_DEFAULT_TICKS 0 +#define LITMUS_ALIGNED_TICKS 1 +#define LITMUS_STAGGERED_TICKS 2 extern int tick_init_highres(void); extern int tick_program_event(ktime_t expires, int force); extern void tick_setup_sched_timer(void); diff --git a/include/linux/uaccess.h b/include/linux/uaccess.h index 975c963..6ae0ff9 100644 --- a/include/linux/uaccess.h +++ b/include/linux/uaccess.h @@ -84,4 +84,20 @@ static inline unsigned long __copy_from_user_nocache(void *to, ret; \ }) +/* This is a naive attempt at a write version of the above native Linux macro. + */ +#define poke_kernel_address(val, addr) \ + ({ \ + long ret; \ + mm_segment_t old_fs = get_fs(); \ + \ + set_fs(KERNEL_DS); \ + pagefault_disable(); \ + ret = __put_user(val, (__force typeof(val) __user *)(addr)); \ + pagefault_enable(); \ + set_fs(old_fs); \ + ret; \ + }) + + #endif /* __LINUX_UACCESS_H__ */ diff --git a/include/litmus/edf_common.h b/include/litmus/edf_common.h new file mode 100644 index 0000000..4dff77a --- /dev/null +++ b/include/litmus/edf_common.h @@ -0,0 +1,25 @@ +/* EDF common data structures and utility functions shared by all EDF + * based scheduler plugins + */ + +/* CLEANUP: Add comments and make it less messy. + * + */ + +#ifndef __UNC_EDF_COMMON_H__ +#define __UNC_EDF_COMMON_H__ + +#include + + +void edf_domain_init(rt_domain_t* rt, check_resched_needed_t resched, + release_job_t release); + +int edf_higher_prio(struct task_struct* first, + struct task_struct* second); + +int edf_preemption_needed(rt_domain_t* rt, struct task_struct *t); + +int edf_set_hp_task(struct pi_semaphore *sem); +int edf_set_hp_cpu_task(struct pi_semaphore *sem, int cpu); +#endif diff --git a/include/litmus/fdso.h b/include/litmus/fdso.h new file mode 100644 index 0000000..286e10f --- /dev/null +++ b/include/litmus/fdso.h @@ -0,0 +1,69 @@ +/* fdso.h - file descriptor attached shared objects + * + * (c) 2007 B. Brandenburg, LITMUS^RT project + */ + +#ifndef _LINUX_FDSO_H_ +#define _LINUX_FDSO_H_ + +#include +#include + +#include + +#define MAX_OBJECT_DESCRIPTORS 32 + +typedef enum { + MIN_OBJ_TYPE = 0, + + FMLP_SEM = 0, + SRP_SEM = 1, + + MAX_OBJ_TYPE = 1 +} obj_type_t; + +struct inode_obj_id { + struct list_head list; + atomic_t count; + struct inode* inode; + + obj_type_t type; + void* obj; + unsigned int id; +}; + + +struct od_table_entry { + unsigned int used; + + struct inode_obj_id* obj; + void* extra; +}; + +struct fdso_ops { + void* (*create) (void); + void (*destroy)(void*); + int (*open) (struct od_table_entry*, void* __user); + int (*close) (struct od_table_entry*); +}; + +/* translate a userspace supplied od into the raw table entry + * returns NULL if od is invalid + */ +struct od_table_entry* __od_lookup(int od); + +/* translate a userspace supplied od into the associated object + * returns NULL if od is invalid + */ +static inline void* od_lookup(int od, obj_type_t type) +{ + struct od_table_entry* e = __od_lookup(od); + return e && e->obj->type == type ? e->obj->obj : NULL; +} + +#define lookup_fmlp_sem(od)((struct pi_semaphore*) od_lookup(od, FMLP_SEM)) +#define lookup_srp_sem(od) ((struct srp_semaphore*) od_lookup(od, SRP_SEM)) +#define lookup_ics(od) ((struct ics*) od_lookup(od, ICS_ID)) + + +#endif diff --git a/include/litmus/feather_buffer.h b/include/litmus/feather_buffer.h new file mode 100644 index 0000000..6c18277 --- /dev/null +++ b/include/litmus/feather_buffer.h @@ -0,0 +1,94 @@ +#ifndef _FEATHER_BUFFER_H_ +#define _FEATHER_BUFFER_H_ + +/* requires UINT_MAX and memcpy */ + +#define SLOT_FREE 0 +#define SLOT_BUSY 1 +#define SLOT_READY 2 + +struct ft_buffer { + unsigned int slot_count; + unsigned int slot_size; + + int free_count; + unsigned int write_idx; + unsigned int read_idx; + + char* slots; + void* buffer_mem; + unsigned int failed_writes; +}; + +static inline int init_ft_buffer(struct ft_buffer* buf, + unsigned int slot_count, + unsigned int slot_size, + char* slots, + void* buffer_mem) +{ + int i = 0; + if (!slot_count || UINT_MAX % slot_count != slot_count - 1) { + /* The slot count must divide UNIT_MAX + 1 so that when it + * wraps around the index correctly points to 0. + */ + return 0; + } else { + buf->slot_count = slot_count; + buf->slot_size = slot_size; + buf->slots = slots; + buf->buffer_mem = buffer_mem; + buf->free_count = slot_count; + buf->write_idx = 0; + buf->read_idx = 0; + buf->failed_writes = 0; + for (i = 0; i < slot_count; i++) + buf->slots[i] = SLOT_FREE; + return 1; + } +} + +static inline int ft_buffer_start_write(struct ft_buffer* buf, void **ptr) +{ + int free = fetch_and_dec(&buf->free_count); + unsigned int idx; + if (free <= 0) { + fetch_and_inc(&buf->free_count); + *ptr = 0; + fetch_and_inc(&buf->failed_writes); + return 0; + } else { + idx = fetch_and_inc((int*) &buf->write_idx) % buf->slot_count; + buf->slots[idx] = SLOT_BUSY; + *ptr = ((char*) buf->buffer_mem) + idx * buf->slot_size; + return 1; + } +} + +static inline void ft_buffer_finish_write(struct ft_buffer* buf, void *ptr) +{ + unsigned int idx = ((char*) ptr - (char*) buf->buffer_mem) / buf->slot_size; + buf->slots[idx] = SLOT_READY; +} + + +/* exclusive reader access is assumed */ +static inline int ft_buffer_read(struct ft_buffer* buf, void* dest) +{ + unsigned int idx; + if (buf->free_count == buf->slot_count) + /* nothing available */ + return 0; + idx = buf->read_idx % buf->slot_count; + if (buf->slots[idx] == SLOT_READY) { + memcpy(dest, ((char*) buf->buffer_mem) + idx * buf->slot_size, + buf->slot_size); + buf->slots[idx] = SLOT_FREE; + buf->read_idx++; + fetch_and_inc(&buf->free_count); + return 1; + } else + return 0; +} + + +#endif diff --git a/include/litmus/feather_trace.h b/include/litmus/feather_trace.h new file mode 100644 index 0000000..f8fb7ba --- /dev/null +++ b/include/litmus/feather_trace.h @@ -0,0 +1,37 @@ +#ifndef _FEATHER_TRACE_H_ +#define _FEATHER_TRACE_H_ + +#include + +int ft_enable_event(unsigned long id); +int ft_disable_event(unsigned long id); +int ft_is_event_enabled(unsigned long id); +int ft_disable_all_events(void); + +#ifndef __ARCH_HAS_FEATHER_TRACE +/* provide default implementation */ + +#define feather_callback + +#define MAX_EVENTS 1024 + +extern int ft_events[MAX_EVENTS]; + +#define ft_event(id, callback) \ + if (ft_events[id]) callback(); + +#define ft_event0(id, callback) \ + if (ft_events[id]) callback(id); + +#define ft_event1(id, callback, param) \ + if (ft_events[id]) callback(id, param); + +#define ft_event2(id, callback, param, param2) \ + if (ft_events[id]) callback(id, param, param2); + +#define ft_event3(id, callback, p, p2, p3) \ + if (ft_events[id]) callback(id, p, p2, p3); +#endif + + +#endif diff --git a/include/litmus/heap.h b/include/litmus/heap.h new file mode 100644 index 0000000..b26804f --- /dev/null +++ b/include/litmus/heap.h @@ -0,0 +1,301 @@ +/* heaps.h -- Binomial Heaps + * + * (c) 2008 Bjoern Brandenburg + */ + +#ifndef HEAP_H +#define HEAP_H + +#define NOT_IN_HEAP UINT_MAX + +struct heap_node { + struct heap_node* parent; + struct heap_node* next; + struct heap_node* child; + + unsigned int degree; + void* value; + struct heap_node** ref; +}; + +struct heap { + struct heap_node* head; + /* We cache the minimum of the heap. + * This speeds up repeated peek operations. + */ + struct heap_node* min; +}; + +typedef int (*heap_prio_t)(struct heap_node* a, struct heap_node* b); + +static inline void heap_init(struct heap* heap) +{ + heap->head = NULL; + heap->head = NULL; +} + +static inline void heap_node_init(struct heap_node** _h, void* value) +{ + struct heap_node* h = *_h; + h->parent = NULL; + h->next = NULL; + h->child = NULL; + h->degree = NOT_IN_HEAP; + h->value = value; + h->ref = _h; +} + +static inline int heap_node_in_heap(struct heap_node* h) +{ + return h->degree != NOT_IN_HEAP; +} + +static inline int heap_empty(struct heap* heap) +{ + return heap->head == NULL && heap->min == NULL; +} + +/* make child a subtree of root */ +static inline void __heap_link(struct heap_node* root, + struct heap_node* child) +{ + child->parent = root; + child->next = root->child; + root->child = child; + root->degree++; +} + +/* merge root lists */ +static inline struct heap_node* __heap_merge(struct heap_node* a, + struct heap_node* b) +{ + struct heap_node* head = NULL; + struct heap_node** pos = &head; + + while (a && b) { + if (a->degree < b->degree) { + *pos = a; + a = a->next; + } else { + *pos = b; + b = b->next; + } + pos = &(*pos)->next; + } + if (a) + *pos = a; + else + *pos = b; + return head; +} + +/* reverse a linked list of nodes. also clears parent pointer */ +static inline struct heap_node* __heap_reverse(struct heap_node* h) +{ + struct heap_node* tail = NULL; + struct heap_node* next; + + if (!h) + return h; + + h->parent = NULL; + while (h->next) { + next = h->next; + h->next = tail; + tail = h; + h = next; + h->parent = NULL; + } + h->next = tail; + return h; +} + +static inline void __heap_min(heap_prio_t higher_prio, struct heap* heap, + struct heap_node** prev, struct heap_node** node) +{ + struct heap_node *_prev, *cur; + *prev = NULL; + + if (!heap->head) { + *node = NULL; + return; + } + + *node = heap->head; + _prev = heap->head; + cur = heap->head->next; + while (cur) { + if (higher_prio(cur, *node)) { + *node = cur; + *prev = _prev; + } + _prev = cur; + cur = cur->next; + } +} + +static inline void __heap_union(heap_prio_t higher_prio, struct heap* heap, + struct heap_node* h2) +{ + struct heap_node* h1; + struct heap_node *prev, *x, *next; + if (!h2) + return; + h1 = heap->head; + if (!h1) { + heap->head = h2; + return; + } + h1 = __heap_merge(h1, h2); + prev = NULL; + x = h1; + next = x->next; + while (next) { + if (x->degree != next->degree || + (next->next && next->next->degree == x->degree)) { + /* nothing to do, advance */ + prev = x; + x = next; + } else if (higher_prio(x, next)) { + /* x becomes the root of next */ + x->next = next->next; + __heap_link(x, next); + } else { + /* next becomes the root of x */ + if (prev) + prev->next = next; + else + h1 = next; + __heap_link(next, x); + x = next; + } + next = x->next; + } + heap->head = h1; +} + +static inline struct heap_node* __heap_extract_min(heap_prio_t higher_prio, + struct heap* heap) +{ + struct heap_node *prev, *node; + __heap_min(higher_prio, heap, &prev, &node); + if (!node) + return NULL; + if (prev) + prev->next = node->next; + else + heap->head = node->next; + __heap_union(higher_prio, heap, __heap_reverse(node->child)); + return node; +} + +/* insert (and reinitialize) a node into the heap */ +static inline void heap_insert(heap_prio_t higher_prio, struct heap* heap, + struct heap_node* node) +{ + struct heap_node *min; + node->child = NULL; + node->parent = NULL; + node->next = NULL; + node->degree = 0; + if (heap->min && higher_prio(node, heap->min)) { + /* swap min cache */ + min = heap->min; + min->child = NULL; + min->parent = NULL; + min->next = NULL; + min->degree = 0; + __heap_union(higher_prio, heap, min); + heap->min = node; + } else + __heap_union(higher_prio, heap, node); +} + +static inline void __uncache_min(heap_prio_t higher_prio, struct heap* heap) +{ + struct heap_node* min; + if (heap->min) { + min = heap->min; + heap->min = NULL; + heap_insert(higher_prio, heap, min); + } +} + +/* merge addition into target */ +static inline void heap_union(heap_prio_t higher_prio, + struct heap* target, struct heap* addition) +{ + /* first insert any cached minima, if necessary */ + __uncache_min(higher_prio, target); + __uncache_min(higher_prio, addition); + __heap_union(higher_prio, target, addition->head); + /* this is a destructive merge */ + addition->head = NULL; +} + +static inline struct heap_node* heap_peek(heap_prio_t higher_prio, + struct heap* heap) +{ + if (!heap->min) + heap->min = __heap_extract_min(higher_prio, heap); + return heap->min; +} + +static inline struct heap_node* heap_take(heap_prio_t higher_prio, + struct heap* heap) +{ + struct heap_node *node; + if (!heap->min) + heap->min = __heap_extract_min(higher_prio, heap); + node = heap->min; + heap->min = NULL; + if (node) + node->degree = NOT_IN_HEAP; + return node; +} + +static inline void heap_delete(heap_prio_t higher_prio, struct heap* heap, + struct heap_node* node) +{ + struct heap_node *parent, *prev, *pos; + struct heap_node** tmp_ref; + void* tmp; + + if (heap->min != node) { + /* bubble up */ + parent = node->parent; + while (parent) { + /* swap parent and node */ + tmp = parent->value; + parent->value = node->value; + node->value = tmp; + /* swap references */ + *(parent->ref) = node; + *(node->ref) = parent; + tmp_ref = parent->ref; + parent->ref = node->ref; + node->ref = tmp_ref; + /* step up */ + node = parent; + parent = node->parent; + } + /* now delete: + * first find prev */ + prev = NULL; + pos = heap->head; + while (pos != node) { + prev = pos; + pos = pos->next; + } + /* we have prev, now remove node */ + if (prev) + prev->next = node->next; + else + heap->head = node->next; + __heap_union(higher_prio, heap, __heap_reverse(node->child)); + } else + heap->min = NULL; + node->degree = NOT_IN_HEAP; +} + +#endif diff --git a/include/litmus/jobs.h b/include/litmus/jobs.h new file mode 100644 index 0000000..9bd361e --- /dev/null +++ b/include/litmus/jobs.h @@ -0,0 +1,9 @@ +#ifndef __LITMUS_JOBS_H__ +#define __LITMUS_JOBS_H__ + +void prepare_for_next_period(struct task_struct *t); +void release_at(struct task_struct *t, lt_t start); +long complete_job(void); + +#endif + diff --git a/include/litmus/litmus.h b/include/litmus/litmus.h new file mode 100644 index 0000000..de2a3c2 --- /dev/null +++ b/include/litmus/litmus.h @@ -0,0 +1,205 @@ +/* + * Constant definitions related to + * scheduling policy. + */ + +#ifndef _LINUX_LITMUS_H_ +#define _LINUX_LITMUS_H_ + +#include +#include + +/* RT mode start time */ +extern volatile unsigned long rt_start_time; + +extern atomic_t __log_seq_no; + +#define TRACE(fmt, args...) \ + sched_trace_log_message("%d P%d: " fmt, atomic_add_return(1, &__log_seq_no), \ + raw_smp_processor_id(), ## args) + +#define TRACE_TASK(t, fmt, args...) \ + TRACE("(%s/%d) " fmt, (t)->comm, (t)->pid, ##args) + +#define TRACE_CUR(fmt, args...) \ + TRACE_TASK(current, fmt, ## args) + +#define TRACE_BUG_ON(cond) \ + do { if (cond) TRACE("BUG_ON(%s) at %s:%d " \ + "called from %p current=%s/%d state=%d " \ + "flags=%x partition=%d cpu=%d rtflags=%d"\ + " job=%u knp=%d timeslice=%u\n", \ + #cond, __FILE__, __LINE__, __builtin_return_address(0), current->comm, \ + current->pid, current->state, current->flags, \ + get_partition(current), smp_processor_id(), get_rt_flags(current), \ + current->rt_param.job_params.job_no, current->rt_param.kernel_np, \ + current->time_slice\ + ); } while(0); + + +/* in_list - is a given list_head queued on some list? + */ +static inline int in_list(struct list_head* list) +{ + return !( /* case 1: deleted */ + (list->next == LIST_POISON1 && + list->prev == LIST_POISON2) + || + /* case 2: initialized */ + (list->next == list && + list->prev == list) + ); +} + +typedef int (*list_cmp_t)(struct list_head*, struct list_head*); + +static inline unsigned int list_insert(struct list_head* new, + struct list_head* head, + list_cmp_t order_before) +{ + struct list_head *pos; + unsigned int passed = 0; + + BUG_ON(!new); + + /* find a spot where the new entry is less than the next */ + list_for_each(pos, head) { + if (unlikely(order_before(new, pos))) { + /* pos is not less than new, thus insert here */ + __list_add(new, pos->prev, pos); + goto out; + } + passed++; + } + /* if we get to this point either the list is empty or every entry + * queued element is less than new. + * Let's add new to the end. */ + list_add_tail(new, head); + out: + return passed; +} + +void list_qsort(struct list_head* list, list_cmp_t less_than); + + +#define RT_PREEMPTIVE 0x2050 /* = NP */ +#define RT_NON_PREEMPTIVE 0x4e50 /* = P */ +#define RT_EXIT_NP_REQUESTED 0x5251 /* = RQ */ + + +/* kill naughty tasks + */ +void scheduler_signal(struct task_struct *t, unsigned int signal); +void send_scheduler_signals(void); +void np_mem_kill(struct task_struct *t); + +void litmus_fork(struct task_struct *tsk); +void litmus_exec(void); +/* clean up real-time state of a task */ +void exit_litmus(struct task_struct *dead_tsk); + +long litmus_admit_task(struct task_struct *tsk); +void litmus_exit_task(struct task_struct *tsk); + +#define is_realtime(t) ((t)->policy == SCHED_LITMUS) +#define rt_transition_pending(t) \ + ((t)->rt_param.transition_pending) + +#define tsk_rt(t) (&(t)->rt_param) + +/* Realtime utility macros */ +#define get_rt_flags(t) (tsk_rt(t)->flags) +#define set_rt_flags(t,f) (tsk_rt(t)->flags=(f)) +#define get_exec_cost(t) (tsk_rt(t)->task_params.exec_cost) +#define get_exec_time(t) (tsk_rt(t)->job_params.exec_time) +#define get_rt_period(t) (tsk_rt(t)->task_params.period) +#define get_partition(t) (tsk_rt(t)->task_params.cpu) +#define get_deadline(t) (tsk_rt(t)->job_params.deadline) +#define get_class(t) (tsk_rt(t)->task_params.cls) + +inline static int budget_exhausted(struct task_struct* t) +{ + return get_exec_time(t) >= get_exec_cost(t); +} + + +#define is_hrt(t) \ + (tsk_rt(t)->task_params.class == RT_CLASS_HARD) +#define is_srt(t) \ + (tsk_rt(t)->task_params.class == RT_CLASS_SOFT) +#define is_be(t) \ + (tsk_rt(t)->task_params.class == RT_CLASS_BEST_EFFORT) + +#define get_release(t) (tsk_rt(t)->job_params.release) + +/* Our notion of time within LITMUS: kernel monotonic time. */ +static inline lt_t litmus_clock(void) +{ + return ktime_to_ns(ktime_get()); +} + +/* A macro to convert from nanoseconds to ktime_t. */ +#define ns_to_ktime(t) ktime_add_ns(ktime_set(0, 0), t) + +/* The high-resolution release timer for a task. */ +#define release_timer(t) (tsk_rt(t)->release_timer) + +/* The high-resolution release timer for a task. */ +#define get_domain(t) (tsk_rt(t)->domain) + +/* Honor the flag in the preempt_count variable that is set + * when scheduling is in progress. + */ +#define is_running(t) \ + ((t)->state == TASK_RUNNING || \ + task_thread_info(t)->preempt_count & PREEMPT_ACTIVE) + +#define is_blocked(t) \ + (!is_running(t)) +#define is_released(t, now) \ + (lt_before_eq(get_release(t), now)) +#define is_tardy(t, now) \ + (lt_before_eq(tsk_rt(t)->job_params.deadline, now)) + +/* real-time comparison macros */ +#define earlier_deadline(a, b) (lt_before(\ + (a)->rt_param.job_params.deadline,\ + (b)->rt_param.job_params.deadline)) +#define earlier_release(a, b) (lt_before(\ + (a)->rt_param.job_params.release,\ + (b)->rt_param.job_params.release)) + +#define make_np(t) do {t->rt_param.kernel_np++;} while(0); +#define take_np(t) do {t->rt_param.kernel_np--;} while(0); + +#ifdef CONFIG_SRP +void srp_ceiling_block(void); +#else +#define srp_ceiling_block() /* nothing */ +#endif + +#define heap2task(hn) ((struct task_struct*) hn->value) + + +#ifdef CONFIG_NP_SECTION +/* returns 1 if task t has registered np flag and set it to RT_NON_PREEMPTIVE + */ +int is_np(struct task_struct *t); + +/* request that the task should call sys_exit_np() + */ +void request_exit_np(struct task_struct *t); + +#else + +static inline int is_np(struct task_struct *t) +{ + return tsk_rt(t)->kernel_np; +} + +#define request_exit_np(t) + +#endif + + +#endif diff --git a/include/litmus/norqlock.h b/include/litmus/norqlock.h new file mode 100644 index 0000000..e4c1d06 --- /dev/null +++ b/include/litmus/norqlock.h @@ -0,0 +1,26 @@ +#ifndef NORQLOCK_H +#define NORQLOCK_H + +typedef void (*work_t)(unsigned long arg); + +struct no_rqlock_work { + int active; + work_t work; + unsigned long arg; + struct no_rqlock_work* next; +}; + +void init_no_rqlock_work(struct no_rqlock_work* w, work_t work, + unsigned long arg); + +void __do_without_rqlock(struct no_rqlock_work *work); + +static inline void do_without_rqlock(struct no_rqlock_work *work) +{ + if (!test_and_set_bit(0, (void*)&work->active)) + __do_without_rqlock(work); +} + +void tick_no_rqlock(void); + +#endif diff --git a/include/litmus/rt_domain.h b/include/litmus/rt_domain.h new file mode 100644 index 0000000..47cd123 --- /dev/null +++ b/include/litmus/rt_domain.h @@ -0,0 +1,127 @@ +/* CLEANUP: Add comments and make it less messy. + * + */ + +#ifndef __UNC_RT_DOMAIN_H__ +#define __UNC_RT_DOMAIN_H__ + +#include +#include + +struct _rt_domain; + +typedef int (*check_resched_needed_t)(struct _rt_domain *rt); +typedef void (*release_job_t)(struct task_struct *t, struct _rt_domain *rt); + +typedef struct _rt_domain { + struct no_rqlock_work arm_timers; + + /* runnable rt tasks are in here */ + spinlock_t ready_lock; + struct heap ready_queue; + + /* real-time tasks waiting for release are in here */ + spinlock_t release_lock; + struct list_head release_queue; + + /* how do we check if we need to kick another CPU? */ + check_resched_needed_t check_resched; + + /* how do we release a job? */ + release_job_t release_job; + + /* how are tasks ordered in the ready queue? */ + heap_prio_t order; +} rt_domain_t; + +static inline struct task_struct* __next_ready(rt_domain_t* rt) +{ + struct heap_node *hn = heap_peek(rt->order, &rt->ready_queue); + if (hn) + return heap2task(hn); + else + return NULL; +} + +void rt_domain_init(rt_domain_t *rt, heap_prio_t order, + check_resched_needed_t check, + release_job_t relase); + +void __add_ready(rt_domain_t* rt, struct task_struct *new); +void __add_release(rt_domain_t* rt, struct task_struct *task); + +static inline struct task_struct* __take_ready(rt_domain_t* rt) +{ + struct heap_node* hn = heap_take(rt->order, &rt->ready_queue); + if (hn) + return heap2task(hn); + else + return NULL; +} + +static inline struct task_struct* __peek_ready(rt_domain_t* rt) +{ + struct heap_node* hn = heap_peek(rt->order, &rt->ready_queue); + if (hn) + return heap2task(hn); + else + return NULL; +} + +static inline int is_queued(struct task_struct *t) +{ + return heap_node_in_heap(tsk_rt(t)->heap_node); +} + +static inline void remove(rt_domain_t* rt, struct task_struct *t) +{ + heap_delete(rt->order, &rt->ready_queue, tsk_rt(t)->heap_node); +} + +static inline void add_ready(rt_domain_t* rt, struct task_struct *new) +{ + unsigned long flags; + /* first we need the write lock for rt_ready_queue */ + spin_lock_irqsave(&rt->ready_lock, flags); + __add_ready(rt, new); + spin_unlock_irqrestore(&rt->ready_lock, flags); +} + +static inline struct task_struct* take_ready(rt_domain_t* rt) +{ + unsigned long flags; + struct task_struct* ret; + /* first we need the write lock for rt_ready_queue */ + spin_lock_irqsave(&rt->ready_lock, flags); + ret = __take_ready(rt); + spin_unlock_irqrestore(&rt->ready_lock, flags); + return ret; +} + + +static inline void add_release(rt_domain_t* rt, struct task_struct *task) +{ + unsigned long flags; + /* first we need the write lock for rt_ready_queue */ + spin_lock_irqsave(&rt->release_lock, flags); + __add_release(rt, task); + spin_unlock_irqrestore(&rt->release_lock, flags); +} + +static inline int __jobs_pending(rt_domain_t* rt) +{ + return !heap_empty(&rt->ready_queue); +} + +static inline int jobs_pending(rt_domain_t* rt) +{ + unsigned long flags; + int ret; + /* first we need the write lock for rt_ready_queue */ + spin_lock_irqsave(&rt->ready_lock, flags); + ret = !heap_empty(&rt->ready_queue); + spin_unlock_irqrestore(&rt->ready_lock, flags); + return ret; +} + +#endif diff --git a/include/litmus/rt_param.h b/include/litmus/rt_param.h new file mode 100644 index 0000000..7bb5684 --- /dev/null +++ b/include/litmus/rt_param.h @@ -0,0 +1,170 @@ +/* + * Definition of the scheduler plugin interface. + * + */ +#ifndef _LINUX_RT_PARAM_H_ +#define _LINUX_RT_PARAM_H_ + +/* Litmus time type. */ +typedef unsigned long long lt_t; + +static inline int lt_after(lt_t a, lt_t b) +{ + return ((long long) b) - ((long long) a) < 0; +} +#define lt_before(a, b) lt_after(b, a) + +static inline int lt_after_eq(lt_t a, lt_t b) +{ + return ((long long) a) - ((long long) b) >= 0; +} +#define lt_before_eq(a, b) lt_after_eq(b, a) + +/* different types of clients */ +typedef enum { + RT_CLASS_HARD, + RT_CLASS_SOFT, + RT_CLASS_BEST_EFFORT +} task_class_t; + +struct rt_task { + lt_t exec_cost; + lt_t period; + lt_t phase; + unsigned int cpu; + task_class_t cls; +}; + +/* don't export internal data structures to user space (liblitmus) */ +#ifdef __KERNEL__ + +struct _rt_domain; +struct heap_node; + +struct rt_job { + /* Time instant the the job was or will be released. */ + lt_t release; + /* What is the current deadline? */ + lt_t deadline; + + /* How much service has this job received so far? */ + lt_t exec_time; + + /* Which job is this. This is used to let user space + * specify which job to wait for, which is important if jobs + * overrun. If we just call sys_sleep_next_period() then we + * will unintentionally miss jobs after an overrun. + * + * Increase this sequence number when a job is released. + */ + unsigned int job_no; + + /* when did this job start executing? */ + lt_t exec_start; +}; + + +struct pfair_param; + +/* RT task parameters for scheduling extensions + * These parameters are inherited during clone and therefore must + * be explicitly set up before the task set is launched. + */ +struct rt_param { + /* is the task sleeping? */ + unsigned int flags:8; + + /* do we need to check for srp blocking? */ + unsigned int srp_non_recurse:1; + + /* user controlled parameters */ + struct rt_task task_params; + + /* timing parameters */ + struct rt_job job_params; + + /* task representing the current "inherited" task + * priority, assigned by inherit_priority and + * return priority in the scheduler plugins. + * could point to self if PI does not result in + * an increased task priority. + */ + struct task_struct* inh_task; + + /* Don't just dereference this pointer in kernel space! + * It might very well point to junk or nothing at all. + * NULL indicates that the task has not requested any non-preemptable + * section support. + * Not inherited upon fork. + */ + short* np_flag; + + /* For the FMLP under PSN-EDF, it is required to make the task + * non-preemptive from kernel space. In order not to interfere with + * user space, this counter indicates the kernel space np setting. + * kernel_np > 0 => task is non-preemptive + */ + unsigned int kernel_np; + + /* This field can be used by plugins to store where the task + * is currently scheduled. It is the responsibility of the + * plugin to avoid race conditions. + * + * This used by GSN-EDF and PFAIR. + */ + volatile int scheduled_on; + + /* Is the stack of the task currently in use? This is updated by + * the LITMUS core. + * + * Be careful to avoid deadlocks! + */ + volatile int stack_in_use; + + /* This field can be used by plugins to store where the task + * is currently linked. It is the responsibility of the plugin + * to avoid race conditions. + * + * Used by GSN-EDF. + */ + volatile int linked_on; + + /* PFAIR/PD^2 state. Allocated on demand. */ + struct pfair_param* pfair; + + /* Fields saved before BE->RT transition. + */ + int old_policy; + int old_prio; + + /* The high-resolution timer used to control its release. */ + struct hrtimer release_timer; + + /* ready queue for this task */ + struct _rt_domain* domain; + + /* heap element for this task + * + * Warning: Don't statically allocate this node. The heap + * implementation swaps these between tasks, thus after + * dequeuing from a heap you may end up with a different node + * then the one you had when enqueuing the task. For the same + * reason, don't obtain and store references to this node + * other than this pointer (which is updated by the heap + * implementation). + */ + struct heap_node* heap_node; + + /* Used by rt_domain to queue task in release list. + */ + struct list_head list; +}; + +/* Possible RT flags */ +#define RT_F_RUNNING 0x00000000 +#define RT_F_SLEEP 0x00000001 +#define RT_F_EXIT_SEM 0x00000008 + +#endif + +#endif diff --git a/include/litmus/sched_plugin.h b/include/litmus/sched_plugin.h new file mode 100644 index 0000000..aba7522 --- /dev/null +++ b/include/litmus/sched_plugin.h @@ -0,0 +1,144 @@ +/* + * Definition of the scheduler plugin interface. + * + */ +#ifndef _LINUX_SCHED_PLUGIN_H_ +#define _LINUX_SCHED_PLUGIN_H_ + +#include + +/* struct for semaphore with priority inheritance */ +struct pi_semaphore { + atomic_t count; + int sleepers; + wait_queue_head_t wait; + union { + /* highest-prio holder/waiter */ + struct task_struct *task; + struct task_struct* cpu_task[NR_CPUS]; + } hp; + /* current lock holder */ + struct task_struct *holder; +}; + + +/********************* scheduler invocation ******************/ + +/* Plugin-specific realtime tick handler */ +typedef void (*scheduler_tick_t) (struct task_struct *cur); +/* Novell make sched decision function */ +typedef struct task_struct* (*schedule_t)(struct task_struct * prev); +/* Clean up after the task switch has occured. + * This function is called after every (even non-rt) task switch. + */ +typedef void (*finish_switch_t)(struct task_struct *prev); + + +/********************* task state changes ********************/ + +/* Called to setup a new real-time task. + * Release the first job, enqueue, etc. + * Task may already be running. + */ +typedef void (*task_new_t) (struct task_struct *task, + int on_rq, + int running); + +/* Called to re-introduce a task after blocking. + * Can potentially be called multiple times. + */ +typedef void (*task_wake_up_t) (struct task_struct *task); +/* called to notify the plugin of a blocking real-time task + * it will only be called for real-time tasks and before schedule is called */ +typedef void (*task_block_t) (struct task_struct *task); +/* Called when a real-time task exits or changes to a different scheduling + * class. + * Free any allocated resources + */ +typedef void (*task_exit_t) (struct task_struct *); + +/* Called when the new_owner is released from the wait queue + * it should now inherit the priority from sem, _before_ it gets readded + * to any queue + */ +typedef long (*inherit_priority_t) (struct pi_semaphore *sem, + struct task_struct *new_owner); + +/* Called when the current task releases a semahpore where it might have + * inherited a piority from + */ +typedef long (*return_priority_t) (struct pi_semaphore *sem); + +/* Called when a task tries to acquire a semaphore and fails. Check if its + * priority is higher than that of the current holder. + */ +typedef long (*pi_block_t) (struct pi_semaphore *sem, struct task_struct *t); + + +/********************* sys call backends ********************/ +/* This function causes the caller to sleep until the next release */ +typedef long (*complete_job_t) (void); + +typedef long (*admit_task_t)(struct task_struct* tsk); + +typedef void (*release_at_t)(struct task_struct *t, lt_t start); + +struct sched_plugin { + struct list_head list; + /* basic info */ + char *plugin_name; +#ifdef CONFIG_SRP + unsigned int srp_active; +#endif + + /* scheduler invocation */ + scheduler_tick_t tick; + schedule_t schedule; + finish_switch_t finish_switch; + + /* syscall backend */ + complete_job_t complete_job; + release_at_t release_at; + + /* task state changes */ + admit_task_t admit_task; + + task_new_t task_new; + task_wake_up_t task_wake_up; + task_block_t task_block; + task_exit_t task_exit; + +#ifdef CONFIG_FMLP + /* priority inheritance */ + unsigned int fmlp_active; + inherit_priority_t inherit_priority; + return_priority_t return_priority; + pi_block_t pi_block; +#endif +} __attribute__ ((__aligned__(SMP_CACHE_BYTES))); + + +extern struct sched_plugin *litmus; + +int register_sched_plugin(struct sched_plugin* plugin); +struct sched_plugin* find_sched_plugin(const char* name); +int print_sched_plugins(char* buf, int max); + +static inline int srp_active(void) +{ +#ifdef CONFIG_SRP + return litmus->srp_active; +#else + return 0; +#endif +} +static inline int fmlp_active(void) +{ +#ifdef CONFIG_FMLP + return litmus->fmlp_active; +#else + return 0; +#endif +} + +#endif diff --git a/include/litmus/sched_trace.h b/include/litmus/sched_trace.h new file mode 100644 index 0000000..60dcbfb --- /dev/null +++ b/include/litmus/sched_trace.h @@ -0,0 +1,31 @@ +/* sched_trace.h -- record scheduler events to a byte stream for offline analysis. + */ +#ifndef _LINUX_SCHED_TRACE_H_ +#define _LINUX_SCHED_TRACE_H_ + +#include + +/* dummies, need to be re-implemented */ + +/* used in sched.c */ +#define sched_trace_task_arrival(t) +#define sched_trace_task_departure(t) +#define sched_trace_task_preemption(t, by) +#define sched_trace_task_scheduled(t) + +/* used in scheduler plugins */ +#define sched_trace_job_release(t) +#define sched_trace_job_completion(t) + + +#ifdef CONFIG_SCHED_DEBUG_TRACE +void sched_trace_log_message(const char* fmt, ...); + +#else + +#define sched_trace_log_message(fmt, ...) + +#endif + + +#endif diff --git a/include/litmus/trace.h b/include/litmus/trace.h new file mode 100644 index 0000000..2c8e141 --- /dev/null +++ b/include/litmus/trace.h @@ -0,0 +1,107 @@ +#ifndef _SYS_TRACE_H_ +#define _SYS_TRACE_H_ + +#ifdef CONFIG_FEATHER_TRACE + +#include +#include + + +/*********************** TIMESTAMPS ************************/ + +enum task_type_marker { + TSK_BE, + TSK_RT, + TSK_UNKNOWN +}; + +struct timestamp { + uint64_t timestamp; + uint32_t seq_no; + uint8_t cpu; + uint8_t event; + uint8_t task_type; +}; + + +/* buffer holding time stamps - will be provided by driver */ +extern struct ft_buffer* trace_ts_buf; + +/* tracing callbacks */ +feather_callback void save_timestamp(unsigned long event); +feather_callback void save_timestamp_def(unsigned long event, unsigned long type); +feather_callback void save_timestamp_task(unsigned long event, unsigned long t_ptr); + +#define TIMESTAMP(id) ft_event0(id, save_timestamp) + +#define DTIMESTAMP(id, def) ft_event1(id, save_timestamp_def, def) + +#define TTIMESTAMP(id, task) ft_event1(id, save_timestamp_task, (unsigned long) task) + +#else /* !CONFIG_FEATHER_TRACE */ + +#define TIMESTAMP(id) /* no tracing */ + +#define DTIMESTAMP(id, def) /* no tracing */ + +#define TTIMESTAMP(id, task) /* no tracing */ + +#endif + + +/* Convention for timestamps + * ========================= + * + * In order to process the trace files with a common tool, we use the following + * convention to measure execution times: The end time id of a code segment is + * always the next number after the start time event id. + */ + +#define TS_SCHED_START DTIMESTAMP(100, TSK_UNKNOWN) /* we only + * care + * about + * next */ +#define TS_SCHED_END(t) TTIMESTAMP(101, t) +#define TS_SCHED2_START(t) TTIMESTAMP(102, t) +#define TS_SCHED2_END(t) TTIMESTAMP(103, t) + +#define TS_CXS_START(t) TTIMESTAMP(104, t) +#define TS_CXS_END(t) TTIMESTAMP(105, t) + +#define TS_RELEASE_START DTIMESTAMP(106, TSK_RT) +#define TS_RELEASE_END DTIMESTAMP(107, TSK_RT) + +#define TS_TICK_START(t) TTIMESTAMP(110, t) +#define TS_TICK_END(t) TTIMESTAMP(111, t) + + +#define TS_PLUGIN_SCHED_START /* TIMESTAMP(120) */ /* currently unused */ +#define TS_PLUGIN_SCHED_END /* TIMESTAMP(121) */ + +#define TS_PLUGIN_TICK_START /* TIMESTAMP(130) */ +#define TS_PLUGIN_TICK_END /* TIMESTAMP(131) */ + +#define TS_ENTER_NP_START TIMESTAMP(140) +#define TS_ENTER_NP_END TIMESTAMP(141) + +#define TS_EXIT_NP_START TIMESTAMP(150) +#define TS_EXIT_NP_END TIMESTAMP(151) + +#define TS_SRP_UP_START TIMESTAMP(160) +#define TS_SRP_UP_END TIMESTAMP(161) +#define TS_SRP_DOWN_START TIMESTAMP(162) +#define TS_SRP_DOWN_END TIMESTAMP(163) + +#define TS_PI_UP_START TIMESTAMP(170) +#define TS_PI_UP_END TIMESTAMP(171) +#define TS_PI_DOWN_START TIMESTAMP(172) +#define TS_PI_DOWN_END TIMESTAMP(173) + +#define TS_FIFO_UP_START TIMESTAMP(180) +#define TS_FIFO_UP_END TIMESTAMP(181) +#define TS_FIFO_DOWN_START TIMESTAMP(182) +#define TS_FIFO_DOWN_END TIMESTAMP(183) + + + +#endif /* !_SYS_TRACE_H_ */ diff --git a/include/litmus/unistd.h b/include/litmus/unistd.h new file mode 100644 index 0000000..8224235 --- /dev/null +++ b/include/litmus/unistd.h @@ -0,0 +1,20 @@ + +#define __LSC(x) (__NR_LITMUS + x) + +#define __NR_set_rt_task_param __LSC(0) +#define __NR_get_rt_task_param __LSC(1) +#define __NR_sleep_next_period __LSC(2) +#define __NR_register_np_flag __LSC(3) +#define __NR_exit_np __LSC(4) +#define __NR_od_open __LSC(5) +#define __NR_od_close __LSC(6) +#define __NR_fmlp_down __LSC(7) +#define __NR_fmlp_up __LSC(8) +#define __NR_srp_down __LSC(9) +#define __NR_srp_up __LSC(10) +#define __NR_query_job_no __LSC(11) +#define __NR_wait_for_job_release __LSC(12) +#define __NR_wait_for_ts_release __LSC(13) +#define __NR_release_ts __LSC(14) + +#define NR_litmus_syscalls 15 diff --git a/kernel/exit.c b/kernel/exit.c index 549c055..bc313b7 100644 --- a/kernel/exit.c +++ b/kernel/exit.c @@ -52,6 +52,8 @@ extern void sem_exit (void); +extern void exit_od_table(struct task_struct* t); + static void exit_mm(struct task_struct * tsk); static void __unhash_process(struct task_struct *p) @@ -987,6 +989,8 @@ fastcall NORET_TYPE void do_exit(long code) if (unlikely(tsk->audit_context)) audit_free(tsk); + exit_od_table(tsk); + tsk->exit_code = code; taskstats_exit(tsk, group_dead); diff --git a/kernel/fork.c b/kernel/fork.c index 8dd8ff2..4c322d4 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -59,6 +59,9 @@ #include #include +#include +#include + /* * Protected counters by write_lock_irq(&tasklist_lock) */ @@ -121,6 +124,8 @@ void __put_task_struct(struct task_struct *tsk) WARN_ON(atomic_read(&tsk->usage)); WARN_ON(tsk == current); + exit_litmus(tsk); + security_task_free(tsk); free_uid(tsk->user); put_group_info(tsk->group_info); @@ -182,6 +187,9 @@ static struct task_struct *dup_task_struct(struct task_struct *orig) *tsk = *orig; tsk->stack = ti; + /* Don't let the new task be a real-time task. */ + memset(&tsk->rt_param, 0, sizeof(struct rt_task)); + err = prop_local_init_single(&tsk->dirties); if (err) { free_thread_info(ti); diff --git a/kernel/printk.c b/kernel/printk.c index 89011bf..9eb2dc5 100644 --- a/kernel/printk.c +++ b/kernel/printk.c @@ -54,6 +54,12 @@ int console_printk[4] = { DEFAULT_CONSOLE_LOGLEVEL, /* default_console_loglevel */ }; +/* divert printk() messages when we have a LITMUS^RT + * debug listener + */ +#include +int trace_override = 0; + /* * Low level drivers may need that to know if they can schedule in * their unblank() callback or not. So let's export it. @@ -652,6 +658,8 @@ asmlinkage int vprintk(const char *fmt, va_list args) /* Emit the output into the temporary buffer */ printed_len = vscnprintf(printk_buf, sizeof(printk_buf), fmt, args); + if (trace_override) + TRACE("%s", printk_buf); /* * Copy the output into log_buf. If the caller didn't provide @@ -932,7 +940,7 @@ int is_console_locked(void) void wake_up_klogd(void) { - if (!oops_in_progress && waitqueue_active(&log_wait)) + if (!trace_override && !oops_in_progress && waitqueue_active(&log_wait)) wake_up_interruptible(&log_wait); } diff --git a/kernel/sched.c b/kernel/sched.c index e76b11c..9ee07ba 100644 --- a/kernel/sched.c +++ b/kernel/sched.c @@ -67,6 +67,10 @@ #include #include +#include + +#include + /* * Scheduler clock - returns current time in nanosec units. * This is default implementation. @@ -324,6 +328,8 @@ struct rq { atomic_t nr_iowait; + struct task_struct* litmus_next; + #ifdef CONFIG_SMP struct sched_domain *sd; @@ -875,11 +881,12 @@ static inline void cpuacct_charge(struct task_struct *tsk, u64 cputime) {} #include "sched_idletask.c" #include "sched_fair.c" #include "sched_rt.c" +#include "../litmus/sched_litmus.c" #ifdef CONFIG_SCHED_DEBUG # include "sched_debug.c" #endif -#define sched_class_highest (&rt_sched_class) +#define sched_class_highest (&litmus_sched_class) /* * Update delta_exec, delta_fair fields for rq. @@ -1516,6 +1523,8 @@ static int try_to_wake_up(struct task_struct *p, unsigned int state, int sync) int new_cpu; #endif + if (is_realtime(p)) + TRACE_TASK(p, "try_to_wake_up()\n"); rq = task_rq_lock(p, &flags); old_state = p->state; if (!(old_state & state)) @@ -1529,7 +1538,7 @@ static int try_to_wake_up(struct task_struct *p, unsigned int state, int sync) this_cpu = smp_processor_id(); #ifdef CONFIG_SMP - if (unlikely(task_running(rq, p))) + if (unlikely(task_running(rq, p) || is_realtime(p))) goto out_activate; new_cpu = cpu; @@ -1650,8 +1659,10 @@ out_activate: out_running: p->state = TASK_RUNNING; out: + if (is_realtime(p)) + TRACE_TASK(p, "try_to_wake_up() done, p->state=%d\n", p->state); task_rq_unlock(rq, &flags); - + tick_no_rqlock(); return success; } @@ -1890,6 +1901,8 @@ static void finish_task_switch(struct rq *rq, struct task_struct *prev) */ prev_state = prev->state; finish_arch_switch(prev); + litmus->finish_switch(prev); + prev->rt_param.stack_in_use = NO_CPU; finish_lock_switch(rq, prev); fire_sched_in_preempt_notifiers(current); if (mm) @@ -3480,6 +3493,7 @@ void scheduler_tick(void) struct task_struct *curr = rq->curr; u64 next_tick = rq->tick_timestamp + TICK_NSEC; + TS_TICK_START(current); spin_lock(&rq->lock); __update_rq_clock(rq); /* @@ -3491,12 +3505,17 @@ void scheduler_tick(void) update_cpu_load(rq); if (curr != rq->idle) /* FIXME: needed? */ curr->sched_class->task_tick(rq, curr); + TS_PLUGIN_TICK_START; + litmus_tick(rq, curr); + TS_PLUGIN_TICK_END; spin_unlock(&rq->lock); #ifdef CONFIG_SMP rq->idle_at_tick = idle_cpu(cpu); - trigger_load_balance(rq, cpu); + if (!is_realtime(current)) + trigger_load_balance(rq, cpu); #endif + TS_TICK_END(current); } #if defined(CONFIG_PREEMPT) && defined(CONFIG_DEBUG_PREEMPT) @@ -3594,11 +3613,13 @@ pick_next_task(struct rq *rq, struct task_struct *prev) * Optimization: we know that if all tasks are in * the fair class we can call that function directly: */ - if (likely(rq->nr_running == rq->cfs.nr_running)) { + /* Don't do that for LITMUS. + if (likely(rq->nr_running == rq->cfs.nr_running)) { p = fair_sched_class.pick_next_task(rq); if (likely(p)) return p; } + */ class = sched_class_highest; for ( ; ; ) { @@ -3633,6 +3654,7 @@ need_resched: release_kernel_lock(prev); need_resched_nonpreemptible: + TS_SCHED_START; schedule_debug(prev); @@ -3643,6 +3665,9 @@ need_resched_nonpreemptible: __update_rq_clock(rq); spin_lock(&rq->lock); clear_tsk_need_resched(prev); + TS_PLUGIN_SCHED_START; + litmus_schedule(rq, prev); + TS_PLUGIN_SCHED_END; if (prev->state && !(preempt_count() & PREEMPT_ACTIVE)) { if (unlikely((prev->state & TASK_INTERRUPTIBLE) && @@ -3667,18 +3692,32 @@ need_resched_nonpreemptible: rq->curr = next; ++*switch_count; + TS_SCHED_END(next); + TS_CXS_START(next); context_switch(rq, prev, next); /* unlocks the rq */ - } else + TS_CXS_END(current); + } else { + TS_SCHED_END(prev); spin_unlock_irq(&rq->lock); + } + TS_SCHED2_START(current); + + tick_no_rqlock(); if (unlikely(reacquire_kernel_lock(current) < 0)) { cpu = smp_processor_id(); rq = cpu_rq(cpu); + TS_SCHED2_END(current); goto need_resched_nonpreemptible; } preempt_enable_no_resched(); - if (unlikely(test_thread_flag(TIF_NEED_RESCHED))) + if (unlikely(test_thread_flag(TIF_NEED_RESCHED))) { + TS_SCHED2_END(current); goto need_resched; + } + TS_SCHED2_END(current); + if (srp_active()) + srp_ceiling_block(); } EXPORT_SYMBOL(schedule); @@ -3886,6 +3925,18 @@ void complete_all(struct completion *x) } EXPORT_SYMBOL(complete_all); +void complete_n(struct completion *x, int n) +{ + unsigned long flags; + + spin_lock_irqsave(&x->wait.lock, flags); + x->done += n; + __wake_up_common(&x->wait, TASK_UNINTERRUPTIBLE | TASK_INTERRUPTIBLE, + n, 0, NULL); + spin_unlock_irqrestore(&x->wait.lock, flags); +} +EXPORT_SYMBOL(complete_n); + static inline long __sched do_wait_for_common(struct completion *x, long timeout, int state) { @@ -4236,6 +4287,9 @@ __setscheduler(struct rq *rq, struct task_struct *p, int policy, int prio) case SCHED_RR: p->sched_class = &rt_sched_class; break; + case SCHED_LITMUS: + p->sched_class = &litmus_sched_class; + break; } p->rt_priority = prio; @@ -4268,7 +4322,7 @@ recheck: policy = oldpolicy = p->policy; else if (policy != SCHED_FIFO && policy != SCHED_RR && policy != SCHED_NORMAL && policy != SCHED_BATCH && - policy != SCHED_IDLE) + policy != SCHED_IDLE && policy != SCHED_LITMUS) return -EINVAL; /* * Valid priorities for SCHED_FIFO and SCHED_RR are @@ -4282,6 +4336,9 @@ recheck: if (rt_policy(policy) != (param->sched_priority != 0)) return -EINVAL; + if (policy == SCHED_LITMUS && policy == p->policy) + return -EINVAL; + /* * Allow unprivileged RT tasks to decrease priority: */ @@ -4316,6 +4373,12 @@ recheck: return -EPERM; } + if (policy == SCHED_LITMUS) { + retval = litmus_admit_task(p); + if (retval) + return retval; + } + retval = security_task_setscheduler(p, policy, param); if (retval) return retval; @@ -4345,9 +4408,17 @@ recheck: p->sched_class->put_prev_task(rq, p); } + if (p->policy == SCHED_LITMUS) + litmus_exit_task(p); + oldprio = p->prio; __setscheduler(rq, p, policy, param->sched_priority); + if (policy == SCHED_LITMUS) { + p->rt_param.stack_in_use = running ? rq->cpu : NO_CPU; + litmus->task_new(p, on_rq, running); + } + if (on_rq) { if (running) p->sched_class->set_curr_task(rq); @@ -4364,6 +4435,7 @@ recheck: check_preempt_curr(rq, p); } } + __task_rq_unlock(rq); spin_unlock_irqrestore(&p->pi_lock, flags); @@ -4494,10 +4566,11 @@ long sched_setaffinity(pid_t pid, cpumask_t new_mask) read_lock(&tasklist_lock); p = find_process_by_pid(pid); - if (!p) { + if (!p || is_realtime(p)) { + /* LITMUS tasks don't get to do this, transition to BE first */ read_unlock(&tasklist_lock); mutex_unlock(&sched_hotcpu_mutex); - return -ESRCH; + return p ? -EPERM : -ESRCH; } /* diff --git a/kernel/sched_fair.c b/kernel/sched_fair.c index da7c061..de30496 100644 --- a/kernel/sched_fair.c +++ b/kernel/sched_fair.c @@ -845,7 +845,7 @@ static void check_preempt_wakeup(struct rq *rq, struct task_struct *p) struct sched_entity *se = &curr->se, *pse = &p->se; unsigned long gran; - if (unlikely(rt_prio(p->prio))) { + if (unlikely(rt_prio(p->prio) || p->policy == SCHED_LITMUS)) { update_rq_clock(rq); update_curr(cfs_rq); resched_task(curr); diff --git a/kernel/sched_rt.c b/kernel/sched_rt.c index 9ba3daa..c7c938c 100644 --- a/kernel/sched_rt.c +++ b/kernel/sched_rt.c @@ -70,7 +70,7 @@ yield_task_rt(struct rq *rq) */ static void check_preempt_curr_rt(struct rq *rq, struct task_struct *p) { - if (p->prio < rq->curr->prio) + if (p->prio < rq->curr->prio || p->policy == SCHED_LITMUS) resched_task(rq->curr); } diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c index cb89fa8..3b1936f 100644 --- a/kernel/time/tick-sched.c +++ b/kernel/time/tick-sched.c @@ -568,6 +568,22 @@ static enum hrtimer_restart tick_sched_timer(struct hrtimer *timer) } /** + * tick_set_quanta_type - get the quanta type as a boot option + * Default is standard setup with ticks staggered over first + * half of tick period. + */ +int quanta_type = LINUX_DEFAULT_TICKS; +static int __init tick_set_quanta_type(char *str) +{ + if (strcmp("aligned", str) == 0) + quanta_type = LITMUS_ALIGNED_TICKS; + else if (strcmp("staggered", str) == 0) + quanta_type = LITMUS_STAGGERED_TICKS; + return 1; +} +__setup("quanta=", tick_set_quanta_type); + +/** * tick_setup_sched_timer - setup the tick emulation timer */ void tick_setup_sched_timer(void) @@ -585,9 +601,24 @@ void tick_setup_sched_timer(void) /* Get the next period (per cpu) */ ts->sched_timer.expires = tick_init_jiffy_update(); - offset = ktime_to_ns(tick_period) >> 1; - do_div(offset, num_possible_cpus()); - offset *= smp_processor_id(); + + /* Offset must be set correctly to achieve desired quanta type. */ + switch (quanta_type) { + case LITMUS_ALIGNED_TICKS: + offset = 0; + break; + case LITMUS_STAGGERED_TICKS: + offset = ktime_to_ns(tick_period); + do_div(offset, num_possible_cpus()); + offset *= smp_processor_id(); + break; + default: + offset = ktime_to_ns(tick_period) >> 1; + do_div(offset, num_possible_cpus()); + offset *= smp_processor_id(); + } + + /* Add correct offset to expiration time. */ ts->sched_timer.expires = ktime_add_ns(ts->sched_timer.expires, offset); for (;;) { diff --git a/litmus/Kconfig b/litmus/Kconfig new file mode 100644 index 0000000..9a2ab90 --- /dev/null +++ b/litmus/Kconfig @@ -0,0 +1,78 @@ +menu "LITMUS^RT" + +menu "Real-Time Synchronization" + +config NP_SECTION + bool "Non-preemptive section support" + depends on !SPARC64 + default n + help + Include support for flag-based non-preemptive section signaling + from userspace. + + (currently broken on SPARC64) + + Say Yes if you want FMLP short critical section synchronization support. + + +config SRP + bool "Stack Resource Policy (SRP)" + default n + help + Include support for Baker's Stack Resource Policy. + + Say Yes if you want FMLP local long critical section synchronization support. + +config FMLP + bool "FMLP support" + depends on NP_SECTION + default n + help + Include support for deterministic multiprocessor real-time + synchronization support. + + Say Yes if you want FMLP long critical section synchronization support. + +endmenu + +menu "Tracing" + +config SCHED_TASK_TRACE + bool "Trace real-time tasks" + default y + help + Include support for the sched_trace_XXX() tracing functions. This + allows the collection of real-time task events such as job + completions, job releases, early completions, etc. This results in a + small overhead in the scheduling code. Disable if the overhead is not + acceptable (e.g., benchmarking). + + Say Yes for debugging. + Say No for overhead tracing. + +config SCHED_DEBUG_TRACE + bool "TRACE() debugging" + default y + help + Include support for sched_trace_log_messageg(), which is used to + implement TRACE(). If disabled, no TRACE() messages will be included + in the kernel, and no overheads due to debugging statements will be + incurred by the scheduler. Disable if the overhead is not acceptable + (e.g. benchmarking). + + Say Yes for debugging. + Say No for overhead tracing. + +config FEATHER_TRACE + bool "Feather-Trace Instrumentation Support" + default y + help + Include Feather-Trace trace points. Currently not supported on + sparc64. + + Say Yes for overhead tracing. + + +endmenu + +endmenu diff --git a/litmus/Makefile b/litmus/Makefile new file mode 100644 index 0000000..5452038 --- /dev/null +++ b/litmus/Makefile @@ -0,0 +1,14 @@ +# +# Makefile for LITMUS^RT +# + +obj-y = sched_plugin.o litmus.o sched_trace.o \ + edf_common.o jobs.o \ + rt_domain.o fdso.o sync.o \ + fmlp.o srp.o norqlock.o \ + sched_gsn_edf.o \ + sched_psn_edf.o \ + sched_cedf.o \ + sched_pfair.o + +obj-$(CONFIG_FEATHER_TRACE) += trace.o ft_event.o diff --git a/litmus/edf_common.c b/litmus/edf_common.c new file mode 100644 index 0000000..d7567ac --- /dev/null +++ b/litmus/edf_common.c @@ -0,0 +1,94 @@ +/* + * kernel/edf_common.c + * + * Common functions for EDF based scheduler. + */ + +#include +#include +#include + +#include +#include +#include + + +#include + +/* edf_higher_prio - returns true if first has a higher EDF priority + * than second. Deadline ties are broken by PID. + * + * first first must not be NULL and a real-time task. + * second may be NULL or a non-rt task. + */ +int edf_higher_prio(struct task_struct* first, + struct task_struct* second) +{ + struct task_struct *first_task = first; + struct task_struct *second_task = second; + + /* Check for inherited priorities. Change task + * used for comparison in such a case. + */ + if (first && first->rt_param.inh_task) + first_task = first->rt_param.inh_task; + if (second && second->rt_param.inh_task) + second_task = second->rt_param.inh_task; + + return + /* does the second task exist and is it a real-time task? If + * not, the first task (which is a RT task) has higher + * priority. + */ + !second_task || !is_realtime(second_task) || + + /* is the deadline of the first task earlier? + * Then it has higher priority. + */ + earlier_deadline(first_task, second_task) || + + /* Do we have a deadline tie? + * Then break by PID. + */ + (get_deadline(first_task) == get_deadline(second_task) && + (first_task->pid < second_task->pid || + + /* If the PIDs are the same then the task with the inherited + * priority wins. + */ + (first_task->pid == second_task->pid && + !second->rt_param.inh_task))); +} + +int edf_ready_order(struct heap_node* a, struct heap_node* b) +{ + return edf_higher_prio(heap2task(a), heap2task(b)); +} + +void edf_domain_init(rt_domain_t* rt, check_resched_needed_t resched, + release_job_t release) +{ + rt_domain_init(rt, edf_ready_order, resched, release); +} + +/* need_to_preempt - check whether the task t needs to be preempted + * call only with irqs disabled and with ready_lock acquired + * THIS DOES NOT TAKE NON-PREEMPTIVE SECTIONS INTO ACCOUNT! + */ +int edf_preemption_needed(rt_domain_t* rt, struct task_struct *t) +{ + /* we need the read lock for edf_ready_queue */ + /* no need to preempt if there is nothing pending */ + if (!__jobs_pending(rt)) + return 0; + /* we need to reschedule if t doesn't exist */ + if (!t) + return 1; + + /* NOTE: We cannot check for non-preemptibility since we + * don't know what address space we're currently in. + */ + + /* make sure to get non-rt stuff out of the way */ + return !is_realtime(t) || edf_higher_prio(__next_ready(rt), t); +} diff --git a/litmus/fdso.c b/litmus/fdso.c new file mode 100644 index 0000000..81ab0af --- /dev/null +++ b/litmus/fdso.c @@ -0,0 +1,282 @@ +/* fdso.c - file descriptor attached shared objects + * + * (c) 2007 B. Brandenburg, LITMUS^RT project + * + * Notes: + * - objects descriptor (OD) tables are not cloned during a fork. + * - objects are created on-demand, and freed after the last reference + * is dropped. + * - for now, object types are hard coded. + * - As long as we have live objects, we keep a reference to the inode. + */ + +#include +#include +#include +#include +#include + +#include + +extern struct fdso_ops fmlp_sem_ops; +extern struct fdso_ops srp_sem_ops; + +static const struct fdso_ops* fdso_ops[] = { + &fmlp_sem_ops, + &srp_sem_ops, +}; + +static void* fdso_create(obj_type_t type) +{ + if (fdso_ops[type]->create) + return fdso_ops[type]->create(); + else + return NULL; +} + +static void fdso_destroy(obj_type_t type, void* obj) +{ + fdso_ops[type]->destroy(obj); +} + +static int fdso_open(struct od_table_entry* entry, void* __user config) +{ + if (fdso_ops[entry->obj->type]->open) + return fdso_ops[entry->obj->type]->open(entry, config); + else + return 0; +} + +static int fdso_close(struct od_table_entry* entry) +{ + if (fdso_ops[entry->obj->type]->close) + return fdso_ops[entry->obj->type]->close(entry); + else + return 0; +} + +/* inode must be locked already */ +static struct inode_obj_id* alloc_inode_obj(struct inode* inode, + obj_type_t type, + unsigned int id) +{ + struct inode_obj_id* obj; + void* raw_obj; + + raw_obj = fdso_create(type); + if (!raw_obj) + return NULL; + + obj = kmalloc(sizeof(struct inode_obj_id), GFP_KERNEL); + if (!obj) + return NULL; + INIT_LIST_HEAD(&obj->list); + atomic_set(&obj->count, 1); + obj->type = type; + obj->id = id; + obj->obj = raw_obj; + obj->inode = inode; + + list_add(&obj->list, &inode->i_obj_list); + atomic_inc(&inode->i_count); + + printk(KERN_DEBUG "alloc_inode_obj(%p, %d, %d): object created\n", inode, type, id); + return obj; +} + +/* inode must be locked already */ +static struct inode_obj_id* get_inode_obj(struct inode* inode, + obj_type_t type, + unsigned int id) +{ + struct list_head* pos; + struct inode_obj_id* obj = NULL; + + list_for_each(pos, &inode->i_obj_list) { + obj = list_entry(pos, struct inode_obj_id, list); + if (obj->id == id && obj->type == type) { + atomic_inc(&obj->count); + return obj; + } + } + printk(KERN_DEBUG "get_inode_obj(%p, %d, %d): couldn't find object\n", inode, type, id); + return NULL; +} + + +static void put_inode_obj(struct inode_obj_id* obj) +{ + struct inode* inode; + int let_go = 0; + + inode = obj->inode; + if (atomic_dec_and_test(&obj->count)) { + + mutex_lock(&inode->i_obj_mutex); + /* no new references can be obtained */ + if (!atomic_read(&obj->count)) { + list_del(&obj->list); + fdso_destroy(obj->type, obj->obj); + kfree(obj); + let_go = 1; + } + mutex_unlock(&inode->i_obj_mutex); + if (let_go) + iput(inode); + } +} + +static struct od_table_entry* get_od_entry(struct task_struct* t) +{ + struct od_table_entry* table; + int i; + + + table = t->od_table; + if (!table) { + table = (struct od_table_entry*) + kzalloc(sizeof(struct od_table_entry) * + MAX_OBJECT_DESCRIPTORS, GFP_KERNEL); + t->od_table = table; + } + + for (i = 0; table && i < MAX_OBJECT_DESCRIPTORS; i++) + if (!table[i].used) { + table[i].used = 1; + return table + i; + } + return NULL; +} + +static int put_od_entry(struct od_table_entry* od) +{ + put_inode_obj(od->obj); + od->used = 0; + return 0; +} + +void exit_od_table(struct task_struct* t) +{ + int i; + + if (t->od_table) { + for (i = 0; i < MAX_OBJECT_DESCRIPTORS; i++) + if (t->od_table[i].used) + put_od_entry(t->od_table + i); + kfree(t->od_table); + t->od_table = NULL; + } +} + +static int do_sys_od_open(struct file* file, obj_type_t type, int id, + void* __user config) +{ + int idx = 0, err; + struct inode* inode; + struct inode_obj_id* obj = NULL; + struct od_table_entry* entry; + + inode = file->f_dentry->d_inode; + + entry = get_od_entry(current); + if (!entry) + return -ENOMEM; + + mutex_lock(&inode->i_obj_mutex); + obj = get_inode_obj(inode, type, id); + if (!obj) + obj = alloc_inode_obj(inode, type, id); + if (!obj) { + idx = -ENOMEM; + entry->used = 0; + } else { + entry->obj = obj; + entry->extra = NULL; + idx = entry - current->od_table; + } + + mutex_unlock(&inode->i_obj_mutex); + + err = fdso_open(entry, config); + if (err < 0) { + /* The class rejected the open call. + * We need to clean up and tell user space. + */ + put_od_entry(entry); + idx = err; + } + + return idx; +} + + +struct od_table_entry* __od_lookup(int od) +{ + struct task_struct *t = current; + + if (!t->od_table) + return NULL; + if (od < 0 || od >= MAX_OBJECT_DESCRIPTORS) + return NULL; + if (!t->od_table[od].used) + return NULL; + return t->od_table + od; +} + + +asmlinkage int sys_od_open(int fd, int type, int obj_id, void* __user config) +{ + int ret = 0; + struct file* file; + + /* + 1) get file from fd, get inode from file + 2) lock inode + 3) try to lookup object + 4) if not present create and enqueue object, inc inode refcnt + 5) increment refcnt of object + 6) alloc od_table_entry, setup ptrs + 7) unlock inode + 8) return offset in od_table as OD + */ + + if (type < MIN_OBJ_TYPE || type > MAX_OBJ_TYPE) { + ret = -EINVAL; + goto out; + } + + file = fget(fd); + if (!file) { + ret = -EBADF; + goto out; + } + + ret = do_sys_od_open(file, type, obj_id, config); + + fput(file); + +out: + return ret; +} + + +asmlinkage int sys_od_close(int od) +{ + int ret = -EINVAL; + struct task_struct *t = current; + + if (od < 0 || od >= MAX_OBJECT_DESCRIPTORS) + return ret; + + if (!t->od_table || !t->od_table[od].used) + return ret; + + + /* give the class a chance to reject the close + */ + ret = fdso_close(t->od_table + od); + if (ret == 0) + ret = put_od_entry(t->od_table + od); + + return ret; +} diff --git a/litmus/fmlp.c b/litmus/fmlp.c new file mode 100644 index 0000000..f34eeea --- /dev/null +++ b/litmus/fmlp.c @@ -0,0 +1,262 @@ +/* + * FMLP implementation. + * Much of the code here is borrowed from include/asm-i386/semaphore.h. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include + +#ifdef CONFIG_FMLP + +static void* create_fmlp_semaphore(void) +{ + struct pi_semaphore* sem; + int i; + + sem = kmalloc(sizeof(struct pi_semaphore), GFP_KERNEL); + if (!sem) + return NULL; + atomic_set(&sem->count, 1); + sem->sleepers = 0; + init_waitqueue_head(&sem->wait); + sem->hp.task = NULL; + sem->holder = NULL; + for (i = 0; i < NR_CPUS; i++) + sem->hp.cpu_task[i] = NULL; + return sem; +} + +static int open_fmlp_semaphore(struct od_table_entry* entry, void* __user arg) +{ + if (!fmlp_active()) + return -EBUSY; + return 0; +} + +static void destroy_fmlp_semaphore(void* sem) +{ + /* XXX assert invariants */ + kfree(sem); +} + +struct fdso_ops fmlp_sem_ops = { + .create = create_fmlp_semaphore, + .open = open_fmlp_semaphore, + .destroy = destroy_fmlp_semaphore +}; + +struct wq_pair { + struct task_struct* tsk; + struct pi_semaphore* sem; +}; + +static int rt_pi_wake_up(wait_queue_t *wait, unsigned mode, int sync, + void *key) +{ + struct wq_pair* wqp = (struct wq_pair*) wait->private; + set_rt_flags(wqp->tsk, RT_F_EXIT_SEM); + litmus->inherit_priority(wqp->sem, wqp->tsk); + TRACE_TASK(wqp->tsk, + "woken up by rt_pi_wake_up() (RT_F_SEM_EXIT, PI)\n"); + /* point to task for default_wake_function() */ + wait->private = wqp->tsk; + default_wake_function(wait, mode, sync, key); + + /* Always return true since we know that if we encountered a task + * that was already running the wake_up raced with the schedule in + * rt_pi_down(). In that case the task in rt_pi_down() will be scheduled + * immediately and own the lock. We must not wake up another task in + * any case. + */ + return 1; +} + +/* caller is responsible for locking */ +int edf_set_hp_task(struct pi_semaphore *sem) +{ + struct list_head *tmp, *next; + struct task_struct *queued; + int ret = 0; + + sem->hp.task = NULL; + list_for_each_safe(tmp, next, &sem->wait.task_list) { + queued = ((struct wq_pair*) + list_entry(tmp, wait_queue_t, + task_list)->private)->tsk; + + /* Compare task prios, find high prio task. */ + if (edf_higher_prio(queued, sem->hp.task)) { + sem->hp.task = queued; + ret = 1; + } + } + return ret; +} + +/* caller is responsible for locking */ +int edf_set_hp_cpu_task(struct pi_semaphore *sem, int cpu) +{ + struct list_head *tmp, *next; + struct task_struct *queued; + int ret = 0; + + sem->hp.cpu_task[cpu] = NULL; + list_for_each_safe(tmp, next, &sem->wait.task_list) { + queued = ((struct wq_pair*) + list_entry(tmp, wait_queue_t, + task_list)->private)->tsk; + + /* Compare task prios, find high prio task. */ + if (get_partition(queued) == cpu && + edf_higher_prio(queued, sem->hp.cpu_task[cpu])) { + sem->hp.cpu_task[cpu] = queued; + ret = 1; + } + } + return ret; +} + +static int do_fmlp_down(struct pi_semaphore* sem) +{ + unsigned long flags; + struct task_struct *tsk = current; + struct wq_pair pair; + int suspended = 1; + wait_queue_t wait = { + .private = &pair, + .func = rt_pi_wake_up, + .task_list = {NULL, NULL} + }; + + pair.tsk = tsk; + pair.sem = sem; + spin_lock_irqsave(&sem->wait.lock, flags); + + if (atomic_dec_return(&sem->count) < 0 || + waitqueue_active(&sem->wait)) { + /* we need to suspend */ + tsk->state = TASK_UNINTERRUPTIBLE; + add_wait_queue_exclusive_locked(&sem->wait, &wait); + + TRACE_CUR("suspends on PI lock %p\n", sem); + litmus->pi_block(sem, tsk); + + /* release lock before sleeping */ + spin_unlock_irqrestore(&sem->wait.lock, flags); + + TS_PI_DOWN_END; + preempt_enable_no_resched(); + + + /* we depend on the FIFO order + * Thus, we don't need to recheck when we wake up, we + * are guaranteed to have the lock since there is only one + * wake up per release + */ + schedule(); + + TRACE_CUR("woke up, now owns PI lock %p\n", sem); + + /* try_to_wake_up() set our state to TASK_RUNNING, + * all we need to do is to remove our wait queue entry + */ + remove_wait_queue(&sem->wait, &wait); + } else { + /* no priority inheritance necessary, since there are no queued + * tasks. + */ + suspended = 0; + TRACE_CUR("acquired PI lock %p, no contention\n", sem); + sem->holder = tsk; + sem->hp.task = tsk; + litmus->inherit_priority(sem, tsk); + spin_unlock_irqrestore(&sem->wait.lock, flags); + } + return suspended; +} + +static void do_fmlp_up(struct pi_semaphore* sem) +{ + unsigned long flags; + + spin_lock_irqsave(&sem->wait.lock, flags); + + TRACE_CUR("releases PI lock %p\n", sem); + litmus->return_priority(sem); + sem->holder = NULL; + if (atomic_inc_return(&sem->count) < 1) + /* there is a task queued */ + wake_up_locked(&sem->wait); + + spin_unlock_irqrestore(&sem->wait.lock, flags); +} + +asmlinkage long sys_fmlp_down(int sem_od) +{ + long ret = 0; + struct pi_semaphore * sem; + int suspended = 0; + + preempt_disable(); + TS_PI_DOWN_START; + + sem = lookup_fmlp_sem(sem_od); + if (sem) + suspended = do_fmlp_down(sem); + else + ret = -EINVAL; + + if (!suspended) { + TS_PI_DOWN_END; + preempt_enable(); + } + + return ret; +} + +asmlinkage long sys_fmlp_up(int sem_od) +{ + long ret = 0; + struct pi_semaphore * sem; + + preempt_disable(); + TS_PI_UP_START; + + sem = lookup_fmlp_sem(sem_od); + if (sem) + do_fmlp_up(sem); + else + ret = -EINVAL; + + + TS_PI_UP_END; + preempt_enable(); + + return ret; +} + +#else + +struct fdso_ops fmlp_sem_ops = {}; + +asmlinkage long sys_fmlp_down(int sem_od) +{ + return -ENOSYS; +} + +asmlinkage long sys_fmlp_up(int sem_od) +{ + return -ENOSYS; +} + +#endif diff --git a/litmus/ft_event.c b/litmus/ft_event.c new file mode 100644 index 0000000..6084b6d --- /dev/null +++ b/litmus/ft_event.c @@ -0,0 +1,43 @@ +#include + +#include + +#ifndef __ARCH_HAS_FEATHER_TRACE +/* provide dummy implementation */ + +int ft_events[MAX_EVENTS]; + +int ft_enable_event(unsigned long id) +{ + if (id < MAX_EVENTS) { + ft_events[id]++; + return 1; + } else + return 0; +} + +int ft_disable_event(unsigned long id) +{ + if (id < MAX_EVENTS && ft_events[id]) { + ft_events[id]--; + return 1; + } else + return 0; +} + +int ft_disable_all_events(void) +{ + int i; + + for (i = 0; i < MAX_EVENTS; i++) + ft_events[i] = 0; + + return MAX_EVENTS; +} + +int ft_is_event_enabled(unsigned long id) +{ + return id < MAX_EVENTS && ft_events[id]; +} + +#endif diff --git a/litmus/jobs.c b/litmus/jobs.c new file mode 100644 index 0000000..e294bc5 --- /dev/null +++ b/litmus/jobs.c @@ -0,0 +1,43 @@ +/* litmus/jobs.c - common job control code + */ + +#include + +#include +#include + +void prepare_for_next_period(struct task_struct *t) +{ + BUG_ON(!t); + /* prepare next release */ + t->rt_param.job_params.release = t->rt_param.job_params.deadline; + t->rt_param.job_params.deadline += get_rt_period(t); + t->rt_param.job_params.exec_time = 0; + /* update job sequence number */ + t->rt_param.job_params.job_no++; + + /* don't confuse Linux */ + t->time_slice = 1; +} + +void release_at(struct task_struct *t, lt_t start) +{ + t->rt_param.job_params.deadline = start; + prepare_for_next_period(t); + set_rt_flags(t, RT_F_RUNNING); +} + + +/* + * Deactivate current task until the beginning of the next period. + */ +long complete_job(void) +{ + /* Mark that we do not excute anymore */ + set_rt_flags(current, RT_F_SLEEP); + /* call schedule, this will return when a new job arrives + * it also takes care of preparing for the next release + */ + schedule(); + return 0; +} diff --git a/litmus/litmus.c b/litmus/litmus.c new file mode 100644 index 0000000..979985e --- /dev/null +++ b/litmus/litmus.c @@ -0,0 +1,826 @@ +/* litmus.c -- Implementation of the LITMUS syscalls, the LITMUS intialization code, + * and the procfs interface.. + */ +#include +#include +#include + +#include +#include +#include + +#include +#include +#include + +#include + +#include + +/* Number of RT tasks that exist in the system */ +atomic_t rt_task_count = ATOMIC_INIT(0); +static DEFINE_SPINLOCK(task_transition_lock); + +/* Give log messages sequential IDs. */ +atomic_t __log_seq_no = ATOMIC_INIT(0); + +/* To send signals from the scheduler + * Must drop locks first. + */ +static LIST_HEAD(sched_sig_list); +static DEFINE_SPINLOCK(sched_sig_list_lock); + +static struct kmem_cache * heap_node_cache; + +/* + * sys_set_task_rt_param + * @pid: Pid of the task which scheduling parameters must be changed + * @param: New real-time extension parameters such as the execution cost and + * period + * Syscall for manipulating with task rt extension params + * Returns EFAULT if param is NULL. + * ESRCH if pid is not corrsponding + * to a valid task. + * EINVAL if either period or execution cost is <=0 + * EPERM if pid is a real-time task + * 0 if success + * + * Only non-real-time tasks may be configured with this system call + * to avoid races with the scheduler. In practice, this means that a + * task's parameters must be set _before_ calling sys_prepare_rt_task() + */ +asmlinkage long sys_set_rt_task_param(pid_t pid, struct rt_task __user * param) +{ + struct rt_task tp; + struct task_struct *target; + int retval = -EINVAL; + + printk("Setting up rt task parameters for process %d.\n", pid); + + if (pid < 0 || param == 0) { + goto out; + } + if (copy_from_user(&tp, param, sizeof(tp))) { + retval = -EFAULT; + goto out; + } + + /* Task search and manipulation must be protected */ + read_lock_irq(&tasklist_lock); + if (!(target = find_task_by_pid(pid))) { + retval = -ESRCH; + goto out_unlock; + } + + if (is_realtime(target)) { + /* The task is already a real-time task. + * We cannot not allow parameter changes at this point. + */ + retval = -EBUSY; + goto out_unlock; + } + + if (tp.exec_cost <= 0) + goto out_unlock; + if (tp.period <= 0) + goto out_unlock; + if (!cpu_online(tp.cpu)) + goto out_unlock; + if (tp.period < tp.exec_cost) + { + printk(KERN_INFO "litmus: real-time task %d rejected " + "because wcet > period\n", pid); + goto out_unlock; + } + + target->rt_param.task_params = tp; + + retval = 0; + out_unlock: + read_unlock_irq(&tasklist_lock); + out: + return retval; +} + +/* Getter of task's RT params + * returns EINVAL if param or pid is NULL + * returns ESRCH if pid does not correspond to a valid task + * returns EFAULT if copying of parameters has failed. + */ +asmlinkage long sys_get_rt_task_param(pid_t pid, struct rt_task __user * param) +{ + int retval = -EINVAL; + struct task_struct *source; + struct rt_task lp; + if (param == 0 || pid < 0) + goto out; + read_lock(&tasklist_lock); + if (!(source = find_task_by_pid(pid))) { + retval = -ESRCH; + goto out_unlock; + } + lp = source->rt_param.task_params; + read_unlock(&tasklist_lock); + /* Do copying outside the lock */ + retval = + copy_to_user(param, &lp, sizeof(lp)) ? -EFAULT : 0; + return retval; + out_unlock: + read_unlock(&tasklist_lock); + out: + return retval; + +} + +/* + * This is the crucial function for periodic task implementation, + * It checks if a task is periodic, checks if such kind of sleep + * is permitted and calls plugin-specific sleep, which puts the + * task into a wait array. + * returns 0 on successful wakeup + * returns EPERM if current conditions do not permit such sleep + * returns EINVAL if current task is not able to go to sleep + */ +asmlinkage long sys_complete_job(void) +{ + int retval = -EPERM; + if (!is_realtime(current)) { + retval = -EINVAL; + goto out; + } + /* Task with negative or zero period cannot sleep */ + if (get_rt_period(current) <= 0) { + retval = -EINVAL; + goto out; + } + /* The plugin has to put the task into an + * appropriate queue and call schedule + */ + retval = litmus->complete_job(); + out: + return retval; +} + +/* This is an "improved" version of sys_complete_job that + * addresses the problem of unintentionally missing a job after + * an overrun. + * + * returns 0 on successful wakeup + * returns EPERM if current conditions do not permit such sleep + * returns EINVAL if current task is not able to go to sleep + */ +asmlinkage long sys_wait_for_job_release(unsigned int job) +{ + int retval = -EPERM; + if (!is_realtime(current)) { + retval = -EINVAL; + goto out; + } + + /* Task with negative or zero period cannot sleep */ + if (get_rt_period(current) <= 0) { + retval = -EINVAL; + goto out; + } + + retval = 0; + + /* first wait until we have "reached" the desired job + * + * This implementation has at least two problems: + * + * 1) It doesn't gracefully handle the wrap around of + * job_no. Since LITMUS is a prototype, this is not much + * of a problem right now. + * + * 2) It is theoretically racy if a job release occurs + * between checking job_no and calling sleep_next_period(). + * A proper solution would requiring adding another callback + * in the plugin structure and testing the condition with + * interrupts disabled. + * + * FIXME: At least problem 2 should be taken care of eventually. + */ + while (!retval && job > current->rt_param.job_params.job_no) + /* If the last job overran then job <= job_no and we + * don't send the task to sleep. + */ + retval = litmus->complete_job(); + out: + return retval; +} + +/* This is a helper syscall to query the current job sequence number. + * + * returns 0 on successful query + * returns EPERM if task is not a real-time task. + * returns EFAULT if &job is not a valid pointer. + */ +asmlinkage long sys_query_job_no(unsigned int __user *job) +{ + int retval = -EPERM; + if (is_realtime(current)) + retval = put_user(current->rt_param.job_params.job_no, job); + + return retval; +} + +struct sched_sig { + struct list_head list; + struct task_struct* task; + unsigned int signal:31; + int force:1; +}; + +static void __scheduler_signal(struct task_struct *t, unsigned int signo, + int force) +{ + struct sched_sig* sig; + + sig = kmalloc(GFP_ATOMIC, sizeof(struct sched_sig)); + if (!sig) { + TRACE_TASK(t, "dropping signal: %u\n", t); + return; + } + + spin_lock(&sched_sig_list_lock); + + sig->signal = signo; + sig->force = force; + sig->task = t; + get_task_struct(t); + list_add(&sig->list, &sched_sig_list); + + spin_unlock(&sched_sig_list_lock); +} + +void scheduler_signal(struct task_struct *t, unsigned int signo) +{ + __scheduler_signal(t, signo, 0); +} + +void force_scheduler_signal(struct task_struct *t, unsigned int signo) +{ + __scheduler_signal(t, signo, 1); +} + +/* FIXME: get rid of the locking and do this on a per-processor basis */ +void send_scheduler_signals(void) +{ + unsigned long flags; + struct list_head *p, *extra; + struct siginfo info; + struct sched_sig* sig; + struct task_struct* t; + struct list_head claimed; + + if (spin_trylock_irqsave(&sched_sig_list_lock, flags)) { + if (list_empty(&sched_sig_list)) + p = NULL; + else { + p = sched_sig_list.next; + list_del(&sched_sig_list); + INIT_LIST_HEAD(&sched_sig_list); + } + spin_unlock_irqrestore(&sched_sig_list_lock, flags); + + /* abort if there are no signals */ + if (!p) + return; + + /* take signal list we just obtained */ + list_add(&claimed, p); + + list_for_each_safe(p, extra, &claimed) { + list_del(p); + sig = list_entry(p, struct sched_sig, list); + t = sig->task; + info.si_signo = sig->signal; + info.si_errno = 0; + info.si_code = SI_KERNEL; + info.si_pid = 1; + info.si_uid = 0; + TRACE("sending signal %d to %d\n", info.si_signo, + t->pid); + if (sig->force) + force_sig_info(sig->signal, &info, t); + else + send_sig_info(sig->signal, &info, t); + put_task_struct(t); + kfree(sig); + } + } + +} + +#ifdef CONFIG_NP_SECTION + +static inline void np_mem_error(struct task_struct* t, const char* reason) +{ + if (t->state != TASK_DEAD && !(t->flags & PF_EXITING)) { + TRACE("np section: %s => %s/%d killed\n", + reason, t->comm, t->pid); + force_scheduler_signal(t, SIGKILL); + } +} + +/* sys_register_np_flag() allows real-time tasks to register an + * np section indicator. + * returns 0 if the flag was successfully registered + * returns EINVAL if current task is not a real-time task + * returns EFAULT if *flag couldn't be written + */ +asmlinkage long sys_register_np_flag(short __user *flag) +{ + int retval = -EINVAL; + short test_val = RT_PREEMPTIVE; + + /* avoid races with the scheduler */ + preempt_disable(); + TRACE("reg_np_flag(%p) for %s/%d\n", flag, + current->comm, current->pid); + + /* Let's first try to write to the address. + * That way it is initialized and any bugs + * involving dangling pointers will caught + * early. + * NULL indicates disabling np section support + * and should not be tested. + */ + if (flag) + retval = poke_kernel_address(test_val, flag); + else + retval = 0; + TRACE("reg_np_flag: retval=%d\n", retval); + if (unlikely(0 != retval)) + np_mem_error(current, "np flag: not writable"); + else + /* the pointer is ok */ + current->rt_param.np_flag = flag; + + preempt_enable(); + return retval; +} + + +void request_exit_np(struct task_struct *t) +{ + int ret; + short flag; + + /* We can only do this if t is actually currently scheduled on this CPU + * because otherwise we are in the wrong address space. Thus make sure + * to check. + */ + BUG_ON(t != current); + + if (unlikely(!is_realtime(t) || !t->rt_param.np_flag)) { + TRACE_TASK(t, "request_exit_np(): BAD TASK!\n"); + return; + } + + flag = RT_EXIT_NP_REQUESTED; + ret = poke_kernel_address(flag, t->rt_param.np_flag + 1); + TRACE("request_exit_np(%s/%d)\n", t->comm, t->pid); + if (unlikely(0 != ret)) + np_mem_error(current, "request_exit_np(): flag not writable"); + +} + + +int is_np(struct task_struct* t) +{ + int ret; + unsigned short flag = 0x5858; /* = XX, looks nicer in debug*/ + + BUG_ON(t != current); + + if (unlikely(t->rt_param.kernel_np)) + return 1; + else if (unlikely(t->rt_param.np_flag == NULL) || + t->flags & PF_EXITING || + t->state == TASK_DEAD) + return 0; + else { + /* This is the tricky part. The process has registered a + * non-preemptive section marker. We now need to check whether + * it is set to to NON_PREEMPTIVE. Along the way we could + * discover that the pointer points to an unmapped region (=> + * kill the task) or that the location contains some garbage + * value (=> also kill the task). Killing the task in any case + * forces userspace to play nicely. Any bugs will be discovered + * immediately. + */ + ret = probe_kernel_address(t->rt_param.np_flag, flag); + if (0 == ret && (flag == RT_NON_PREEMPTIVE || + flag == RT_PREEMPTIVE)) + return flag != RT_PREEMPTIVE; + else { + /* either we could not read from the address or + * it contained garbage => kill the process + * FIXME: Should we cause a SEGFAULT instead? + */ + TRACE("is_np: ret=%d flag=%c%c (%x)\n", ret, + flag & 0xff, (flag >> 8) & 0xff, flag); + np_mem_error(t, "is_np() could not read"); + return 0; + } + } +} + +/* + * sys_exit_np() allows real-time tasks to signal that it left a + * non-preemptable section. It will be called after the kernel requested a + * callback in the preemption indicator flag. + * returns 0 if the signal was valid and processed. + * returns EINVAL if current task is not a real-time task + */ +asmlinkage long sys_exit_np(void) +{ + int retval = -EINVAL; + + TS_EXIT_NP_START; + + if (!is_realtime(current)) + goto out; + + TRACE("sys_exit_np(%s/%d)\n", current->comm, current->pid); + /* force rescheduling so that we can be preempted */ + set_tsk_need_resched(current); + retval = 0; + out: + + TS_EXIT_NP_END; + return retval; +} + +#else /* !CONFIG_NP_SECTION */ + +asmlinkage long sys_register_np_flag(short __user *flag) +{ + return -ENOSYS; +} + +asmlinkage long sys_exit_np(void) +{ + return -ENOSYS; +} + +#endif /* CONFIG_NP_SECTION */ + + +/* p is a real-time task. Re-init its state as a best-effort task. */ +static void reinit_litmus_state(struct task_struct* p, int restore) +{ + struct rt_task user_config = {}; + __user short *np_flag = NULL; + + if (restore) { + /* Safe user-space provided configuration data. */ + user_config = p->rt_param.task_params; + np_flag = p->rt_param.np_flag; + } + + /* We probably should not be inheriting any task's priority + * at this point in time. + */ + WARN_ON(p->rt_param.inh_task); + + /* We need to restore the priority of the task. */ +// __setscheduler(p, p->rt_param.old_policy, p->rt_param.old_prio); + + /* Cleanup everything else. */ + memset(&p->rt_param, 0, sizeof(struct rt_task)); + + /* Restore preserved fields. */ + if (restore) { + p->rt_param.task_params = user_config; + p->rt_param.np_flag = np_flag; + } +} + +long litmus_admit_task(struct task_struct* tsk) +{ + long retval; + long flags; + + BUG_ON(is_realtime(tsk)); + + if (get_rt_period(tsk) == 0 || + get_exec_cost(tsk) > get_rt_period(tsk)) { + TRACE_TASK(tsk, "litmus admit: invalid task parameters " + "(%lu, %lu)\n", + get_exec_cost(tsk), get_rt_period(tsk)); + return -EINVAL; + } + + if (!cpu_online(get_partition(tsk))) + { + TRACE_TASK(tsk, "litmus admit: cpu %d is not online\n", + get_partition(tsk)); + return -EINVAL; + } + + INIT_LIST_HEAD(&tsk_rt(tsk)->list); + + /* avoid scheduler plugin changing underneath us */ + spin_lock_irqsave(&task_transition_lock, flags); + retval = litmus->admit_task(tsk); + + /* allocate heap node for this task */ + tsk_rt(tsk)->heap_node = kmem_cache_alloc(heap_node_cache, GFP_ATOMIC); + if (!tsk_rt(tsk)->heap_node) + retval = -ENOMEM; + else + heap_node_init(&tsk_rt(tsk)->heap_node, tsk); + + if (!retval) + atomic_inc(&rt_task_count); + + spin_unlock_irqrestore(&task_transition_lock, flags); + + return retval; + +} + +void litmus_exit_task(struct task_struct* tsk) +{ + if (is_realtime(tsk)) { + litmus->task_exit(tsk); + BUG_ON(heap_node_in_heap(tsk_rt(tsk)->heap_node)); + kmem_cache_free(heap_node_cache, tsk_rt(tsk)->heap_node); + atomic_dec(&rt_task_count); + reinit_litmus_state(tsk, 1); + } +} + +/* Switching a plugin in use is tricky. + * We must watch out that no real-time tasks exists + * (and that none is created in parallel) and that the plugin is not + * currently in use on any processor (in theory). + * + * For now, we don't enforce the second part since it is unlikely to cause + * any trouble by itself as long as we don't unload modules. + */ +int switch_sched_plugin(struct sched_plugin* plugin) +{ + long flags; + int ret = 0; + + BUG_ON(!plugin); + + /* stop task transitions */ + spin_lock_irqsave(&task_transition_lock, flags); + + /* don't switch if there are active real-time tasks */ + if (atomic_read(&rt_task_count) == 0) { + printk(KERN_INFO "Switching to LITMUS^RT plugin %s.\n", plugin->plugin_name); + litmus = plugin; + } else + ret = -EBUSY; + + spin_unlock_irqrestore(&task_transition_lock, flags); + return ret; +} + +/* Called upon fork. + * p is the newly forked task. + */ +void litmus_fork(struct task_struct* p) +{ + if (is_realtime(p)) + /* clean out any litmus related state, don't preserve anything*/ + reinit_litmus_state(p, 0); +} + +/* Called upon execve(). + * current is doing the exec. + * Don't let address space specific stuff leak. + */ +void litmus_exec(void) +{ + struct task_struct* p = current; + + if (is_realtime(p)) { + WARN_ON(p->rt_param.inh_task); + p->rt_param.np_flag = NULL; + } +} + +void exit_litmus(struct task_struct *dead_tsk) +{ + if (is_realtime(dead_tsk)) + litmus_exit_task(dead_tsk); +} + + +void list_qsort(struct list_head* list, list_cmp_t less_than) +{ + struct list_head lt; + struct list_head geq; + struct list_head *pos, *extra, *pivot; + int n_lt = 0, n_geq = 0; + BUG_ON(!list); + + if (list->next == list) + return; + + INIT_LIST_HEAD(<); + INIT_LIST_HEAD(&geq); + + pivot = list->next; + list_del(pivot); + list_for_each_safe(pos, extra, list) { + list_del(pos); + if (less_than(pos, pivot)) { + list_add(pos, <); + n_lt++; + } else { + list_add(pos, &geq); + n_geq++; + } + } + if (n_lt < n_geq) { + list_qsort(<, less_than); + list_qsort(&geq, less_than); + } else { + list_qsort(&geq, less_than); + list_qsort(<, less_than); + } + list_splice(&geq, list); + list_add(pivot, list); + list_splice(<, list); +} + +#ifdef CONFIG_MAGIC_SYSRQ +int sys_kill(int pid, int sig); + +static void sysrq_handle_kill_rt_tasks(int key, struct tty_struct *tty) +{ + struct task_struct *t; + read_lock(&tasklist_lock); + for_each_process(t) { + if (is_realtime(t)) { + sys_kill(t->pid, SIGKILL); + } + } + read_unlock(&tasklist_lock); +} + +static struct sysrq_key_op sysrq_kill_rt_tasks_op = { + .handler = sysrq_handle_kill_rt_tasks, + .help_msg = "Quit-rt-tasks", + .action_msg = "sent SIGKILL to all real-time tasks", +}; +#endif + +static int proc_read_stats(char *page, char **start, + off_t off, int count, + int *eof, void *data) +{ + int len; + + len = snprintf(page, PAGE_SIZE, + "real-time task count = %d\n", + atomic_read(&rt_task_count)); + return len; +} + +static int proc_read_plugins(char *page, char **start, + off_t off, int count, + int *eof, void *data) +{ + int len; + + len = print_sched_plugins(page, PAGE_SIZE); + return len; +} + +static int proc_read_curr(char *page, char **start, + off_t off, int count, + int *eof, void *data) +{ + int len; + + len = snprintf(page, PAGE_SIZE, "%s\n", litmus->plugin_name); + return len; +} + +static int proc_write_curr(struct file *file, + const char *buffer, + unsigned long count, + void *data) +{ + int len, ret; + char name[65]; + struct sched_plugin* found; + + if(count > 64) + len = 64; + else + len = count; + + if(copy_from_user(name, buffer, len)) + return -EFAULT; + + name[len] = '\0'; + /* chomp name */ + if (len > 1 && name[len - 1] == '\n') + name[len - 1] = '\0'; + + found = find_sched_plugin(name); + + if (found) { + ret = switch_sched_plugin(found); + if (ret != 0) + printk(KERN_INFO "Could not switch plugin: %d\n", ret); + } else + printk(KERN_INFO "Plugin '%s' is unknown.\n", name); + + return len; +} + + +static struct proc_dir_entry *litmus_dir = NULL, + *curr_file = NULL, + *stat_file = NULL, + *plugs_file = NULL; + +static int __init init_litmus_proc(void) +{ + litmus_dir = proc_mkdir("litmus", NULL); + if (!litmus_dir) { + printk(KERN_ERR "Could not allocate LITMUS^RT procfs entry.\n"); + return -ENOMEM; + } + litmus_dir->owner = THIS_MODULE; + + curr_file = create_proc_entry("active_plugin", + 0644, litmus_dir); + if (!curr_file) { + printk(KERN_ERR "Could not allocate active_plugin " + "procfs entry.\n"); + return -ENOMEM; + } + curr_file->owner = THIS_MODULE; + curr_file->read_proc = proc_read_curr; + curr_file->write_proc = proc_write_curr; + + stat_file = create_proc_read_entry("stats", 0444, litmus_dir, + proc_read_stats, NULL); + + plugs_file = create_proc_read_entry("plugins", 0444, litmus_dir, + proc_read_plugins, NULL); + + return 0; +} + +static void exit_litmus_proc(void) +{ + if (plugs_file) + remove_proc_entry("plugins", litmus_dir); + if (stat_file) + remove_proc_entry("stats", litmus_dir); + if (curr_file) + remove_proc_entry("active_plugin", litmus_dir); + if (litmus_dir) + remove_proc_entry("litmus", NULL); +} + +extern struct sched_plugin linux_sched_plugin; + +static int __init _init_litmus(void) +{ + /* Common initializers, + * mode change lock is used to enforce single mode change + * operation. + */ + printk("Starting LITMUS^RT kernel\n"); + + register_sched_plugin(&linux_sched_plugin); + + heap_node_cache = KMEM_CACHE(heap_node, 0); + if (!heap_node_cache) + return -ENOMEM; + +#ifdef CONFIG_MAGIC_SYSRQ + /* offer some debugging help */ + if (!register_sysrq_key('q', &sysrq_kill_rt_tasks_op)) + printk("Registered kill rt tasks magic sysrq.\n"); + else + printk("Could not register kill rt tasks magic sysrq.\n"); +#endif + + init_litmus_proc(); + + return 0; +} + +static void _exit_litmus(void) +{ + exit_litmus_proc(); + kmem_cache_destroy(heap_node_cache); +} + +module_init(_init_litmus); +module_exit(_exit_litmus); diff --git a/litmus/norqlock.c b/litmus/norqlock.c new file mode 100644 index 0000000..11f85d3 --- /dev/null +++ b/litmus/norqlock.c @@ -0,0 +1,56 @@ +#include +#include +#include +#include +#include + +#include + +struct worklist { + struct no_rqlock_work* next; +}; + +static DEFINE_PER_CPU(struct worklist, norq_worklist) = {NULL}; + +void init_no_rqlock_work(struct no_rqlock_work* w, work_t work, + unsigned long arg) +{ + w->active = 0; + w->work = work; + w->arg = arg; + w->next = NULL; +} + +void __do_without_rqlock(struct no_rqlock_work *work) +{ + long flags; + struct worklist* wl; + + local_irq_save(flags); + wl = &__get_cpu_var(norq_worklist); + work->next = wl->next; + wl->next = work; + local_irq_restore(flags); +} + +void tick_no_rqlock(void) +{ + long flags; + struct no_rqlock_work *todo, *next; + + local_irq_save(flags); + + next = __get_cpu_var(norq_worklist).next; + __get_cpu_var(norq_worklist).next = NULL; + + while (next) { + todo = next; + next = next->next; + todo->next = NULL; + smp_mb__before_clear_bit(); + clear_bit(0, (void*) &todo->active); + todo->work(todo->arg); + } + + local_irq_restore(flags); +} diff --git a/litmus/rt_domain.c b/litmus/rt_domain.c new file mode 100644 index 0000000..2880308 --- /dev/null +++ b/litmus/rt_domain.c @@ -0,0 +1,138 @@ +/* + * kernel/rt_domain.c + * + * LITMUS real-time infrastructure. This file contains the + * functions that manipulate RT domains. RT domains are an abstraction + * of a ready queue and a release queue. + */ + +#include +#include +#include + +#include +#include +#include + +#include + +#include + +static int dummy_resched(rt_domain_t *rt) +{ + return 0; +} + +static int dummy_order(struct heap_node* a, struct heap_node* b) +{ + return 0; +} + +/* default implementation: use default lock */ +static void default_release_job(struct task_struct* t, rt_domain_t* rt) +{ + add_ready(rt, t); +} + +static enum hrtimer_restart release_job_timer(struct hrtimer *timer) +{ + struct task_struct *t; + + TS_RELEASE_START; + + t = container_of(timer, struct task_struct, + rt_param.release_timer); + + get_domain(t)->release_job(t, get_domain(t)); + + TS_RELEASE_END; + + return HRTIMER_NORESTART; +} + +static void setup_job_release_timer(struct task_struct *task) +{ + hrtimer_init(&release_timer(task), CLOCK_MONOTONIC, HRTIMER_MODE_ABS); + release_timer(task).function = release_job_timer; +#ifdef CONFIG_HIGH_RES_TIMERS + release_timer(task).cb_mode = HRTIMER_CB_IRQSAFE_NO_RESTART; +#endif + /* Expiration time of timer is release time of task. */ + release_timer(task).expires = ns_to_ktime(get_release(task)); + + TRACE_TASK(task, "arming release timer rel=%llu at %llu\n", + get_release(task), litmus_clock()); + + hrtimer_start(&release_timer(task), release_timer(task).expires, + HRTIMER_MODE_ABS); +} + +static void arm_release_timers(unsigned long _rt) +{ + rt_domain_t *rt = (rt_domain_t*) _rt; + unsigned long flags; + struct list_head alt; + struct list_head *pos, *safe; + struct task_struct* t; + + spin_lock_irqsave(&rt->release_lock, flags); + list_replace_init(&rt->release_queue, &alt); + spin_unlock_irqrestore(&rt->release_lock, flags); + + list_for_each_safe(pos, safe, &alt) { + t = list_entry(pos, struct task_struct, rt_param.list); + list_del(pos); + setup_job_release_timer(t); + } +} + + +void rt_domain_init(rt_domain_t *rt, + heap_prio_t order, + check_resched_needed_t check, + release_job_t release + ) +{ + BUG_ON(!rt); + if (!check) + check = dummy_resched; + if (!release) + release = default_release_job; + if (!order) + order = dummy_order; + heap_init(&rt->ready_queue); + INIT_LIST_HEAD(&rt->release_queue); + spin_lock_init(&rt->ready_lock); + spin_lock_init(&rt->release_lock); + rt->check_resched = check; + rt->release_job = release; + rt->order = order; + init_no_rqlock_work(&rt->arm_timers, arm_release_timers, (unsigned long) rt); +} + +/* add_ready - add a real-time task to the rt ready queue. It must be runnable. + * @new: the newly released task + */ +void __add_ready(rt_domain_t* rt, struct task_struct *new) +{ + TRACE("rt: adding %s/%d (%llu, %llu) rel=%llu to ready queue at %llu\n", + new->comm, new->pid, get_exec_cost(new), get_rt_period(new), + get_release(new), litmus_clock()); + + BUG_ON(heap_node_in_heap(tsk_rt(new)->heap_node)); + + heap_insert(rt->order, &rt->ready_queue, tsk_rt(new)->heap_node); + rt->check_resched(rt); +} + +/* add_release - add a real-time task to the rt release queue. + * @task: the sleeping task + */ +void __add_release(rt_domain_t* rt, struct task_struct *task) +{ + TRACE_TASK(task, "add_release(), rel=%llu\n", get_release(task)); + list_add(&tsk_rt(task)->list, &rt->release_queue); + task->rt_param.domain = rt; + do_without_rqlock(&rt->arm_timers); +} + diff --git a/litmus/sched_cedf.c b/litmus/sched_cedf.c new file mode 100755 index 0000000..2ac14cd --- /dev/null +++ b/litmus/sched_cedf.c @@ -0,0 +1,717 @@ +/* + * kernel/sched_cedf.c + * + * Implementation of the Clustered EDF (C-EDF) scheduling algorithm. + * Linking is included so that support for synchronization (e.g., through + * the implementation of a "CSN-EDF" algorithm) can be added later if desired. + * + * This version uses the simple approach and serializes all scheduling + * decisions by the use of a queue lock. This is probably not the + * best way to do it, but it should suffice for now. + */ + +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include + +/* Overview of C-EDF operations. + * + * link_task_to_cpu(T, cpu) - Low-level operation to update the linkage + * structure (NOT the actually scheduled + * task). If there is another linked task To + * already it will set To->linked_on = NO_CPU + * (thereby removing its association with this + * CPU). However, it will not requeue the + * previously linked task (if any). It will set + * T's state to RT_F_RUNNING and check whether + * it is already running somewhere else. If T + * is scheduled somewhere else it will link + * it to that CPU instead (and pull the linked + * task to cpu). T may be NULL. + * + * unlink(T) - Unlink removes T from all scheduler data + * structures. If it is linked to some CPU it + * will link NULL to that CPU. If it is + * currently queued in the cedf queue for + * a partition, it will be removed from + * the rt_domain. It is safe to call + * unlink(T) if T is not linked. T may not + * be NULL. + * + * requeue(T) - Requeue will insert T into the appropriate + * queue. If the system is in real-time mode and + * the T is released already, it will go into the + * ready queue. If the system is not in + * real-time mode is T, then T will go into the + * release queue. If T's release time is in the + * future, it will go into the release + * queue. That means that T's release time/job + * no/etc. has to be updated before requeue(T) is + * called. It is not safe to call requeue(T) + * when T is already queued. T may not be NULL. + * + * cedf_job_arrival(T) - This is the catch-all function when T enters + * the system after either a suspension or at a + * job release. It will queue T (which means it + * is not safe to call cedf_job_arrival(T) if + * T is already queued) and then check whether a + * preemption is necessary. If a preemption is + * necessary it will update the linkage + * accordingly and cause scheduled to be called + * (either with an IPI or need_resched). It is + * safe to call cedf_job_arrival(T) if T's + * next job has not been actually released yet + * (release time in the future). T will be put + * on the release queue in that case. + * + * job_completion(T) - Take care of everything that needs to be done + * to prepare T for its next release and place + * it in the right queue with + * cedf_job_arrival(). + * + * + * When we now that T is linked to CPU then link_task_to_cpu(NULL, CPU) is + * equivalent to unlink(T). Note that if you unlink a task from a CPU none of + * the functions will automatically propagate pending task from the ready queue + * to a linked task. This is the job of the calling function ( by means of + * __take_ready). + */ + +/* cpu_entry_t - maintain the linked and scheduled state + */ +typedef struct { + int cpu; + struct task_struct* linked; /* only RT tasks */ + struct task_struct* scheduled; /* only RT tasks */ + struct list_head list; + atomic_t will_schedule; /* prevent unneeded IPIs */ +} cpu_entry_t; +DEFINE_PER_CPU(cpu_entry_t, cedf_cpu_entries); + +cpu_entry_t* cedf_cpu_entries_array[NR_CPUS]; + +#define set_will_schedule() \ + (atomic_set(&__get_cpu_var(cedf_cpu_entries).will_schedule, 1)) +#define clear_will_schedule() \ + (atomic_set(&__get_cpu_var(cedf_cpu_entries).will_schedule, 0)) +#define test_will_schedule(cpu) \ + (atomic_read(&per_cpu(cedf_cpu_entries, cpu).will_schedule)) + +#define NO_CPU 0xffffffff + +/* Cluster size -- currently four. This is a variable to allow for + * the possibility of changing the cluster size online in the future. + */ +int cluster_size = 4; + +typedef struct { + rt_domain_t domain; + int first_cpu; + int last_cpu; + + /* the cpus queue themselves according to priority in here */ + struct list_head cedf_cpu_queue; + + /* per-partition spinlock: protects the domain and + * serializes scheduling decisions + */ +#define slock domain.ready_lock +} cedf_domain_t; + +DEFINE_PER_CPU(cedf_domain_t*, cedf_domains) = NULL; + +cedf_domain_t* cedf_domains_array[NR_CPUS]; + + +/* These are defined similarly to partitioning, except that a + * tasks partition is any cpu of the cluster to which it + * is assigned, typically the lowest-numbered cpu. + */ +#define local_edf (&__get_cpu_var(cedf_domains)->domain) +#define local_cedf __get_cpu_var(cedf_domains) +#define remote_edf(cpu) (&per_cpu(cedf_domains, cpu)->domain) +#define remote_cedf(cpu) per_cpu(cedf_domains, cpu) +#define task_edf(task) remote_edf(get_partition(task)) +#define task_cedf(task) remote_cedf(get_partition(task)) + +/* update_cpu_position - Move the cpu entry to the correct place to maintain + * order in the cpu queue. Caller must hold cedf lock. + * + * This really should be a heap. + */ +static void update_cpu_position(cpu_entry_t *entry) +{ + cpu_entry_t *other; + struct list_head *cedf_cpu_queue = + &(remote_cedf(entry->cpu))->cedf_cpu_queue; + struct list_head *pos; + + BUG_ON(!cedf_cpu_queue); + + if (likely(in_list(&entry->list))) + list_del(&entry->list); + /* if we do not execute real-time jobs we just move + * to the end of the queue + */ + if (entry->linked) { + list_for_each(pos, cedf_cpu_queue) { + other = list_entry(pos, cpu_entry_t, list); + if (edf_higher_prio(entry->linked, other->linked)) { + __list_add(&entry->list, pos->prev, pos); + return; + } + } + } + /* if we get this far we have the lowest priority job */ + list_add_tail(&entry->list, cedf_cpu_queue); +} + +/* link_task_to_cpu - Update the link of a CPU. + * Handles the case where the to-be-linked task is already + * scheduled on a different CPU. + */ +static noinline void link_task_to_cpu(struct task_struct* linked, + cpu_entry_t *entry) +{ + cpu_entry_t *sched; + struct task_struct* tmp; + int on_cpu; + + BUG_ON(linked && !is_realtime(linked)); + + /* Cannot link task to a CPU that doesn't belong to its partition... */ + BUG_ON(linked && remote_cedf(entry->cpu) != task_cedf(linked)); + + /* Currently linked task is set to be unlinked. */ + if (entry->linked) { + entry->linked->rt_param.linked_on = NO_CPU; + } + + /* Link new task to CPU. */ + if (linked) { + set_rt_flags(linked, RT_F_RUNNING); + /* handle task is already scheduled somewhere! */ + on_cpu = linked->rt_param.scheduled_on; + if (on_cpu != NO_CPU) { + sched = &per_cpu(cedf_cpu_entries, on_cpu); + /* this should only happen if not linked already */ + BUG_ON(sched->linked == linked); + + /* If we are already scheduled on the CPU to which we + * wanted to link, we don't need to do the swap -- + * we just link ourselves to the CPU and depend on + * the caller to get things right. + */ + if (entry != sched) { + tmp = sched->linked; + linked->rt_param.linked_on = sched->cpu; + sched->linked = linked; + update_cpu_position(sched); + linked = tmp; + } + } + if (linked) /* might be NULL due to swap */ + linked->rt_param.linked_on = entry->cpu; + } + entry->linked = linked; + + if (entry->linked) + TRACE_TASK(entry->linked, "linked to CPU %d, state:%d\n", + entry->cpu, entry->linked->state); + else + TRACE("NULL linked to CPU %d\n", entry->cpu); + + update_cpu_position(entry); +} + +/* unlink - Make sure a task is not linked any longer to an entry + * where it was linked before. Must hold cedf_lock. + */ +static noinline void unlink(struct task_struct* t) +{ + cpu_entry_t *entry; + + if (unlikely(!t)) { + TRACE_BUG_ON(!t); + return; + } + + if (t->rt_param.linked_on != NO_CPU) { + /* unlink */ + entry = &per_cpu(cedf_cpu_entries, t->rt_param.linked_on); + t->rt_param.linked_on = NO_CPU; + link_task_to_cpu(NULL, entry); + } else if (is_queued(t)) { + /* This is an interesting situation: t is scheduled, + * but was just recently unlinked. It cannot be + * linked anywhere else (because then it would have + * been relinked to this CPU), thus it must be in some + * queue. We must remove it from the list in this + * case. + */ + remove(task_edf(t), t); + } +} + + +/* preempt - force a CPU to reschedule + */ +static noinline void preempt(cpu_entry_t *entry) +{ + /* We cannot make the is_np() decision here if it is a remote CPU + * because requesting exit_np() requires that we currently use the + * address space of the task. Thus, in the remote case we just send + * the IPI and let schedule() handle the problem. + */ + + if (smp_processor_id() == entry->cpu) { + if (entry->scheduled && is_np(entry->scheduled)) + request_exit_np(entry->scheduled); + else + set_tsk_need_resched(current); + } else + /* in case that it is a remote CPU we have to defer the + * the decision to the remote CPU + * FIXME: We could save a few IPI's here if we leave the flag + * set when we are waiting for a np_exit(). + */ + if (!test_will_schedule(entry->cpu)) + smp_send_reschedule(entry->cpu); +} + +/* requeue - Put an unlinked task into c-edf domain. + * Caller must hold cedf_lock. + */ +static noinline void requeue(struct task_struct* task) +{ + cedf_domain_t* cedf; + rt_domain_t* edf; + + BUG_ON(!task); + /* sanity check rt_list before insertion */ + BUG_ON(is_queued(task)); + + /* Get correct real-time domain. */ + cedf = task_cedf(task); + edf = &cedf->domain; + + if (get_rt_flags(task) == RT_F_SLEEP) { + /* this task has expired + * _schedule has already taken care of updating + * the release and + * deadline. We just must check if it has been released. + */ + if (is_released(task, litmus_clock())) + __add_ready(edf, task); + else { + /* it has got to wait */ + add_release(edf, task); + } + + } else + /* this is a forced preemption + * thus the task stays in the ready_queue + * we only must make it available to others + */ + __add_ready(edf, task); +} + +/* cedf_job_arrival: task is either resumed or released */ +static noinline void cedf_job_arrival(struct task_struct* task) +{ + cpu_entry_t* last; + cedf_domain_t* cedf; + rt_domain_t* edf; + struct list_head *cedf_cpu_queue; + + BUG_ON(!task); + + /* Get correct real-time domain. */ + cedf = task_cedf(task); + edf = &cedf->domain; + cedf_cpu_queue = &cedf->cedf_cpu_queue; + + BUG_ON(!cedf); + BUG_ON(!edf); + BUG_ON(!cedf_cpu_queue); + BUG_ON(list_empty(cedf_cpu_queue)); + + /* first queue arriving job */ + requeue(task); + + /* then check for any necessary preemptions */ + last = list_entry(cedf_cpu_queue->prev, cpu_entry_t, list); + if (edf_preemption_needed(edf, last->linked)) { + /* preemption necessary */ + task = __take_ready(edf); + TRACE("job_arrival: task %d linked to %d, state:%d\n", + task->pid, last->cpu, task->state); + if (last->linked) + requeue(last->linked); + + link_task_to_cpu(task, last); + preempt(last); + } +} + +/* check for current job releases */ +static void cedf_job_release(struct task_struct* t, rt_domain_t* _) +{ + cedf_domain_t* cedf = task_cedf(t); + unsigned long flags; + + BUG_ON(!t); + BUG_ON(!cedf); + + spin_lock_irqsave(&cedf->slock, flags); + sched_trace_job_release(queued); + cedf_job_arrival(t); + spin_unlock_irqrestore(&cedf->slock, flags); +} + +/* cedf_tick - this function is called for every local timer + * interrupt. + * + * checks whether the current task has expired and checks + * whether we need to preempt it if it has not expired + */ +static void cedf_tick(struct task_struct* t) +{ + BUG_ON(!t); + + if (is_realtime(t) && budget_exhausted(t)) { + if (!is_np(t)) { + /* np tasks will be preempted when they become + * preemptable again + */ + set_tsk_need_resched(t); + set_will_schedule(); + TRACE("cedf_scheduler_tick: " + "%d is preemptable (state:%d) " + " => FORCE_RESCHED\n", t->pid, t->state); + } else { + TRACE("cedf_scheduler_tick: " + "%d is non-preemptable (state:%d), " + "preemption delayed.\n", t->pid, t->state); + request_exit_np(t); + } + } +} + +/* caller holds cedf_lock */ +static noinline void job_completion(struct task_struct *t) +{ + BUG_ON(!t); + + sched_trace_job_completion(t); + + TRACE_TASK(t, "job_completion(). [state:%d]\n", t->state); + + /* set flags */ + set_rt_flags(t, RT_F_SLEEP); + /* prepare for next period */ + prepare_for_next_period(t); + /* unlink */ + unlink(t); + /* requeue + * But don't requeue a blocking task. */ + if (is_running(t)) + cedf_job_arrival(t); +} + +/* Getting schedule() right is a bit tricky. schedule() may not make any + * assumptions on the state of the current task since it may be called for a + * number of reasons. The reasons include a scheduler_tick() determined that it + * was necessary, because sys_exit_np() was called, because some Linux + * subsystem determined so, or even (in t