diff --git a/arch/x86/kernel/syscall_table_32.S b/arch/x86/kernel/syscall_table_32.S index 3770290..57d5b3e 100644 --- a/arch/x86/kernel/syscall_table_32.S +++ b/arch/x86/kernel/syscall_table_32.S @@ -352,3 +352,4 @@ ENTRY(sys_call_table) .long sys_wait_for_ts_release .long sys_release_ts .long sys_null_call + .long sys_set_rt_task_mc_param diff --git a/include/linux/hrtimer.h b/include/linux/hrtimer.h index 76da541..0315d04 100644 --- a/include/linux/hrtimer.h +++ b/include/linux/hrtimer.h @@ -378,6 +378,8 @@ extern void hrtimer_start_on_info_init(struct hrtimer_start_on_info *info); extern int hrtimer_start_on(int cpu, struct hrtimer_start_on_info *info, struct hrtimer *timer, ktime_t time, const enum hrtimer_mode mode); +extern int hrtimer_pull_cancel(int cpu, struct hrtimer *timer, + struct hrtimer_start_on_info *info); #endif extern int hrtimer_cancel(struct hrtimer *timer); diff --git a/include/litmus/budget.h b/include/litmus/budget.h index 732530e..ff18d89e 100644 --- a/include/litmus/budget.h +++ b/include/litmus/budget.h @@ -1,8 +1,35 @@ #ifndef _LITMUS_BUDGET_H_ #define _LITMUS_BUDGET_H_ -/* Update the per-processor enforcement timer (arm/reproram/cancel) for - * the next task. */ +/** + * update_enforcement_timer() - Update per-processor enforcement timer for + * the next scheduled task. + * + * If @t is not NULL and has a precisely enforced budget, the timer will be + * armed to trigger a reschedule when the budget is exhausted. Otherwise, + * the timer will be cancelled. +*/ void update_enforcement_timer(struct task_struct* t); +/* True if a task's server has progressed farther than the task + * itself. This happens when budget enforcement has caused a task to be + * booted off until the next period. + */ +#define behind_server(t)\ + (lt_before((t)->rt_param.job_params.real_release, get_release(t))) + +/** + * server_release() - Prepare the task server parameters for the next period. + * The server for @t is what is actually executed from the schedulers + * perspective. + */ +void server_release(struct task_struct *t); + +/** + * task_release() - Prepare actual task parameters for the next period. + * The actual task parameters for @t, real_deadline and real_release, are + * the deadline and release from the tasks perspective. We only record these + * so that we can write them to feather trace. + */ +void task_release(struct task_struct *t); #endif diff --git a/include/litmus/ce_domain.h b/include/litmus/ce_domain.h new file mode 100644 index 0000000..5d5fdf7 --- /dev/null +++ b/include/litmus/ce_domain.h @@ -0,0 +1,27 @@ +#ifndef _LITMUS_CE_DOMAIN_H +#define _LITMUS_CE_DOMAIN_H + +/* + * Functions that the MC plugin needs to call through a domain pointer. + */ +void ce_requeue(domain_t*, struct task_struct*); +struct task_struct* ce_peek_and_take_ready(domain_t*); +int ce_higher_prio(struct task_struct*, struct task_struct*); + +#ifdef CONFIG_MERGE_TIMERS +typedef void (*ce_timer_callback_t)(struct rt_event*); +#else +typedef enum hrtimer_restart (*ce_timer_callback_t)(struct hrtimer*); +#endif + +void ce_domain_init(domain_t*, + raw_spinlock_t*, + requeue_t, + peek_ready_t, + take_ready_t, + preempt_needed_t, + task_prio_t, + struct ce_dom_data*, + const int, + ce_timer_callback_t); +#endif diff --git a/include/litmus/domain.h b/include/litmus/domain.h new file mode 100644 index 0000000..d16ed18 --- /dev/null +++ b/include/litmus/domain.h @@ -0,0 +1,50 @@ +/** + * --Todo-- + * Naming: this should become rt_domain while the old rt_domain should be + * changed to sd_domain (sporadic) or pd_domain (periodic). + * task_new: need to add and use this method + */ +#ifndef _LITMUS_DOMAIN_H_ +#define _LITMUS_DOMAIN_H_ + +struct domain; + +typedef void (*requeue_t)(struct domain*, struct task_struct*); +typedef void (*remove_t)(struct domain*, struct task_struct*); +typedef struct task_struct* (*peek_ready_t)(struct domain*); +typedef struct task_struct* (*take_ready_t)(struct domain*); +typedef int (*preempt_needed_t)(struct domain*, struct task_struct*); +typedef int (*task_prio_t)(struct task_struct*, struct task_struct*); + +typedef struct domain { + raw_spinlock_t* lock; /* for coarse serialization */ + struct list_head list; /* list membership */ + void* data; /* implementation-specific data */ + char* name; /* for debugging */ + + /* add a task to the domain */ + requeue_t requeue; + /* prevent a task from being returned by the domain */ + remove_t remove; + /* return next ready task */ + peek_ready_t peek_ready; + /* remove and return next ready task */ + take_ready_t take_ready; + /* return true if the domain has a task which should preempt the + * task given + */ + preempt_needed_t preempt_needed; + /* for tasks within this domain, returns true if the first has + * has a higher priority than the second + */ + task_prio_t higher_prio; +} domain_t; + +void domain_init(domain_t *dom, + raw_spinlock_t *lock, + requeue_t requeue, + peek_ready_t peek_ready, + take_ready_t take_ready, + preempt_needed_t preempt_needed, + task_prio_t priority); +#endif diff --git a/include/litmus/event_group.h b/include/litmus/event_group.h new file mode 100644 index 0000000..b0654e0 --- /dev/null +++ b/include/litmus/event_group.h @@ -0,0 +1,91 @@ +#ifndef _LINUX_EVENT_QUEUE_H_ +#define _LINUX_EVENT_QUEUE_H_ + +#define EVENT_QUEUE_SLOTS 127 /* prime */ + +#define NUM_EVENT_PRIORITIES 4 /* num crit levels really */ + +struct rt_event; +typedef void (*fire_event_t)(struct rt_event *e); + +struct event_group { + lt_t res; + int cpu; + struct list_head event_queue[EVENT_QUEUE_SLOTS]; + raw_spinlock_t queue_lock; +}; + +/** + * A group of actions to fire at a given time + */ +struct event_list { + /* Use multiple list heads so that inserts are O(1) */ + struct list_head events[NUM_EVENT_PRIORITIES]; + + /* For timer firing */ + lt_t fire_time; + struct hrtimer timer; + struct hrtimer_start_on_info info; + + struct list_head queue_node; /* For event_queue */ + struct event_group* group; /* For callback */ +}; + +/** + * A single action to fire at a time + */ +struct rt_event { + /* Function to call on event expiration */ + fire_event_t function; + /* Priority of this event (lower is better) */ + int prio; + + /* For membership in the event_list */ + struct list_head events_node; + /* To avoid runtime allocation. This is NOT necessarily + * the event_list containing this event. This is just a + * pre-allocated event list which can be used for merging + * events. + */ + struct event_list* event_list; + /* Pointer set by add_event() so that we can cancel this event + * without knowing what group it is in (don't touch it). + */ + struct event_group* _event_group; +}; + +/** + * add_event() - Add timer to event group. + * @group Group with which to merge event. If NULL, use the event + * group of whatever CPU currently executing on. + * @e Event to be fired at a specific time + * @time Time to fire event + */ +void add_event(struct event_group* group, struct rt_event* e, lt_t time); + +/** + * cancel_event() - Remove event from the group. + */ +void cancel_event(struct rt_event*); + +/** + * init_event() - Create an event. + * @e Event to create + * @prio Priority of the event (lower is better) + * @function Function to fire when event expires + * @el Pre-allocated event list for timer merging + */ +void init_event(struct rt_event* e, int prio, fire_event_t function, + struct event_list *el); + +struct event_list* event_list_alloc(int); +void event_list_free(struct event_list *el); + +/** + * get_event_group_for() - Get the event group for a CPU. + * @cpu The CPU to get the event group for. Use NO_CPU to get the + * event group of the CPU that the call is executing on. + */ +struct event_group *get_event_group_for(const int cpu); + +#endif diff --git a/include/litmus/litmus.h b/include/litmus/litmus.h index e7769ca..4322d59 100644 --- a/include/litmus/litmus.h +++ b/include/litmus/litmus.h @@ -26,7 +26,7 @@ static inline int in_list(struct list_head* list) ); } -struct task_struct* __waitqueue_remove_first(wait_queue_head_t *wq); +struct task_struct* waitqueue_first(wait_queue_head_t *wq); #define NO_CPU 0xffffffff @@ -55,6 +55,7 @@ void litmus_exit_task(struct task_struct *tsk); #define get_deadline(t) (tsk_rt(t)->job_params.deadline) #define get_release(t) (tsk_rt(t)->job_params.release) #define get_class(t) (tsk_rt(t)->task_params.cls) +#define get_task_domain(t) (tsk_rt(t)->_domain) #define is_priority_boosted(t) (tsk_rt(t)->priority_boosted) #define get_boost_start(t) (tsk_rt(t)->boost_start_time) @@ -128,6 +129,16 @@ void srp_ceiling_block(void); #define bheap2task(hn) ((struct task_struct*) hn->value) +static inline struct control_page* get_control_page(struct task_struct *t) +{ + return tsk_rt(t)->ctrl_page; +} + +static inline int has_control_page(struct task_struct* t) +{ + return tsk_rt(t)->ctrl_page != NULL; +} + #ifdef CONFIG_NP_SECTION static inline int is_kernel_np(struct task_struct *t) @@ -243,4 +254,8 @@ static inline quanta_t time2quanta(lt_t time, enum round round) /* By how much is cpu staggered behind CPU 0? */ u64 cpu_stagger_offset(int cpu); +#define TS_SYSCALL_IN_START \ + if (has_control_page(current)) \ + __TS_SYSCALL_IN_START(&get_control_page(current)->ts_syscall_start); + #endif diff --git a/include/litmus/preempt.h b/include/litmus/preempt.h index 260c6fe..29bf9bc 100644 --- a/include/litmus/preempt.h +++ b/include/litmus/preempt.h @@ -8,11 +8,13 @@ #include + extern DEFINE_PER_CPU_SHARED_ALIGNED(atomic_t, resched_state); #ifdef CONFIG_DEBUG_KERNEL const char* sched_state_name(int s); -#define TRACE_STATE(fmt, args...) TRACE("SCHED_STATE " fmt, args) +//#define TRACE_STATE(fmt, args...) TRACE("SCHED_STATE " fmt, args) +#define TRACE_STATE(fmt, args...) /* ignore */ #else #define TRACE_STATE(fmt, args...) /* ignore */ #endif diff --git a/include/litmus/rt_domain.h b/include/litmus/rt_domain.h index ac24929..f98d47c 100644 --- a/include/litmus/rt_domain.h +++ b/include/litmus/rt_domain.h @@ -6,6 +6,8 @@ #define __UNC_RT_DOMAIN_H__ #include +#include +#include #define RELEASE_QUEUE_SLOTS 127 /* prime */ @@ -29,7 +31,10 @@ typedef struct _rt_domain { raw_spinlock_t release_lock; struct release_queue release_queue; -#ifdef CONFIG_RELEASE_MASTER +#if defined(CONFIG_MERGE_TIMERS) + struct event_group* event_group; + int prio; +#elif defined(CONFIG_RELEASE_MASTER) int release_master; #endif @@ -53,13 +58,18 @@ struct release_heap { lt_t release_time; /* all tasks to be released at release_time */ struct bheap heap; + +#ifdef CONFIG_MERGE_TIMERS + /* used to merge timer calls */ + struct rt_event event; +#else /* used to trigger the release */ struct hrtimer timer; - #ifdef CONFIG_RELEASE_MASTER /* used to delegate releases */ struct hrtimer_start_on_info info; #endif +#endif /* required for the timer callback */ rt_domain_t* dom; }; @@ -76,7 +86,15 @@ static inline struct task_struct* __next_ready(rt_domain_t* rt) void rt_domain_init(rt_domain_t *rt, bheap_prio_t order, check_resched_needed_t check, - release_jobs_t relase); + release_jobs_t release); + +void pd_domain_init(domain_t *dom, + rt_domain_t *rt, + bheap_prio_t order, + check_resched_needed_t check, + release_jobs_t release, + preempt_needed_t preempt_needed, + task_prio_t priority); void __add_ready(rt_domain_t* rt, struct task_struct *new); void __merge_ready(rt_domain_t* rt, struct bheap *tasks); diff --git a/include/litmus/rt_param.h b/include/litmus/rt_param.h index 5de422c..55419d4 100644 --- a/include/litmus/rt_param.h +++ b/include/litmus/rt_param.h @@ -63,6 +63,9 @@ struct control_page { * its non-preemptive section? */ int delayed_preemption; + /* locking overhead tracing: time stamp prior to system call */ + uint64_t ts_syscall_start; /* Feather-Trace cycles */ + /* to be extended */ }; @@ -72,6 +75,7 @@ struct control_page { struct _rt_domain; struct bheap_node; struct release_heap; +struct domain; struct rt_job { /* Time instant the the job was or will be released. */ @@ -79,6 +83,9 @@ struct rt_job { /* What is the current deadline? */ lt_t deadline; + lt_t real_release; + lt_t real_deadline; + /* How much service has this job received so far? */ lt_t exec_time; @@ -93,6 +100,9 @@ struct rt_job { }; struct pfair_param; +#ifdef CONFIG_PLUGIN_MC +struct mc_data; +#endif /* RT task parameters for scheduling extensions * These parameters are inherited during clone and therefore must @@ -115,6 +125,14 @@ struct rt_param { lt_t boost_start_time; #endif +#ifdef CONFIG_PLUGIN_MC + /* mixed criticality specific data */ + struct mc_data *mc_data; +#endif +#ifdef CONFIG_MERGE_TIMERS + struct rt_event *event; +#endif + /* user controlled parameters */ struct rt_task task_params; @@ -169,6 +187,9 @@ struct rt_param { int old_policy; int old_prio; + /* TODO: rename */ + struct domain *_domain; + /* ready queue for this task */ struct _rt_domain* domain; diff --git a/include/litmus/sched_mc.h b/include/litmus/sched_mc.h new file mode 100644 index 0000000..740cc11 --- /dev/null +++ b/include/litmus/sched_mc.h @@ -0,0 +1,134 @@ +#ifndef _LINUX_SCHED_MC_H_ +#define _LINUX_SCHED_MC_H_ + +/* criticality levels */ +enum crit_level { + /* probably don't need to assign these (paranoid) */ + CRIT_LEVEL_A = 0, + CRIT_LEVEL_B = 1, + CRIT_LEVEL_C = 2, + NUM_CRIT_LEVELS = 3, +}; + +struct mc_task { + enum crit_level crit; + int lvl_a_id; + int lvl_a_eligible; +}; + +struct mc_job { + int is_ghost:1; + lt_t ghost_budget; +}; + +#ifdef __KERNEL__ +/* + * These are used only in the kernel. Userspace programs like RTSpin won't see + * them. + */ +struct mc_data { + struct mc_task mc_task; + struct mc_job mc_job; +}; + +#define tsk_mc_data(t) (tsk_rt(t)->mc_data) +#define tsk_mc_crit(t) (tsk_mc_data(t)->mc_task.crit) +#define is_ghost(t) (tsk_mc_data(t)->mc_job.is_ghost) + +#define TS "(%s/%d:%d:%s)" +#define TA(t) (t) ? tsk_mc_data(t) ? is_ghost(t) ? "ghost" : t->comm \ + : t->comm : "NULL", \ + (t) ? t->pid : 1, \ + (t) ? t->rt_param.job_params.job_no : 1, \ + (t && get_task_domain(t)) ? get_task_domain(t)->name : "" +#define STRACE(fmt, args...) \ + sched_trace_log_message("%d P%d [%s@%s:%d]: " fmt, \ + TRACE_ARGS, ## args) +#define TRACE_MC_TASK(t, fmt, args...) \ + STRACE(TS " " fmt, TA(t), ##args) + +/* + * The MC-CE scheduler uses this as domain data. + */ +struct ce_dom_data { + int cpu; + struct task_struct *scheduled, *should_schedule; +#ifdef CONFIG_MERGE_TIMERS + struct rt_event event; +#else + struct hrtimer_start_on_info timer_info; + struct hrtimer timer; +#endif +}; + +/** + * enum crit_state - Logically add / remove CPUs from criticality levels. + * + * Global crit levels need to use a two step process to remove CPUs so + * that the CPUs can be removed without holding domain locks. + * + * @CS_ACTIVE The criticality entry can run a task + * @CS_ACTIVATE The criticality entry can run a task, but hasn't had its + * position updated in a global heap. Set with ONLY CPU lock. + * @CS_REMOVE The criticality entry is logically removed, but hasn't had its + * position adjusted in a global heap. This should be set when + * ONLY the CPU state is locked. + * @CS_REMOVED The criticality entry has been removed from the crit level + */ +enum crit_state { CS_ACTIVE, CS_ACTIVATE, CS_REMOVE, CS_REMOVED }; + +/** + * struct crit_entry - State of a CPU within each criticality level system. + * @level Criticality level of this entry + * @linked Logically running task, ghost or regular + * @domain Domain from which to draw tasks + * @usable False if a higher criticality task is running + * @event For ghost task budget enforcement (merge timers) + * @timer For ghost task budget enforcement (not merge timers) + * @node Used to sort crit_entries by preemptability in global domains + */ +struct crit_entry { + enum crit_level level; + struct task_struct* linked; + struct domain* domain; + enum crit_state state; +#ifdef CONFIG_MERGE_TIMERS + struct rt_event event; +#else + struct hrtimer timer; +#endif + struct bheap_node* node; +}; + +/** + * struct domain_data - Wrap domains with related CPU state + * @domain A domain for a criticality level + * @heap The preemptable heap of crit entries (for global domains) + * @crit_entry The crit entry for this domain (for partitioned domains) + */ +struct domain_data { + struct domain domain; + struct bheap* heap; + struct crit_entry* crit_entry; +}; + +/* + * Functions that are used with the MC-CE plugin. + */ +long mc_ce_set_domains(const int, struct domain_data*[]); +unsigned int mc_ce_get_expected_job(const int, const int); + +/* + * These functions are (lazily) inserted into the MC plugin code so that it + * manipulates the MC-CE state. + */ +long mc_ce_admit_task_common(struct task_struct*); +void mc_ce_task_exit_common(struct task_struct*); +lt_t mc_ce_timer_callback_common(domain_t*); +void mc_ce_release_at_common(struct task_struct*, lt_t); +long mc_ce_activate_plugin_common(void); +long mc_ce_deactivate_plugin_common(void); + +#endif /* __KERNEL__ */ + +#endif diff --git a/include/litmus/sched_plugin.h b/include/litmus/sched_plugin.h index 6e7cabd..32c2397 100644 --- a/include/litmus/sched_plugin.h +++ b/include/litmus/sched_plugin.h @@ -11,6 +11,8 @@ #include #endif +struct litmus_lock; + /************************ setup/tear down ********************/ typedef long (*activate_plugin_t) (void); diff --git a/include/litmus/sched_trace.h b/include/litmus/sched_trace.h index 7ca34cb..5cc05be 100644 --- a/include/litmus/sched_trace.h +++ b/include/litmus/sched_trace.h @@ -24,7 +24,8 @@ struct st_param_data { /* regular params */ u32 phase; u8 partition; u8 class; - u8 __unused[2]; + u8 level; + u8 __unused[1]; }; struct st_release_data { /* A job is was/is going to be released. */ @@ -71,8 +72,8 @@ struct st_resume_data { /* A task resumes. */ struct st_action_data { u64 when; - u8 action; - u8 __unused[7]; + u32 action; + u8 __unused[4]; }; struct st_sys_release_data { @@ -193,6 +194,12 @@ feather_callback void do_sched_trace_sys_release(unsigned long id, SCHED_TRACE(SCHED_TRACE_BASE_ID + 10, do_sched_trace_sys_release, when) +#define QT_START lt_t _qt_start = litmus_clock() +#define QT_END \ + sched_trace_log_message("%d P%d [%s@%s:%d]: Took %llu\n\n", \ + TRACE_ARGS, litmus_clock() - _qt_start) + + #define sched_trace_quantum_boundary() /* NOT IMPLEMENTED */ #endif /* __KERNEL__ */ diff --git a/include/litmus/trace.h b/include/litmus/trace.h index 05f4872..e0c8471 100644 --- a/include/litmus/trace.h +++ b/include/litmus/trace.h @@ -12,7 +12,8 @@ enum task_type_marker { TSK_BE, TSK_RT, - TSK_UNKNOWN + TSK_UNKNOWN, + TSK_LVLA }; struct timestamp { @@ -28,7 +29,8 @@ feather_callback void save_timestamp(unsigned long event); feather_callback void save_timestamp_def(unsigned long event, unsigned long type); feather_callback void save_timestamp_task(unsigned long event, unsigned long t_ptr); feather_callback void save_timestamp_cpu(unsigned long event, unsigned long cpu); - +feather_callback void save_task_latency(unsigned long event, unsigned long when_ptr); +feather_callback void save_timestamp_time(unsigned long event, unsigned long time_ptr); #define TIMESTAMP(id) ft_event0(id, save_timestamp) @@ -40,6 +42,14 @@ feather_callback void save_timestamp_cpu(unsigned long event, unsigned long cpu) #define CTIMESTAMP(id, cpu) \ ft_event1(id, save_timestamp_cpu, (unsigned long) cpu) +#define LTIMESTAMP(id, task) \ + ft_event1(id, save_task_latency, (unsigned long) task) + +#define TIMESTAMP_TIME(id, time_ptr) \ + ft_event1(id, save_timestamp_time, (unsigned long) time_ptr) + +#define TIMESTAMP_PID(id) ft_event0(id, save_timestamp_pid) + #else /* !CONFIG_SCHED_OVERHEAD_TRACE */ #define TIMESTAMP(id) /* no tracing */ @@ -50,6 +60,12 @@ feather_callback void save_timestamp_cpu(unsigned long event, unsigned long cpu) #define CTIMESTAMP(id, cpu) /* no tracing */ +#define LTIMESTAMP(id, when_ptr) /* no tracing */ + +#define TIMESTAMP_TIME(id, time_ptr) /* no tracing */ + +#define TIMESTAMP_PID(id) /* no tracing */ + #endif @@ -61,6 +77,21 @@ feather_callback void save_timestamp_cpu(unsigned long event, unsigned long cpu) * always the next number after the start time event id. */ +#define __TS_SYSCALL_IN_START(p) TIMESTAMP_TIME(10, p) +#define TS_SYSCALL_IN_END TIMESTAMP_PID(11) + +#define TS_SYSCALL_OUT_START TIMESTAMP_PID(20) +#define TS_SYSCALL_OUT_END TIMESTAMP_PID(21) + +#define TS_LOCK_START TIMESTAMP_PID(30) +#define TS_LOCK_END TIMESTAMP_PID(31) + +#define TS_LOCK_SUSPEND TIMESTAMP_PID(38) +#define TS_LOCK_RESUME TIMESTAMP_PID(39) + +#define TS_UNLOCK_START TIMESTAMP_PID(40) +#define TS_UNLOCK_END TIMESTAMP_PID(41) + #define TS_SCHED_START DTIMESTAMP(100, TSK_UNKNOWN) /* we only * care * about @@ -78,6 +109,13 @@ feather_callback void save_timestamp_cpu(unsigned long event, unsigned long cpu) #define TS_TICK_START(t) TTIMESTAMP(110, t) #define TS_TICK_END(t) TTIMESTAMP(111, t) +#define TS_LVLA_RELEASE_START DTIMESTAMP(112, TSK_RT) +#define TS_LVLA_RELEASE_END DTIMESTAMP(113, TSK_RT) + +#define TS_LVLA_SCHED_START DTIMESTAMP(114, TSK_UNKNOWN) +#define TS_LVLA_SCHED_END_ID 115 +#define TS_LVLA_SCHED_END(t) TTIMESTAMP(TS_LVLA_SCHED_END_ID, t) + #define TS_PLUGIN_SCHED_START /* TIMESTAMP(120) */ /* currently unused */ #define TS_PLUGIN_SCHED_END /* TIMESTAMP(121) */ @@ -91,13 +129,9 @@ feather_callback void save_timestamp_cpu(unsigned long event, unsigned long cpu) #define TS_EXIT_NP_START TIMESTAMP(150) #define TS_EXIT_NP_END TIMESTAMP(151) -#define TS_LOCK_START TIMESTAMP(170) -#define TS_LOCK_END TIMESTAMP(171) -#define TS_UNLOCK_START TIMESTAMP(172) -#define TS_UNLOCK_END TIMESTAMP(173) - #define TS_SEND_RESCHED_START(c) CTIMESTAMP(190, c) #define TS_SEND_RESCHED_END DTIMESTAMP(191, TSK_UNKNOWN) +#define TS_RELEASE_LATENCY(when) LTIMESTAMP(208, &(when)) #endif /* !_SYS_TRACE_H_ */ diff --git a/include/litmus/unistd_32.h b/include/litmus/unistd_32.h index 94264c2..71be3cd 100644 --- a/include/litmus/unistd_32.h +++ b/include/litmus/unistd_32.h @@ -17,5 +17,6 @@ #define __NR_wait_for_ts_release __LSC(9) #define __NR_release_ts __LSC(10) #define __NR_null_call __LSC(11) +#define __NR_set_rt_task_mc_param __LSC(12) -#define NR_litmus_syscalls 12 +#define NR_litmus_syscalls 13 diff --git a/include/litmus/unistd_64.h b/include/litmus/unistd_64.h index d5ced0d..95cb744 100644 --- a/include/litmus/unistd_64.h +++ b/include/litmus/unistd_64.h @@ -29,5 +29,7 @@ __SYSCALL(__NR_wait_for_ts_release, sys_wait_for_ts_release) __SYSCALL(__NR_release_ts, sys_release_ts) #define __NR_null_call __LSC(11) __SYSCALL(__NR_null_call, sys_null_call) +#define __NR_set_rt_task_mc_param __LSC(12) +__SYSCALL(__NR_set_rt_task_mc_param, sys_set_rt_task_mc_param) -#define NR_litmus_syscalls 12 +#define NR_litmus_syscalls 13 diff --git a/kernel/hrtimer.c b/kernel/hrtimer.c index cb49883..d6a15dd 100644 --- a/kernel/hrtimer.c +++ b/kernel/hrtimer.c @@ -1053,6 +1053,7 @@ void hrtimer_start_on_info_init(struct hrtimer_start_on_info *info) { memset(info, 0, sizeof(struct hrtimer_start_on_info)); atomic_set(&info->state, HRTIMER_START_ON_INACTIVE); + INIT_LIST_HEAD(&info->list); } /** @@ -1071,12 +1072,32 @@ void hrtimer_pull(void) list_for_each_safe(pos, safe, &list) { info = list_entry(pos, struct hrtimer_start_on_info, list); TRACE("pulled timer 0x%x\n", info->timer); - list_del(pos); - hrtimer_start(info->timer, info->time, info->mode); + list_del_init(pos); + if (!info->timer) continue; + if (atomic_read(&info->state) != HRTIMER_START_ON_INACTIVE) + hrtimer_start(info->timer, info->time, info->mode); + if (atomic_read(&info->state) == HRTIMER_START_ON_INACTIVE) + hrtimer_cancel(info->timer); } } /** + * hrtimer_pull_cancel - Cancel a remote timer pull + */ +int hrtimer_pull_cancel(int cpu, struct hrtimer *timer, + struct hrtimer_start_on_info *info) +{ + struct hrtimer_cpu_base *base = &per_cpu(hrtimer_bases, cpu); + + raw_spin_lock(&base->lock); + list_del_init(&info->list); + raw_spin_unlock(&base->lock); + + atomic_set(&info->state, HRTIMER_START_ON_INACTIVE); + return hrtimer_try_to_cancel(timer); +} + +/** * hrtimer_start_on - trigger timer arming on remote cpu * @cpu: remote cpu * @info: save timer information for enqueuing on remote cpu @@ -1085,8 +1106,8 @@ void hrtimer_pull(void) * @mode: timer mode */ int hrtimer_start_on(int cpu, struct hrtimer_start_on_info* info, - struct hrtimer *timer, ktime_t time, - const enum hrtimer_mode mode) + struct hrtimer *timer, ktime_t time, + const enum hrtimer_mode mode) { unsigned long flags; struct hrtimer_cpu_base* base; @@ -1118,7 +1139,8 @@ int hrtimer_start_on(int cpu, struct hrtimer_start_on_info* info, __hrtimer_start_range_ns(info->timer, info->time, 0, info->mode, 0); } else { - TRACE("hrtimer_start_on: pulling to remote CPU\n"); + TRACE("hrtimer_start_on: pulling 0x%x to remote CPU\n", + info->timer); base = &per_cpu(hrtimer_bases, cpu); raw_spin_lock_irqsave(&base->lock, flags); was_empty = list_empty(&base->to_pull); diff --git a/kernel/sched.c b/kernel/sched.c index c5d7750..71907dc 100644 --- a/kernel/sched.c +++ b/kernel/sched.c @@ -3810,6 +3810,7 @@ need_resched: release_kernel_lock(prev); need_resched_nonpreemptible: TS_SCHED_START; + TS_LVLA_SCHED_START; sched_trace_task_switch_away(prev); schedule_debug(prev); @@ -3859,6 +3860,7 @@ need_resched_nonpreemptible: rq->curr = next; ++*switch_count; + TS_LVLA_SCHED_END(next); TS_SCHED_END(next); TS_CXS_START(next); context_switch(rq, prev, next); /* unlocks the rq */ diff --git a/litmus/Kconfig b/litmus/Kconfig index ad8dc83..d629a28 100644 --- a/litmus/Kconfig +++ b/litmus/Kconfig @@ -23,16 +23,67 @@ config PLUGIN_PFAIR If unsure, say Yes. +config MERGE_TIMERS + bool "Timer-merging Support" + depends on HIGH_RES_TIMERS + default y + help + Include support for merging timers. + +config MERGE_TIMERS_WINDOW + int "Timer-merging Window (in nanoseconds)" + depends on MERGE_TIMERS + default 1000 + help + Window within which seperate timers may be merged. + config RELEASE_MASTER bool "Release-master Support" depends on ARCH_HAS_SEND_PULL_TIMERS default n help - Allow one processor to act as a dedicated interrupt processor - that services all timer interrupts, but that does not schedule - real-time tasks. See RTSS'09 paper for details - (http://www.cs.unc.edu/~anderson/papers.html). - Currently only supported by GSN-EDF. + In GSN-EDF, allow one processor to act as a dedicated interrupt + processor that services all timer interrupts, but that does not schedule + real-time tasks. See RTSS'09 paper for details + (http://www.cs.unc.edu/~anderson/papers.html). + +menu "Mixed Criticality" + +config PLUGIN_MC + bool "Mixed Criticality Scheduler" + depends on X86 && SYSFS + default y + help + Include the mixed criticality scheduler. This plugin depends + on the global release-master processor for its _REDIRECT and + _RELEASE_MASTER options. + + If unsure, say Yes. + +config PLUGIN_MC_LEVEL_A_MAX_TASKS + int "Maximum level A tasks" + depends on PLUGIN_MC + range 1 128 + default 32 + help + The maximum number of level A tasks allowed (per-cpu) in level A. + +config PLUGIN_MC_RELEASE_MASTER + bool "Release-master support for MC" + depends on PLUGIN_MC && RELEASE_MASTER + default y + help + Send all timer interrupts to the system-wide release-master CPU. + +config PLUGIN_MC_REDIRECT + bool "Redirect Work to Release-master" + depends on PLUGIN_MC && RELEASE_MASTER + default y + help + Allow processors to send work involving global state to the + release-master cpu in order to avoid excess overheads during + partitioned decisions. +endmenu endmenu diff --git a/litmus/Makefile b/litmus/Makefile index ad9936e..b6ca7ab 100644 --- a/litmus/Makefile +++ b/litmus/Makefile @@ -16,11 +16,12 @@ obj-y = sched_plugin.o litmus.o \ srp.o \ bheap.o \ ctrldev.o \ - sched_gsn_edf.o \ - sched_psn_edf.o + domain.o obj-$(CONFIG_PLUGIN_CEDF) += sched_cedf.o obj-$(CONFIG_PLUGIN_PFAIR) += sched_pfair.o +obj-$(CONFIG_PLUGIN_MC) += sched_mc.o sched_mc_ce.o ce_domain.o +obj-$(CONFIG_MERGE_TIMERS) += event_group.o obj-$(CONFIG_FEATHER_TRACE) += ft_event.o ftdev.o obj-$(CONFIG_SCHED_TASK_TRACE) += sched_task_trace.o diff --git a/litmus/bheap.c b/litmus/bheap.c index 528af97..42122d8 100644 --- a/litmus/bheap.c +++ b/litmus/bheap.c @@ -248,13 +248,14 @@ int bheap_decrease(bheap_prio_t higher_prio, struct bheap_node* node) void bheap_delete(bheap_prio_t higher_prio, struct bheap* heap, struct bheap_node* node) { - struct bheap_node *parent, *prev, *pos; + struct bheap_node *parent, *prev, *pos, *old; struct bheap_node** tmp_ref; void* tmp; if (heap->min != node) { /* bubble up */ parent = node->parent; + old = node; while (parent) { /* swap parent and node */ tmp = parent->value; diff --git a/litmus/budget.c b/litmus/budget.c index 310e9a3..93945e0 100644 --- a/litmus/budget.c +++ b/litmus/budget.c @@ -4,6 +4,8 @@ #include #include +#include +#include struct enforcement_timer { /* The enforcement timer is used to accurately police @@ -108,4 +110,22 @@ static int __init init_budget_enforcement(void) return 0; } +void task_release(struct task_struct *t) +{ + t->rt_param.job_params.real_release = t->rt_param.job_params.real_deadline; + t->rt_param.job_params.real_deadline += get_rt_period(t); + t->rt_param.job_params.job_no++; + sched_trace_task_release(t); +} + +void server_release(struct task_struct *t) +{ + lt_t now = litmus_clock(); + t->rt_param.job_params.exec_time = 0; + t->rt_param.job_params.release = t->rt_param.job_params.deadline; + t->rt_param.job_params.deadline += get_rt_period(t); + /* don't confuse linux */ + t->rt.time_slice = 1; +} + module_init(init_budget_enforcement); diff --git a/litmus/ce_domain.c b/litmus/ce_domain.c new file mode 100644 index 0000000..b2c5d4e --- /dev/null +++ b/litmus/ce_domain.c @@ -0,0 +1,102 @@ +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +/* + * Called for: + * task_new + * job_completion + * wake_up + */ +void ce_requeue(domain_t *dom, struct task_struct *ts) +{ + const struct ce_dom_data *ce_data = dom->data; + const int idx = tsk_mc_data(ts)->mc_task.lvl_a_id; + const unsigned int just_finished = tsk_rt(ts)->job_params.job_no; + const unsigned int expected_job = + mc_ce_get_expected_job(ce_data->cpu, idx); + const int asleep = RT_F_SLEEP == get_rt_flags(ts); + + TRACE_MC_TASK(ts, "entered ce_requeue. asleep: %d just_finished: %3u " + "expected_job: %3u\n", + asleep, just_finished, expected_job); + + tsk_mc_data(ts)->mc_task.lvl_a_eligible = 1; + + /* When coming from job completion, the task will be asleep. */ + if (asleep && just_finished < expected_job) { + TRACE_MC_TASK(ts, "appears behind\n"); + } else if (asleep && expected_job < just_finished) { + TRACE_MC_TASK(ts, "job %u completed in expected job %u which " + "seems too early\n", just_finished, + expected_job); + } +} + +/* + * + */ +void ce_remove(domain_t *dom, struct task_struct *ts) +{ + tsk_mc_data(ts)->mc_task.lvl_a_eligible = 0; +} + +/* + * ce_take_ready and ce_peek_ready + */ +struct task_struct* ce_peek_and_take_ready(domain_t *dom) +{ + const struct ce_dom_data *ce_data = dom->data; + struct task_struct *ret = NULL, *sched = ce_data->should_schedule; + const int exists = NULL != sched; + const int blocked = exists && !is_running(sched); + const int elig = exists && tsk_mc_data(sched) && + tsk_mc_data(sched)->mc_task.lvl_a_eligible; + + /* Return the task we should schedule if it is not blocked or sleeping. */ + if (exists && !blocked && elig) + ret = sched; + return ret; +} + +int ce_higher_prio(struct task_struct *a, struct task_struct *b) +{ + const domain_t *dom = get_task_domain(a); + const struct ce_dom_data *ce_data = dom->data; + return (a != b && a == ce_data->should_schedule); +} + +void ce_domain_init(domain_t *dom, + raw_spinlock_t *lock, + requeue_t requeue, + peek_ready_t peek_ready, + take_ready_t take_ready, + preempt_needed_t preempt_needed, + task_prio_t task_prio, + struct ce_dom_data *dom_data, + const int cpu, + ce_timer_callback_t ce_timer_callback) +{ + domain_init(dom, lock, requeue, peek_ready, take_ready, preempt_needed, + task_prio); + dom->data = dom_data; + dom->remove = ce_remove; + dom_data->cpu = cpu; +#ifdef CONFIG_MERGE_TIMERS + init_event(&dom_data->event, CRIT_LEVEL_A, ce_timer_callback, + event_list_alloc(GFP_ATOMIC)); +#else + hrtimer_start_on_info_init(&dom_data->timer_info); + hrtimer_init(&dom_data->timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS); + dom_data->timer.function = ce_timer_callback; +#endif +} diff --git a/litmus/domain.c b/litmus/domain.c new file mode 100644 index 0000000..4dc3649 --- /dev/null +++ b/litmus/domain.c @@ -0,0 +1,21 @@ +#include +#include + +#include + +void domain_init(domain_t *dom, + raw_spinlock_t *lock, + requeue_t requeue, + peek_ready_t peek_ready, + take_ready_t take_ready, + preempt_needed_t preempt_needed, + task_prio_t priority) +{ + INIT_LIST_HEAD(&dom->list); + dom->lock = lock; + dom->requeue = requeue; + dom->peek_ready = peek_ready; + dom->take_ready = take_ready; + dom->preempt_needed = preempt_needed; + dom->higher_prio = priority; +} diff --git a/litmus/event_group.c b/litmus/event_group.c new file mode 100644 index 0000000..478698a --- /dev/null +++ b/litmus/event_group.c @@ -0,0 +1,334 @@ +#include +#include +#include + +#include +#include +#include +#include + +#if 1 +#define VTRACE(fmt, args...) \ +sched_trace_log_message("%d P%d [%s@%s:%d]: " fmt, \ + TRACE_ARGS, ## args) +#else +#define VTRACE(fmt, args...) +#endif + +/* + * Return event_queue slot for the given time. + */ +static unsigned int time2slot(lt_t time) +{ + return (unsigned int) time2quanta(time, FLOOR) % EVENT_QUEUE_SLOTS; +} + +/* + * Executes events from an event_list in priority order. + * Events can requeue themselves when they are called. + */ +static enum hrtimer_restart on_timer(struct hrtimer *timer) +{ + int prio, num; + unsigned long flags; + struct event_list *el; + struct rt_event *e; + struct list_head *pos, events[NUM_EVENT_PRIORITIES]; + raw_spinlock_t *queue_lock; + + el = container_of(timer, struct event_list, timer); + queue_lock = &el->group->queue_lock; + + raw_spin_lock_irqsave(queue_lock, flags); + + /* Remove event_list from hashtable so that no more events + * are added to it. + */ + VTRACE("Removing event list 0x%x\n", el); + list_del_init(&el->queue_node); + + /* Copy over events so that the event_list can re-used when the lock + * is released. + */ + VTRACE("Emptying event list 0x%x\n", el); + for (prio = 0; prio < NUM_EVENT_PRIORITIES; prio++) { + list_replace_init(&el->events[prio], &events[prio]); + } + + for (prio = 0; prio < NUM_EVENT_PRIORITIES; prio++) { + /* Fire events. Complicated loop is used so that events + * in the list can be canceled (removed) while other events are + * executing. + */ + for (pos = events[prio].next, num = 0; + prefetch(pos->next), events[prio].next != &events[prio]; + pos = events[prio].next, num++) { + + e = list_entry(pos, struct rt_event, events_node); + list_del_init(pos); + raw_spin_unlock_irqrestore(queue_lock, flags); + + VTRACE("Dequeueing event 0x%x with prio %d from 0x%x\n", + e, e->prio, el); + e->function(e); + + raw_spin_lock_irqsave(queue_lock, flags); + } + } + raw_spin_unlock_irqrestore(queue_lock, flags); + + VTRACE("Exhausted %d events from list 0x%x\n", num, el); + + return HRTIMER_NORESTART; +} + +/* + * Return event_list for the given event and time. If no event_list + * is being used yet and use_event_heap is 1, will create the list + * and return it. Otherwise it will return NULL. + */ +static struct event_list* get_event_list(struct event_group *group, + struct rt_event *e, + lt_t fire, + int use_event_list) +{ + struct list_head* pos; + struct event_list *el = NULL, *tmp; + unsigned int slot = time2slot(fire); + int remaining = 300; + + VTRACE("Getting list for time %llu, event 0x%x\n", fire, e); + + /* Initialize pos for the case that the list is empty */ + pos = group->event_queue[slot].next; + list_for_each(pos, &group->event_queue[slot]) { + BUG_ON(remaining-- < 0); + tmp = list_entry(pos, struct event_list, queue_node); + if (lt_after_eq(fire, tmp->fire_time) && + lt_before(fire, tmp->fire_time + group->res)) { + VTRACE("Found match 0x%x at time %llu\n", + tmp, tmp->fire_time); + el = tmp; + break; + } else if (lt_before(fire, tmp->fire_time)) { + /* We need to insert a new node since el is + * already in the future + */ + VTRACE("Time %llu was before %llu\n", + fire, tmp->fire_time); + break; + } else { + VTRACE("Time %llu was after %llu\n", + fire, tmp->fire_time + group->res); + } + } + if (!el && use_event_list) { + /* Use pre-allocated list */ + tmp = e->event_list; + tmp->fire_time = fire; + tmp->group = group; + /* Add to queue */ + VTRACE("Using list 0x%x for priority %d and time %llu\n", + tmp, e->prio, fire); + BUG_ON(!list_empty(&tmp->queue_node)); + list_add(&tmp->queue_node, pos->prev); + el = tmp; + } + return el; +} + +/* + * Prepare a release list for a new set of events. + */ +static void reinit_event_list(struct event_group *group, struct rt_event *e) +{ + int prio, t_ret; + struct event_list *el = e->event_list; + + VTRACE("Reinitting list 0x%x for event 0x%x\n", el, e); + + /* Cancel timer */ + t_ret = hrtimer_pull_cancel(group->cpu, &el->timer, &el->info); + BUG_ON(t_ret == 1); + if (t_ret == -1) { + /* The on_timer callback is running for this list */ + VTRACE("Timer is running concurrently!\n"); + } + /* Clear event lists */ + for (prio = 0; prio < NUM_EVENT_PRIORITIES; prio++) + INIT_LIST_HEAD(&el->events[prio]); +} + +/** + * add_event() - Add timer to event group. + */ +void add_event(struct event_group *group, struct rt_event *e, lt_t fire) +{ + struct event_list *el; + int in_use; + + VTRACE("Adding event 0x%x with priority %d for time %llu\n", + e, e->prio, fire); + + /* A NULL group means use the group of the currently executing CPU */ + if (NULL == group) + group = get_event_group_for(NO_CPU); + /* Saving the group is important for cancellations */ + e->_event_group = group; + + raw_spin_lock(&group->queue_lock); + el = get_event_list(group, e, fire, 0); + if (!el) { + /* Use our own, but drop lock first */ + raw_spin_unlock(&group->queue_lock); + reinit_event_list(group, e); + raw_spin_lock(&group->queue_lock); + el = get_event_list(group, e, fire, 1); + } + + /* Add event to sorted list */ + VTRACE("Inserting event 0x%x at end of event_list 0x%x\n", e, el); + list_add(&e->events_node, &el->events[e->prio]); + raw_spin_unlock(&group->queue_lock); + + /* Arm timer if we are the owner */ + if (el == e->event_list) { + VTRACE("Arming timer on event 0x%x for %llu\n", e, fire); + in_use = hrtimer_start_on(group->cpu, &el->info, + &el->timer, ns_to_ktime(el->fire_time), + HRTIMER_MODE_ABS_PINNED); + BUG_ON(in_use); + } else { + VTRACE("Not my timer @%llu\n", fire); + } +} + +/** + * cancel_event() - Remove event from the group. + */ +void cancel_event(struct rt_event *e) +{ + int prio, cancel; + struct rt_event *swap, *entry; + struct event_list *tmp; + struct event_group *group; + struct list_head *list, *pos; + + VTRACE("Canceling event 0x%x with priority %d\n", e, e->prio); + group = e->_event_group; + if (!group) return; + + raw_spin_lock(&group->queue_lock); + + /* Relies on the fact that an event_list's owner is ALWAYS present + * as one of the event_list's events. + */ + for (prio = 0, cancel = 0, swap = NULL; + prio < NUM_EVENT_PRIORITIES && !swap; + prio++) { + + list = &e->event_list->events[prio]; + cancel |= !list_empty(list); + + /* Find any element which is not the event_list's owner */ + list_for_each(pos, list) { + entry = list_entry(pos, struct rt_event, events_node); + if (entry != e) { + swap = entry; + break; + } + } + } + + if (swap) { + /* Give the other guy ownership of the event_list */ + VTRACE("Swapping list 0x%x with event 0x%x event list 0x%x\n", + e->event_list, swap, swap->event_list); + tmp = swap->event_list; + swap->event_list = e->event_list; + BUG_ON(!tmp); + e->event_list = tmp; + } else if (cancel) { + /* Cancel the event_list we own */ + hrtimer_pull_cancel(group->cpu, + &e->event_list->timer, + &e->event_list->info); + list_del_init(&e->event_list->queue_node); + } + /* Remove ourselves from any list we may be a part of */ + list_del_init(&e->events_node); + e->_event_group = NULL; + + raw_spin_unlock(&group->queue_lock); +} + +struct kmem_cache *event_list_cache; + +struct event_list* event_list_alloc(int gfp_flags) +{ + int prio; + struct event_list *el = kmem_cache_alloc(event_list_cache, gfp_flags); + if (el) { + hrtimer_init(&el->timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS); + INIT_LIST_HEAD(&el->queue_node); + el->timer.function = on_timer; + hrtimer_start_on_info_init(&el->info); + for (prio = 0; prio < NUM_EVENT_PRIORITIES; prio++) + INIT_LIST_HEAD(&el->events[prio]); + } else { + VTRACE("Failed to allocate event list!\n"); + printk(KERN_CRIT "Failed to allocate event list.\n"); + BUG(); + } + return el; +} + +void init_event(struct rt_event *e, int prio, fire_event_t function, + struct event_list *el) +{ + e->prio = prio; + e->function = function; + e->event_list = el; + e->_event_group = NULL; + INIT_LIST_HEAD(&e->events_node); +} + +/** + * init_event_group() - Prepare group for events. + * @group Group to prepare + * @res Timer resolution. Two events of @res distance will be merged + * @cpu Cpu on which to fire timers + */ +static void init_event_group(struct event_group *group, lt_t res, int cpu) +{ + int i; + VTRACE("Creating group with resolution %llu on CPU %d", res, cpu); + group->res = res; + group->cpu = cpu; + for (i = 0; i < EVENT_QUEUE_SLOTS; i++) + INIT_LIST_HEAD(&group->event_queue[i]); + raw_spin_lock_init(&group->queue_lock); +} + + +DEFINE_PER_CPU(struct event_group, _event_groups); + +struct event_group *get_event_group_for(const int cpu) +{ + return &per_cpu(_event_groups, + (NO_CPU == cpu) ? smp_processor_id() : cpu); +} + +static int __init _init_event_groups(void) +{ + int cpu; + printk("Initializing LITMUS^RT event groups.\n"); + + for_each_online_cpu(cpu) { + init_event_group(get_event_group_for(cpu), + CONFIG_MERGE_TIMERS_WINDOW, cpu); + } + return 0; +} + +module_init(_init_event_groups); diff --git a/litmus/ftdev.c b/litmus/ftdev.c index 4a4b2e3..e282f8a 100644 --- a/litmus/ftdev.c +++ b/litmus/ftdev.c @@ -229,13 +229,20 @@ static ssize_t ftdev_read(struct file *filp, * here with copied data because that data would get * lost if the task is interrupted (e.g., killed). */ + mutex_unlock(&ftdm->lock); set_current_state(TASK_INTERRUPTIBLE); + schedule_timeout(50); + if (signal_pending(current)) { if (err == 0) /* nothing read yet, signal problem */ err = -ERESTARTSYS; - break; + goto out; + } + if (mutex_lock_interruptible(&ftdm->lock)) { + err = -ERESTARTSYS; + goto out; } } else if (copied < 0) { /* page fault */ @@ -250,64 +257,47 @@ out: return err; } -typedef uint32_t cmd_t; - -static ssize_t ftdev_write(struct file *filp, const char __user *from, - size_t len, loff_t *f_pos) +static long ftdev_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) { + long err = -ENOIOCTLCMD; struct ftdev_minor* ftdm = filp->private_data; - ssize_t err = -EINVAL; - cmd_t cmd; - cmd_t id; - - if (len % sizeof(cmd) || len < 2 * sizeof(cmd)) - goto out; - - if (copy_from_user(&cmd, from, sizeof(cmd))) { - err = -EFAULT; - goto out; - } - len -= sizeof(cmd); - from += sizeof(cmd); - - if (cmd != FTDEV_ENABLE_CMD && cmd != FTDEV_DISABLE_CMD) - goto out; if (mutex_lock_interruptible(&ftdm->lock)) { err = -ERESTARTSYS; goto out; } - err = sizeof(cmd); - while (len) { - if (copy_from_user(&id, from, sizeof(cmd))) { - err = -EFAULT; - goto out_unlock; - } - /* FIXME: check id against list of acceptable events */ - len -= sizeof(cmd); - from += sizeof(cmd); - if (cmd == FTDEV_DISABLE_CMD) - deactivate(&ftdm->events, id); - else if (activate(&ftdm->events, id) != 0) { + /* FIXME: check id against list of acceptable events */ + + switch (cmd) { + case FTDEV_ENABLE_CMD: + if (activate(&ftdm->events, arg)) err = -ENOMEM; - goto out_unlock; - } - err += sizeof(cmd); - } + else + err = 0; + break; + + case FTDEV_DISABLE_CMD: + deactivate(&ftdm->events, arg); + err = 0; + break; + + default: + printk(KERN_DEBUG "ftdev: strange ioctl (%u, %lu)\n", cmd, arg); + }; -out_unlock: mutex_unlock(&ftdm->lock); out: return err; } + struct file_operations ftdev_fops = { .owner = THIS_MODULE, .open = ftdev_open, .release = ftdev_release, - .write = ftdev_write, .read = ftdev_read, + .unlocked_ioctl = ftdev_ioctl, }; int ftdev_init( struct ftdev* ftdev, struct module* owner, diff --git a/litmus/jobs.c b/litmus/jobs.c index 36e3146..10a42db 100644 --- a/litmus/jobs.c +++ b/litmus/jobs.c @@ -10,8 +10,11 @@ void prepare_for_next_period(struct task_struct *t) { BUG_ON(!t); /* prepare next release */ + t->rt_param.job_params.release = t->rt_param.job_params.deadline; + t->rt_param.job_params.real_release = t->rt_param.job_params.release; t->rt_param.job_params.deadline += get_rt_period(t); + t->rt_param.job_params.real_deadline = t->rt_param.job_params.deadline; t->rt_param.job_params.exec_time = 0; /* update job sequence number */ t->rt_param.job_params.job_no++; diff --git a/litmus/litmus.c b/litmus/litmus.c index 26938ac..f4d676c 100644 --- a/litmus/litmus.c +++ b/litmus/litmus.c @@ -13,10 +13,19 @@ #include #include #include +#include #include #include #include +#ifdef CONFIG_PLUGIN_MC +#include +#include +#include +#else +struct mc_task; +#endif + /* Number of RT tasks that exist in the system */ atomic_t rt_task_count = ATOMIC_INIT(0); static DEFINE_RAW_SPINLOCK(task_transition_lock); @@ -31,8 +40,16 @@ atomic_t __log_seq_no = ATOMIC_INIT(0); atomic_t release_master_cpu = ATOMIC_INIT(NO_CPU); #endif -static struct kmem_cache * bheap_node_cache; -extern struct kmem_cache * release_heap_cache; +static struct kmem_cache *bheap_node_cache; +extern struct kmem_cache *release_heap_cache; + +#ifdef CONFIG_MERGE_TIMERS +extern struct kmem_cache *event_list_cache; +#endif + +#ifdef CONFIG_PLUGIN_MC +static struct kmem_cache *mc_data_cache; +#endif struct bheap_node* bheap_node_alloc(int gfp_flags) { @@ -110,14 +127,6 @@ asmlinkage long sys_set_rt_task_param(pid_t pid, struct rt_task __user * param) "because wcet > period\n", pid); goto out_unlock; } - if ( tp.cls != RT_CLASS_HARD && - tp.cls != RT_CLASS_SOFT && - tp.cls != RT_CLASS_BEST_EFFORT) - { - printk(KERN_INFO "litmus: real-time task %d rejected " - "because its class is invalid\n"); - goto out_unlock; - } if (tp.budget_policy != NO_ENFORCEMENT && tp.budget_policy != QUANTUM_ENFORCEMENT && tp.budget_policy != PRECISE_ENFORCEMENT) @@ -282,6 +291,79 @@ asmlinkage long sys_null_call(cycles_t __user *ts) return ret; } +#ifdef CONFIG_PLUGIN_MC +asmlinkage long sys_set_rt_task_mc_param(pid_t pid, struct mc_task __user *param) +{ + struct mc_task mc; + struct mc_data *mc_data; + struct task_struct *target; + int retval = -EINVAL; + + printk("Setting up mixed-criticality task parameters for process %d.\n", + pid); + + if (pid < 0 || param == 0) { + goto out; + } + if (copy_from_user(&mc, param, sizeof(mc))) { + retval = -EFAULT; + goto out; + } + + /* Task search and manipulation must be protected */ + read_lock_irq(&tasklist_lock); + if (!(target = find_task_by_vpid(pid))) { + retval = -ESRCH; + goto out_unlock; + } + + if (is_realtime(target)) { + /* The task is already a real-time task. + * We cannot not allow parameter changes at this point. + */ + retval = -EBUSY; + goto out_unlock; + } + + /* check parameters passed in are valid */ + if (mc.crit < CRIT_LEVEL_A || mc.crit >= NUM_CRIT_LEVELS) { + printk(KERN_WARNING "litmus: real-time task %d rejected because " + "of invalid criticality level\n", pid); + goto out_unlock; + } + if (CRIT_LEVEL_A == mc.crit && + (mc.lvl_a_id < 0 || + mc.lvl_a_id >= CONFIG_PLUGIN_MC_LEVEL_A_MAX_TASKS)) { + printk(KERN_WARNING "litmus: real-time task %d rejected because " + "of invalid level A id\n", pid); + goto out_unlock; + } + + mc_data = tsk_rt(target)->mc_data; + if (!mc_data) { + mc_data = kmem_cache_alloc(mc_data_cache, GFP_ATOMIC); + if (!mc_data) { + retval = -ENOMEM; + goto out_unlock; + } + tsk_rt(target)->mc_data = mc_data; + } + mc_data->mc_task = mc; + + retval = 0; +out_unlock: + read_unlock_irq(&tasklist_lock); +out: + return retval; +} +#else +asmlinkage long sys_set_rt_task_mc_param(pid_t pid, struct mc_task __user *param) +{ + /* don't allow this syscall if the plugin is not enabled */ + return -EINVAL; +} +#endif + /* p is a real-time task. Re-init its state as a best-effort task. */ static void reinit_litmus_state(struct task_struct* p, int restore) { @@ -487,6 +569,15 @@ void exit_litmus(struct task_struct *dead_tsk) free_page((unsigned long) tsk_rt(dead_tsk)->ctrl_page); } +#ifdef CONFIG_PLUGIN_MC + /* The MC-setup syscall might succeed and allocate mc_data, but the + * task may not exit in real-time mode, and that memory will leak. + * Check and free it here. + */ + if (tsk_rt(dead_tsk)->mc_data) + kmem_cache_free(mc_data_cache, tsk_rt(dead_tsk)->mc_data); +#endif + /* main cleanup only for RT tasks */ if (is_realtime(dead_tsk)) litmus_exit_task(dead_tsk); @@ -527,8 +618,14 @@ static int __init _init_litmus(void) register_sched_plugin(&linux_sched_plugin); - bheap_node_cache = KMEM_CACHE(bheap_node, SLAB_PANIC); + bheap_node_cache = KMEM_CACHE(bheap_node, SLAB_PANIC); release_heap_cache = KMEM_CACHE(release_heap, SLAB_PANIC); +#ifdef CONFIG_MERGE_TIMERS + event_list_cache = KMEM_CACHE(event_list, SLAB_PANIC); +#endif +#ifdef CONFIG_PLUGIN_MC + mc_data_cache = KMEM_CACHE(mc_data, SLAB_PANIC); +#endif #ifdef CONFIG_MAGIC_SYSRQ /* offer some debugging help */ @@ -548,6 +645,12 @@ static void _exit_litmus(void) exit_litmus_proc(); kmem_cache_destroy(bheap_node_cache); kmem_cache_destroy(release_heap_cache); +#ifdef CONFIG_MERGE_TIMERS + kmem_cache_destroy(event_list_cache); +#endif +#ifdef CONFIG_PLUGIN_MC + kmem_cache_destroy(mc_data_cache); +#endif } module_init(_init_litmus); diff --git a/litmus/locking.c b/litmus/locking.c index 2693f1a..bc41508 100644 --- a/litmus/locking.c +++ b/litmus/locking.c @@ -1,3 +1,5 @@ +#include +#include #include #ifdef CONFIG_LITMUS_LOCKING @@ -69,6 +71,10 @@ asmlinkage long sys_litmus_lock(int lock_od) struct od_table_entry* entry; struct litmus_lock* l; + TS_SYSCALL_IN_START; + + TS_SYSCALL_IN_END; + TS_LOCK_START; entry = get_entry_for_od(lock_od); @@ -80,7 +86,9 @@ asmlinkage long sys_litmus_lock(int lock_od) /* Note: task my have been suspended or preempted in between! Take * this into account when computing overheads. */ - TS_UNLOCK_END; + TS_LOCK_END; + + TS_SYSCALL_OUT_START; return err; } @@ -91,6 +99,10 @@ asmlinkage long sys_litmus_unlock(int lock_od) struct od_table_entry* entry; struct litmus_lock* l; + TS_SYSCALL_IN_START; + + TS_SYSCALL_IN_END; + TS_UNLOCK_START; entry = get_entry_for_od(lock_od); @@ -104,21 +116,21 @@ asmlinkage long sys_litmus_unlock(int lock_od) * account when computing overheads. */ TS_UNLOCK_END; + TS_SYSCALL_OUT_START; + return err; } -struct task_struct* __waitqueue_remove_first(wait_queue_head_t *wq) +struct task_struct* waitqueue_first(wait_queue_head_t *wq) { - wait_queue_t* q; - struct task_struct* t = NULL; + wait_queue_t *q; if (waitqueue_active(wq)) { q = list_entry(wq->task_list.next, wait_queue_t, task_list); - t = (struct task_struct*) q->private; - __remove_wait_queue(wq, q); - } - return(t); + return (struct task_struct*) q->private; + } else + return NULL; } diff --git a/litmus/preempt.c b/litmus/preempt.c index ebe2e34..40e2686 100644 --- a/litmus/preempt.c +++ b/litmus/preempt.c @@ -6,6 +6,8 @@ /* The rescheduling state of each processor. */ DEFINE_PER_CPU_SHARED_ALIGNED(atomic_t, resched_state); +#define TRACE_TASK(t, fmt, args...) +#define TRACE(fmt, args...) void sched_state_will_schedule(struct task_struct* tsk) { diff --git a/litmus/rt_domain.c b/litmus/rt_domain.c index 81a5ac1..5296665 100644 --- a/litmus/rt_domain.c +++ b/litmus/rt_domain.c @@ -12,13 +12,11 @@ #include #include +#include #include #include - #include - #include - #include /* Uncomment when debugging timer races... */ @@ -51,34 +49,36 @@ static unsigned int time2slot(lt_t time) return (unsigned int) time2quanta(time, FLOOR) % RELEASE_QUEUE_SLOTS; } -static enum hrtimer_restart on_release_timer(struct hrtimer *timer) +static void do_release(struct release_heap *rh) { unsigned long flags; - struct release_heap* rh; - - VTRACE("on_release_timer(0x%p) starts.\n", timer); - TS_RELEASE_START; - rh = container_of(timer, struct release_heap, timer); - raw_spin_lock_irqsave(&rh->dom->release_lock, flags); VTRACE("CB has the release_lock 0x%p\n", &rh->dom->release_lock); /* remove from release queue */ - list_del(&rh->list); + list_del_init(&rh->list); raw_spin_unlock_irqrestore(&rh->dom->release_lock, flags); VTRACE("CB returned release_lock 0x%p\n", &rh->dom->release_lock); /* call release callback */ rh->dom->release_jobs(rh->dom, &rh->heap); - /* WARNING: rh can be referenced from other CPUs from now on. */ TS_RELEASE_END; +} - VTRACE("on_release_timer(0x%p) ends.\n", timer); - - return HRTIMER_NORESTART; +#ifdef CONFIG_MERGE_TIMERS +static void on_release(struct rt_event *e) +{ + do_release(container_of(e, struct release_heap, event)); } +#else +static enum hrtimer_restart on_release(struct hrtimer *timer) +{ + do_release(container_of(timer, struct release_heap, timer)); + return HRTIMER_NORESTART; +} +#endif /* allocated in litmus.c */ struct kmem_cache * release_heap_cache; @@ -86,19 +86,35 @@ struct kmem_cache * release_heap_cache; struct release_heap* release_heap_alloc(int gfp_flags) { struct release_heap* rh; - rh= kmem_cache_alloc(release_heap_cache, gfp_flags); + rh = kmem_cache_alloc(release_heap_cache, gfp_flags); if (rh) { +#ifdef CONFIG_MERGE_TIMERS + init_event(&rh->event, 0, on_release, + event_list_alloc(GFP_ATOMIC)); +#else /* initialize timer */ hrtimer_init(&rh->timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS); - rh->timer.function = on_release_timer; + rh->timer.function = on_release; +#endif } return rh; } +#ifdef CONFIG_MERGE_TIMERS +extern struct kmem_cache *event_list_cache; +#endif + void release_heap_free(struct release_heap* rh) { /* make sure timer is no longer in use */ +#ifdef CONFIG_MERGE_TIMERS + if (rh->dom) { + cancel_event(&rh->event); + kmem_cache_free(event_list_cache, rh->event.event_list); + } +#else hrtimer_cancel(&rh->timer); +#endif kmem_cache_free(release_heap_cache, rh); } @@ -147,13 +163,17 @@ static struct release_heap* get_release_heap(rt_domain_t *rt, return heap; } -static void reinit_release_heap(struct task_struct* t) +static void reinit_release_heap(rt_domain_t *rt, struct task_struct* t) { struct release_heap* rh; /* use pre-allocated release heap */ rh = tsk_rt(t)->rel_heap; +#ifdef CONFIG_MERGE_TIMERS + rh->event.prio = rt->prio; + cancel_event(&rh->event); +#else /* Make sure it is safe to use. The timer callback could still * be executing on another CPU; hrtimer_cancel() will wait * until the timer callback has completed. However, under no @@ -165,13 +185,50 @@ static void reinit_release_heap(struct task_struct* t) */ BUG_ON(hrtimer_cancel(&rh->timer)); +#ifdef CONFIG_RELEASE_MASTER + atomic_set(&rh->info.state, HRTIMER_START_ON_INACTIVE); +#endif +#endif /* initialize */ bheap_init(&rh->heap); + +} + #ifdef CONFIG_RELEASE_MASTER - atomic_set(&rh->info.state, HRTIMER_START_ON_INACTIVE); +static void arm_release_timer_on(struct release_heap *rh, int target_cpu) +#else +static void arm_release_timer(struct release_heap *rh) +#endif +{ +#ifdef CONFIG_MERGE_TIMERS + add_event(rh->dom->event_group, &rh->event, rh->release_time); +#else + VTRACE("arming timer 0x%p\n", &rh->timer); + /* we cannot arm the timer using hrtimer_start() + * as it may deadlock on rq->lock + * PINNED mode is ok on both local and remote CPU + */ + +#ifdef CONFIG_RELEASE_MASTER + if (rh->dom->release_master == NO_CPU && target_cpu == NO_CPU) +#endif + __hrtimer_start_range_ns(&rh->timer, + ns_to_ktime(rh->release_time), + 0, HRTIMER_MODE_ABS_PINNED, 0); +#ifdef CONFIG_RELEASE_MASTER + else + hrtimer_start_on(/* target_cpu overrides release master */ + (target_cpu != NO_CPU ? + target_cpu : rh->dom->release_master), + &rh->info, &rh->timer, + ns_to_ktime(rh->release_time), + HRTIMER_MODE_ABS_PINNED); +#endif #endif } -/* arm_release_timer() - start local release timer or trigger + + +/* setup_release() - start local release timer or trigger * remote timer (pull timer) * * Called by add_release() with: @@ -179,10 +236,10 @@ static void reinit_release_heap(struct task_struct* t) * - IRQ disabled */ #ifdef CONFIG_RELEASE_MASTER -#define arm_release_timer(t) arm_release_timer_on((t), NO_CPU) -static void arm_release_timer_on(rt_domain_t *_rt , int target_cpu) +#define setup_release(t) setup_release_on((t), NO_CPU) +static void setup_release_on(rt_domain_t *_rt , int target_cpu) #else -static void arm_release_timer(rt_domain_t *_rt) +static void setup_release(rt_domain_t *_rt) #endif { rt_domain_t *rt = _rt; @@ -191,14 +248,13 @@ static void arm_release_timer(rt_domain_t *_rt) struct task_struct* t; struct release_heap* rh; - VTRACE("arm_release_timer() at %llu\n", litmus_clock()); + VTRACE("setup_release() at %llu\n", litmus_clock()); list_replace_init(&rt->tobe_released, &list); list_for_each_safe(pos, safe, &list) { /* pick task of work list */ t = list_entry(pos, struct task_struct, rt_param.list); - sched_trace_task_release(t); - list_del(pos); + list_del_init(pos); /* put into release heap while holding release_lock */ raw_spin_lock(&rt->release_lock); @@ -211,7 +267,7 @@ static void arm_release_timer(rt_domain_t *_rt) VTRACE_TASK(t, "Dropped release_lock 0x%p\n", &rt->release_lock); - reinit_release_heap(t); + reinit_release_heap(rt, t); VTRACE_TASK(t, "release_heap ready\n"); raw_spin_lock(&rt->release_lock); @@ -221,7 +277,7 @@ static void arm_release_timer(rt_domain_t *_rt) rh = get_release_heap(rt, t, 1); } bheap_insert(rt->order, &rh->heap, tsk_rt(t)->heap_node); - VTRACE_TASK(t, "arm_release_timer(): added to release heap\n"); + VTRACE_TASK(t, "setup_release(): added to release heap\n"); raw_spin_unlock(&rt->release_lock); VTRACE_TASK(t, "Returned the release_lock 0x%p\n", &rt->release_lock); @@ -231,39 +287,19 @@ static void arm_release_timer(rt_domain_t *_rt) * this release_heap anyway). */ if (rh == tsk_rt(t)->rel_heap) { - VTRACE_TASK(t, "arming timer 0x%p\n", &rh->timer); - /* we cannot arm the timer using hrtimer_start() - * as it may deadlock on rq->lock - * - * PINNED mode is ok on both local and remote CPU - */ -#ifdef CONFIG_RELEASE_MASTER - if (rt->release_master == NO_CPU && - target_cpu == NO_CPU) -#endif - __hrtimer_start_range_ns(&rh->timer, - ns_to_ktime(rh->release_time), - 0, HRTIMER_MODE_ABS_PINNED, 0); #ifdef CONFIG_RELEASE_MASTER - else - hrtimer_start_on( - /* target_cpu overrides release master */ - (target_cpu != NO_CPU ? - target_cpu : rt->release_master), - &rh->info, &rh->timer, - ns_to_ktime(rh->release_time), - HRTIMER_MODE_ABS_PINNED); + arm_release_timer_on(rh, target_cpu); +#else + arm_release_timer(rh); #endif - } else - VTRACE_TASK(t, "0x%p is not my timer\n", &rh->timer); + } } } void rt_domain_init(rt_domain_t *rt, bheap_prio_t order, check_resched_needed_t check, - release_jobs_t release - ) + release_jobs_t release) { int i; @@ -275,7 +311,7 @@ void rt_domain_init(rt_domain_t *rt, if (!order) order = dummy_order; -#ifdef CONFIG_RELEASE_MASTER +#if defined(CONFIG_RELEASE_MASTER) && !defined(CONFIG_MERGE_TIMERS) rt->release_master = NO_CPU; #endif @@ -298,12 +334,13 @@ void rt_domain_init(rt_domain_t *rt, */ void __add_ready(rt_domain_t* rt, struct task_struct *new) { - TRACE("rt: adding %s/%d (%llu, %llu) rel=%llu to ready queue at %llu\n", + VTRACE("rt: adding %s/%d (%llu, %llu) rel=%llu to ready queue at %llu\n", new->comm, new->pid, get_exec_cost(new), get_rt_period(new), get_release(new), litmus_clock()); BUG_ON(bheap_node_in_heap(tsk_rt(new)->heap_node)); + new->rt_param.domain = rt; bheap_insert(rt->order, &rt->ready_queue, tsk_rt(new)->heap_node); rt->check_resched(rt); } @@ -322,7 +359,7 @@ void __merge_ready(rt_domain_t* rt, struct bheap* tasks) void __add_release_on(rt_domain_t* rt, struct task_struct *task, int target_cpu) { - TRACE_TASK(task, "add_release_on(), rel=%llu, target=%d\n", + VTRACE_TASK(task, "add_release_on(), rel=%llu, target=%d\n", get_release(task), target_cpu); list_add(&tsk_rt(task)->list, &rt->tobe_released); task->rt_param.domain = rt; @@ -330,7 +367,7 @@ void __add_release_on(rt_domain_t* rt, struct task_struct *task, /* start release timer */ TS_SCHED2_START(task); - arm_release_timer_on(rt, target_cpu); + setup_release_on(rt, target_cpu); TS_SCHED2_END(task); } @@ -341,15 +378,88 @@ void __add_release_on(rt_domain_t* rt, struct task_struct *task, */ void __add_release(rt_domain_t* rt, struct task_struct *task) { - TRACE_TASK(task, "add_release(), rel=%llu\n", get_release(task)); + VTRACE_TASK(task, "add_release(), rel=%llu\n", get_release(task)); list_add(&tsk_rt(task)->list, &rt->tobe_released); task->rt_param.domain = rt; /* start release timer */ TS_SCHED2_START(task); - arm_release_timer(rt); + setup_release(rt); TS_SCHED2_END(task); } +/****************************************************************************** + * domain_t wrapper + ******************************************************************************/ + +/* pd_requeue - calls underlying rt_domain add methods. + * If the task is not yet released, it is inserted into the rt_domain + * ready queue. Otherwise, it is queued for release. + * + * Assumes the caller already holds dom->lock. + */ +static void pd_requeue(domain_t *dom, struct task_struct *task) +{ + rt_domain_t *domain = (rt_domain_t*)dom->data; + + TRACE_TASK(task, "Requeueing\n"); + BUG_ON(!task || !is_realtime(task)); + BUG_ON(is_queued(task)); + BUG_ON(get_task_domain(task) != dom); + + if (is_released(task, litmus_clock())) { + __add_ready(domain, task); + VTRACE("rt: adding %s/%d (%llu, %llu) rel=%llu to ready queue at %llu\n", + task->comm, task->pid, get_exec_cost(task), get_rt_period(task), + get_release(task), litmus_clock()); + } else { + /* task has to wait for next release */ + VTRACE_TASK(task, "add release(), rel=%llu\n", get_release(task)); + add_release(domain, task); + } + +} + +/* pd_take_ready - removes and returns the next ready task from the rt_domain + * + * Assumes the caller already holds dom->lock. + */ +static struct task_struct* pd_take_ready(domain_t *dom) +{ + return __take_ready((rt_domain_t*)dom->data); + } + +/* pd_peek_ready - returns the head of the rt_domain ready queue + * + * Assumes the caller already holds dom->lock. + */ +static struct task_struct* pd_peek_ready(domain_t *dom) +{ + return __next_ready((rt_domain_t*)dom->data); +} + +static void pd_remove(domain_t *dom, struct task_struct *task) +{ + if (is_queued(task)) + remove((rt_domain_t*)dom->data, task); +} + +/* pd_domain_init - create a generic domain wrapper for an rt_domain + */ +void pd_domain_init(domain_t *dom, + rt_domain_t *domain, + bheap_prio_t order, + check_resched_needed_t check, + release_jobs_t release, + preempt_needed_t preempt_needed, + task_prio_t priority) +{ + rt_domain_init(domain, order, check, release); + domain_init(dom, &domain->ready_lock, + pd_requeue, pd_peek_ready, pd_take_ready, + preempt_needed, priority); + dom->remove = pd_remove; + dom->data = domain; +} diff --git a/litmus/sched_gsn_edf.c b/litmus/sched_gsn_edf.c index 3092797..c5c9600 100644 --- a/litmus/sched_gsn_edf.c +++ b/litmus/sched_gsn_edf.c @@ -374,10 +374,8 @@ static struct task_struct* gsnedf_schedule(struct task_struct * prev) /* Bail out early if we are the release master. * The release master never schedules any real-time tasks. */ - if (gsnedf.release_master == entry->cpu) { - sched_state_task_picked(); + if (gsnedf.release_master == entry->cpu) return NULL; - } #endif raw_spin_lock(&gsnedf_lock); @@ -778,6 +776,8 @@ int gsnedf_fmlp_lock(struct litmus_lock* l) * ->owner. We can thus check it without acquiring the spin * lock. */ BUG_ON(sem->owner != t); + + remove_wait_queue(&sem->wait, &wait); } else { /* it's ours now */ sem->owner = t; @@ -803,7 +803,7 @@ int gsnedf_fmlp_unlock(struct litmus_lock* l) } /* check if there are jobs waiting for this resource */ - next = __waitqueue_remove_first(&sem->wait); + next = waitqueue_first(&sem->wait); if (next) { /* next becomes the resouce holder */ sem->owner = next; diff --git a/litmus/sched_litmus.c b/litmus/sched_litmus.c index e695289..0687be0 100644 --- a/litmus/sched_litmus.c +++ b/litmus/sched_litmus.c @@ -194,6 +194,9 @@ static void dequeue_task_litmus(struct rq *rq, struct task_struct *p, static void yield_task_litmus(struct rq *rq) { + TS_SYSCALL_IN_START; + + TS_SYSCALL_OUT_END; BUG_ON(rq->curr != current); /* sched_yield() is called to trigger delayed preemptions. * Thus, mark the current task as needing to be rescheduled. @@ -202,6 +205,8 @@ static void yield_task_litmus(struct rq *rq) */ clear_exit_np(current); litmus_reschedule_local(); + + TS_SYSCALL_OUT_START; } /* Plugins are responsible for this. diff --git a/litmus/sched_mc.c b/litmus/sched_mc.c new file mode 100644 index 0000000..43b0731 --- /dev/null +++ b/litmus/sched_mc.c @@ -0,0 +1,1297 @@ +/** + * litmus/sched_mc.c + * + * Implementation of the Mixed Criticality scheduling algorithm. + * + * (Per Mollison, Erickson, Anderson, Baruah, Scoredos 2010) + * + * Absolute first: relative time spent doing different parts of release + * and scheduling overhead needs to be measured and graphed. + * + * Domain locks should be more fine-grained. There is no reason to hold the + * ready-queue lock when adding a task to the release-queue. + * + * The levels should be converted to linked-lists so that they are more + * adaptable and need not be identical on all processors. + * + * The interaction between remove_from_all and other concurrent operations + * should be re-examined. If a job_completion and a preemption happen + * simultaneously, a task could be requeued, removed, then requeued again. + * + * Level-C tasks should be able to swap CPUs a-la GSN-EDF. They should also + * try and swap with the last CPU they were on. This could be complicated for + * ghost tasks. + * + * Locking for timer-merging could be infinitely more fine-grained. A second + * hash could select a lock to use based on queue slot. This approach might + * also help with add_release in rt_domains. + * + * It should be possible to reserve a CPU for ftdumping. + * + * The real_deadline business seems sloppy. + * + * The amount of data in the header file should be cut down. The use of the + * header file in general needs to be re-examined. + * + * The plugin needs to be modified so that it doesn't freeze when it is + * deactivated in a VM. + * + * The locking in check_for_preempt is not fine-grained enough. + * + * The size of the structures could be smaller. Debugging info might be + * excessive as things currently stand. + * + * The macro can_requeue has been expanded too much. Anything beyond + * scheduled_on is a hack! + * + * Domain names (rt_domain) are still clumsy. + * + * Should BE be moved into the kernel? This will require benchmarking. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +/** + * struct cpu_entry - State of a CPU for the entire MC system + * @cpu CPU id + * @scheduled Task that is physically running + * @linked Task that should be running / is logically running + * @lock For serialization + * @crit_entries Array of CPU state per criticality level + * @redir List of redirected work for this CPU. + * @redir_lock Lock for @redir. + * @event_group Event group for timer merging. + */ +struct cpu_entry { + int cpu; + struct task_struct* scheduled; + struct task_struct* will_schedule; + struct task_struct* linked; + raw_spinlock_t lock; + struct crit_entry crit_entries[NUM_CRIT_LEVELS]; +#ifdef CONFIG_PLUGIN_MC_REDIRECT + struct list_head redir; + raw_spinlock_t redir_lock; +#endif +#ifdef CONFIG_MERGE_TIMERS + struct event_group *event_group; +#endif +}; + +DEFINE_PER_CPU(struct cpu_entry, cpus); +#ifdef CONFIG_RELEASE_MASTER +static int interrupt_cpu; +#endif + +#define domain_data(dom) (container_of(dom, struct domain_data, domain)) +#define is_global(dom) (domain_data(dom)->heap) +#define is_global_task(t) (is_global(get_task_domain(t))) +#define can_use(ce) \ + ((ce)->state == CS_ACTIVE || (ce->state == CS_ACTIVATE)) +#define can_requeue(t) \ + ((t)->rt_param.linked_on == NO_CPU && /* Not linked anywhere */ \ + !is_queued(t) && /* Not gonna be linked */ \ + (!is_global_task(t) || (t)->rt_param.scheduled_on == NO_CPU)) +#define entry_level(e) \ + (((e)->linked) ? tsk_mc_crit((e)->linked) : NUM_CRIT_LEVELS - 1) +#define crit_cpu(ce) \ + (container_of((void*)((ce) - (ce)->level), struct cpu_entry, crit_entries)) +#define get_crit_entry_for(cpu, level) (&per_cpu(cpus, cpu).crit_entries[level]) +#define TRACE_ENTRY(e, fmt, args...) \ + STRACE("P%d, linked=" TS " " fmt, e->cpu, TA(e->linked), ##args) +#define TRACE_CRIT_ENTRY(ce, fmt, args...) \ + STRACE("%s P%d, linked=" TS " " fmt, \ + (ce)->domain->name, crit_cpu(ce)->cpu, TA((ce)->linked), ##args) + +/* + * Sort CPUs within a global domain's heap. + */ +static int cpu_lower_prio(struct bheap_node *a, struct bheap_node *b) +{ + struct domain *domain; + struct crit_entry *first, *second; + struct task_struct *first_link, *second_link; + + first = a->value; + second = b->value; + first_link = first->linked; + second_link = second->linked; + + if (first->state == CS_REMOVED || second->state == CS_REMOVED) { + /* Removed entries go at the back of the heap */ + return first->state != CS_REMOVED && + second->state != CS_REMOVED; + } else if (!first_link || !second_link) { + /* Entry with nothing scheduled is lowest priority */ + return second_link && !first_link; + } else { + /* Sort by deadlines of tasks */ + domain = get_task_domain(first_link); + return domain->higher_prio(second_link, first_link); + } +} + +/* + * Return true if the domain has a higher priority ready task. The @curr + * task must belong to the domain. + */ +static int mc_preempt_needed(struct domain *dom, struct task_struct* curr) +{ + struct task_struct *next = dom->peek_ready(dom); + if (!next || !curr) { + return next && !curr; + } else { + BUG_ON(tsk_mc_crit(next) != tsk_mc_crit(curr)); + return get_task_domain(next)->higher_prio(next, curr); + } +} + +/* + * Update crit entry position in a global heap. Caller must hold + * @ce's domain lock. + */ +static inline void update_crit_position(struct crit_entry *ce) +{ + struct bheap *heap; + if (is_global(ce->domain)) { + heap = domain_data(ce->domain)->heap; + BUG_ON(!heap); + BUG_ON(!bheap_node_in_heap(ce->node)); + bheap_delete(cpu_lower_prio, heap, ce->node); + bheap_insert(cpu_lower_prio, heap, ce->node); + } +} + +/* + * Update crit entry position in a global heap if it has been marked + * for update. Caller must hold @ce's domain lock. + */ +static void fix_crit_position(struct crit_entry *ce) +{ + if (is_global(ce->domain)) { + if (CS_ACTIVATE == ce->state) { + ce->state = CS_ACTIVE; + update_crit_position(ce); + } else if (CS_REMOVE == ce->state) { + ce->state = CS_REMOVED; + update_crit_position(ce); + } + } +} + +/* + * Return next CPU which should preempted or NULL if the domain has no + * preemptable CPUs. Caller must hold the @dom lock. + */ +static inline struct crit_entry* lowest_prio_cpu(struct domain *dom) +{ + struct bheap *heap = domain_data(dom)->heap; + struct bheap_node* hn; + struct crit_entry *ce, *res = NULL; + do { + hn = bheap_peek(cpu_lower_prio, heap); + ce = (hn) ? hn->value : NULL; + if (ce) { + if (ce->state == CS_ACTIVE) + res = ce; + else if (ce->state == CS_REMOVED) + ce = NULL; + else + fix_crit_position(ce); + } + } while (ce && !res); + return res; +} + +/* + * Cancel ghost timer. + */ +static inline void cancel_ghost(struct crit_entry *ce) +{ +#ifdef CONFIG_MERGE_TIMERS + cancel_event(&ce->event); +#else + hrtimer_try_to_cancel(&ce->timer); +#endif +} + +/* + * Arm ghost timer. Will merge timers if the option is specified. + */ +static inline void arm_ghost(struct crit_entry *ce, lt_t fire) +{ +#ifdef CONFIG_MERGE_TIMERS + add_event(crit_cpu(ce)->event_group, &ce->event, fire); +#else + __hrtimer_start_range_ns(&ce->timer, + ns_to_ktime(fire), + 0 /* delta */, + HRTIMER_MODE_ABS_PINNED, + 0 /* no wakeup */); +#endif +} + +/* + * Time accounting for ghost tasks. + * Must be called before a decision is made involving the task's budget. + */ +static void update_ghost_time(struct task_struct *p) +{ + u64 clock = litmus_clock(); + u64 delta = clock - p->se.exec_start; + BUG_ON(!is_ghost(p)); + if (unlikely ((s64)delta < 0)) { + delta = 0; + TRACE_MC_TASK(p, "WARNING: negative time delta\n"); + } + if (tsk_mc_data(p)->mc_job.ghost_budget <= delta) { + TRACE_MC_TASK(p, "Ghost job could have ended\n"); + tsk_mc_data(p)->mc_job.ghost_budget = 0; + p->se.exec_start = clock; + } else { + TRACE_MC_TASK(p, "Ghost job updated, but didn't finish\n"); + tsk_mc_data(p)->mc_job.ghost_budget -= delta; + p->se.exec_start = clock; + } +} + +/** + * link_task_to_crit() - Logically run a task at a criticality level. + * Caller must hold @ce's CPU lock. + */ +static void link_task_to_crit(struct crit_entry *ce, + struct task_struct *task) +{ + lt_t when_to_fire; + + TRACE_CRIT_ENTRY(ce, "Linking " TS "\n", TA(task)); + BUG_ON(!can_use(ce) && task); + BUG_ON(task && tsk_rt(task)->linked_on != NO_CPU); + BUG_ON(task && is_global(ce->domain) && + !bheap_node_in_heap(ce->node)); + + /* Unlink last task */ + if (ce->linked) { + TRACE_MC_TASK(ce->linked, "Unlinking\n"); + ce->linked->rt_param.linked_on = NO_CPU; + if (is_ghost(ce->linked)) { + cancel_ghost(ce); + if (tsk_mc_data(ce->linked)->mc_job.ghost_budget > 0) { + /* Job isn't finished, so do accounting */ + update_ghost_time(ce->linked); + } + } + } + + /* Actually link task */ + ce->linked = task; + if (task) { + task->rt_param.linked_on = crit_cpu(ce)->cpu; + if (is_ghost(task) && CRIT_LEVEL_A != tsk_mc_crit(task)) { + /* There is a level-A timer that will force a + * preemption, so we don't set this for level-A + * tasks. Otherwise reset the budget timer. + */ + task->se.exec_start = litmus_clock(); + when_to_fire = task->se.exec_start + + tsk_mc_data(task)->mc_job.ghost_budget; + arm_ghost(ce, when_to_fire); + } + } +} + +static void check_for_preempt(struct domain*); + +/** + * job_arrival() - Called when a task re-enters the system. + * Caller must hold no locks. + */ +static void job_arrival(struct task_struct *task) +{ + struct domain *dom = get_task_domain(task); + + TRACE_MC_TASK(task, "Job arriving\n"); + BUG_ON(!task); + + raw_spin_lock(dom->lock); + if (can_requeue(task)) { + BUG_ON(task->rt_param.linked_on != NO_CPU); + dom->requeue(dom, task); + check_for_preempt(dom); + } else { + /* If a global task is scheduled on one cpu, it CANNOT + * be requeued into a global domain. Another cpu might + * dequeue the global task before it is descheduled, + * causing the system to crash when the task is scheduled + * in two places simultaneously. + */ + TRACE_MC_TASK(task, "Delayed arrival of scheduled task\n"); + } + raw_spin_unlock(dom->lock); +} + +/** + * low_prio_arrival() - If CONFIG_PLUGIN_MC_REDIRECT is enabled, will + * redirect a lower priority job_arrival work to the interrupt_cpu. + */ +static void low_prio_arrival(struct task_struct *task) +{ + struct cpu_entry *entry; + + /* Race conditions! */ + if (!can_requeue(task)) return; + +#ifdef CONFIG_PLUGIN_MC_REDIRECT + if (!is_global_task(task)) + goto arrive; + if (smp_processor_id() != interrupt_cpu) { + entry = &__get_cpu_var(cpus); + raw_spin_lock(&entry->redir_lock); + TRACE_MC_TASK(task, "Adding to redirect queue\n"); + list_add(&tsk_rt(task)->list, &entry->redir); + raw_spin_unlock(&entry->redir_lock); + litmus_reschedule(interrupt_cpu); + } else +#endif + { +arrive: + job_arrival(task); + } +} + +#ifdef CONFIG_PLUGIN_MC_REDIRECT +/** + * fix_global_levels() - Execute redirected job arrivals on this cpu. + */ +static void fix_global_levels(void) +{ + int c; + struct cpu_entry *e; + struct list_head *pos, *safe; + struct task_struct *t; + + STRACE("Fixing global levels\n"); + for_each_online_cpu(c) { + e = &per_cpu(cpus, c); + raw_spin_lock(&e->redir_lock); + list_for_each_safe(pos, safe, &e->redir) { + t = list_entry(pos, struct task_struct, rt_param.list); + BUG_ON(!t); + TRACE_MC_TASK(t, "Dequeued redirected job\n"); + list_del_init(pos); + job_arrival(t); + } + raw_spin_unlock(&e->redir_lock); + } +} +#endif + +/** + * link_task_to_cpu() - Logically run a task on a CPU. + * The task must first have been linked to one of the CPU's crit_entries. + * Caller must hold the entry lock. + */ +static void link_task_to_cpu(struct cpu_entry *entry, struct task_struct *task) +{ + int i = entry_level(entry); + struct crit_entry *ce; + TRACE_MC_TASK(task, "Linking to P%d\n", entry->cpu); + BUG_ON(task && tsk_rt(task)->linked_on != entry->cpu); + BUG_ON(task && is_ghost(task)); + + if (task){ + set_rt_flags(task, RT_F_RUNNING); + } + entry->linked = task; + + /* Higher criticality crit entries are now usable */ + for (; i < entry_level(entry) + 1; i++) { + ce = &entry->crit_entries[i]; + if (!can_use(ce)) { + ce->state = CS_ACTIVATE; + } + } +} + +/** + * preempt() - Preempt a logically running task with a higher priority one. + * @dom Domain from which to draw higher priority task + * @ce CPU criticality level to preempt + * + * Caller must hold the lock for @dom and @ce's CPU lock. + */ +static void preempt(struct domain *dom, struct crit_entry *ce) +{ + struct task_struct *task = dom->take_ready(dom); + struct cpu_entry *entry = crit_cpu(ce); + struct task_struct *old = ce->linked; + + BUG_ON(!task); + TRACE_CRIT_ENTRY(ce, "Preempted by " TS "\n", TA(task)); + + /* Per-domain preemption */ + link_task_to_crit(ce, task); + if (old && can_requeue(old)) { + dom->requeue(dom, old); + } + update_crit_position(ce); + + /* Preempt actual execution if this is a running task */ + if (!is_ghost(task)) { + link_task_to_cpu(entry, task); + preempt_if_preemptable(entry->scheduled, entry->cpu); + } else if (old && old == entry->linked) { + /* Preempted a running task with a ghost job. Null needs to be + * running. + */ + link_task_to_cpu(entry, NULL); + preempt_if_preemptable(entry->scheduled, entry->cpu); + } +} + +/** + * update_crit_levels() - Update criticality entries for the new cpu state. + * This should be called after a new task has been linked to @entry. + * The caller must hold the @entry->lock, but this method will release it. + */ +static void update_crit_levels(struct cpu_entry *entry) +{ + int i, global_preempted; + struct crit_entry *ce; + struct task_struct *readmit[NUM_CRIT_LEVELS]; + enum crit_level level = entry_level(entry); + + /* Remove lower priority tasks from the entry */ + for (i = level + 1; i < NUM_CRIT_LEVELS; i++) { + ce = &entry->crit_entries[i]; + + global_preempted = ce->linked && + /* This task is running on a cpu */ + ce->linked->rt_param.scheduled_on == entry->cpu && + /* But it was preempted */ + ce->linked != entry->linked && + /* And it is an eligible global task */ + !is_ghost(ce->linked) && is_global(ce->domain); + + /* Do not readmit global tasks which are preempted! These can't + * ever be re-admitted until they are descheduled for reasons + * explained in job_arrival. + */ + readmit[i] = (!global_preempted) ? ce->linked : NULL; + + ce->state = CS_REMOVE; + if (ce->linked) + link_task_to_crit(ce, NULL); + } + /* Need to unlock so we can access domains */ + raw_spin_unlock(&entry->lock); + + /* Re-admit tasks to the system */ + for (i = level + 1; i < NUM_CRIT_LEVELS; i++) { + ce = &entry->crit_entries[i]; + if (readmit[i]) { + low_prio_arrival(readmit[i]); + } + } +} + +/** + * check_for_preempt() - Causes a preemption if higher-priority tasks are ready. + * Caller must hold domain lock. + * Makes gigantic nasty assumption that there is 1 global criticality level, + * and it is the last one in each list, so it doesn't call update_crit.. + */ +static void check_for_preempt(struct domain *dom) +{ + int recheck = 1; + struct cpu_entry *entry; + struct crit_entry *ce; + + if (is_global(dom)) { + /* Loop until we find a non-preemptable CPU */ + while ((ce = lowest_prio_cpu(dom)) && recheck) { + entry = crit_cpu(ce); + recheck = 1; + + /* Cache next task */ + dom->peek_ready(dom); + + raw_spin_lock(&entry->lock); + if (!can_use(ce)) + /* CPU disabled while locking! */ + fix_crit_position(ce); + else if (dom->preempt_needed(dom, ce->linked)) + /* Success! Check for more preemptions */ + preempt(dom, ce); + else { + /* Failure! */ + recheck = 0; + TRACE_CRIT_ENTRY(ce, "Stopped global check\n"); + } + raw_spin_unlock(&entry->lock); + } + } else /* Partitioned */ { + ce = domain_data(dom)->crit_entry; + entry = crit_cpu(ce); + + /* Cache next task */ + dom->peek_ready(dom); + + raw_spin_lock(&entry->lock); + if (can_use(ce) && dom->preempt_needed(dom, ce->linked)) { + preempt(dom, ce); + update_crit_levels(entry); + } else { + raw_spin_unlock(&entry->lock); + } + } +} + +/** + * remove_from_all() - Logically remove a task from all structures. + * Caller must hold no locks. + */ +static void remove_from_all(struct task_struct* task) +{ + int update = 0; + struct cpu_entry *entry; + struct crit_entry *ce; + struct domain *dom = get_task_domain(task); + + TRACE_MC_TASK(task, "Removing from everything\n"); + BUG_ON(!task); + + raw_spin_lock(dom->lock); + + /* Remove the task from any CPU state */ + if (task->rt_param.linked_on != NO_CPU) { + entry = &per_cpu(cpus, task->rt_param.linked_on); + raw_spin_lock(&entry->lock); + + /* Unlink only if task is still linked post lock */ + ce = &entry->crit_entries[tsk_mc_crit(task)]; + if (task->rt_param.linked_on != NO_CPU) { + BUG_ON(ce->linked != task); + link_task_to_crit(ce, NULL); + update_crit_position(ce); + if (!is_ghost(task) && entry->linked == task) { + update = 1; + link_task_to_cpu(entry, NULL); + } + } else { + TRACE_MC_TASK(task, "Unlinked before we got lock!\n"); + } + if (update) + update_crit_levels(entry); + else + raw_spin_unlock(&entry->lock); + } else { + TRACE_MC_TASK(task, "Not linked to anything\n"); + } + + /* Ensure the task isn't returned by its domain */ + dom->remove(dom, task); + + raw_spin_unlock(dom->lock); +} + +/** + * job_completion() - Update task state and re-enter it into the system. + * Converts tasks which have completed their execution early into ghost jobs. + * Caller must hold no locks. + */ +static void job_completion(struct task_struct *task, int forced) +{ + int behind; + TRACE_MC_TASK(task, "Completed\n"); + + /* Logically stop the task execution */ + set_rt_flags(task, RT_F_SLEEP); + remove_from_all(task); + + /* Level-A tasks cannot ever get behind */ + behind = tsk_mc_crit(task) != CRIT_LEVEL_A && behind_server(task); + + if (!forced && !is_ghost(task)) { + /* Task voluntarily ceased execution. Move on to next period */ + task_release(task); + sched_trace_task_completion(task, forced); + + /* Convert to ghost job */ + tsk_mc_data(task)->mc_job.ghost_budget = budget_remaining(task); + tsk_mc_data(task)->mc_job.is_ghost = 1; + } + + /* If the task has no ghost budget, convert back from ghost. + * If the task is behind, undo ghost conversion so that it + * can catch up. + */ + if (behind || tsk_mc_data(task)->mc_job.ghost_budget == 0) { + TRACE_MC_TASK(task, "Not a ghost task\n"); + tsk_mc_data(task)->mc_job.is_ghost = 0; + tsk_mc_data(task)->mc_job.ghost_budget = 0; + } + + /* If server has run out of budget, wait until next release */ + if (budget_exhausted(task)) { + server_release(task); + } + + /* Requeue non-blocking tasks */ + if (is_running(task)) + job_arrival(task); +} + +/** + * mc_ghost_exhausted() - Complete logically running ghost task. + */ +#ifdef CONFIG_MERGE_TIMERS +static void mc_ghost_exhausted(struct rt_event *e) +{ + struct crit_entry *ce = container_of(e, struct crit_entry, event); +#else +static enum hrtimer_restart mc_ghost_exhausted(struct hrtimer *timer) +{ + struct crit_entry *ce = container_of(timer, struct crit_entry, timer); +#endif + + unsigned long flags; + struct task_struct *tmp = NULL; + + local_irq_save(flags); + TRACE("Ghost exhausted\n"); + TRACE_CRIT_ENTRY(ce, "Firing here\n"); + + /* Due to race conditions, we cannot just set the linked + * task's budget to 0 as it may no longer be the task + * for which this timer was armed. Instead, update the running + * task time and see if this causes exhaustion. + */ + raw_spin_lock(&crit_cpu(ce)->lock); + if (ce->linked && is_ghost(ce->linked)) { + update_ghost_time(ce->linked); + if (tsk_mc_data(ce->linked)->mc_job.ghost_budget == 0) { + tmp = ce->linked; + } + } + raw_spin_unlock(&crit_cpu(ce)->lock); + + if (tmp) + job_completion(tmp, 0); + + local_irq_restore(flags); +#ifndef CONFIG_MERGE_TIMERS + return HRTIMER_NORESTART; +#endif +} + +/* + * The MC-CE common timer callback code for merged and non-merged timers. + * Returns the next time the timer should fire. + */ +static lt_t __ce_timer_function(struct ce_dom_data *ce_data) +{ + struct crit_entry *ce = get_crit_entry_for(ce_data->cpu, CRIT_LEVEL_A); + struct domain *dom = ce->domain; + struct task_struct *old_link = NULL; + lt_t next_timer_abs; + + TRACE("MC level-A timer callback for CPU %d\n", ce_data->cpu); + + raw_spin_lock(dom->lock); + + raw_spin_lock(&crit_cpu(ce)->lock); + if (ce->linked && + ce->linked == ce_data->should_schedule && + is_ghost(ce->linked)) + { + old_link = ce->linked; + tsk_mc_data(ce->linked)->mc_job.ghost_budget = 0; + link_task_to_crit(ce, NULL); + } + raw_spin_unlock(&crit_cpu(ce)->lock); + + next_timer_abs = mc_ce_timer_callback_common(dom); + + /* Job completion will check for preemptions by means of calling job + * arrival if the task is not blocked */ + if (NULL != old_link) { + STRACE("old_link " TS " so will call job completion\n", TA(old_link)); + raw_spin_unlock(dom->lock); + job_completion(old_link, 0); + } else { + STRACE("old_link was null, so will call check for preempt\n"); + raw_spin_unlock(dom->lock); + check_for_preempt(dom); + } + return next_timer_abs; +} + +#ifdef CONFIG_MERGE_TIMERS +static void ce_timer_function(struct rt_event *e) +{ + struct ce_dom_data *ce_data = + container_of(e, struct ce_dom_data, event); + unsigned long flags; + lt_t next_timer_abs; + + TS_LVLA_RELEASE_START; + + local_irq_save(flags); + next_timer_abs = __ce_timer_function(ce_data); + add_event(per_cpu(cpus, ce_data->cpu).event_group, e, next_timer_abs); + local_irq_restore(flags); + + TS_LVLA_RELEASE_END; +} +#else /* else to CONFIG_MERGE_TIMERS */ +static enum hrtimer_restart ce_timer_function(struct hrtimer *timer) +{ + struct ce_dom_data *ce_data = + container_of(timer, struct ce_dom_data, timer); + unsigned long flags; + lt_t next_timer_abs; + + TS_LVLA_RELEASE_START; + + local_irq_save(flags); + next_timer_abs = __ce_timer_function(ce_data); + hrtimer_set_expires(timer, ns_to_ktime(next_timer_abs)); + local_irq_restore(flags); + + TS_LVLA_RELEASE_END; + + return HRTIMER_RESTART; +} +#endif /* CONFIG_MERGE_TIMERS */ + + +/** + * mc_release_jobs() - Add heap of tasks to the system, check for preemptions. + */ +static void mc_release_jobs(rt_domain_t* rt, struct bheap* tasks) +{ + unsigned long flags; + struct task_struct *first = bheap_peek(rt->order, tasks)->value; + struct domain *dom = get_task_domain(first); + + raw_spin_lock_irqsave(dom->lock, flags); + TRACE(TS "Jobs released\n", TA(first)); + __merge_ready(rt, tasks); + check_for_preempt(dom); + raw_spin_unlock_irqrestore(dom->lock, flags); +} + +/** + * ms_task_new() - Setup new mixed-criticality task. + * Assumes that there are no partitioned domains after level B. + */ +static void mc_task_new(struct task_struct *t, int on_rq, int running) +{ + unsigned long flags; + struct cpu_entry* entry; + enum crit_level level = tsk_mc_crit(t); + + local_irq_save(flags); + TRACE("New mixed criticality task %d\n", t->pid); + + /* Assign domain */ + if (level < CRIT_LEVEL_C) + entry = &per_cpu(cpus, get_partition(t)); + else + entry = &per_cpu(cpus, task_cpu(t)); + t->rt_param._domain = entry->crit_entries[level].domain; + + /* Setup job params */ + release_at(t, litmus_clock()); + tsk_mc_data(t)->mc_job.ghost_budget = 0; + tsk_mc_data(t)->mc_job.is_ghost = 0; + if (running) { + BUG_ON(entry->scheduled); + entry->scheduled = t; + tsk_rt(t)->scheduled_on = entry->cpu; + } else { + t->rt_param.scheduled_on = NO_CPU; + } + t->rt_param.linked_on = NO_CPU; + + job_arrival(t); + + local_irq_restore(flags); +} + +/** + * mc_task_new() - Add task back into its domain check for preemptions. + */ +static void mc_task_wake_up(struct task_struct *task) +{ + unsigned long flags; + lt_t now = litmus_clock(); + local_irq_save(flags); + + TRACE(TS " wakes up\n", TA(task)); + if (is_tardy(task, now)) { + /* Task missed its last release */ + release_at(task, now); + sched_trace_task_release(task); + } + if (!is_ghost(task)) + job_arrival(task); + + local_irq_restore(flags); +} + +/** + * mc_task_block() - Remove task from state to prevent it being run anywhere. + */ +static void mc_task_block(struct task_struct *task) +{ + unsigned long flags; + local_irq_save(flags); + TRACE(TS " blocks\n", TA(task)); + remove_from_all(task); + local_irq_restore(flags); +} + +/** + * mc_task_exit() - Remove task from the system. + */ +static void mc_task_exit(struct task_struct *task) +{ + unsigned long flags; + local_irq_save(flags); + BUG_ON(!is_realtime(task)); + TRACE(TS " RIP\n", TA(task)); + + remove_from_all(task); + if (tsk_rt(task)->scheduled_on != NO_CPU) { + per_cpu(cpus, tsk_rt(task)->scheduled_on).scheduled = NULL; + tsk_rt(task)->scheduled_on = NO_CPU; + } + + if (CRIT_LEVEL_A == tsk_mc_crit(task)) + mc_ce_task_exit_common(task); + + local_irq_restore(flags); +} + +/** + * mc_admit_task() - Return true if the task is valid. + * Assumes there are no partitioned levels after level B. + */ +static long mc_admit_task(struct task_struct* task) +{ + const enum crit_level crit = tsk_mc_crit(task); + long ret; + if (!tsk_mc_data(task)) { + printk(KERN_WARNING "Tried to admit task with no criticality " + "level\n"); + ret = -EINVAL; + goto out; + } + if (crit < CRIT_LEVEL_C && get_partition(task) == NO_CPU) { + printk(KERN_WARNING "Tried to admit partitioned task with no " + "partition\n"); + ret = -EINVAL; + goto out; + } + if (crit == CRIT_LEVEL_A) { + ret = mc_ce_admit_task_common(task); + if (ret) + goto out; + } + printk(KERN_INFO "Admitted task with criticality level %d\n", + tsk_mc_crit(task)); + ret = 0; +out: + return ret; +} + +/** + * mc_schedule() - Return next task which should be scheduled. + */ +static struct task_struct* mc_schedule(struct task_struct* prev) +{ + unsigned long flags; + struct domain *dom; + struct crit_entry *ce; + struct cpu_entry* entry = &__get_cpu_var(cpus); + int i, out_of_time, sleep, preempt, exists, blocks, global, lower; + struct task_struct *dtask = NULL, *ready_task = NULL, *next = NULL; + + local_irq_save(flags); + + /* Litmus gave up because it couldn't access the stack of the CPU + * on which will_schedule was migrating from. Requeue it. + * This really only happens in VMs. + */ + if (entry->will_schedule && entry->will_schedule != prev) { + entry->will_schedule->rt_param.scheduled_on = NO_CPU; + low_prio_arrival(entry->will_schedule); + } + + raw_spin_lock(&entry->lock); + + /* Sanity checking */ + BUG_ON(entry->scheduled && entry->scheduled != prev); + BUG_ON(entry->scheduled && !is_realtime(prev)); + BUG_ON(is_realtime(prev) && !entry->scheduled); + + /* Determine state */ + exists = entry->scheduled != NULL; + blocks = exists && !is_running(entry->scheduled); + out_of_time = exists && budget_enforced(entry->scheduled) && + budget_exhausted(entry->scheduled); + sleep = exists && get_rt_flags(entry->scheduled) == RT_F_SLEEP; + global = exists && is_global_task(entry->scheduled); + preempt = entry->scheduled != entry->linked; + lower = exists && preempt && entry->linked && + tsk_mc_crit(entry->scheduled) > tsk_mc_crit(entry->linked); + + TRACE(TS " blocks:%d out_of_time:%d sleep:%d preempt:%d\n", + TA(prev), blocks, out_of_time, sleep, preempt); + + if (exists) + prev->rt_param.scheduled_on = NO_CPU; + + raw_spin_unlock(&entry->lock); + + +#ifdef CONFIG_PLUGIN_MC_REDIRECT + if (smp_processor_id() == interrupt_cpu) + fix_global_levels(); +#endif + + /* If a task blocks we have no choice but to reschedule */ + if (blocks) + remove_from_all(entry->scheduled); + /* Any task which exhausts its budget or sleeps waiting for its next + * period completes unless its execution has been forcibly stopped. + */ + if ((out_of_time || sleep) && !blocks)/* && !preempt)*/ + job_completion(entry->scheduled, !sleep); + /* Global scheduled tasks must wait for a deschedule before they + * can rejoin the global state. Rejoin them here. + */ + else if (global && preempt && !blocks) { + if (lower) + low_prio_arrival(entry->scheduled); + else + job_arrival(entry->scheduled); + } + + /* Pick next task if none is linked */ + raw_spin_lock(&entry->lock); + for (i = 0; i < NUM_CRIT_LEVELS && !entry->linked; i++) { + ce = &entry->crit_entries[i]; + dom = ce->domain; + + /* Swap locks. We cannot acquire a domain lock while + * holding an entry lock or deadlocks will happen. + */ + raw_spin_unlock(&entry->lock); + raw_spin_lock(dom->lock); + + /* Do domain stuff before grabbing CPU locks */ + dtask = dom->peek_ready(dom); + fix_crit_position(ce); + + raw_spin_lock(&entry->lock); + + if (!entry->linked && !ce->linked && dtask && can_use(ce)) { + dom->take_ready(dom); + link_task_to_crit(ce, dtask); + update_crit_position(ce); + ready_task = (is_ghost(dtask)) ? NULL : dtask; + + /* Task found! */ + if (ready_task) { + link_task_to_cpu(entry, ready_task); + raw_spin_unlock(dom->lock); + update_crit_levels(entry); + raw_spin_lock(&entry->lock); + continue; + } + } + raw_spin_unlock(dom->lock); + } + + /* Schedule next task */ + next = entry->linked; + if (entry->linked) + entry->linked->rt_param.scheduled_on = entry->cpu; + entry->will_schedule = entry->linked; + sched_state_task_picked(); + + raw_spin_unlock(&entry->lock); + local_irq_restore(flags); + if (next) { + TRACE_MC_TASK(next, "Picked this task\n"); + } else if (exists && !next) + TRACE_ENTRY(entry, "Becomes idle at %llu\n", litmus_clock()); + return next; +} + +void mc_finish_switch(struct task_struct *prev) +{ + struct cpu_entry* entry = &__get_cpu_var(cpus); + entry->scheduled = is_realtime(current) ? current : NULL; + TRACE_TASK(prev, "Switched away from to " TS "\n", + TA(entry->scheduled)); +} + +/* + * This is the plugin's release at function, called by the release task-set + * system call. Other places in the file use the generic LITMUS release_at(), + * which is not this. + */ +void mc_release_at(struct task_struct *ts, lt_t start) +{ + /* hack so that we can have CE timers start at the right time */ + if (CRIT_LEVEL_A == tsk_mc_crit(ts)) + mc_ce_release_at_common(ts, start); + else + release_at(ts, start); +} + +long mc_deactivate_plugin(void) +{ + return mc_ce_deactivate_plugin_common(); +} + +/* ************************************************************************** + * Initialization + * ************************************************************************** */ + +/* Initialize values here so that they are allocated with the module + * and destroyed when the module is unloaded. + */ + +/* LVL-A */ +DEFINE_PER_CPU(struct domain_data, _mc_crit_a); +DEFINE_PER_CPU(raw_spinlock_t, _mc_crit_a_lock); +DEFINE_PER_CPU(struct ce_dom_data, _mc_crit_a_ce_data); +/* LVL-B */ +DEFINE_PER_CPU(struct domain_data, _mc_crit_b); +DEFINE_PER_CPU(rt_domain_t, _mc_crit_b_rt); +/* LVL-C */ +static struct domain_data _mc_crit_c; +static rt_domain_t _mc_crit_c_rt; +struct bheap _mc_heap_c; +struct bheap_node _mc_nodes_c[NR_CPUS]; + +static long mc_activate_plugin(void) +{ + struct domain_data *dom_data; + struct domain *dom; + struct domain_data *our_domains[NR_CPUS]; + int cpu, n = 0; + long ret; + +#ifdef CONFIG_RELEASE_MASTER + interrupt_cpu = atomic_read(&release_master_cpu); +#if defined(CONFIG_PLUGIN_MC_REDIRECT) || defined(CONFIG_PLUGIN_MC_RELEASE_MASTER) + if (NO_CPU == interrupt_cpu) { + printk(KERN_ERR "LITMUS-MC: need a release master\n"); + ret = -EINVAL; + goto out; + } +#endif +#endif + + for_each_online_cpu(cpu) { + BUG_ON(NR_CPUS <= n); + dom = per_cpu(cpus, cpu).crit_entries[CRIT_LEVEL_A].domain; + dom_data = domain_data(dom); + our_domains[cpu] = dom_data; +#if defined(CONFIG_MERGE_TIMERS) && defined(CONFIG_PLUGIN_MC_RELEASE_MASTER) + per_cpu(cpus, cpu).event_group = + get_event_group_for(interrupt_cpu); +#elif defined(CONFIG_MERGE_TIMERS) && !defined(CONFIG_PLUGIN_MC_RELEASE_MASTER) + per_cpu(cpus, cpu).event_group = get_event_group_for(cpu); +#endif + n++; + } + ret = mc_ce_set_domains(n, our_domains); + if (ret) + goto out; + ret = mc_ce_activate_plugin_common(); +out: + return ret; +} + +static struct sched_plugin mc_plugin __cacheline_aligned_in_smp = { + .plugin_name = "MC", + .task_new = mc_task_new, + .complete_job = complete_job, + .task_exit = mc_task_exit, + .schedule = mc_schedule, + .task_wake_up = mc_task_wake_up, + .task_block = mc_task_block, + .admit_task = mc_admit_task, + .activate_plugin = mc_activate_plugin, + .release_at = mc_release_at, + .deactivate_plugin = mc_deactivate_plugin, + .finish_switch = mc_finish_switch, +}; + +static void init_crit_entry(struct crit_entry *ce, enum crit_level level, + struct domain_data *dom_data, + struct bheap_node *node) +{ + ce->level = level; + ce->linked = NULL; + ce->node = node; + ce->domain = &dom_data->domain; + ce->state = CS_ACTIVE; +#ifdef CONFIG_MERGE_TIMERS + init_event(&ce->event, level, mc_ghost_exhausted, + event_list_alloc(GFP_ATOMIC)); +#else + hrtimer_init(&ce->timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS); + ce->timer.function = mc_ghost_exhausted; +#endif + +} + +static void init_local_domain(struct cpu_entry *entry, struct domain_data *dom_data, + enum crit_level level) +{ + dom_data->heap = NULL; + dom_data->crit_entry = &entry->crit_entries[level]; + init_crit_entry(dom_data->crit_entry, level, dom_data, NULL); +} + +static void init_global_domain(struct domain_data *dom_data, enum crit_level level, + struct bheap *heap, struct bheap_node *nodes) +{ + int cpu; + struct cpu_entry *entry; + struct crit_entry *ce; + struct bheap_node *node; + + dom_data->crit_entry = NULL; + dom_data->heap = heap; + bheap_init(heap); + + for_each_online_cpu(cpu) { + entry = &per_cpu(cpus, cpu); + node = &nodes[cpu]; + ce = &entry->crit_entries[level]; + init_crit_entry(ce, level, dom_data, node); + bheap_node_init(&ce->node, ce); + bheap_insert(cpu_lower_prio, heap, node); + } +} + +static inline void init_edf_domain(struct domain *dom, rt_domain_t *rt, + int prio, int is_partitioned, int cpu) +{ + pd_domain_init(dom, rt, edf_ready_order, NULL, + mc_release_jobs, mc_preempt_needed, + edf_higher_prio); +#if defined(CONFIG_PLUGIN_MC_RELEASE_MASTER) && defined(CONFIG_MERGE_TIMERS) + /* All timers are on one CPU and release-master is using the event + * merging interface as well. */ + BUG_ON(NO_CPU == interrupt_cpu); + rt->event_group = get_event_group_for(interrupt_cpu); + rt->prio = prio; +#elif defined(CONFIG_PLUGIN_MC_RELEASE_MASTER) && !defined(CONFIG_MERGE_TIMERS) + /* Using release master, but not merging timers. */ + rt->release_master = interrupt_cpu; +#elif !defined(CONFIG_PLUGIN_MC_RELEASE_MASTER) && defined(CONFIG_MERGE_TIMERS) + /* Merge the timers, but don't move them to the release master. */ + if (is_partitioned) { + rt->event_group = get_event_group_for(cpu); + } else { + /* Global timers will be added to the event groups that code is + * executing on when add_event() is called. + */ + rt->event_group = NULL; + } + rt->prio = prio; +#endif +} + +struct domain_data *ce_domain_for(int); +static int __init init_mc(void) +{ + int cpu; + struct cpu_entry *entry; + struct domain_data *dom_data; + rt_domain_t *rt; + raw_spinlock_t *a_dom_lock, *b_dom_lock, *c_dom_lock; /* For lock debugger */ + struct ce_dom_data *ce_data; + + for_each_online_cpu(cpu) { + entry = &per_cpu(cpus, cpu); + + /* CPU */ + entry->cpu = cpu; + entry->scheduled = NULL; + entry->linked = NULL; + + raw_spin_lock_init(&entry->lock); + +#ifdef CONFIG_PLUGIN_MC_REDIRECT + raw_spin_lock_init(&entry->redir_lock); + INIT_LIST_HEAD(&entry->redir); +#endif + + /* CRIT_LEVEL_A */ + dom_data = &per_cpu(_mc_crit_a, cpu); + ce_data = &per_cpu(_mc_crit_a_ce_data, cpu); + a_dom_lock = &per_cpu(_mc_crit_a_lock, cpu); + raw_spin_lock_init(a_dom_lock); + ce_domain_init(&dom_data->domain, + a_dom_lock, ce_requeue, ce_peek_and_take_ready, + ce_peek_and_take_ready, mc_preempt_needed, + ce_higher_prio, ce_data, cpu, + ce_timer_function); + init_local_domain(entry, dom_data, CRIT_LEVEL_A); + dom_data->domain.name = "LVL-A"; + + /* CRIT_LEVEL_B */ + dom_data = &per_cpu(_mc_crit_b, cpu); + rt = &per_cpu(_mc_crit_b_rt, cpu); + init_local_domain(entry, dom_data, CRIT_LEVEL_B); + init_edf_domain(&dom_data->domain, rt, CRIT_LEVEL_B, 1, cpu); + b_dom_lock = dom_data->domain.lock; + raw_spin_lock_init(b_dom_lock); + dom_data->domain.name = "LVL-B"; + } + + /* CRIT_LEVEL_C */ + init_global_domain(&_mc_crit_c, CRIT_LEVEL_C, + &_mc_heap_c, _mc_nodes_c); + init_edf_domain(&_mc_crit_c.domain, &_mc_crit_c_rt, CRIT_LEVEL_C, + 0, NO_CPU); + c_dom_lock = _mc_crit_c.domain.lock; + raw_spin_lock_init(c_dom_lock); + _mc_crit_c.domain.name = "LVL-C"; + + return register_sched_plugin(&mc_plugin); +} + +module_init(init_mc); diff --git a/litmus/sched_mc_ce.c b/litmus/sched_mc_ce.c new file mode 100644 index 0000000..af02dfd --- /dev/null +++ b/litmus/sched_mc_ce.c @@ -0,0 +1,1052 @@ +/** + * litmus/sched_mc_ce.c + * + * The Cyclic Executive (CE) scheduler used by the mixed criticality scheduling + * algorithm. + */ + +#include +#include + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static struct sched_plugin mc_ce_plugin __cacheline_aligned_in_smp; + +#define using_linux_plugin() (litmus == &linux_sched_plugin) + +/* get a reference to struct domain for a CPU */ +#define get_domain_for(cpu) (&per_cpu(domains, cpu)->domain) + +#define get_pid_table(cpu) (&per_cpu(ce_pid_table, cpu)) +#define get_pid_entry(cpu, idx) (&(get_pid_table(cpu)->entries[idx])) + +static atomic_t start_time_set = ATOMIC_INIT(-1); +static atomic64_t start_time = ATOMIC64_INIT(0); +static struct proc_dir_entry *mc_ce_dir = NULL, *ce_file = NULL; + +/* + * Cache the budget along with the struct PID for a task so that we don't need + * to fetch its task_struct every time we check to see what should be + * scheduled. + */ +struct ce_pid_entry { + struct pid *pid; + lt_t budget; + /* accumulated (summed) budgets, including this one */ + lt_t acc_time; + unsigned int expected_job; +}; + +/* + * Each CPU needs a mapping of level A ID (integer) to struct pid so that we + * can get its task struct. + */ +struct ce_pid_table { + struct ce_pid_entry entries[CONFIG_PLUGIN_MC_LEVEL_A_MAX_TASKS]; + int num_pid_entries; + lt_t cycle_time; +}; + +DEFINE_PER_CPU(struct ce_pid_table, ce_pid_table); + +/* + * How we get the domain for a given CPU locally. Set with the + * mc_ce_set_domains function. Must be done before activating plugins. Be + * careful when using domains as a variable elsewhere in this file. + */ + +DEFINE_PER_CPU(struct domain_data*, domains); + +/* + * The domains and other data used by the MC-CE plugin when it runs alone. + */ +DEFINE_PER_CPU(struct domain_data, _mc_ce_doms); +DEFINE_PER_CPU(struct ce_dom_data, _mc_ce_dom_data); +DEFINE_PER_CPU(raw_spinlock_t, _mc_ce_dom_locks); + +#ifdef CONFIG_PLUGIN_MC_RELEASE_MASTER +static int interrupt_cpu; +#endif + +long mc_ce_set_domains(const int n, struct domain_data *domains_in[]) +{ + const int max = (NR_CPUS < n) ? NR_CPUS : n; + struct domain_data *new_dom = NULL; + int i, ret; + if (!using_linux_plugin()) { + printk(KERN_WARNING "can't set MC-CE domains when not using " + "Linux scheduler.\n"); + ret = -EINVAL; + goto out; + } + for (i = 0; i < max; ++i) { + new_dom = domains_in[i]; + per_cpu(domains, i) = new_dom; + } + ret = 0; +out: + return ret; +} + +unsigned int mc_ce_get_expected_job(const int cpu, const int idx) +{ + const struct ce_pid_table *pid_table = get_pid_table(cpu); + BUG_ON(0 > cpu); + BUG_ON(0 > idx); + BUG_ON(pid_table->num_pid_entries <= idx); + return pid_table->entries[idx].expected_job; +} + +/* + * Get the offset into the cycle taking the start time into account. + */ +static inline lt_t get_cycle_offset(const lt_t when, const lt_t cycle_time) +{ + long long st = atomic64_read(&start_time); + lt_t offset = (when - st) % cycle_time; + TRACE("when: %llu cycle_time: %llu start_time: %lld offset %llu\n", + when, cycle_time, st, offset); + return offset; +} + +/* + * The user land job completion call will set the RT_F_SLEEP flag and then + * call schedule. This function is used when schedule sleeps a task. + * + * Do not call prepare_for_next_period on Level-A tasks! + */ +static void mc_ce_job_completion(struct domain *dom, struct task_struct *ts) +{ + const int cpu = task_cpu(ts); + const int idx = tsk_mc_data(ts)->mc_task.lvl_a_id; + const struct ce_pid_entry *pid_entry = get_pid_entry(cpu, idx); + unsigned int just_finished; + + TRACE_TASK(ts, "Completed\n"); + + sched_trace_task_completion(ts, 0); + /* post-increment is important here */ + just_finished = (tsk_rt(ts)->job_params.job_no)++; + + /* Job completes in expected window: everything is normal. + * Job completes in an earlier window: BUG(), that's wrong. + * Job completes in a later window: The job is behind. + */ + if (just_finished < pid_entry->expected_job) { + /* this job is already released because it's running behind */ + set_rt_flags(ts, RT_F_RUNNING); + TRACE_TASK(ts, "appears behind: the expected job is %u but " + "job %u just completed\n", + pid_entry->expected_job, just_finished); + } else if (pid_entry->expected_job < just_finished) { + printk(KERN_CRIT "job %u completed in expected job %u which " + "seems too early\n", just_finished, + pid_entry->expected_job); + BUG(); + } +} + + +/* + * Return the index into the PID entries table of what to schedule next. + * Don't call if the table is empty. Assumes the caller has the domain lock. + * The offset parameter is the offset into the cycle. + * + * TODO Currently O(n) in the number of tasks on the CPU. Binary search? + */ +static int mc_ce_schedule_at(const struct domain *dom, lt_t offset) +{ + const struct ce_dom_data *ce_data = dom->data; + struct ce_pid_table *pid_table = get_pid_table(ce_data->cpu); + const struct ce_pid_entry *pid_entry = NULL; + int idx; + + BUG_ON(pid_table->cycle_time < 1); + BUG_ON(pid_table->num_pid_entries < 1); + + for (idx = 0; idx < pid_table->num_pid_entries; ++idx) { + pid_entry = &pid_table->entries[idx]; + if (offset < pid_entry->acc_time) { + /* found task to schedule in this window */ + break; + } + } + /* can only happen if cycle_time is not right */ + BUG_ON(pid_entry->acc_time > pid_table->cycle_time); + TRACE("schedule at returning task %d for CPU %d\n", idx, ce_data->cpu); + return idx; +} + +static struct task_struct *mc_ce_schedule(struct task_struct *prev) +{ + struct domain *dom = get_domain_for(smp_processor_id()); + struct ce_dom_data *ce_data = dom->data; + struct task_struct *next = NULL; + int exists, sleep, should_sched_exists, should_sched_blocked, + should_sched_asleep; + + raw_spin_lock(dom->lock); + + /* sanity checking */ + BUG_ON(ce_data->scheduled && ce_data->scheduled != prev); + BUG_ON(ce_data->scheduled && !is_realtime(prev)); + BUG_ON(is_realtime(prev) && !ce_data->scheduled); + + exists = NULL != ce_data->scheduled; + sleep = exists && RT_F_SLEEP == get_rt_flags(ce_data->scheduled); + + TRACE("exists: %d, sleep: %d\n", exists, sleep); + + if (sleep) + mc_ce_job_completion(dom, ce_data->scheduled); + + /* these checks must go after the call to mc_ce_job_completion in case + * a late task needs to be scheduled again right away and its the only + * task on a core + */ + should_sched_exists = NULL != ce_data->should_schedule; + should_sched_blocked = should_sched_exists && + !is_running(ce_data->should_schedule); + should_sched_asleep = should_sched_exists && + RT_F_SLEEP == get_rt_flags(ce_data->should_schedule); + + TRACE("should_sched_exists: %d, should_sched_blocked: %d, " + "should_sched_asleep: %d\n", should_sched_exists, + should_sched_blocked, should_sched_asleep); + + if (should_sched_exists && !should_sched_blocked && + !should_sched_asleep) { + /* + * schedule the task that should be executing in the cyclic + * schedule if it is not blocked and not sleeping + */ + next = ce_data->should_schedule; + } + sched_state_task_picked(); + raw_spin_unlock(dom->lock); + return next; +} + +static void mc_ce_finish_switch(struct task_struct *prev) +{ + struct domain *dom = get_domain_for(smp_processor_id()); + struct ce_dom_data *ce_data = dom->data; + + TRACE("finish switch\n"); + + if (is_realtime(current) && CRIT_LEVEL_A == tsk_mc_crit(current)) + ce_data->scheduled = current; + else + ce_data->scheduled = NULL; +} + +/* + * Admit task called to see if this task is permitted to enter the system. + * Here we look up the task's PID structure and save it in the proper slot on + * the CPU this task will run on. + */ +long mc_ce_admit_task_common(struct task_struct *ts) +{ + struct domain *dom = get_domain_for(get_partition(ts)); + struct ce_dom_data *ce_data = dom->data; + struct mc_data *mcd = tsk_mc_data(ts); + struct pid *pid = NULL; + long retval = -EINVAL; + const int lvl_a_id = mcd->mc_task.lvl_a_id; + struct ce_pid_table *pid_table = get_pid_table(ce_data->cpu); + + BUG_ON(get_partition(ts) != ce_data->cpu); + + /* check the task has migrated to the right CPU (like in sched_cedf) */ + if (task_cpu(ts) != get_partition(ts)) { + printk(KERN_INFO "litmus: %d admitted on CPU %d but want %d ", + ts->pid, task_cpu(ts), get_partition(ts)); + goto out; + } + + /* only level A tasks can be CE */ + if (!mcd || CRIT_LEVEL_A != tsk_mc_crit(ts)) { + printk(KERN_INFO "litmus: non-MC or non level A task %d\n", + ts->pid); + goto out; + } + + /* try and get the task's PID structure */ + pid = get_task_pid(ts, PIDTYPE_PID); + if (IS_ERR_OR_NULL(pid)) { + printk(KERN_INFO "litmus: couldn't get pid struct for %d\n", + ts->pid); + goto out; + } + + if (lvl_a_id >= pid_table->num_pid_entries) { + printk(KERN_INFO "litmus: level A id greater than expected " + "number of tasks %d for %d cpu %d\n", + pid_table->num_pid_entries, ts->pid, + get_partition(ts)); + goto out_put_pid; + } + if (pid_table->entries[lvl_a_id].pid) { + printk(KERN_INFO "litmus: have saved pid info id: %d cpu: %d\n", + lvl_a_id, get_partition(ts)); + goto out_put_pid; + } + if (get_exec_cost(ts) >= pid_table->entries[lvl_a_id].budget) { + printk(KERN_INFO "litmus: execution cost %llu is larger than " + "the budget %llu\n", + get_exec_cost(ts), + pid_table->entries[lvl_a_id].budget); + goto out_put_pid; + } + pid_table->entries[lvl_a_id].pid = pid; + retval = 0; + /* don't call put_pid if we are successful */ + goto out; + +out_put_pid: + put_pid(pid); +out: + return retval; +} + +static long mc_ce_admit_task(struct task_struct *ts) +{ + struct domain *dom = get_domain_for(get_partition(ts)); + unsigned long flags, retval; + raw_spin_lock_irqsave(dom->lock, flags); + retval = mc_ce_admit_task_common(ts); + raw_spin_unlock_irqrestore(dom->lock, flags); + return retval; +} + +/* + * Called to set up a new real-time task (after the admit_task callback). + * At this point the task's struct PID is already hooked up on the destination + * CPU. The task may already be running. + */ +static void mc_ce_task_new(struct task_struct *ts, int on_rq, int running) +{ + const int cpu = task_cpu(ts); + struct domain *dom = get_domain_for(cpu); + struct ce_dom_data *ce_data = dom->data; + struct ce_pid_table *pid_table = get_pid_table(cpu); + struct pid *pid_should_be_running; + struct ce_pid_entry *pid_entry; + unsigned long flags; + int idx, should_be_running; + lt_t offset; + + raw_spin_lock_irqsave(dom->lock, flags); + pid_entry = get_pid_entry(cpu, tsk_mc_data(ts)->mc_task.lvl_a_id); + /* initialize some task state */ + set_rt_flags(ts, RT_F_RUNNING); + + /* have to call mc_ce_schedule_at because the task only gets a PID + * entry after calling admit_task */ + offset = get_cycle_offset(litmus_clock(), pid_table->cycle_time); + idx = mc_ce_schedule_at(dom, offset); + pid_should_be_running = get_pid_entry(cpu, idx)->pid; + rcu_read_lock(); + should_be_running = (ts == pid_task(pid_should_be_running, PIDTYPE_PID)); + rcu_read_unlock(); + if (running) { + /* admit task checks that the task is not on the wrong CPU */ + BUG_ON(task_cpu(ts) != get_partition(ts)); + BUG_ON(ce_data->scheduled); + ce_data->scheduled = ts; + + if (should_be_running) + ce_data->should_schedule = ts; + else + preempt_if_preemptable(ce_data->scheduled, ce_data->cpu); + } else if (!running && should_be_running) { + ce_data->should_schedule = ts; + preempt_if_preemptable(ce_data->scheduled, ce_data->cpu); + } + raw_spin_unlock_irqrestore(dom->lock, flags); +} + +/* + * Called to re-introduce a task after blocking. + * Can potentailly be called multiple times. + */ +static void mc_ce_task_wake_up(struct task_struct *ts) +{ + struct domain *dom = get_domain_for(get_partition(ts)); + struct ce_dom_data *ce_data = dom->data; + unsigned long flags; + + TRACE_TASK(ts, "wake up\n"); + + raw_spin_lock_irqsave(dom->lock, flags); + if (ts == ce_data->should_schedule && ts != ce_data->scheduled) + preempt_if_preemptable(ts, ce_data->cpu); + raw_spin_unlock_irqrestore(dom->lock, flags); +} + +/* + * Called to notify the plugin of a blocking real-time tasks. Only called for + * real-time tasks and before schedule is called. + */ +static void mc_ce_task_block(struct task_struct *ts) +{ + /* nothing to do because it will be taken care of in schedule */ + TRACE_TASK(ts, "blocked\n"); +} + +/* + * Called when a task switches from RT mode back to normal mode. + */ +void mc_ce_task_exit_common(struct task_struct *ts) +{ + struct domain *dom = get_domain_for(get_partition(ts)); + struct ce_dom_data *ce_data = dom->data; + unsigned long flags; + struct pid *pid; + const int lvl_a_id = tsk_mc_data(ts)->mc_task.lvl_a_id; + struct ce_pid_table *pid_table = get_pid_table(ce_data->cpu); + + BUG_ON(CRIT_LEVEL_A != tsk_mc_crit(ts)); + BUG_ON(lvl_a_id >= pid_table->num_pid_entries); + + raw_spin_lock_irqsave(dom->lock, flags); + pid = pid_table->entries[lvl_a_id].pid; + BUG_ON(!pid); + put_pid(pid); + pid_table->entries[lvl_a_id].pid = NULL; + if (ce_data->scheduled == ts) + ce_data->scheduled = NULL; + if (ce_data->should_schedule == ts) + ce_data->should_schedule = NULL; + raw_spin_unlock_irqrestore(dom->lock, flags); +} + +/*********************************************************** + * Timer stuff + **********************************************************/ + +/* + * Returns the next absolute time that the timer should fire. + */ +lt_t mc_ce_timer_callback_common(struct domain *dom) +{ + /* relative and absolute times for cycles */ + lt_t now, offset_rel, cycle_start_abs, next_timer_abs; + struct task_struct *should_schedule; + struct ce_pid_table *pid_table; + struct ce_pid_entry *pid_entry; + struct ce_dom_data *ce_data; + int idx, budget_overrun; + + ce_data = dom->data; + pid_table = get_pid_table(ce_data->cpu); + + /* Based off of the current time, figure out the offset into the cycle + * and the cycle's start time, and determine what should be scheduled. + */ + now = litmus_clock(); + offset_rel = get_cycle_offset(now, pid_table->cycle_time); + cycle_start_abs = now - offset_rel; + idx = mc_ce_schedule_at(dom, offset_rel); + pid_entry = get_pid_entry(ce_data->cpu, idx); + next_timer_abs = cycle_start_abs + pid_entry->acc_time; + + STRACE("timer: now: %llu offset_rel: %llu cycle_start_abs: %llu " + "next_timer_abs: %llu\n", now, offset_rel, + cycle_start_abs, next_timer_abs); + + /* get the task_struct (pid_task can accept a NULL) */ + rcu_read_lock(); + should_schedule = pid_task(pid_entry->pid, PIDTYPE_PID); + rcu_read_unlock(); + ce_data->should_schedule = should_schedule; + + if (should_schedule && 0 == atomic_read(&start_time_set)) { + /* + * If jobs are not overrunning their budgets, then this + * should not happen. + */ + pid_entry->expected_job++; + budget_overrun = pid_entry->expected_job != + tsk_rt(should_schedule)->job_params.job_no; + if (budget_overrun) + TRACE_MC_TASK(should_schedule, + "timer expected job number: %u " + "but current job: %u\n", + pid_entry->expected_job, + tsk_rt(should_schedule)->job_params.job_no); + } + + if (ce_data->should_schedule) { + tsk_rt(should_schedule)->job_params.deadline = + cycle_start_abs + pid_entry->acc_time; + tsk_rt(should_schedule)->job_params.release = + tsk_rt(should_schedule)->job_params.deadline - + pid_entry->budget; + tsk_rt(should_schedule)->job_params.exec_time = 0; + sched_trace_task_release(should_schedule); + set_rt_flags(ce_data->should_schedule, RT_F_RUNNING); + } + return next_timer_abs; +} + +/* + * What to do when a timer fires. The timer should only be armed if the number + * of PID entries is positive. + */ +#ifdef CONFIG_MERGE_TIMERS +static void mc_ce_timer_callback(struct rt_event *e) +#else +static enum hrtimer_restart mc_ce_timer_callback(struct hrtimer *timer) +#endif +{ + struct ce_dom_data *ce_data; + unsigned long flags; + struct domain *dom; + lt_t next_timer_abs; +#ifdef CONFIG_MERGE_TIMERS + struct event_group *event_group; + ce_data = container_of(e, struct ce_dom_data, event); + /* use the same CPU the callbacking is executing on by passing NO_CPU */ + event_group = get_event_group_for(NO_CPU); +#else /* CONFIG_MERGE_TIMERS */ + ce_data = container_of(timer, struct ce_dom_data, timer); +#endif + dom = get_domain_for(ce_data->cpu); + + TRACE("timer callback on CPU %d (before lock)\n", ce_data->cpu); + + raw_spin_lock_irqsave(dom->lock, flags); + next_timer_abs = mc_ce_timer_callback_common(dom); + + /* setup an event or timer for the next release in the CE schedule */ +#ifdef CONFIG_MERGE_TIMERS + add_event(event_group, e, next_timer_abs); +#else + hrtimer_set_expires(timer, ns_to_ktime(next_timer_abs)); +#endif + + if (ce_data->scheduled != ce_data->should_schedule) + preempt_if_preemptable(ce_data->scheduled, ce_data->cpu); + + raw_spin_unlock_irqrestore(dom->lock, flags); + +#ifndef CONFIG_MERGE_TIMERS + return HRTIMER_RESTART; +#endif +} + +/* + * Cancel timers on all CPUs. Returns 1 if any were active. + */ +static int cancel_all_timers(void) +{ + struct ce_dom_data *ce_data; + struct domain *dom; + int cpu, ret = 0; +#ifndef CONFIG_MERGE_TIMERS + int cancel_res; +#endif + + TRACE("cancel all timers\n"); + + for_each_online_cpu(cpu) { + dom = get_domain_for(cpu); + ce_data = dom->data; + ce_data->should_schedule = NULL; +#ifdef CONFIG_MERGE_TIMERS + cancel_event(&ce_data->event); +#else + cancel_res = hrtimer_cancel(&ce_data->timer); + atomic_set(&ce_data->timer_info.state, + HRTIMER_START_ON_INACTIVE); + ret = ret || cancel_res; +#endif + } + return ret; +} + +/* + * Arm all timers so that they start at the new value of start time. + * Any CPU without CE PID entries won't have a timer armed. + * All timers should be canceled before calling this. + */ +static void arm_all_timers(void) +{ + struct domain *dom; + struct ce_dom_data *ce_data; + struct ce_pid_table *pid_table; + int cpu, idx, cpu_for_timer; + const lt_t start = atomic64_read(&start_time); + + TRACE("arm all timers\n"); + + for_each_online_cpu(cpu) { + dom = get_domain_for(cpu); + ce_data = dom->data; + pid_table = get_pid_table(cpu); + if (0 == pid_table->num_pid_entries) + continue; + for (idx = 0; idx < pid_table->num_pid_entries; idx++) { + pid_table->entries[idx].expected_job = 0; + } +#ifdef CONFIG_PLUGIN_MC_RELEASE_MASTER + cpu_for_timer = interrupt_cpu; +#else + cpu_for_timer = cpu; +#endif + +#ifdef CONFIG_MERGE_TIMERS + add_event(get_event_group_for(cpu_for_timer), + &ce_data->event, start); +#else + hrtimer_start_on(cpu_for_timer, &ce_data->timer_info, + &ce_data->timer, ns_to_ktime(start), + HRTIMER_MODE_ABS_PINNED); +#endif + } +} + +/* + * There are no real releases in the CE, but the task release syscall will + * call this. We can re-set our notion of the CE period start to make + * the schedule look pretty. + */ +void mc_ce_release_at_common(struct task_struct *ts, lt_t start) +{ + TRACE_TASK(ts, "release at\n"); + if (atomic_inc_and_test(&start_time_set)) { + /* in this case, we won the race */ + cancel_all_timers(); + atomic64_set(&start_time, start); + arm_all_timers(); + } else + atomic_dec(&start_time_set); +} + +long mc_ce_activate_plugin_common(void) +{ + struct ce_dom_data *ce_data; + struct domain *dom; + long ret; + int cpu; + +#ifdef CONFIG_PLUGIN_MC_RELEASE_MASTER + interrupt_cpu = atomic_read(&release_master_cpu); + if (NO_CPU == interrupt_cpu) { + printk(KERN_ERR "LITMUS: MC-CE needs a release master\n"); + ret = -EINVAL; + goto out; + } +#endif + + for_each_online_cpu(cpu) { + dom = get_domain_for(cpu); + ce_data = dom->data; + ce_data->scheduled = NULL; + ce_data->should_schedule = NULL; + } + + atomic_set(&start_time_set, -1); + atomic64_set(&start_time, litmus_clock()); + /* may not want to arm timers on activation, just after release */ + arm_all_timers(); + ret = 0; +out: + return ret; +} + +static long mc_ce_activate_plugin(void) +{ + struct domain_data *our_domains[NR_CPUS]; + int cpu, n = 0; + long ret; + + for_each_online_cpu(cpu) { + BUG_ON(NR_CPUS <= n); + our_domains[cpu] = &per_cpu(_mc_ce_doms, cpu); + n++; + } + ret = mc_ce_set_domains(n, our_domains); + if (ret) + goto out; + ret = mc_ce_activate_plugin_common(); +out: + return ret; +} + +static void clear_pid_entries(void) +{ + struct ce_pid_table *pid_table = NULL; + int cpu, entry; + + for_each_online_cpu(cpu) { + pid_table = get_pid_table(cpu); + pid_table->num_pid_entries = 0; + pid_table->cycle_time = 0; + for (entry = 0; entry < CONFIG_PLUGIN_MC_LEVEL_A_MAX_TASKS; + ++entry) { + if (NULL != pid_table->entries[entry].pid) { + put_pid(pid_table->entries[entry].pid); + pid_table->entries[entry].pid = NULL; + } + pid_table->entries[entry].budget = 0; + pid_table->entries[entry].acc_time = 0; + pid_table->entries[entry].expected_job = 0; + } + } +} + +long mc_ce_deactivate_plugin_common(void) +{ + int cpu; + cancel_all_timers(); + for_each_online_cpu(cpu) { + per_cpu(domains, cpu) = NULL; + } + return 0; +} + +/* Plugin object */ +static struct sched_plugin mc_ce_plugin __cacheline_aligned_in_smp = { + .plugin_name = "MC-CE", + .admit_task = mc_ce_admit_task, + .task_new = mc_ce_task_new, + .complete_job = complete_job, + .release_at = mc_ce_release_at_common, + .task_exit = mc_ce_task_exit_common, + .schedule = mc_ce_schedule, + .finish_switch = mc_ce_finish_switch, + .task_wake_up = mc_ce_task_wake_up, + .task_block = mc_ce_task_block, + .activate_plugin = mc_ce_activate_plugin, + .deactivate_plugin = mc_ce_deactivate_plugin_common, +}; + +static int setup_proc(void); +static int __init init_sched_mc_ce(void) +{ + raw_spinlock_t *ce_lock; + struct domain_data *dom_data; + struct domain *dom; + int cpu, err; + + for_each_online_cpu(cpu) { + per_cpu(domains, cpu) = NULL; + ce_lock = &per_cpu(_mc_ce_dom_locks, cpu); + raw_spin_lock_init(ce_lock); + dom_data = &per_cpu(_mc_ce_doms, cpu); + dom = &dom_data->domain; + ce_domain_init(dom, ce_lock, NULL, NULL, NULL, NULL, NULL, + &per_cpu(_mc_ce_dom_data, cpu), cpu, + mc_ce_timer_callback); + } + clear_pid_entries(); + err = setup_proc(); + if (!err) + err = register_sched_plugin(&mc_ce_plugin); + return err; +} + +#define BUF_SIZE PAGE_SIZE +static int write_into_proc(char *proc_buf, const int proc_size, char *fmt, ...) +{ + static char buf[BUF_SIZE]; + int n; + va_list args; + + /* When writing to procfs, we don't care about the trailing null that + * is not included in the count returned by vscnprintf. + */ + va_start(args, fmt); + n = vsnprintf(buf, BUF_SIZE, fmt, args); + va_end(args); + if (BUF_SIZE <= n || proc_size <= n) { + /* too big for formatting buffer or proc (less null byte) */ + n = -EINVAL; + goto out; + } + memcpy(proc_buf, buf, n); +out: + return n; +} +#undef BUF_SIZE + +/* + * Writes a PID entry to the procfs. + * + * @page buffer to write into. + * @count bytes available in the buffer + */ +#define PID_SPACE 15 +#define TASK_INFO_BUF (PID_SPACE + TASK_COMM_LEN) +static int write_pid_entry(char *page, const int count, const int cpu, + const int task, struct ce_pid_entry *pid_entry) +{ + static char task_info[TASK_INFO_BUF]; + struct task_struct *ts; + int n = 0, err, ti_n; + char *ti_b; + + if (pid_entry->pid) { + rcu_read_lock(); + ts = pid_task(pid_entry->pid, PIDTYPE_PID); + rcu_read_unlock(); + + /* get some information about the task */ + if (ts) { + ti_b = task_info; + ti_n = snprintf(ti_b, PID_SPACE, "%d", ts->pid); + if (PID_SPACE <= ti_n) + ti_n = PID_SPACE - 1; + ti_b += ti_n; + *ti_b = ' '; /* nuke the null byte */ + ti_b++; + get_task_comm(ti_b, ts); + } else { + strncpy(task_info, "pid_task() failed :(", + TASK_INFO_BUF); + } + + } else + strncpy(task_info, "no", TASK_INFO_BUF); + task_info[TASK_INFO_BUF - 1] = '\0'; /* just to be sure */ + + err = write_into_proc(page + n, count - n, "# task: %s\n", task_info); + if (err < 0) { + n = -ENOSPC; + goto out; + } + n += err; + err = write_into_proc(page + n, count - n, "%d, %d, %llu\n", + cpu, task, pid_entry->budget); + if (err < 0) { + n = -ENOSPC; + goto out; + } + n += err; +out: + return n; +} +#undef PID_SPACE +#undef TASK_INFO_BUF + +/* + * Called when the user-land reads from proc. + */ +static int proc_read_ce_file(char *page, char **start, off_t off, int count, + int *eof, void *data) +{ + int n = 0, err, cpu, t; + struct ce_pid_table *pid_table; + + if (off > 0) { + printk(KERN_INFO "litmus: MC-CE called read with off > 0\n"); + goto out; + } + + for_each_online_cpu(cpu) { + pid_table = get_pid_table(cpu); + for (t = 0; t < pid_table->num_pid_entries; ++t) { + err = write_pid_entry(page + n, count - n, + cpu, t, get_pid_entry(cpu, t)); + if (err < 0) { + n = -ENOSPC; + goto out; + } + n += err; + } + } +out: + *eof = 1; + return n; +} + +/* + * Skip a commented line. + */ +static int skip_comment(const char *buf, const unsigned long max) +{ + unsigned long i = 0; + const char *c = buf; + if (0 == max || !c || *c != '#') + return 0; + ++c; ++i; + for (; i < max; ++i) { + if (*c == '\n') { + ++c; ++i; + break; + } + ++c; + } + return i; +} + +/* a budget of 5 milliseconds is probably reasonable */ +#define BUDGET_THRESHOLD 5000000ULL +static int setup_pid_entry(const int cpu, const int task, const lt_t budget) +{ + struct ce_pid_table *pid_table = get_pid_table(cpu); + struct ce_pid_entry *new_entry = NULL; + int err = 0; + + /* check the inputs */ + if (cpu < 0 || NR_CPUS <= cpu || task < 0 || + CONFIG_PLUGIN_MC_LEVEL_A_MAX_TASKS <= task || + budget < 1) { + printk(KERN_INFO "litmus: bad cpu, task ID, or budget sent to " + "MC-CE proc\n"); + err = -EINVAL; + goto out; + } + /* check for small budgets */ + if (BUDGET_THRESHOLD > budget) { + printk(KERN_CRIT "litmus: you gave a small budget for an " + "MC-CE task; that might be an issue.\n"); + } + /* check that we have space for a new entry */ + if (CONFIG_PLUGIN_MC_LEVEL_A_MAX_TASKS <= pid_table->num_pid_entries) { + printk(KERN_INFO "litmus: too many MC-CE tasks for cpu " + "%d\n", cpu); + err = -EINVAL; + goto out; + } + /* add the new entry */ + new_entry = get_pid_entry(cpu, pid_table->num_pid_entries); + BUG_ON(NULL != new_entry->pid); + new_entry->budget = budget; + new_entry->acc_time = pid_table->cycle_time + budget; + /* update the domain entry */ + pid_table->cycle_time += budget; + pid_table->num_pid_entries++; +out: + return err; +} +#undef BUDGET_THRESHOLD + +/* + * Called when the user-land writes to proc. + * + * Error checking is quite minimal. Format is: + * , , + */ +#define PROCFS_MAX_SIZE PAGE_SIZE +static int proc_write_ce_file(struct file *file, const char __user *buffer, + unsigned long count, void *data) +{ + static char kbuf[PROCFS_MAX_SIZE]; + char *c = kbuf, *c_skipped; + int cpu, task, cnt = 0, chars_read, converted, err; + lt_t budget; + + if (!using_linux_plugin()) { + printk(KERN_INFO "litmus: can only edit MC-CE proc under Linux " + "plugin\n"); + cnt = -EINVAL; + goto out; + } + + if (count > PROCFS_MAX_SIZE) { + printk(KERN_INFO "litmus: MC-CE procfs got too many bytes " + "from user-space.\n"); + cnt = -EINVAL; + goto out; + } + + if (copy_from_user(kbuf, buffer, count)) { + printk(KERN_INFO "litmus: couldn't copy from user %s\n", + __FUNCTION__); + cnt = -EFAULT; + goto out; + } + clear_pid_entries(); + while (cnt < count) { + c_skipped = skip_spaces(c); + if (c_skipped != c) { + chars_read = c_skipped - c; + cnt += chars_read; + c += chars_read; + continue; + } + if (*c == '#') { + chars_read = skip_comment(c, count - cnt); + cnt += chars_read; + c += chars_read; + continue; + } + converted = sscanf(c, "%d, %d, %llu%n", &cpu, &task, &budget, + &chars_read); + if (3 != converted) { + printk(KERN_INFO "litmus: MC-CE procfs expected three " + "arguments, but got %d.\n", converted); + cnt = -EINVAL; + goto out; + } + cnt += chars_read; + c += chars_read; + err = setup_pid_entry(cpu, task, budget); + if (err) { + cnt = -EINVAL; + goto out; + } + } +out: + return cnt; +} +#undef PROCFS_MAX_SIZE + +#define CE_FILE_PROC_NAME "ce_file" +static void tear_down_proc(void) +{ + if (ce_file) + remove_proc_entry(CE_FILE_PROC_NAME, mc_ce_dir); + if (mc_ce_dir) + remove_plugin_proc_dir(&mc_ce_plugin); +} + +static int setup_proc(void) +{ + int err; + err = make_plugin_proc_dir(&mc_ce_plugin, &mc_ce_dir); + if (err) { + printk(KERN_ERR "could not create MC-CE procfs dir.\n"); + goto out; + } + ce_file = create_proc_entry(CE_FILE_PROC_NAME, 0644, mc_ce_dir); + if (!ce_file) { + printk(KERN_ERR "could not create MC-CE procfs file.\n"); + err = -EIO; + goto out_remove_proc; + } + ce_file->read_proc = proc_read_ce_file; + ce_file->write_proc = proc_write_ce_file; + goto out; +out_remove_proc: + tear_down_proc(); +out: + return err; +} +#undef CE_FILE_PROC_NAME + +static void clean_sched_mc_ce(void) +{ + tear_down_proc(); +} + +module_init(init_sched_mc_ce); +module_exit(clean_sched_mc_ce); diff --git a/litmus/sched_psn_edf.c b/litmus/sched_psn_edf.c index 71c0240..abb06fa 100644 --- a/litmus/sched_psn_edf.c +++ b/litmus/sched_psn_edf.c @@ -442,6 +442,10 @@ int psnedf_fmlp_lock(struct litmus_lock* l) * ->owner. We can thus check it without acquiring the spin * lock. */ BUG_ON(sem->owner != t); + + /* FIXME: could we punt the dequeuing to the previous job, + * which is holding the spinlock anyway? */ + remove_wait_queue(&sem->wait, &wait); } else { /* it's ours now */ sem->owner = t; @@ -474,7 +478,7 @@ int psnedf_fmlp_unlock(struct litmus_lock* l) unboost_priority(t); /* check if there are jobs waiting for this resource */ - next = __waitqueue_remove_first(&sem->wait); + next = waitqueue_first(&sem->wait); if (next) { /* boost next job */ boost_priority(next); diff --git a/litmus/sched_task_trace.c b/litmus/sched_task_trace.c index 5ef8d09..f923280 100644 --- a/litmus/sched_task_trace.c +++ b/litmus/sched_task_trace.c @@ -14,7 +14,11 @@ #include #include #include +#include +#include +#include +#include #define NO_EVENTS (1 << CONFIG_SCHED_TASK_TRACE_SHIFT) @@ -132,6 +136,7 @@ feather_callback void do_sched_trace_task_param(unsigned long id, unsigned long rec->data.param.phase = get_rt_phase(t); rec->data.param.partition = get_partition(t); rec->data.param.class = get_class(t); + rec->data.param.level = (tsk_mc_data(t) ? tsk_mc_crit(t) : -1); put_record(rec); } } @@ -141,8 +146,8 @@ feather_callback void do_sched_trace_task_release(unsigned long id, unsigned lon struct task_struct *t = (struct task_struct*) _task; struct st_event_record* rec = get_record(ST_RELEASE, t); if (rec) { - rec->data.release.release = get_release(t); - rec->data.release.deadline = get_deadline(t); + rec->data.release.release = tsk_rt(t)->job_params.real_release; + rec->data.release.deadline = tsk_rt(t)->job_params.real_deadline; put_record(rec); } } diff --git a/litmus/trace.c b/litmus/trace.c index e7ea1c2..1d4593a 100644 --- a/litmus/trace.c +++ b/litmus/trace.c @@ -5,6 +5,10 @@ #include #include +#include +#include +#include + /******************************************************************************/ /* Allocation */ /******************************************************************************/ @@ -37,6 +41,36 @@ static inline void __save_timestamp(unsigned long event, __save_timestamp_cpu(event, type, raw_smp_processor_id()); } +/* hack: fake timestamp to user-reported time, and record parts of the PID */ +feather_callback void save_timestamp_time(unsigned long event, unsigned long ptr) +{ + uint64_t* time = (uint64_t*) ptr; + unsigned int seq_no; + struct timestamp *ts; + seq_no = fetch_and_inc((int *) &ts_seq_no); + if (ft_buffer_start_write(trace_ts_buf, (void**) &ts)) { + ts->event = event; + ts->timestamp = *time; + ts->seq_no = seq_no; + /* type takes lowest byte of PID */ + ts->task_type = (uint8_t) current->pid; + /* cpu takes second-lowest byte of PID*/ + ts->cpu = (uint8_t) (current->pid >> 8); + + ft_buffer_finish_write(trace_ts_buf, ts); + } +} + +feather_callback void save_timestamp_pid(unsigned long event) +{ + /* Abuse existing fields to partially export PID. */ + __save_timestamp_cpu(event, + /* type takes lowest byte of PID */ + (uint8_t) current->pid, + /* cpu takes second-lowest byte of PID*/ + (uint8_t) (current->pid >> 8)); +} + feather_callback void save_timestamp(unsigned long event) { __save_timestamp(event, TSK_UNKNOWN); @@ -51,8 +85,15 @@ feather_callback void save_timestamp_def(unsigned long event, feather_callback void save_timestamp_task(unsigned long event, unsigned long t_ptr) { - int rt = is_realtime((struct task_struct *) t_ptr); - __save_timestamp(event, rt ? TSK_RT : TSK_BE); + struct task_struct *ts = (struct task_struct*) t_ptr; + int rt = is_realtime(ts); + uint8_t type = rt ? TSK_RT : TSK_BE; + + if (TS_LVLA_SCHED_END_ID == event) { + if (rt && CRIT_LEVEL_A == tsk_mc_crit(ts)) + type = TSK_LVLA; + } + __save_timestamp(event, type); } feather_callback void save_timestamp_cpu(unsigned long event,