From 7dbc4a842d3bcfa755ba82cae46171d0098d4c2c Mon Sep 17 00:00:00 2001 From: Jonathan Herman Date: Wed, 26 Jan 2011 17:47:49 -0500 Subject: [PATCH 01/24] Added support for tracing arbitrary actions. --- include/litmus/sched_trace.h | 25 ++++++++++++++++++++----- litmus/sched_task_trace.c | 14 ++++++++++++++ 2 files changed, 34 insertions(+), 5 deletions(-) diff --git a/include/litmus/sched_trace.h b/include/litmus/sched_trace.h index a5f7373..f7b2189 100644 --- a/include/litmus/sched_trace.h +++ b/include/litmus/sched_trace.h @@ -34,7 +34,7 @@ struct st_release_data { /* A job is was/is going to be released. */ struct st_assigned_data { /* A job was asigned to a CPU. */ u64 when; u8 target; /* Where should it execute? */ - u8 __unused[3]; + u8 __unused[7]; }; struct st_switch_to_data { /* A process was switched to on a given CPU. */ @@ -54,7 +54,7 @@ struct st_completion_data { /* A job completed. */ * next task automatically; set to 0 otherwise. */ u8 __uflags:7; - u8 __unused[3]; + u8 __unused[7]; }; struct st_block_data { /* A task blocks. */ @@ -67,6 +67,12 @@ struct st_resume_data { /* A task resumes. */ u64 __unused; }; +struct st_action_data { + u64 when; + u8 action; + u8 __unused[7]; +}; + struct st_sys_release_data { u64 when; u64 release; @@ -85,7 +91,8 @@ typedef enum { ST_COMPLETION, ST_BLOCK, ST_RESUME, - ST_SYS_RELEASE, + ST_ACTION, + ST_SYS_RELEASE } st_event_record_type_t; struct st_event_record { @@ -102,8 +109,8 @@ struct st_event_record { DATA(completion); DATA(block); DATA(resume); + DATA(action); DATA(sys_release); - } data; }; @@ -140,8 +147,12 @@ feather_callback void do_sched_trace_task_block(unsigned long id, struct task_struct* task); feather_callback void do_sched_trace_task_resume(unsigned long id, struct task_struct* task); +feather_callback void do_sched_trace_action(unsigned long id, + struct task_struct* task, + unsigned long action); feather_callback void do_sched_trace_sys_release(unsigned long id, lt_t* start); + #endif #else @@ -172,9 +183,13 @@ feather_callback void do_sched_trace_sys_release(unsigned long id, SCHED_TRACE(SCHED_TRACE_BASE_ID + 7, do_sched_trace_task_block, t) #define sched_trace_task_resume(t) \ SCHED_TRACE(SCHED_TRACE_BASE_ID + 8, do_sched_trace_task_resume, t) +#define sched_trace_action(t, action) \ + SCHED_TRACE2(SCHED_TRACE_BASE_ID + 9, do_sched_trace_action, t, \ + (unsigned long) action); /* when is a pointer, it does not need an explicit cast to unsigned long */ #define sched_trace_sys_release(when) \ - SCHED_TRACE(SCHED_TRACE_BASE_ID + 9, do_sched_trace_sys_release, when) + SCHED_TRACE(SCHED_TRACE_BASE_ID + 10, do_sched_trace_sys_release, when) + #define sched_trace_quantum_boundary() /* NOT IMPLEMENTED */ diff --git a/litmus/sched_task_trace.c b/litmus/sched_task_trace.c index a15b25d..bd49005 100644 --- a/litmus/sched_task_trace.c +++ b/litmus/sched_task_trace.c @@ -224,3 +224,17 @@ feather_callback void do_sched_trace_sys_release(unsigned long id, put_record(rec); } } + +feather_callback void do_sched_trace_action(unsigned long id, + unsigned long _task, + unsigned long action) +{ + struct task_struct *t = (struct task_struct*) _task; + struct st_event_record* rec = get_record(ST_ACTION, t); + + if (rec) { + rec->data.action.when = now(); + rec->data.action.action = action; + put_record(rec); + } +} -- 1.7.2.3 From 3d8eb93db513bd9caa982f27fee8156405fac754 Mon Sep 17 00:00:00 2001 From: Bjoern B. Brandenburg Date: Wed, 26 Jan 2011 20:36:49 -0500 Subject: [PATCH 02/24] Litmus core: add copy_and_chomp() helper We read in a line from userspace and remove the trailing newline in a number of places. This function extracts the common code to avoid future duplication. --- include/litmus/litmus_proc.h | 6 ++++ litmus/litmus_proc.c | 66 ++++++++++++++++++++++++------------------ 2 files changed, 44 insertions(+), 28 deletions(-) diff --git a/include/litmus/litmus_proc.h b/include/litmus/litmus_proc.h index fbc0082..6800e72 100644 --- a/include/litmus/litmus_proc.h +++ b/include/litmus/litmus_proc.h @@ -17,3 +17,9 @@ long make_plugin_proc_dir(struct sched_plugin* plugin, */ void remove_plugin_proc_dir(struct sched_plugin* plugin); + +/* Copy at most size-1 bytes from ubuf into kbuf, null-terminate buf, and + * remove a '\n' if present. Returns the number of bytes that were read or + * -EFAULT. */ +int copy_and_chomp(char *kbuf, unsigned long ksize, + __user const char* ubuf, unsigned long ulength); diff --git a/litmus/litmus_proc.c b/litmus/litmus_proc.c index 81ea5c3..e3f3f11 100644 --- a/litmus/litmus_proc.c +++ b/litmus/litmus_proc.c @@ -69,18 +69,9 @@ static int proc_write_curr(struct file *file, char name[65]; struct sched_plugin* found; - if(count > 64) - len = 64; - else - len = count; - - if(copy_from_user(name, buffer, len)) - return -EFAULT; - - name[len] = '\0'; - /* chomp name */ - if (len > 1 && name[len - 1] == '\n') - name[len - 1] = '\0'; + len = copy_and_chomp(name, sizeof(name), buffer, count); + if (len < 0) + return len; found = find_sched_plugin(name); @@ -113,36 +104,28 @@ static int proc_write_release_master(struct file *file, unsigned long count, void *data) { - int cpu, err, online = 0; + int cpu, err, len, online = 0; char msg[64]; - if (count > 63) - return -EINVAL; - - if (copy_from_user(msg, buffer, count)) - return -EFAULT; + len = copy_and_chomp(msg, sizeof(msg), buffer, count); - /* terminate */ - msg[count] = '\0'; - /* chomp */ - if (count > 1 && msg[count - 1] == '\n') - msg[count - 1] = '\0'; + if (len < 0) + return len; - if (strcmp(msg, "NO_CPU") == 0) { + if (strcmp(msg, "NO_CPU") == 0) atomic_set(&release_master_cpu, NO_CPU); - return count; - } else { + else { err = sscanf(msg, "%d", &cpu); if (err == 1 && cpu >= 0 && (online = cpu_online(cpu))) { atomic_set(&release_master_cpu, cpu); - return count; } else { TRACE("invalid release master: '%s' " "(err:%d cpu:%d online:%d)\n", msg, err, cpu, online); - return -EINVAL; + len = -EINVAL; } } + return len; } #endif @@ -257,3 +240,30 @@ void remove_plugin_proc_dir(struct sched_plugin* plugin) } remove_proc_entry(plugin->plugin_name, plugs_dir); } + + + +/* misc. I/O helper functions */ + +int copy_and_chomp(char *kbuf, unsigned long ksize, + __user const char* ubuf, unsigned long ulength) +{ + /* caller must provide buffer space */ + BUG_ON(!ksize); + + ksize--; /* leave space for null byte */ + + if (ksize > ulength) + ksize = ulength; + + if(copy_from_user(kbuf, ubuf, ksize)) + return -EFAULT; + + kbuf[ksize] = '\0'; + + /* chomp kbuf */ + if (ksize > 0 && kbuf[ksize - 1] == '\n') + kbuf[ksize - 1] = '\0'; + + return ksize; +} -- 1.7.2.3 From 904531a6321964579ab0972a8833616e97dbf582 Mon Sep 17 00:00:00 2001 From: Bjoern B. Brandenburg Date: Sat, 29 Jan 2011 20:31:57 -0500 Subject: [PATCH 03/24] bugfix: don't let children stay Litmus real-time tasks It has always been LITMUS^RT policy that children of real-time tasks may not skip the admissions test, etc. This used to be enforced, but was apparently dropped during some port. This commit re-introduces this policy. This fixes a kernel panic that occurred when "real-time children" exited without proper initilization. --- kernel/sched.c | 3 ++- litmus/litmus.c | 9 ++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/kernel/sched.c b/kernel/sched.c index 1b13c8e..5beefb2 100644 --- a/kernel/sched.c +++ b/kernel/sched.c @@ -2557,7 +2557,8 @@ void sched_fork(struct task_struct *p, int clone_flags) * Revert to default priority/policy on fork if requested. */ if (unlikely(p->sched_reset_on_fork)) { - if (p->policy == SCHED_FIFO || p->policy == SCHED_RR) { + if (p->policy == SCHED_FIFO || p->policy == SCHED_RR || + p->policy == SCHED_LITMUS) { p->policy = SCHED_NORMAL; p->normal_prio = p->static_prio; } diff --git a/litmus/litmus.c b/litmus/litmus.c index 8efd3f9..11ccaaf 100644 --- a/litmus/litmus.c +++ b/litmus/litmus.c @@ -292,9 +292,6 @@ static void reinit_litmus_state(struct task_struct* p, int restore) */ WARN_ON(p->rt_param.inh_task); - /* We need to restore the priority of the task. */ -// __setscheduler(p, p->rt_param.old_policy, p->rt_param.old_prio); XXX why is this commented? - /* Cleanup everything else. */ memset(&p->rt_param, 0, sizeof(p->rt_param)); @@ -437,10 +434,12 @@ out: */ void litmus_fork(struct task_struct* p) { - if (is_realtime(p)) + if (is_realtime(p)) { /* clean out any litmus related state, don't preserve anything */ reinit_litmus_state(p, 0); - else + /* Don't let the child be a real-time task. */ + p->sched_reset_on_fork = 1; + } else /* non-rt tasks might have ctrl_page set */ tsk_rt(p)->ctrl_page = NULL; -- 1.7.2.3 From 3cb35a8d90658bd8fb6f9b4f60eb7f97d0643313 Mon Sep 17 00:00:00 2001 From: Jonathan Herman Date: Sun, 30 Jan 2011 15:10:49 -0500 Subject: [PATCH 04/24] Added task class to feather trace param record. --- include/litmus/sched_trace.h | 4 +++- litmus/sched_task_trace.c | 1 + 2 files changed, 4 insertions(+), 1 deletions(-) diff --git a/include/litmus/sched_trace.h b/include/litmus/sched_trace.h index f7b2189..7ca34cb 100644 --- a/include/litmus/sched_trace.h +++ b/include/litmus/sched_trace.h @@ -23,7 +23,8 @@ struct st_param_data { /* regular params */ u32 period; u32 phase; u8 partition; - u8 __unused[3]; + u8 class; + u8 __unused[2]; }; struct st_release_data { /* A job is was/is going to be released. */ @@ -40,6 +41,7 @@ struct st_assigned_data { /* A job was asigned to a CPU. */ struct st_switch_to_data { /* A process was switched to on a given CPU. */ u64 when; /* When did this occur? */ u32 exec_time; /* Time the current job has executed. */ + u8 __unused[4]; }; diff --git a/litmus/sched_task_trace.c b/litmus/sched_task_trace.c index bd49005..5ef8d09 100644 --- a/litmus/sched_task_trace.c +++ b/litmus/sched_task_trace.c @@ -131,6 +131,7 @@ feather_callback void do_sched_trace_task_param(unsigned long id, unsigned long rec->data.param.period = get_rt_period(t); rec->data.param.phase = get_rt_phase(t); rec->data.param.partition = get_partition(t); + rec->data.param.class = get_class(t); put_record(rec); } } -- 1.7.2.3 From a0f243fd1d66c3499f88a690e485e94160ac1a8c Mon Sep 17 00:00:00 2001 From: Jonathan Herman Date: Sun, 30 Jan 2011 15:14:20 -0500 Subject: [PATCH 05/24] Fixed is_hrt, is_srt, and is_be macros. --- include/litmus/litmus.h | 6 +++--- 1 files changed, 3 insertions(+), 3 deletions(-) diff --git a/include/litmus/litmus.h b/include/litmus/litmus.h index 2464837..4a774a9 100644 --- a/include/litmus/litmus.h +++ b/include/litmus/litmus.h @@ -74,11 +74,11 @@ inline static lt_t budget_remaining(struct task_struct* t) == PRECISE_ENFORCEMENT) #define is_hrt(t) \ - (tsk_rt(t)->task_params.class == RT_CLASS_HARD) + (tsk_rt(t)->task_params.cls == RT_CLASS_HARD) #define is_srt(t) \ - (tsk_rt(t)->task_params.class == RT_CLASS_SOFT) + (tsk_rt(t)->task_params.cls == RT_CLASS_SOFT) #define is_be(t) \ - (tsk_rt(t)->task_params.class == RT_CLASS_BEST_EFFORT) + (tsk_rt(t)->task_params.cls == RT_CLASS_BEST_EFFORT) /* Our notion of time within LITMUS: kernel monotonic time. */ static inline lt_t litmus_clock(void) -- 1.7.2.3 From fd8ae31c74975c8499983c9831bff2b136b98434 Mon Sep 17 00:00:00 2001 From: Bjoern B. Brandenburg Date: Fri, 28 Jan 2011 11:54:38 -0500 Subject: [PATCH 06/24] fdso: supply object type to constructor and destructor methods Passing the object type explicitly will enable generic lock constructors. --- include/litmus/fdso.h | 4 ++-- litmus/fdso.c | 4 ++-- litmus/fmlp.c | 4 ++-- litmus/srp.c | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/include/litmus/fdso.h b/include/litmus/fdso.h index 61f1b5b..bfab9fd 100644 --- a/include/litmus/fdso.h +++ b/include/litmus/fdso.h @@ -42,8 +42,8 @@ struct od_table_entry { }; struct fdso_ops { - void* (*create) (void); - void (*destroy)(void*); + void* (*create)(obj_type_t type); + void (*destroy)(obj_type_t type, void*); int (*open) (struct od_table_entry*, void* __user); int (*close) (struct od_table_entry*); }; diff --git a/litmus/fdso.c b/litmus/fdso.c index 85be716..3bb331e 100644 --- a/litmus/fdso.c +++ b/litmus/fdso.c @@ -29,14 +29,14 @@ static const struct fdso_ops* fdso_ops[] = { static void* fdso_create(obj_type_t type) { if (fdso_ops[type]->create) - return fdso_ops[type]->create(); + return fdso_ops[type]->create(type); else return NULL; } static void fdso_destroy(obj_type_t type, void* obj) { - fdso_ops[type]->destroy(obj); + fdso_ops[type]->destroy(type, obj); } static int fdso_open(struct od_table_entry* entry, void* __user config) diff --git a/litmus/fmlp.c b/litmus/fmlp.c index a9a6385..1e4d544 100644 --- a/litmus/fmlp.c +++ b/litmus/fmlp.c @@ -20,7 +20,7 @@ #ifdef CONFIG_FMLP -static void* create_fmlp_semaphore(void) +static void* create_fmlp_semaphore(obj_type_t type) { struct pi_semaphore* sem; int i; @@ -45,7 +45,7 @@ static int open_fmlp_semaphore(struct od_table_entry* entry, void* __user arg) return 0; } -static void destroy_fmlp_semaphore(void* sem) +static void destroy_fmlp_semaphore(obj_type_t type, void* sem) { /* XXX assert invariants */ kfree(sem); diff --git a/litmus/srp.c b/litmus/srp.c index cb57759..4601b7d 100644 --- a/litmus/srp.c +++ b/litmus/srp.c @@ -108,7 +108,7 @@ static void srp_add_prio(struct srp* srp, struct srp_priority* prio) } -static void* create_srp_semaphore(void) +static void* create_srp_semaphore(obj_type_t type) { struct srp_semaphore* sem; @@ -152,7 +152,7 @@ static noinline int open_srp_semaphore(struct od_table_entry* entry, void* __use return ret; } -static void destroy_srp_semaphore(void* sem) +static void destroy_srp_semaphore(obj_type_t type, void* sem) { /* XXX invariants */ atomic_dec(&srp_objects_in_use); -- 1.7.2.3 From 2dea9d5e7727b8474981557cbf925687b8f33865 Mon Sep 17 00:00:00 2001 From: Bjoern B. Brandenburg Date: Fri, 28 Jan 2011 12:24:58 -0500 Subject: [PATCH 07/24] Litmus core: change plugin locking interface to generic 'allocate_lock()' As the number of supported locking protocols is expected to rise, hard-coding things like priority inheritance in the plugin interface doesn't scale. Instead, use a new generic lock-ops approach. With this approach, each plugin can define its own protocol implementation (or use a generic one), and plugins can support multiple protocols without having to change the plugin interface for each protocol. --- include/litmus/locking.h | 28 +++++++++++++++++++++++ include/litmus/sched_plugin.h | 50 +++++++--------------------------------- litmus/Kconfig | 23 +++++------------- litmus/sched_plugin.c | 30 +++++------------------ 4 files changed, 51 insertions(+), 80 deletions(-) create mode 100644 include/litmus/locking.h diff --git a/include/litmus/locking.h b/include/litmus/locking.h new file mode 100644 index 0000000..4d7b870 --- /dev/null +++ b/include/litmus/locking.h @@ -0,0 +1,28 @@ +#ifndef LITMUS_LOCKING_H +#define LITMUS_LOCKING_H + +struct litmus_lock_ops; + +/* Generic base struct for LITMUS^RT userspace semaphores. + * This structure should be embedded in protocol-specific semaphores. + */ +struct litmus_lock { + struct litmus_lock_ops *ops; + int type; +}; + +struct litmus_lock_ops { + /* Current task tries to obtain / drop a reference to a lock. + * Optional methods, allowed by default. */ + int (*open)(struct litmus_lock*, void* __user); + int (*close)(struct litmus_lock*); + + /* Current tries to lock/unlock this lock (mandatory methods). */ + int (*lock)(struct litmus_lock*); + int (*unlock)(struct litmus_lock*); + + /* The lock is no longer being referenced (mandatory method). */ + void (*deallocate)(struct litmus_lock*); +}; + +#endif diff --git a/include/litmus/sched_plugin.h b/include/litmus/sched_plugin.h index 2d856d5..27b719a 100644 --- a/include/litmus/sched_plugin.h +++ b/include/litmus/sched_plugin.h @@ -7,19 +7,9 @@ #include -/* struct for semaphore with priority inheritance */ -struct pi_semaphore { - atomic_t count; - int sleepers; - wait_queue_head_t wait; - struct { - /* highest-prio holder/waiter */ - struct task_struct *task; - struct task_struct* cpu_task[NR_CPUS]; - } hp; - /* current lock holder */ - struct task_struct *holder; -}; +#ifdef CONFIG_LITMUS_LOCKING +#include +#endif /************************ setup/tear down ********************/ @@ -63,24 +53,9 @@ typedef void (*task_block_t) (struct task_struct *task); */ typedef void (*task_exit_t) (struct task_struct *); -/* Called when the new_owner is released from the wait queue - * it should now inherit the priority from sem, _before_ it gets readded - * to any queue - */ -typedef long (*inherit_priority_t) (struct pi_semaphore *sem, - struct task_struct *new_owner); - -/* Called when the current task releases a semahpore where it might have - * inherited a piority from - */ -typedef long (*return_priority_t) (struct pi_semaphore *sem); - -/* Called when a task tries to acquire a semaphore and fails. Check if its - * priority is higher than that of the current holder. - */ -typedef long (*pi_block_t) (struct pi_semaphore *sem, struct task_struct *t); - - +/* Called when the current task attempts to create a new lock of a given + * protocol type. */ +typedef long (*allocate_lock_t) (struct litmus_lock **lock, int type); /********************* sys call backends ********************/ @@ -100,10 +75,6 @@ struct sched_plugin { activate_plugin_t activate_plugin; deactivate_plugin_t deactivate_plugin; -#ifdef CONFIG_SRP - unsigned int srp_active; -#endif - /* scheduler invocation */ scheduler_tick_t tick; schedule_t schedule; @@ -121,12 +92,9 @@ struct sched_plugin { task_block_t task_block; task_exit_t task_exit; -#ifdef CONFIG_FMLP - /* priority inheritance */ - unsigned int fmlp_active; - inherit_priority_t inherit_priority; - return_priority_t return_priority; - pi_block_t pi_block; +#ifdef CONFIG_LITMUS_LOCKING + /* locking protocols */ + allocate_lock_t allocate_lock; #endif } __attribute__ ((__aligned__(SMP_CACHE_BYTES))); diff --git a/litmus/Kconfig b/litmus/Kconfig index a2f2678..ad8dc83 100644 --- a/litmus/Kconfig +++ b/litmus/Kconfig @@ -46,28 +46,19 @@ config NP_SECTION Note that plugins still need to explicitly support non-preemptivity. Currently, only GSN-EDF and PSN-EDF have such support. - This is required to support the FMLP. + This is required to support locking protocols such as the FMLP. If disabled, all tasks will be considered preemptable at all times. -config SRP - bool "Stack Resource Policy (SRP)" - default n - help - Include support for Baker's Stack Resource Policy. - - Say Yes if you want FMLP local long critical section - synchronization support. - -config FMLP - bool "FMLP support" +config LITMUS_LOCKING + bool "Support for real-time locking protocols" depends on NP_SECTION default n help - Include support for deterministic multiprocessor real-time - synchronization support. + Enable LITMUS^RT's deterministic multiprocessor real-time + locking protocols. - Say Yes if you want FMLP long critical section - synchronization support. + Say Yes if you want to include locking protocols such as the FMLP and + Baker's SRP. endmenu diff --git a/litmus/sched_plugin.c b/litmus/sched_plugin.c index d912a64..2f8f399 100644 --- a/litmus/sched_plugin.c +++ b/litmus/sched_plugin.c @@ -121,23 +121,11 @@ static long litmus_dummy_deactivate_plugin(void) return 0; } -#ifdef CONFIG_FMLP +#ifdef CONFIG_LITMUS_LOCKING -static long litmus_dummy_inherit_priority(struct pi_semaphore *sem, - struct task_struct *new_owner) +static long litmus_dummy_allocate_lock(struct litmus_lock **lock, int type) { - return -ENOSYS; -} - -static long litmus_dummy_return_priority(struct pi_semaphore *sem) -{ - return -ENOSYS; -} - -static long litmus_dummy_pi_block(struct pi_semaphore *sem, - struct task_struct *new_waiter) -{ - return -ENOSYS; + return -ENXIO; } #endif @@ -158,10 +146,8 @@ struct sched_plugin linux_sched_plugin = { .finish_switch = litmus_dummy_finish_switch, .activate_plugin = litmus_dummy_activate_plugin, .deactivate_plugin = litmus_dummy_deactivate_plugin, -#ifdef CONFIG_FMLP - .inherit_priority = litmus_dummy_inherit_priority, - .return_priority = litmus_dummy_return_priority, - .pi_block = litmus_dummy_pi_block, +#ifdef CONFIG_LITMUS_LOCKING + .allocate_lock = litmus_dummy_allocate_lock, #endif .admit_task = litmus_dummy_admit_task }; @@ -198,10 +184,8 @@ int register_sched_plugin(struct sched_plugin* plugin) CHECK(complete_job); CHECK(activate_plugin); CHECK(deactivate_plugin); -#ifdef CONFIG_FMLP - CHECK(inherit_priority); - CHECK(return_priority); - CHECK(pi_block); +#ifdef CONFIG_LITMUS_LOCKING + CHECK(allocate_lock); #endif CHECK(admit_task); -- 1.7.2.3 From a3db326495d4051bddc657d3b226ad4daa7997c4 Mon Sep 17 00:00:00 2001 From: Bjoern B. Brandenburg Date: Fri, 28 Jan 2011 13:26:15 -0500 Subject: [PATCH 08/24] Litmus core: add generic locking API Provide a unified userspace interface for plugin-specific locking protocols. --- include/litmus/fdso.h | 7 ++- litmus/Makefile | 1 + litmus/fdso.c | 4 +- litmus/locking.c | 123 +++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 130 insertions(+), 5 deletions(-) create mode 100644 litmus/locking.c diff --git a/include/litmus/fdso.h b/include/litmus/fdso.h index bfab9fd..25a292d 100644 --- a/include/litmus/fdso.h +++ b/include/litmus/fdso.h @@ -33,12 +33,13 @@ struct inode_obj_id { unsigned int id; }; +struct fdso_ops; struct od_table_entry { unsigned int used; struct inode_obj_id* obj; - void* extra; + const struct fdso_ops* class; }; struct fdso_ops { @@ -51,14 +52,14 @@ struct fdso_ops { /* translate a userspace supplied od into the raw table entry * returns NULL if od is invalid */ -struct od_table_entry* __od_lookup(int od); +struct od_table_entry* get_entry_for_od(int od); /* translate a userspace supplied od into the associated object * returns NULL if od is invalid */ static inline void* od_lookup(int od, obj_type_t type) { - struct od_table_entry* e = __od_lookup(od); + struct od_table_entry* e = get_entry_for_od(od); return e && e->obj->type == type ? e->obj->obj : NULL; } diff --git a/litmus/Makefile b/litmus/Makefile index b7366b5..4e019d4 100644 --- a/litmus/Makefile +++ b/litmus/Makefile @@ -11,6 +11,7 @@ obj-y = sched_plugin.o litmus.o \ rt_domain.o \ edf_common.o \ fdso.o \ + locking.o \ srp.o \ fmlp.o \ bheap.o \ diff --git a/litmus/fdso.c b/litmus/fdso.c index 3bb331e..faede9b 100644 --- a/litmus/fdso.c +++ b/litmus/fdso.c @@ -190,7 +190,7 @@ static int do_sys_od_open(struct file* file, obj_type_t type, int id, entry->used = 0; } else { entry->obj = obj; - entry->extra = NULL; + entry->class = fdso_ops[type]; idx = entry - current->od_table; } @@ -209,7 +209,7 @@ static int do_sys_od_open(struct file* file, obj_type_t type, int id, } -struct od_table_entry* __od_lookup(int od) +struct od_table_entry* get_entry_for_od(int od) { struct task_struct *t = current; diff --git a/litmus/locking.c b/litmus/locking.c new file mode 100644 index 0000000..848407b --- /dev/null +++ b/litmus/locking.c @@ -0,0 +1,123 @@ +#include + +#ifdef CONFIG_LITMUS_LOCKING + +#include +#include + +static void* create_generic_lock(obj_type_t type); +static int open_generic_lock(struct od_table_entry* entry, void* __user arg); +static int close_generic_lock(struct od_table_entry* entry); +static void destroy_generic_lock(obj_type_t type, void* sem); + +struct fdso_ops generic_lock_ops = { + .create = create_generic_lock, + .open = open_generic_lock, + .close = close_generic_lock, + .destroy = destroy_generic_lock +}; + +static inline bool is_lock(struct od_table_entry* entry) +{ + return entry->class == &generic_lock_ops; +} + +static inline struct litmus_lock* get_lock(struct od_table_entry* entry) +{ + BUG_ON(!is_lock(entry)); + return (struct litmus_lock*) entry->obj->obj; +} + +static void* create_generic_lock(obj_type_t type) +{ + struct litmus_lock* lock; + int err; + + err = litmus->allocate_lock(&lock, type); + if (err == 0) + return lock; + else + return NULL; +} + +static int open_generic_lock(struct od_table_entry* entry, void* __user arg) +{ + struct litmus_lock* lock = get_lock(entry); + if (lock->ops->open) + return lock->ops->open(lock, arg); + else + return 0; /* default: any task can open it */ +} + +static int close_generic_lock(struct od_table_entry* entry) +{ + struct litmus_lock* lock = get_lock(entry); + if (lock->ops->close) + return lock->ops->close(lock); + else + return 0; /* default: closing succeeds */ +} + +static void destroy_generic_lock(obj_type_t type, void* obj) +{ + struct litmus_lock* lock = (struct litmus_lock*) obj; + lock->ops->deallocate(lock); +} + +asmlinkage long sys_litmus_lock(int lock_od) +{ + long err = -EINVAL; + struct od_table_entry* entry; + struct litmus_lock* l; + + TS_PI_DOWN_START; + + entry = get_entry_for_od(lock_od); + if (entry && is_lock(entry)) { + l = get_lock(entry); + err = l->ops->lock(l); + } + + /* Note: task my have been suspended or preempted in between! Take + * this into account when computing overheads. */ + TS_PI_DOWN_END; + + return err; +} + +asmlinkage long sys_litmus_unlock(int lock_od) +{ + long err = -EINVAL; + struct od_table_entry* entry; + struct litmus_lock* l; + + TS_PI_UP_START; + + entry = get_entry_for_od(lock_od); + if (entry && is_lock(entry)) { + l = get_lock(entry); + err = l->ops->lock(l); + } + + /* Note: task my have been preempted in between! Take this into + * account when computing overheads. */ + TS_PI_UP_END; + + return err; +} + +#else + +struct fdso_ops generic_lock_ops = {}; + +asmlinkage long sys_litmus_lock(int sem_od) +{ + return -ENOSYS; +} + +asmlinkage long sys_litmus_unlock(int sem_od) +{ + return -ENOSYS; +} + +#endif -- 1.7.2.3 From cc602187d4466374bca031039e145aa1b89aca96 Mon Sep 17 00:00:00 2001 From: Bjoern B. Brandenburg Date: Fri, 28 Jan 2011 16:41:16 -0500 Subject: [PATCH 09/24] Litmus core: replace FMLP & SRP system calls with generic syscalls This renders the FMLP and SRP unfunctional until they are ported to the new locking API. --- arch/arm/kernel/calls.S | 10 ++---- arch/x86/kernel/syscall_table_32.S | 6 +-- include/litmus/sched_plugin.h | 17 ---------- include/litmus/unistd_32.h | 18 +++++------ include/litmus/unistd_64.h | 24 ++++++-------- kernel/sched.c | 3 +- litmus/fmlp.c | 54 -------------------------------- litmus/locking.c | 4 ++- litmus/sched_gsn_edf.c | 8 +---- litmus/sched_psn_edf.c | 11 +------ litmus/srp.c | 59 ------------------------------------ 11 files changed, 30 insertions(+), 184 deletions(-) diff --git a/arch/arm/kernel/calls.S b/arch/arm/kernel/calls.S index 584a683..b99087a 100644 --- a/arch/arm/kernel/calls.S +++ b/arch/arm/kernel/calls.S @@ -384,14 +384,12 @@ CALL(sys_complete_job) CALL(sys_od_open) CALL(sys_od_close) -/* 375 */ CALL(sys_fmlp_down) - CALL(sys_fmlp_up) - CALL(sys_srp_down) - CALL(sys_srp_up) +/* 375 */ CALL(sys_litmus_lock) + CALL(sys_litmus_unlock) CALL(sys_query_job_no) -/* 380 */ CALL(sys_wait_for_job_release) + CALL(sys_wait_for_job_release) CALL(sys_wait_for_ts_release) - CALL(sys_release_ts) +/* 380 */ CALL(sys_release_ts) CALL(sys_null_call) #ifndef syscalls_counted .equ syscalls_padding, ((NR_syscalls + 3) & ~3) - NR_syscalls diff --git a/arch/x86/kernel/syscall_table_32.S b/arch/x86/kernel/syscall_table_32.S index d78c5ed..3770290 100644 --- a/arch/x86/kernel/syscall_table_32.S +++ b/arch/x86/kernel/syscall_table_32.S @@ -345,10 +345,8 @@ ENTRY(sys_call_table) .long sys_complete_job .long sys_od_open .long sys_od_close - .long sys_fmlp_down - .long sys_fmlp_up - .long sys_srp_down - .long sys_srp_up + .long sys_litmus_lock + .long sys_litmus_unlock .long sys_query_job_no .long sys_wait_for_job_release .long sys_wait_for_ts_release diff --git a/include/litmus/sched_plugin.h b/include/litmus/sched_plugin.h index 27b719a..8a3ed6d 100644 --- a/include/litmus/sched_plugin.h +++ b/include/litmus/sched_plugin.h @@ -105,23 +105,6 @@ int register_sched_plugin(struct sched_plugin* plugin); struct sched_plugin* find_sched_plugin(const char* name); int print_sched_plugins(char* buf, int max); -static inline int srp_active(void) -{ -#ifdef CONFIG_SRP - return litmus->srp_active; -#else - return 0; -#endif -} -static inline int fmlp_active(void) -{ -#ifdef CONFIG_FMLP - return litmus->fmlp_active; -#else - return 0; -#endif -} - extern struct sched_plugin linux_sched_plugin; #endif diff --git a/include/litmus/unistd_32.h b/include/litmus/unistd_32.h index dbddc65..94264c2 100644 --- a/include/litmus/unistd_32.h +++ b/include/litmus/unistd_32.h @@ -10,14 +10,12 @@ #define __NR_complete_job __LSC(2) #define __NR_od_open __LSC(3) #define __NR_od_close __LSC(4) -#define __NR_fmlp_down __LSC(5) -#define __NR_fmlp_up __LSC(6) -#define __NR_srp_down __LSC(7) -#define __NR_srp_up __LSC(8) -#define __NR_query_job_no __LSC(9) -#define __NR_wait_for_job_release __LSC(10) -#define __NR_wait_for_ts_release __LSC(11) -#define __NR_release_ts __LSC(12) -#define __NR_null_call __LSC(13) +#define __NR_litmus_lock __LSC(5) +#define __NR_litmus_unlock __LSC(6) +#define __NR_query_job_no __LSC(7) +#define __NR_wait_for_job_release __LSC(8) +#define __NR_wait_for_ts_release __LSC(9) +#define __NR_release_ts __LSC(10) +#define __NR_null_call __LSC(11) -#define NR_litmus_syscalls 14 +#define NR_litmus_syscalls 12 diff --git a/include/litmus/unistd_64.h b/include/litmus/unistd_64.h index f0618e7..d5ced0d 100644 --- a/include/litmus/unistd_64.h +++ b/include/litmus/unistd_64.h @@ -15,23 +15,19 @@ __SYSCALL(__NR_complete_job, sys_complete_job) __SYSCALL(__NR_od_open, sys_od_open) #define __NR_od_close __LSC(4) __SYSCALL(__NR_od_close, sys_od_close) -#define __NR_fmlp_down __LSC(5) -__SYSCALL(__NR_fmlp_down, sys_fmlp_down) -#define __NR_fmlp_up __LSC(6) -__SYSCALL(__NR_fmlp_up, sys_fmlp_up) -#define __NR_srp_down __LSC(7) -__SYSCALL(__NR_srp_down, sys_srp_down) -#define __NR_srp_up __LSC(8) -__SYSCALL(__NR_srp_up, sys_srp_up) -#define __NR_query_job_no __LSC(9) +#define __NR_litmus_lock __LSC(5) +__SYSCALL(__NR_litmus_lock, sys_litmus_lock) +#define __NR_litmus_unlock __LSC(6) +__SYSCALL(__NR_litmus_unlock, sys_litmus_unlock) +#define __NR_query_job_no __LSC(7) __SYSCALL(__NR_query_job_no, sys_query_job_no) -#define __NR_wait_for_job_release __LSC(10) +#define __NR_wait_for_job_release __LSC(8) __SYSCALL(__NR_wait_for_job_release, sys_wait_for_job_release) -#define __NR_wait_for_ts_release __LSC(11) +#define __NR_wait_for_ts_release __LSC(9) __SYSCALL(__NR_wait_for_ts_release, sys_wait_for_ts_release) -#define __NR_release_ts __LSC(12) +#define __NR_release_ts __LSC(10) __SYSCALL(__NR_release_ts, sys_release_ts) -#define __NR_null_call __LSC(13) +#define __NR_null_call __LSC(11) __SYSCALL(__NR_null_call, sys_null_call) -#define NR_litmus_syscalls 14 +#define NR_litmus_syscalls 12 diff --git a/kernel/sched.c b/kernel/sched.c index 5beefb2..c5d7750 100644 --- a/kernel/sched.c +++ b/kernel/sched.c @@ -3887,8 +3887,7 @@ need_resched_nonpreemptible: if (need_resched()) goto need_resched; - if (srp_active()) - srp_ceiling_block(); + srp_ceiling_block(); } EXPORT_SYMBOL(schedule); diff --git a/litmus/fmlp.c b/litmus/fmlp.c index 1e4d544..6e3ddad 100644 --- a/litmus/fmlp.c +++ b/litmus/fmlp.c @@ -207,62 +207,8 @@ static void do_fmlp_up(struct pi_semaphore* sem) spin_unlock_irqrestore(&sem->wait.lock, flags); } -asmlinkage long sys_fmlp_down(int sem_od) -{ - long ret = 0; - struct pi_semaphore * sem; - int suspended = 0; - - preempt_disable(); - TS_PI_DOWN_START; - - sem = lookup_fmlp_sem(sem_od); - if (sem) - suspended = do_fmlp_down(sem); - else - ret = -EINVAL; - - if (!suspended) { - TS_PI_DOWN_END; - preempt_enable(); - } - - return ret; -} - -asmlinkage long sys_fmlp_up(int sem_od) -{ - long ret = 0; - struct pi_semaphore * sem; - - preempt_disable(); - TS_PI_UP_START; - - sem = lookup_fmlp_sem(sem_od); - if (sem) - do_fmlp_up(sem); - else - ret = -EINVAL; - - - TS_PI_UP_END; - preempt_enable(); - - return ret; -} - #else struct fdso_ops fmlp_sem_ops = {}; -asmlinkage long sys_fmlp_down(int sem_od) -{ - return -ENOSYS; -} - -asmlinkage long sys_fmlp_up(int sem_od) -{ - return -ENOSYS; -} - #endif diff --git a/litmus/locking.c b/litmus/locking.c index 848407b..ab64347 100644 --- a/litmus/locking.c +++ b/litmus/locking.c @@ -75,6 +75,7 @@ asmlinkage long sys_litmus_lock(int lock_od) entry = get_entry_for_od(lock_od); if (entry && is_lock(entry)) { l = get_lock(entry); + TRACE_CUR("attempts to lock 0x%p\n", l); err = l->ops->lock(l); } @@ -96,7 +97,8 @@ asmlinkage long sys_litmus_unlock(int lock_od) entry = get_entry_for_od(lock_od); if (entry && is_lock(entry)) { l = get_lock(entry); - err = l->ops->lock(l); + TRACE_CUR("attempts to unlock 0x%p\n", l); + err = l->ops->unlock(l); } /* Note: task my have been preempted in between! Take this into diff --git a/litmus/sched_gsn_edf.c b/litmus/sched_gsn_edf.c index e9c5e53..4ad95db 100644 --- a/litmus/sched_gsn_edf.c +++ b/litmus/sched_gsn_edf.c @@ -594,7 +594,7 @@ static void gsnedf_task_exit(struct task_struct * t) TRACE_TASK(t, "RIP\n"); } -#ifdef CONFIG_FMLP +#if 0 /* Update the queue position of a task that got it's priority boosted via * priority inheritance. */ @@ -795,12 +795,6 @@ static struct sched_plugin gsn_edf_plugin __cacheline_aligned_in_smp = { .schedule = gsnedf_schedule, .task_wake_up = gsnedf_task_wake_up, .task_block = gsnedf_task_block, -#ifdef CONFIG_FMLP - .fmlp_active = 1, - .pi_block = gsnedf_pi_block, - .inherit_priority = gsnedf_inherit_priority, - .return_priority = gsnedf_return_priority, -#endif .admit_task = gsnedf_admit_task, .activate_plugin = gsnedf_activate_plugin, }; diff --git a/litmus/sched_psn_edf.c b/litmus/sched_psn_edf.c index b89823d..01f31e4 100644 --- a/litmus/sched_psn_edf.c +++ b/litmus/sched_psn_edf.c @@ -309,7 +309,7 @@ static void psnedf_task_exit(struct task_struct * t) raw_spin_unlock_irqrestore(&pedf->slock, flags); } -#ifdef CONFIG_FMLP +#if 0 static long psnedf_pi_block(struct pi_semaphore *sem, struct task_struct *new_waiter) { @@ -443,9 +443,6 @@ static long psnedf_admit_task(struct task_struct* tsk) /* Plugin object */ static struct sched_plugin psn_edf_plugin __cacheline_aligned_in_smp = { .plugin_name = "PSN-EDF", -#ifdef CONFIG_SRP - .srp_active = 1, -#endif .tick = psnedf_tick, .task_new = psnedf_task_new, .complete_job = complete_job, @@ -453,12 +450,6 @@ static struct sched_plugin psn_edf_plugin __cacheline_aligned_in_smp = { .schedule = psnedf_schedule, .task_wake_up = psnedf_task_wake_up, .task_block = psnedf_task_block, -#ifdef CONFIG_FMLP - .fmlp_active = 1, - .pi_block = psnedf_pi_block, - .inherit_priority = psnedf_inherit_priority, - .return_priority = psnedf_return_priority, -#endif .admit_task = psnedf_admit_task }; diff --git a/litmus/srp.c b/litmus/srp.c index 4601b7d..b4c171e 100644 --- a/litmus/srp.c +++ b/litmus/srp.c @@ -189,55 +189,6 @@ static void do_srp_up(struct srp_semaphore* sem) wake_up_all(&__get_cpu_var(srp).ceiling_blocked); } -/* Adjust the system-wide priority ceiling if resource is claimed. */ -asmlinkage long sys_srp_down(int sem_od) -{ - int cpu; - int ret = -EINVAL; - struct srp_semaphore* sem; - - /* disabling preemptions is sufficient protection since - * SRP is strictly per CPU and we don't interfere with any - * interrupt handlers - */ - preempt_disable(); - TS_SRP_DOWN_START; - - cpu = smp_processor_id(); - sem = lookup_srp_sem(sem_od); - if (sem && sem->cpu == cpu) { - do_srp_down(sem); - ret = 0; - } - - TS_SRP_DOWN_END; - preempt_enable(); - return ret; -} - -/* Adjust the system-wide priority ceiling if resource is freed. */ -asmlinkage long sys_srp_up(int sem_od) -{ - int cpu; - int ret = -EINVAL; - struct srp_semaphore* sem; - - preempt_disable(); - TS_SRP_UP_START; - - cpu = smp_processor_id(); - sem = lookup_srp_sem(sem_od); - - if (sem && sem->cpu == cpu) { - do_srp_up(sem); - ret = 0; - } - - TS_SRP_UP_END; - preempt_enable(); - return ret; -} - static int srp_wake_up(wait_queue_t *wait, unsigned mode, int sync, void *key) { @@ -303,16 +254,6 @@ void srp_ceiling_block(void) #else -asmlinkage long sys_srp_down(int sem_od) -{ - return -ENOSYS; -} - -asmlinkage long sys_srp_up(int sem_od) -{ - return -ENOSYS; -} - struct fdso_ops srp_sem_ops = {}; #endif -- 1.7.2.3 From e1b81e70c3af9d19d639bc8bdaa5a8fc13bf17a8 Mon Sep 17 00:00:00 2001 From: Bjoern B. Brandenburg Date: Fri, 28 Jan 2011 17:04:58 -0500 Subject: [PATCH 10/24] SRP: port to new generic locking API This re-enables SRP support under PSN-EDF and demonstrates how the new locking API should be used. --- include/litmus/litmus.h | 2 +- include/litmus/srp.h | 28 ++++++ litmus/fdso.c | 4 +- litmus/sched_psn_edf.c | 45 +++++++++- litmus/srp.c | 236 +++++++++++++++++++++++++++-------------------- 5 files changed, 211 insertions(+), 104 deletions(-) create mode 100644 include/litmus/srp.h diff --git a/include/litmus/litmus.h b/include/litmus/litmus.h index 4a774a9..8971b25 100644 --- a/include/litmus/litmus.h +++ b/include/litmus/litmus.h @@ -115,7 +115,7 @@ static inline lt_t litmus_clock(void) void preempt_if_preemptable(struct task_struct* t, int on_cpu); -#ifdef CONFIG_SRP +#ifdef CONFIG_LITMUS_LOCKING void srp_ceiling_block(void); #else #define srp_ceiling_block() /* nothing */ diff --git a/include/litmus/srp.h b/include/litmus/srp.h new file mode 100644 index 0000000..c9a4552 --- /dev/null +++ b/include/litmus/srp.h @@ -0,0 +1,28 @@ +#ifndef LITMUS_SRP_H +#define LITMUS_SRP_H + +struct srp_semaphore; + +struct srp_priority { + struct list_head list; + unsigned int priority; + pid_t pid; +}; +#define list2prio(l) list_entry(l, struct srp_priority, list) + +/* struct for uniprocessor SRP "semaphore" */ +struct srp_semaphore { + struct litmus_lock litmus_lock; + struct srp_priority ceiling; + struct task_struct* owner; + int cpu; /* cpu associated with this "semaphore" and resource */ +}; + +/* map a task to its SRP preemption level priority */ +typedef unsigned int (*srp_prioritization_t)(struct task_struct* t); +/* Must be updated by each plugin that uses SRP.*/ +extern srp_prioritization_t get_srp_prio; + +struct srp_semaphore* allocate_srp_semaphore(void); + +#endif diff --git a/litmus/fdso.c b/litmus/fdso.c index faede9b..209431f 100644 --- a/litmus/fdso.c +++ b/litmus/fdso.c @@ -19,11 +19,11 @@ #include extern struct fdso_ops fmlp_sem_ops; -extern struct fdso_ops srp_sem_ops; +extern struct fdso_ops generic_lock_ops; static const struct fdso_ops* fdso_ops[] = { &fmlp_sem_ops, - &srp_sem_ops, + &generic_lock_ops, /* SRP_SEM */ }; static void* fdso_create(obj_type_t type) diff --git a/litmus/sched_psn_edf.c b/litmus/sched_psn_edf.c index 01f31e4..c1e2796 100644 --- a/litmus/sched_psn_edf.c +++ b/litmus/sched_psn_edf.c @@ -435,6 +435,45 @@ static long psnedf_return_priority(struct pi_semaphore *sem) #endif +#ifdef CONFIG_LITMUS_LOCKING + +#include +#include + +static unsigned int psnedf_get_srp_prio(struct task_struct* t) +{ + /* assumes implicit deadlines */ + return get_rt_period(t); +} + +static long psnedf_activate_plugin(void) +{ + get_srp_prio = psnedf_get_srp_prio; + return 0; +} + +static long psnedf_allocate_lock(struct litmus_lock **lock, int type) +{ + int err = -ENXIO; + struct srp_semaphore* srp; + + switch (type) { + case SRP_SEM: + /* Baker's SRP */ + srp = allocate_srp_semaphore(); + if (srp) { + *lock = &srp->litmus_lock; + err = 0; + } else + err = -ENOMEM; + break; + }; + + return err; +} + +#endif + static long psnedf_admit_task(struct task_struct* tsk) { return task_cpu(tsk) == tsk->rt_param.task_params.cpu ? 0 : -EINVAL; @@ -450,7 +489,11 @@ static struct sched_plugin psn_edf_plugin __cacheline_aligned_in_smp = { .schedule = psnedf_schedule, .task_wake_up = psnedf_task_wake_up, .task_block = psnedf_task_block, - .admit_task = psnedf_admit_task + .admit_task = psnedf_admit_task, +#ifdef CONFIG_LITMUS_LOCKING + .allocate_lock = psnedf_allocate_lock, + .activate_plugin = psnedf_activate_plugin, +#endif }; diff --git a/litmus/srp.c b/litmus/srp.c index b4c171e..2ed4ec1 100644 --- a/litmus/srp.c +++ b/litmus/srp.c @@ -12,42 +12,25 @@ #include -#ifdef CONFIG_SRP +#ifdef CONFIG_LITMUS_LOCKING -struct srp_priority { - struct list_head list; - unsigned int period; - pid_t pid; -}; +#include -#define list2prio(l) list_entry(l, struct srp_priority, list) - -/* SRP task priority comparison function. Smaller periods have highest - * priority, tie-break is PID. Special case: period == 0 <=> no priority - */ -static int srp_higher_prio(struct srp_priority* first, - struct srp_priority* second) -{ - if (!first->period) - return 0; - else - return !second->period || - first->period < second->period || ( - first->period == second->period && - first->pid < second->pid); -} +srp_prioritization_t get_srp_prio; struct srp { struct list_head ceiling; wait_queue_head_t ceiling_blocked; }; +#define system_ceiling(srp) list2prio(srp->ceiling.next) +#define ceiling2sem(c) container_of(c, struct srp_semaphore, ceiling) +#define UNDEF_SEM -2 atomic_t srp_objects_in_use = ATOMIC_INIT(0); DEFINE_PER_CPU(struct srp, srp); - /* Initialize SRP semaphores at boot time. */ static int __init srp_init(void) { @@ -64,30 +47,35 @@ static int __init srp_init(void) } module_init(srp_init); +/* SRP task priority comparison function. Smaller numeric values have higher + * priority, tie-break is PID. Special case: priority == 0 <=> no priority + */ +static int srp_higher_prio(struct srp_priority* first, + struct srp_priority* second) +{ + if (!first->priority) + return 0; + else + return !second->priority || + first->priority < second->priority || ( + first->priority == second->priority && + first->pid < second->pid); +} -#define system_ceiling(srp) list2prio(srp->ceiling.next) - - -#define UNDEF_SEM -2 - - -/* struct for uniprocessor SRP "semaphore" */ -struct srp_semaphore { - struct srp_priority ceiling; - struct task_struct* owner; - int cpu; /* cpu associated with this "semaphore" and resource */ -}; - -#define ceiling2sem(c) container_of(c, struct srp_semaphore, ceiling) static int srp_exceeds_ceiling(struct task_struct* first, struct srp* srp) { - return list_empty(&srp->ceiling) || - get_rt_period(first) < system_ceiling(srp)->period || - (get_rt_period(first) == system_ceiling(srp)->period && - first->pid < system_ceiling(srp)->pid) || - ceiling2sem(system_ceiling(srp))->owner == first; + struct srp_priority prio; + + if (list_empty(&srp->ceiling)) + return 1; + else { + prio.pid = first->pid; + prio.priority = get_srp_prio(first); + return srp_higher_prio(&prio, system_ceiling(srp)) || + ceiling2sem(system_ceiling(srp))->owner == first; + } } static void srp_add_prio(struct srp* srp, struct srp_priority* prio) @@ -108,85 +96,139 @@ static void srp_add_prio(struct srp* srp, struct srp_priority* prio) } -static void* create_srp_semaphore(obj_type_t type) +static int lock_srp_semaphore(struct litmus_lock* l) { - struct srp_semaphore* sem; + struct srp_semaphore* sem = container_of(l, struct srp_semaphore, litmus_lock); - sem = kmalloc(sizeof(*sem), GFP_KERNEL); - if (!sem) - return NULL; + if (!is_realtime(current)) + return -EPERM; - INIT_LIST_HEAD(&sem->ceiling.list); - sem->ceiling.period = 0; - sem->cpu = UNDEF_SEM; - sem->owner = NULL; - atomic_inc(&srp_objects_in_use); - return sem; + preempt_disable(); + + /* Update ceiling. */ + srp_add_prio(&__get_cpu_var(srp), &sem->ceiling); + + /* SRP invariant: all resources available */ + BUG_ON(sem->owner != NULL); + + sem->owner = current; + TRACE_CUR("acquired srp 0x%p\n", sem); + + preempt_enable(); + + return 0; +} + +static int unlock_srp_semaphore(struct litmus_lock* l) +{ + struct srp_semaphore* sem = container_of(l, struct srp_semaphore, litmus_lock); + int err = 0; + + preempt_disable(); + + if (sem->owner != current) { + err = -EINVAL; + } else { + /* Determine new system priority ceiling for this CPU. */ + BUG_ON(!in_list(&sem->ceiling.list)); + + list_del(&sem->ceiling.list); + sem->owner = NULL; + + /* Wake tasks on this CPU, if they exceed current ceiling. */ + TRACE_CUR("released srp 0x%p\n", sem); + wake_up_all(&__get_cpu_var(srp).ceiling_blocked); + } + + preempt_enable(); + return err; } -static noinline int open_srp_semaphore(struct od_table_entry* entry, void* __user arg) +static int open_srp_semaphore(struct litmus_lock* l, void* __user arg) { - struct srp_semaphore* sem = (struct srp_semaphore*) entry->obj->obj; - int ret = 0; + struct srp_semaphore* sem = container_of(l, struct srp_semaphore, litmus_lock); + int err = 0; struct task_struct* t = current; struct srp_priority t_prio; - TRACE("opening SRP semaphore %p, cpu=%d\n", sem, sem->cpu); - if (!srp_active()) - return -EBUSY; + if (!is_realtime(t)) + return -EPERM; - if (sem->cpu == UNDEF_SEM) - sem->cpu = get_partition(t); - else if (sem->cpu != get_partition(t)) - ret = -EPERM; + TRACE_CUR("opening SRP semaphore %p, cpu=%d\n", sem, sem->cpu); - if (ret == 0) { - t_prio.period = get_rt_period(t); - t_prio.pid = t->pid; + preempt_disable(); + + if (sem->owner != NULL) + err = -EBUSY; + + if (err == 0) { + if (sem->cpu == UNDEF_SEM) + sem->cpu = get_partition(t); + else if (sem->cpu != get_partition(t)) + err = -EPERM; + } + + if (err == 0) { + t_prio.priority = get_srp_prio(t); + t_prio.pid = t->pid; if (srp_higher_prio(&t_prio, &sem->ceiling)) { - sem->ceiling.period = t_prio.period; - sem->ceiling.pid = t_prio.pid; + sem->ceiling.priority = t_prio.priority; + sem->ceiling.pid = t_prio.pid; } } - return ret; + preempt_enable(); + + return err; +} + +static int close_srp_semaphore(struct litmus_lock* l) +{ + struct srp_semaphore* sem = container_of(l, struct srp_semaphore, litmus_lock); + int err = 0; + + preempt_disable(); + + if (sem->owner == current) + unlock_srp_semaphore(l); + + preempt_enable(); + + return err; } -static void destroy_srp_semaphore(obj_type_t type, void* sem) +static void deallocate_srp_semaphore(struct litmus_lock* l) { - /* XXX invariants */ + struct srp_semaphore* sem = container_of(l, struct srp_semaphore, litmus_lock); atomic_dec(&srp_objects_in_use); kfree(sem); } -struct fdso_ops srp_sem_ops = { - .create = create_srp_semaphore, - .open = open_srp_semaphore, - .destroy = destroy_srp_semaphore +static struct litmus_lock_ops srp_lock_ops = { + .open = open_srp_semaphore, + .close = close_srp_semaphore, + .lock = lock_srp_semaphore, + .unlock = unlock_srp_semaphore, + .deallocate = deallocate_srp_semaphore, }; - -static void do_srp_down(struct srp_semaphore* sem) +struct srp_semaphore* allocate_srp_semaphore(void) { - /* Update ceiling. */ - srp_add_prio(&__get_cpu_var(srp), &sem->ceiling); - WARN_ON(sem->owner != NULL); - sem->owner = current; - TRACE_CUR("acquired srp 0x%p\n", sem); -} + struct srp_semaphore* sem; -static void do_srp_up(struct srp_semaphore* sem) -{ - /* Determine new system priority ceiling for this CPU. */ - WARN_ON(!in_list(&sem->ceiling.list)); - if (in_list(&sem->ceiling.list)) - list_del(&sem->ceiling.list); + sem = kmalloc(sizeof(*sem), GFP_KERNEL); + if (!sem) + return NULL; - sem->owner = NULL; + INIT_LIST_HEAD(&sem->ceiling.list); + sem->ceiling.priority = 0; + sem->cpu = UNDEF_SEM; + sem->owner = NULL; + + sem->litmus_lock.ops = &srp_lock_ops; - /* Wake tasks on this CPU, if they exceed current ceiling. */ - TRACE_CUR("released srp 0x%p\n", sem); - wake_up_all(&__get_cpu_var(srp).ceiling_blocked); + atomic_inc(&srp_objects_in_use); + return sem; } static int srp_wake_up(wait_queue_t *wait, unsigned mode, int sync, @@ -202,8 +244,6 @@ static int srp_wake_up(wait_queue_t *wait, unsigned mode, int sync, return 0; } - - static void do_ceiling_block(struct task_struct *tsk) { wait_queue_t wait = { @@ -223,6 +263,7 @@ static void do_ceiling_block(struct task_struct *tsk) } /* Wait for current task priority to exceed system-wide priority ceiling. + * FIXME: the hotpath should be inline. */ void srp_ceiling_block(void) { @@ -251,9 +292,4 @@ void srp_ceiling_block(void) preempt_enable(); } - -#else - -struct fdso_ops srp_sem_ops = {}; - #endif -- 1.7.2.3 From fc6482bb7a6a638474565c90159997bd59069297 Mon Sep 17 00:00:00 2001 From: Bjoern B. Brandenburg Date: Fri, 28 Jan 2011 17:30:14 -0500 Subject: [PATCH 11/24] FMLP: remove old implementation --- include/litmus/edf_common.h | 2 - litmus/Makefile | 1 - litmus/fdso.c | 3 +- litmus/fmlp.c | 214 ------------------------------------------- litmus/sched_gsn_edf.c | 155 ------------------------------- litmus/sched_psn_edf.c | 126 ------------------------- 6 files changed, 1 insertions(+), 500 deletions(-) delete mode 100644 litmus/fmlp.c diff --git a/include/litmus/edf_common.h b/include/litmus/edf_common.h index 80d4321..bbaf22e 100644 --- a/include/litmus/edf_common.h +++ b/include/litmus/edf_common.h @@ -22,6 +22,4 @@ int edf_ready_order(struct bheap_node* a, struct bheap_node* b); int edf_preemption_needed(rt_domain_t* rt, struct task_struct *t); -int edf_set_hp_task(struct pi_semaphore *sem); -int edf_set_hp_cpu_task(struct pi_semaphore *sem, int cpu); #endif diff --git a/litmus/Makefile b/litmus/Makefile index 4e019d4..62a20e2 100644 --- a/litmus/Makefile +++ b/litmus/Makefile @@ -13,7 +13,6 @@ obj-y = sched_plugin.o litmus.o \ fdso.o \ locking.o \ srp.o \ - fmlp.o \ bheap.o \ ctrldev.o \ sched_gsn_edf.o \ diff --git a/litmus/fdso.c b/litmus/fdso.c index 209431f..b3a95f1 100644 --- a/litmus/fdso.c +++ b/litmus/fdso.c @@ -18,11 +18,10 @@ #include -extern struct fdso_ops fmlp_sem_ops; extern struct fdso_ops generic_lock_ops; static const struct fdso_ops* fdso_ops[] = { - &fmlp_sem_ops, + &generic_lock_ops, /* FMLP_SEM */ &generic_lock_ops, /* SRP_SEM */ }; diff --git a/litmus/fmlp.c b/litmus/fmlp.c deleted file mode 100644 index 6e3ddad..0000000 --- a/litmus/fmlp.c +++ /dev/null @@ -1,214 +0,0 @@ -/* - * FMLP implementation. - * Much of the code here is borrowed from include/asm-i386/semaphore.h - */ - -#include - -#include -#include -#include -#include - -#include -#include -#include - -#include - -#include - -#ifdef CONFIG_FMLP - -static void* create_fmlp_semaphore(obj_type_t type) -{ - struct pi_semaphore* sem; - int i; - - sem = kmalloc(sizeof(*sem), GFP_KERNEL); - if (!sem) - return NULL; - atomic_set(&sem->count, 1); - sem->sleepers = 0; - init_waitqueue_head(&sem->wait); - sem->hp.task = NULL; - sem->holder = NULL; - for (i = 0; i < NR_CPUS; i++) - sem->hp.cpu_task[i] = NULL; - return sem; -} - -static int open_fmlp_semaphore(struct od_table_entry* entry, void* __user arg) -{ - if (!fmlp_active()) - return -EBUSY; - return 0; -} - -static void destroy_fmlp_semaphore(obj_type_t type, void* sem) -{ - /* XXX assert invariants */ - kfree(sem); -} - -struct fdso_ops fmlp_sem_ops = { - .create = create_fmlp_semaphore, - .open = open_fmlp_semaphore, - .destroy = destroy_fmlp_semaphore -}; - -struct wq_pair { - struct task_struct* tsk; - struct pi_semaphore* sem; -}; - -static int rt_pi_wake_up(wait_queue_t *wait, unsigned mode, int sync, - void *key) -{ - struct wq_pair* wqp = (struct wq_pair*) wait->private; - set_rt_flags(wqp->tsk, RT_F_EXIT_SEM); - litmus->inherit_priority(wqp->sem, wqp->tsk); - TRACE_TASK(wqp->tsk, - "woken up by rt_pi_wake_up() (RT_F_SEM_EXIT, PI)\n"); - /* point to task for default_wake_function() */ - wait->private = wqp->tsk; - default_wake_function(wait, mode, sync, key); - - /* Always return true since we know that if we encountered a task - * that was already running the wake_up raced with the schedule in - * rt_pi_down(). In that case the task in rt_pi_down() will be scheduled - * immediately and own the lock. We must not wake up another task in - * any case. - */ - return 1; -} - -/* caller is responsible for locking */ -int edf_set_hp_task(struct pi_semaphore *sem) -{ - struct list_head *tmp, *next; - struct task_struct *queued; - int ret = 0; - - sem->hp.task = NULL; - list_for_each_safe(tmp, next, &sem->wait.task_list) { - queued = ((struct wq_pair*) - list_entry(tmp, wait_queue_t, - task_list)->private)->tsk; - - /* Compare task prios, find high prio task. */ - if (edf_higher_prio(queued, sem->hp.task)) { - sem->hp.task = queued; - ret = 1; - } - } - return ret; -} - -/* caller is responsible for locking */ -int edf_set_hp_cpu_task(struct pi_semaphore *sem, int cpu) -{ - struct list_head *tmp, *next; - struct task_struct *queued; - int ret = 0; - - sem->hp.cpu_task[cpu] = NULL; - list_for_each_safe(tmp, next, &sem->wait.task_list) { - queued = ((struct wq_pair*) - list_entry(tmp, wait_queue_t, - task_list)->private)->tsk; - - /* Compare task prios, find high prio task. */ - if (get_partition(queued) == cpu && - edf_higher_prio(queued, sem->hp.cpu_task[cpu])) { - sem->hp.cpu_task[cpu] = queued; - ret = 1; - } - } - return ret; -} - -static int do_fmlp_down(struct pi_semaphore* sem) -{ - unsigned long flags; - struct task_struct *tsk = current; - struct wq_pair pair; - int suspended = 1; - wait_queue_t wait = { - .private = &pair, - .func = rt_pi_wake_up, - .task_list = {NULL, NULL} - }; - - pair.tsk = tsk; - pair.sem = sem; - spin_lock_irqsave(&sem->wait.lock, flags); - - if (atomic_dec_return(&sem->count) < 0 || - waitqueue_active(&sem->wait)) { - /* we need to suspend */ - tsk->state = TASK_UNINTERRUPTIBLE; - __add_wait_queue_tail_exclusive(&sem->wait, &wait); - - TRACE_CUR("suspends on PI lock %p\n", sem); - litmus->pi_block(sem, tsk); - - /* release lock before sleeping */ - spin_unlock_irqrestore(&sem->wait.lock, flags); - - TS_PI_DOWN_END; - preempt_enable_no_resched(); - - - /* we depend on the FIFO order - * Thus, we don't need to recheck when we wake up, we - * are guaranteed to have the lock since there is only one - * wake up per release - */ - schedule(); - - TRACE_CUR("woke up, now owns PI lock %p\n", sem); - - /* try_to_wake_up() set our state to TASK_RUNNING, - * all we need to do is to remove our wait queue entry - */ - remove_wait_queue(&sem->wait, &wait); - } else { - /* no priority inheritance necessary, since there are no queued - * tasks. - */ - suspended = 0; - TRACE_CUR("acquired PI lock %p, no contention\n", sem); - sem->holder = tsk; - - /* don't know if we're global or partitioned. */ - sem->hp.task = tsk; - sem->hp.cpu_task[get_partition(tsk)] = tsk; - - litmus->inherit_priority(sem, tsk); - spin_unlock_irqrestore(&sem->wait.lock, flags); - } - return suspended; -} - -static void do_fmlp_up(struct pi_semaphore* sem) -{ - unsigned long flags; - - spin_lock_irqsave(&sem->wait.lock, flags); - - TRACE_CUR("releases PI lock %p\n", sem); - litmus->return_priority(sem); - sem->holder = NULL; - if (atomic_inc_return(&sem->count) < 1) - /* there is a task queued */ - wake_up_locked(&sem->wait); - - spin_unlock_irqrestore(&sem->wait.lock, flags); -} - -#else - -struct fdso_ops fmlp_sem_ops = {}; - -#endif diff --git a/litmus/sched_gsn_edf.c b/litmus/sched_gsn_edf.c index 4ad95db..5de0980 100644 --- a/litmus/sched_gsn_edf.c +++ b/litmus/sched_gsn_edf.c @@ -594,161 +594,6 @@ static void gsnedf_task_exit(struct task_struct * t) TRACE_TASK(t, "RIP\n"); } -#if 0 - -/* Update the queue position of a task that got it's priority boosted via - * priority inheritance. */ -static void update_queue_position(struct task_struct *holder) -{ - /* We don't know whether holder is in the ready queue. It should, but - * on a budget overrun it may already be in a release queue. Hence, - * calling unlink() is not possible since it assumes that the task is - * not in a release queue. However, we can safely check whether - * sem->holder is currently in a queue or scheduled after locking both - * the release and the ready queue lock. */ - - /* Assumption: caller holds gsnedf_lock */ - - int check_preempt = 0; - - if (tsk_rt(holder)->linked_on != NO_CPU) { - TRACE_TASK(holder, "%s: linked on %d\n", - __FUNCTION__, tsk_rt(holder)->linked_on); - /* Holder is scheduled; need to re-order CPUs. - * We can't use heap_decrease() here since - * the cpu_heap is ordered in reverse direction, so - * it is actually an increase. */ - bheap_delete(cpu_lower_prio, &gsnedf_cpu_heap, - gsnedf_cpus[tsk_rt(holder)->linked_on]->hn); - bheap_insert(cpu_lower_prio, &gsnedf_cpu_heap, - gsnedf_cpus[tsk_rt(holder)->linked_on]->hn); - } else { - /* holder may be queued: first stop queue changes */ - raw_spin_lock(&gsnedf.release_lock); - if (is_queued(holder)) { - TRACE_TASK(holder, "%s: is queued\n", - __FUNCTION__); - /* We need to update the position - * of holder in some heap. Note that this - * may be a release heap. */ - check_preempt = - !bheap_decrease(edf_ready_order, - tsk_rt(holder)->heap_node); - } else { - /* Nothing to do: if it is not queued and not linked - * then it is currently being moved by other code - * (e.g., a timer interrupt handler) that will use the - * correct priority when enqueuing the task. */ - TRACE_TASK(holder, "%s: is NOT queued => Done.\n", - __FUNCTION__); - } - raw_spin_unlock(&gsnedf.release_lock); - - /* If holder was enqueued in a release heap, then the following - * preemption check is pointless, but we can't easily detect - * that case. If you want to fix this, then consider that - * simply adding a state flag requires O(n) time to update when - * releasing n tasks, which conflicts with the goal to have - * O(log n) merges. */ - if (check_preempt) { - /* heap_decrease() hit the top level of the heap: make - * sure preemption checks get the right task, not the - * potentially stale cache. */ - bheap_uncache_min(edf_ready_order, - &gsnedf.ready_queue); - check_for_preemptions(); - } - } -} - -static long gsnedf_pi_block(struct pi_semaphore *sem, - struct task_struct *new_waiter) -{ - /* This callback has to handle the situation where a new waiter is - * added to the wait queue of the semaphore. - * - * We must check if has a higher priority than the currently - * highest-priority task, and then potentially reschedule. - */ - - BUG_ON(!new_waiter); - - if (edf_higher_prio(new_waiter, sem->hp.task)) { - TRACE_TASK(new_waiter, " boosts priority via %p\n", sem); - /* called with IRQs disabled */ - raw_spin_lock(&gsnedf_lock); - /* store new highest-priority task */ - sem->hp.task = new_waiter; - if (sem->holder) { - TRACE_TASK(sem->holder, - " holds %p and will inherit from %s/%d\n", - sem, - new_waiter->comm, new_waiter->pid); - /* let holder inherit */ - sem->holder->rt_param.inh_task = new_waiter; - update_queue_position(sem->holder); - } - raw_spin_unlock(&gsnedf_lock); - } - - return 0; -} - -static long gsnedf_inherit_priority(struct pi_semaphore *sem, - struct task_struct *new_owner) -{ - /* We don't need to acquire the gsnedf_lock since at the time of this - * call new_owner isn't actually scheduled yet (it's still sleeping) - * and since the calling function already holds sem->wait.lock, which - * prevents concurrent sem->hp.task changes. - */ - - if (sem->hp.task && sem->hp.task != new_owner) { - new_owner->rt_param.inh_task = sem->hp.task; - TRACE_TASK(new_owner, "inherited priority from %s/%d\n", - sem->hp.task->comm, sem->hp.task->pid); - } else - TRACE_TASK(new_owner, - "cannot inherit priority, " - "no higher priority job waits.\n"); - return 0; -} - -/* This function is called on a semaphore release, and assumes that - * the current task is also the semaphore holder. - */ -static long gsnedf_return_priority(struct pi_semaphore *sem) -{ - struct task_struct* t = current; - int ret = 0; - - /* Find new highest-priority semaphore task - * if holder task is the current hp.task. - * - * Calling function holds sem->wait.lock. - */ - if (t == sem->hp.task) - edf_set_hp_task(sem); - - TRACE_CUR("gsnedf_return_priority for lock %p\n", sem); - - if (t->rt_param.inh_task) { - /* interrupts already disabled by PI code */ - raw_spin_lock(&gsnedf_lock); - - /* Reset inh_task to NULL. */ - t->rt_param.inh_task = NULL; - - /* Check if rescheduling is necessary */ - unlink(t); - gsnedf_job_arrival(t); - raw_spin_unlock(&gsnedf_lock); - } - - return ret; -} - -#endif static long gsnedf_admit_task(struct task_struct* tsk) { diff --git a/litmus/sched_psn_edf.c b/litmus/sched_psn_edf.c index c1e2796..fc64c17 100644 --- a/litmus/sched_psn_edf.c +++ b/litmus/sched_psn_edf.c @@ -309,132 +309,6 @@ static void psnedf_task_exit(struct task_struct * t) raw_spin_unlock_irqrestore(&pedf->slock, flags); } -#if 0 -static long psnedf_pi_block(struct pi_semaphore *sem, - struct task_struct *new_waiter) -{ - psnedf_domain_t* pedf; - rt_domain_t* edf; - struct task_struct* t; - int cpu = get_partition(new_waiter); - - BUG_ON(!new_waiter); - - if (edf_higher_prio(new_waiter, sem->hp.cpu_task[cpu])) { - TRACE_TASK(new_waiter, " boosts priority\n"); - pedf = task_pedf(new_waiter); - edf = task_edf(new_waiter); - - /* interrupts already disabled */ - raw_spin_lock(&pedf->slock); - - /* store new highest-priority task */ - sem->hp.cpu_task[cpu] = new_waiter; - if (sem->holder && - get_partition(sem->holder) == get_partition(new_waiter)) { - /* let holder inherit */ - sem->holder->rt_param.inh_task = new_waiter; - t = sem->holder; - if (is_queued(t)) { - /* queued in domain*/ - remove(edf, t); - /* readd to make priority change take place */ - /* FIXME: this looks outdated */ - if (is_released(t, litmus_clock())) - __add_ready(edf, t); - else - add_release(edf, t); - } - } - - /* check if we need to reschedule */ - if (edf_preemption_needed(edf, current)) - preempt(pedf); - - raw_spin_unlock(&pedf->slock); - } - - return 0; -} - -static long psnedf_inherit_priority(struct pi_semaphore *sem, - struct task_struct *new_owner) -{ - int cpu = get_partition(new_owner); - - new_owner->rt_param.inh_task = sem->hp.cpu_task[cpu]; - if (sem->hp.cpu_task[cpu] && new_owner != sem->hp.cpu_task[cpu]) { - TRACE_TASK(new_owner, - "inherited priority from %s/%d\n", - sem->hp.cpu_task[cpu]->comm, - sem->hp.cpu_task[cpu]->pid); - } else - TRACE_TASK(new_owner, - "cannot inherit priority: " - "no higher priority job waits on this CPU!\n"); - /* make new owner non-preemptable as required by FMLP under - * PSN-EDF. - */ - make_np(new_owner); - return 0; -} - - -/* This function is called on a semaphore release, and assumes that - * the current task is also the semaphore holder. - */ -static long psnedf_return_priority(struct pi_semaphore *sem) -{ - struct task_struct* t = current; - psnedf_domain_t* pedf = task_pedf(t); - rt_domain_t* edf = task_edf(t); - int ret = 0; - int cpu = get_partition(current); - int still_np; - - - /* Find new highest-priority semaphore task - * if holder task is the current hp.cpu_task[cpu]. - * - * Calling function holds sem->wait.lock. - */ - if (t == sem->hp.cpu_task[cpu]) - edf_set_hp_cpu_task(sem, cpu); - - still_np = take_np(current); - - /* Since we don't nest resources, this - * should always be zero */ - BUG_ON(still_np); - - if (current->rt_param.inh_task) { - TRACE_CUR("return priority of %s/%d\n", - current->rt_param.inh_task->comm, - current->rt_param.inh_task->pid); - } else - TRACE_CUR(" no priority to return %p\n", sem); - - - /* Always check for delayed preemptions that might have become - * necessary due to non-preemptive execution. - */ - raw_spin_lock(&pedf->slock); - - /* Reset inh_task to NULL. */ - current->rt_param.inh_task = NULL; - - /* check if we need to reschedule */ - if (edf_preemption_needed(edf, current)) - preempt(pedf); - - raw_spin_unlock(&pedf->slock); - - - return ret; -} - -#endif - #ifdef CONFIG_LITMUS_LOCKING #include -- 1.7.2.3 From e593c9dbe858c82e284ff85e625837ae3ab32f1c Mon Sep 17 00:00:00 2001 From: Bjoern B. Brandenburg Date: Fri, 28 Jan 2011 19:04:08 -0500 Subject: [PATCH 12/24] EDF: support priority boosting While we are at it, simplify edf_higher_prio() a bit. --- include/litmus/litmus.h | 3 +++ include/litmus/rt_param.h | 7 +++++++ litmus/edf_common.c | 38 +++++++++++++++++++++++++++----------- 3 files changed, 37 insertions(+), 11 deletions(-) diff --git a/include/litmus/litmus.h b/include/litmus/litmus.h index 8971b25..b38d39a 100644 --- a/include/litmus/litmus.h +++ b/include/litmus/litmus.h @@ -54,6 +54,9 @@ void litmus_exit_task(struct task_struct *tsk); #define get_release(t) (tsk_rt(t)->job_params.release) #define get_class(t) (tsk_rt(t)->task_params.cls) +#define is_priority_boosted(t) (tsk_rt(t)->priority_boosted) +#define get_boost_start(t) (tsk_rt(t)->boost_start_time) + inline static int budget_exhausted(struct task_struct* t) { return get_exec_time(t) >= get_exec_cost(t); diff --git a/include/litmus/rt_param.h b/include/litmus/rt_param.h index a7a183f..5de422c 100644 --- a/include/litmus/rt_param.h +++ b/include/litmus/rt_param.h @@ -108,6 +108,13 @@ struct rt_param { /* is the task present? (true if it can be scheduled) */ unsigned int present:1; +#ifdef CONFIG_LITMUS_LOCKING + /* Is the task being priority-boosted by a locking protocol? */ + unsigned int priority_boosted:1; + /* If so, when did this start? */ + lt_t boost_start_time; +#endif + /* user controlled parameters */ struct rt_task task_params; diff --git a/litmus/edf_common.c b/litmus/edf_common.c index 06daec6..9b44dc2 100644 --- a/litmus/edf_common.c +++ b/litmus/edf_common.c @@ -33,22 +33,38 @@ int edf_higher_prio(struct task_struct* first, } + /* check for NULL tasks */ + if (!first || !second) + return first && !second; + +#ifdef CONFIG_LITMUS_LOCKING + /* Check for inherited priorities. Change task * used for comparison in such a case. */ - if (first && first->rt_param.inh_task) + if (unlikely(first->rt_param.inh_task)) first_task = first->rt_param.inh_task; - if (second && second->rt_param.inh_task) + if (unlikely(second->rt_param.inh_task)) second_task = second->rt_param.inh_task; - return - /* it has to exist in order to have higher priority */ - first_task && ( - /* does the second task exist and is it a real-time task? If - * not, the first task (which is a RT task) has higher - * priority. - */ - !second_task || !is_realtime(second_task) || + /* Check for priority boosting. Tie-break by start of boosting. + */ + if (unlikely(is_priority_boosted(first_task))) { + /* first_task is boosted, how about second_task? */ + if (!is_priority_boosted(second_task) || + lt_before(get_boost_start(first_task), + get_boost_start(second_task))) + return 1; + else + return 0; + } else if (unlikely(is_priority_boosted(second_task))) + /* second_task is boosted, first is not*/ + return 0; + +#endif + + + return !is_realtime(second_task) || /* is the deadline of the first task earlier? * Then it has higher priority. @@ -65,7 +81,7 @@ int edf_higher_prio(struct task_struct* first, * priority wins. */ (first_task->pid == second_task->pid && - !second->rt_param.inh_task)))); + !second->rt_param.inh_task))); } int edf_ready_order(struct bheap_node* a, struct bheap_node* b) -- 1.7.2.3 From e705aa52df711112d434ccc87ee5fb5838c205a2 Mon Sep 17 00:00:00 2001 From: Bjoern B. Brandenburg Date: Fri, 28 Jan 2011 19:06:11 -0500 Subject: [PATCH 13/24] PSN-EDF: re-implement FMLP support Implement the partitioned FMLP with priority boosting based on the generic lock API. --- include/litmus/litmus.h | 2 + litmus/locking.c | 13 +++ litmus/sched_psn_edf.c | 244 +++++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 253 insertions(+), 6 deletions(-) diff --git a/include/litmus/litmus.h b/include/litmus/litmus.h index b38d39a..94086e2 100644 --- a/include/litmus/litmus.h +++ b/include/litmus/litmus.h @@ -26,6 +26,8 @@ static inline int in_list(struct list_head* list) ); } +struct task_struct* waitqueue_first(wait_queue_head_t *wq); + #define NO_CPU 0xffffffff void litmus_fork(struct task_struct *tsk); diff --git a/litmus/locking.c b/litmus/locking.c index ab64347..d39afae 100644 --- a/litmus/locking.c +++ b/litmus/locking.c @@ -108,6 +108,19 @@ asmlinkage long sys_litmus_unlock(int lock_od) return err; } +struct task_struct* waitqueue_first(wait_queue_head_t *wq) +{ + wait_queue_t *q; + + if (waitqueue_active(wq)) { + q = list_entry(wq->task_list.next, + wait_queue_t, task_list); + return (struct task_struct*) q->private; + } else + return NULL; +} + + #else struct fdso_ops generic_lock_ops = {}; diff --git a/litmus/sched_psn_edf.c b/litmus/sched_psn_edf.c index fc64c17..801bc92 100644 --- a/litmus/sched_psn_edf.c +++ b/litmus/sched_psn_edf.c @@ -71,6 +71,66 @@ static void preempt(psnedf_domain_t *pedf) preempt_if_preemptable(pedf->scheduled, pedf->cpu); } +#ifdef CONFIG_LITMUS_LOCKING + +static void boost_priority(struct task_struct* t) +{ + unsigned long flags; + psnedf_domain_t* pedf = task_pedf(t); + lt_t now; + + raw_spin_lock_irqsave(&pedf->slock, flags); + now = litmus_clock(); + + TRACE_TASK(t, "priority boosted at %llu\n", now); + + tsk_rt(t)->priority_boosted = 1; + tsk_rt(t)->boost_start_time = now; + + if (pedf->scheduled != t) { + /* holder may be queued: first stop queue changes */ + raw_spin_lock(&pedf->domain.release_lock); + if (is_queued(t) && + /* If it is queued, then we need to re-order. */ + bheap_decrease(edf_ready_order, tsk_rt(t)->heap_node) && + /* If we bubbled to the top, then we need to check for preemptions. */ + edf_preemption_needed(&pedf->domain, pedf->scheduled)) + preempt(pedf); + raw_spin_unlock(&pedf->domain.release_lock); + } /* else: nothing to do since the job is not queued while scheduled */ + + raw_spin_unlock_irqrestore(&pedf->slock, flags); +} + +static void unboost_priority(struct task_struct* t) +{ + unsigned long flags; + psnedf_domain_t* pedf = task_pedf(t); + lt_t now; + + raw_spin_lock_irqsave(&pedf->slock, flags); + now = litmus_clock(); + + /* assumption: this only happens when the job is scheduled */ + BUG_ON(pedf->scheduled != t); + + TRACE_TASK(t, "priority restored at %llu\n", now); + + /* priority boosted jobs must be scheduled */ + BUG_ON(pedf->scheduled != t); + + tsk_rt(t)->priority_boosted = 0; + tsk_rt(t)->boost_start_time = 0; + + /* check if this changes anything */ + if (edf_preemption_needed(&pedf->domain, pedf->scheduled)) + preempt(pedf); + + raw_spin_unlock_irqrestore(&pedf->slock, flags); +} + +#endif + /* This check is trivial in partioned systems as we only have to consider * the CPU of the partition. */ @@ -252,15 +312,16 @@ static void psnedf_task_wake_up(struct task_struct *task) TRACE_TASK(task, "wake_up at %llu\n", litmus_clock()); raw_spin_lock_irqsave(&pedf->slock, flags); BUG_ON(is_queued(task)); + now = litmus_clock(); + if (is_tardy(task, now) +#ifdef CONFIG_LITMUS_LOCKING /* We need to take suspensions because of semaphores into * account! If a job resumes after being suspended due to acquiring * a semaphore, it should never be treated as a new job release. - * - * FIXME: This should be done in some more predictable and userspace-controlled way. */ - now = litmus_clock(); - if (is_tardy(task, now) && - get_rt_flags(task) != RT_F_EXIT_SEM) { + && !is_priority_boosted(task) +#endif + ) { /* new sporadic release */ release_at(task, now); sched_trace_task_release(task); @@ -314,6 +375,8 @@ static void psnedf_task_exit(struct task_struct * t) #include #include +/* ******************** SRP support ************************ */ + static unsigned int psnedf_get_srp_prio(struct task_struct* t) { /* assumes implicit deadlines */ @@ -326,14 +389,183 @@ static long psnedf_activate_plugin(void) return 0; } +/* ******************** FMLP support ********************** */ + +/* struct for semaphore with priority inheritance */ +struct fmlp_semaphore { + struct litmus_lock litmus_lock; + + /* current resource holder */ + struct task_struct *owner; + + /* FIFO queue of waiting tasks */ + wait_queue_head_t wait; +}; + +static inline struct fmlp_semaphore* fmlp_from_lock(struct litmus_lock* lock) +{ + return container_of(lock, struct fmlp_semaphore, litmus_lock); +} +int psnedf_fmlp_lock(struct litmus_lock* l) +{ + struct task_struct* t = current; + struct fmlp_semaphore *sem = fmlp_from_lock(l); + wait_queue_t wait; + unsigned long flags; + + if (!is_realtime(t)) + return -EPERM; + + spin_lock_irqsave(&sem->wait.lock, flags); + + if (sem->owner) { + /* resource is not free => must suspend and wait */ + + init_waitqueue_entry(&wait, t); + + /* FIXME: interruptible would be nice some day */ + set_task_state(t, TASK_UNINTERRUPTIBLE); + + __add_wait_queue_tail_exclusive(&sem->wait, &wait); + + /* release lock before sleeping */ + spin_unlock_irqrestore(&sem->wait.lock, flags); + + /* We depend on the FIFO order. Thus, we don't need to recheck + * when we wake up; we are guaranteed to have the lock since + * there is only one wake up per release. + */ + + schedule(); + + /* Since we hold the lock, no other task will change + * ->owner. We can thus check it without acquiring the spin + * lock. */ + BUG_ON(sem->owner != t); + + /* FIXME: could we punt the dequeuing to the previous job, + * which is holding the spinlock anyway? */ + remove_wait_queue(&sem->wait, &wait); + } else { + /* it's ours now */ + sem->owner = t; + + /* mark the task as priority-boosted. */ + boost_priority(t); + + spin_unlock_irqrestore(&sem->wait.lock, flags); + } + + return 0; +} + +int psnedf_fmlp_unlock(struct litmus_lock* l) +{ + struct task_struct *t = current, *next; + struct fmlp_semaphore *sem = fmlp_from_lock(l); + unsigned long flags; + int err = 0; + + spin_lock_irqsave(&sem->wait.lock, flags); + + if (sem->owner != t) { + err = -EINVAL; + goto out; + } + + /* we lose the benefit of priority boosting */ + + unboost_priority(t); + + /* check if there are jobs waiting for this resource */ + next = waitqueue_first(&sem->wait); + if (next) { + /* boost next job */ + boost_priority(next); + + /* next becomes the resouce holder */ + sem->owner = next; + + /* wake up next */ + wake_up_process(next); + } else + /* resource becomes available */ + sem->owner = NULL; + +out: + spin_unlock_irqrestore(&sem->wait.lock, flags); + return err; +} + +int psnedf_fmlp_close(struct litmus_lock* l) +{ + struct task_struct *t = current; + struct fmlp_semaphore *sem = fmlp_from_lock(l); + unsigned long flags; + + int owner; + + spin_lock_irqsave(&sem->wait.lock, flags); + + owner = sem->owner == t; + + spin_unlock_irqrestore(&sem->wait.lock, flags); + + if (owner) + psnedf_fmlp_unlock(l); + + return 0; +} + +void psnedf_fmlp_free(struct litmus_lock* lock) +{ + kfree(fmlp_from_lock(lock)); +} + +static struct litmus_lock_ops psnedf_fmlp_lock_ops = { + .close = psnedf_fmlp_close, + .lock = psnedf_fmlp_lock, + .unlock = psnedf_fmlp_unlock, + .deallocate = psnedf_fmlp_free, +}; + +static struct litmus_lock* psnedf_new_fmlp(void) +{ + struct fmlp_semaphore* sem; + + sem = kmalloc(sizeof(*sem), GFP_KERNEL); + if (!sem) + return NULL; + + sem->owner = NULL; + init_waitqueue_head(&sem->wait); + sem->litmus_lock.ops = &psnedf_fmlp_lock_ops; + + return &sem->litmus_lock; +} + +/* **** lock constructor **** */ + + static long psnedf_allocate_lock(struct litmus_lock **lock, int type) { int err = -ENXIO; struct srp_semaphore* srp; + /* PSN-EDF currently supports the SRP for local resources and the FMLP + * for global resources. */ switch (type) { + case FMLP_SEM: + /* Flexible Multiprocessor Locking Protocol */ + *lock = psnedf_new_fmlp(); + if (*lock) + err = 0; + else + err = -ENOMEM; + break; + case SRP_SEM: - /* Baker's SRP */ + /* Baker's Stack Resource Policy */ srp = allocate_srp_semaphore(); if (srp) { *lock = &srp->litmus_lock; -- 1.7.2.3 From fab768a4cdc49ad7886cac0d0361f8432965a817 Mon Sep 17 00:00:00 2001 From: Bjoern B. Brandenburg Date: Sat, 29 Jan 2011 13:38:24 -0500 Subject: [PATCH 14/24] GSN-EDF: re-implement FMLP support This introduces the global FMLP based on the generic locking layer. --- litmus/sched_gsn_edf.c | 322 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 files changed, 322 insertions(+), 0 deletions(-) diff --git a/litmus/sched_gsn_edf.c b/litmus/sched_gsn_edf.c index 5de0980..c525d43 100644 --- a/litmus/sched_gsn_edf.c +++ b/litmus/sched_gsn_edf.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include @@ -446,6 +447,7 @@ static struct task_struct* gsnedf_schedule(struct task_struct * prev) if (entry->linked) { entry->linked->rt_param.scheduled_on = entry->cpu; next = entry->linked; + TRACE_TASK(next, "scheduled_on = P%d\n", smp_processor_id()); } if (entry->scheduled) { /* not gonna be scheduled soon */ @@ -600,6 +602,323 @@ static long gsnedf_admit_task(struct task_struct* tsk) return 0; } +#ifdef CONFIG_LITMUS_LOCKING + +#include + +/* called with IRQs off */ +static void set_priority_inheritance(struct task_struct* t, struct task_struct* prio_inh) +{ + int linked_on; + int check_preempt = 0; + + raw_spin_lock(&gsnedf_lock); + + TRACE_TASK(t, "inherits priority from %s/%d\n", prio_inh->comm, prio_inh->pid); + tsk_rt(t)->inh_task = prio_inh; + + linked_on = tsk_rt(t)->linked_on; + + /* If it is scheduled, then we need to reorder the CPU heap. */ + if (linked_on != NO_CPU) { + TRACE_TASK(t, "%s: linked on %d\n", + __FUNCTION__, linked_on); + /* Holder is scheduled; need to re-order CPUs. + * We can't use heap_decrease() here since + * the cpu_heap is ordered in reverse direction, so + * it is actually an increase. */ + bheap_delete(cpu_lower_prio, &gsnedf_cpu_heap, + gsnedf_cpus[linked_on]->hn); + bheap_insert(cpu_lower_prio, &gsnedf_cpu_heap, + gsnedf_cpus[linked_on]->hn); + } else { + /* holder may be queued: first stop queue changes */ + raw_spin_lock(&gsnedf.release_lock); + if (is_queued(t)) { + TRACE_TASK(t, "%s: is queued\n", + __FUNCTION__); + /* We need to update the position of holder in some + * heap. Note that this could be a release heap if we + * budget enforcement is used and this job overran. */ + check_preempt = + !bheap_decrease(edf_ready_order, + tsk_rt(t)->heap_node); + } else { + /* Nothing to do: if it is not queued and not linked + * then it is either sleeping or currently being moved + * by other code (e.g., a timer interrupt handler) that + * will use the correct priority when enqueuing the + * task. */ + TRACE_TASK(t, "%s: is NOT queued => Done.\n", + __FUNCTION__); + } + raw_spin_unlock(&gsnedf.release_lock); + + /* If holder was enqueued in a release heap, then the following + * preemption check is pointless, but we can't easily detect + * that case. If you want to fix this, then consider that + * simply adding a state flag requires O(n) time to update when + * releasing n tasks, which conflicts with the goal to have + * O(log n) merges. */ + if (check_preempt) { + /* heap_decrease() hit the top level of the heap: make + * sure preemption checks get the right task, not the + * potentially stale cache. */ + bheap_uncache_min(edf_ready_order, + &gsnedf.ready_queue); + check_for_preemptions(); + } + } + + raw_spin_unlock(&gsnedf_lock); +} + +/* called with IRQs off */ +static void clear_priority_inheritance(struct task_struct* t) +{ + raw_spin_lock(&gsnedf_lock); + + /* A job only stops inheriting a priority when it releases a + * resource. Thus we can make the following assumption.*/ + BUG_ON(tsk_rt(t)->scheduled_on == NO_CPU); + + TRACE_TASK(t, "priority restored\n"); + tsk_rt(t)->inh_task = NULL; + + /* Check if rescheduling is necessary. We can't use heap_decrease() + * since the priority was effectively lowered. */ + unlink(t); + gsnedf_job_arrival(t); + + raw_spin_unlock(&gsnedf_lock); +} + + +/* ******************** FMLP support ********************** */ + +/* struct for semaphore with priority inheritance */ +struct fmlp_semaphore { + struct litmus_lock litmus_lock; + + /* current resource holder */ + struct task_struct *owner; + + /* highest-priority waiter */ + struct task_struct *hp_waiter; + + /* FIFO queue of waiting tasks */ + wait_queue_head_t wait; +}; + +static inline struct fmlp_semaphore* fmlp_from_lock(struct litmus_lock* lock) +{ + return container_of(lock, struct fmlp_semaphore, litmus_lock); +} + +/* caller is responsible for locking */ +struct task_struct* find_hp_waiter(struct fmlp_semaphore *sem, + struct task_struct* skip) +{ + struct list_head *pos; + struct task_struct *queued, *found = NULL; + + list_for_each(pos, &sem->wait.task_list) { + queued = (struct task_struct*) list_entry(pos, wait_queue_t, + task_list)->private; + + /* Compare task prios, find high prio task. */ + if (queued != skip && edf_higher_prio(queued, found)) + found = queued; + } + return found; +} + +int gsnedf_fmlp_lock(struct litmus_lock* l) +{ + struct task_struct* t = current; + struct fmlp_semaphore *sem = fmlp_from_lock(l); + wait_queue_t wait; + unsigned long flags; + + if (!is_realtime(t)) + return -EPERM; + + spin_lock_irqsave(&sem->wait.lock, flags); + + if (sem->owner) { + /* resource is not free => must suspend and wait */ + + init_waitqueue_entry(&wait, t); + + /* FIXME: interruptible would be nice some day */ + set_task_state(t, TASK_UNINTERRUPTIBLE); + + __add_wait_queue_tail_exclusive(&sem->wait, &wait); + + /* check if we need to activate priority inheritance */ + if (edf_higher_prio(t, sem->hp_waiter)) { + sem->hp_waiter = t; + if (edf_higher_prio(t, sem->owner)) + set_priority_inheritance(sem->owner, sem->hp_waiter); + } + + /* release lock before sleeping */ + spin_unlock_irqrestore(&sem->wait.lock, flags); + + /* We depend on the FIFO order. Thus, we don't need to recheck + * when we wake up; we are guaranteed to have the lock since + * there is only one wake up per release. + */ + + schedule(); + + /* Since we hold the lock, no other task will change + * ->owner. We can thus check it without acquiring the spin + * lock. */ + BUG_ON(sem->owner != t); + + remove_wait_queue(&sem->wait, &wait); + } else { + /* it's ours now */ + sem->owner = t; + + spin_unlock_irqrestore(&sem->wait.lock, flags); + } + + return 0; +} + +int gsnedf_fmlp_unlock(struct litmus_lock* l) +{ + struct task_struct *t = current, *next; + struct fmlp_semaphore *sem = fmlp_from_lock(l); + unsigned long flags; + int err = 0; + + spin_lock_irqsave(&sem->wait.lock, flags); + + if (sem->owner != t) { + err = -EINVAL; + goto out; + } + + /* check if there are jobs waiting for this resource */ + next = waitqueue_first(&sem->wait); + if (next) { + /* next becomes the resouce holder */ + sem->owner = next; + TRACE_CUR("lock ownership passed to %s/%d\n", next->comm, next->pid); + + /* determine new hp_waiter if necessary */ + if (next == sem->hp_waiter) { + TRACE_TASK(next, "was highest-prio waiter\n"); + /* next has the highest priority --- it doesn't need to + * inherit. However, we need to make sure that the + * next-highest priority in the queue is reflected in + * hp_waiter. */ + sem->hp_waiter = find_hp_waiter(sem, next); + if (sem->hp_waiter) + TRACE_TASK(sem->hp_waiter, "is new highest-prio waiter\n"); + else + TRACE("no further waiters\n"); + } else { + /* Well, if next is not the highest-priority waiter, + * then it ought to inherit the highest-priority + * waiter's priority. */ + set_priority_inheritance(next, sem->hp_waiter); + } + + /* wake up next */ + wake_up_process(next); + } else + /* becomes available */ + sem->owner = NULL; + + /* we lose the benefit of priority inheritance (if any) */ + if (tsk_rt(t)->inh_task) + clear_priority_inheritance(t); + +out: + spin_unlock_irqrestore(&sem->wait.lock, flags); + + return err; +} + +int gsnedf_fmlp_close(struct litmus_lock* l) +{ + struct task_struct *t = current; + struct fmlp_semaphore *sem = fmlp_from_lock(l); + unsigned long flags; + + int owner; + + spin_lock_irqsave(&sem->wait.lock, flags); + + owner = sem->owner == t; + + spin_unlock_irqrestore(&sem->wait.lock, flags); + + if (owner) + gsnedf_fmlp_unlock(l); + + return 0; +} + +void gsnedf_fmlp_free(struct litmus_lock* lock) +{ + kfree(fmlp_from_lock(lock)); +} + +static struct litmus_lock_ops gsnedf_fmlp_lock_ops = { + .close = gsnedf_fmlp_close, + .lock = gsnedf_fmlp_lock, + .unlock = gsnedf_fmlp_unlock, + .deallocate = gsnedf_fmlp_free, +}; + +static struct litmus_lock* gsnedf_new_fmlp(void) +{ + struct fmlp_semaphore* sem; + + sem = kmalloc(sizeof(*sem), GFP_KERNEL); + if (!sem) + return NULL; + + sem->owner = NULL; + sem->hp_waiter = NULL; + init_waitqueue_head(&sem->wait); + sem->litmus_lock.ops = &gsnedf_fmlp_lock_ops; + + return &sem->litmus_lock; +} + +/* **** lock constructor **** */ + + +static long gsnedf_allocate_lock(struct litmus_lock **lock, int type) +{ + int err = -ENXIO; + + /* GSN-EDF currently only supports the FMLP for global resources. */ + switch (type) { + + case FMLP_SEM: + /* Flexible Multiprocessor Locking Protocol */ + *lock = gsnedf_new_fmlp(); + if (*lock) + err = 0; + else + err = -ENOMEM; + break; + + }; + + return err; +} + +#endif + + static long gsnedf_activate_plugin(void) { int cpu; @@ -642,6 +961,9 @@ static struct sched_plugin gsn_edf_plugin __cacheline_aligned_in_smp = { .task_block = gsnedf_task_block, .admit_task = gsnedf_admit_task, .activate_plugin = gsnedf_activate_plugin, +#ifdef CONFIG_LITMUS_LOCKING + .allocate_lock = gsnedf_allocate_lock, +#endif }; -- 1.7.2.3 From 7f0bd4c213ff8dca0eb3bdd887f5c62c8d30fab5 Mon Sep 17 00:00:00 2001 From: Bjoern B. Brandenburg Date: Sat, 29 Jan 2011 15:50:52 -0500 Subject: [PATCH 15/24] fdso: pass userpsace config argument to object constructor As Glenn pointed out, it is useful for some protocols (e.g., k-exclusion protocols) to know the userspace configuration at object creation time. This patch changes the fdso API to pass the parameter to the object constructor, which is then in turn passed to the lock allocater. The return code from the lock allocater is passed to userspace in return. This also fixes some null pointer dereferences in the FDSO code found by the test suite in liblitmus. --- include/litmus/fdso.h | 2 +- include/litmus/sched_plugin.h | 3 +- litmus/fdso.c | 51 +++++++++++++++++++++++++--------------- litmus/locking.c | 11 ++++----- litmus/sched_gsn_edf.c | 3 +- litmus/sched_plugin.c | 3 +- litmus/sched_psn_edf.c | 3 +- 7 files changed, 46 insertions(+), 30 deletions(-) diff --git a/include/litmus/fdso.h b/include/litmus/fdso.h index 25a292d..caf2a1e 100644 --- a/include/litmus/fdso.h +++ b/include/litmus/fdso.h @@ -43,7 +43,7 @@ struct od_table_entry { }; struct fdso_ops { - void* (*create)(obj_type_t type); + int (*create)(void** obj_ref, obj_type_t type, void* __user); void (*destroy)(obj_type_t type, void*); int (*open) (struct od_table_entry*, void* __user); int (*close) (struct od_table_entry*); diff --git a/include/litmus/sched_plugin.h b/include/litmus/sched_plugin.h index 8a3ed6d..6e7cabd 100644 --- a/include/litmus/sched_plugin.h +++ b/include/litmus/sched_plugin.h @@ -55,7 +55,8 @@ typedef void (*task_exit_t) (struct task_struct *); /* Called when the current task attempts to create a new lock of a given * protocol type. */ -typedef long (*allocate_lock_t) (struct litmus_lock **lock, int type); +typedef long (*allocate_lock_t) (struct litmus_lock **lock, int type, + void* __user config); /********************* sys call backends ********************/ diff --git a/litmus/fdso.c b/litmus/fdso.c index b3a95f1..aa7b384 100644 --- a/litmus/fdso.c +++ b/litmus/fdso.c @@ -25,12 +25,12 @@ static const struct fdso_ops* fdso_ops[] = { &generic_lock_ops, /* SRP_SEM */ }; -static void* fdso_create(obj_type_t type) +static int fdso_create(void** obj_ref, obj_type_t type, void* __user config) { if (fdso_ops[type]->create) - return fdso_ops[type]->create(type); + return fdso_ops[type]->create(obj_ref, type, config); else - return NULL; + return -EINVAL; } static void fdso_destroy(obj_type_t type, void* obj) @@ -55,20 +55,27 @@ static int fdso_close(struct od_table_entry* entry) } /* inode must be locked already */ -static struct inode_obj_id* alloc_inode_obj(struct inode* inode, - obj_type_t type, - unsigned int id) +static int alloc_inode_obj(struct inode_obj_id** obj_ref, + struct inode* inode, + obj_type_t type, + unsigned int id, + void* __user config) { struct inode_obj_id* obj; void* raw_obj; - - raw_obj = fdso_create(type); - if (!raw_obj) - return NULL; + int err; obj = kmalloc(sizeof(*obj), GFP_KERNEL); - if (!obj) - return NULL; + if (!obj) { + return -ENOMEM; + } + + err = fdso_create(&raw_obj, type, config); + if (err != 0) { + kfree(obj); + return err; + } + INIT_LIST_HEAD(&obj->list); atomic_set(&obj->count, 1); obj->type = type; @@ -80,7 +87,9 @@ static struct inode_obj_id* alloc_inode_obj(struct inode* inode, atomic_inc(&inode->i_count); printk(KERN_DEBUG "alloc_inode_obj(%p, %d, %d): object created\n", inode, type, id); - return obj; + + *obj_ref = obj; + return 0; } /* inode must be locked already */ @@ -169,7 +178,7 @@ void exit_od_table(struct task_struct* t) static int do_sys_od_open(struct file* file, obj_type_t type, int id, void* __user config) { - int idx = 0, err; + int idx = 0, err = 0; struct inode* inode; struct inode_obj_id* obj = NULL; struct od_table_entry* entry; @@ -183,9 +192,10 @@ static int do_sys_od_open(struct file* file, obj_type_t type, int id, mutex_lock(&inode->i_obj_mutex); obj = get_inode_obj(inode, type, id); if (!obj) - obj = alloc_inode_obj(inode, type, id); - if (!obj) { - idx = -ENOMEM; + err = alloc_inode_obj(&obj, inode, type, id, config); + if (err != 0) { + obj = NULL; + idx = err; entry->used = 0; } else { entry->obj = obj; @@ -195,12 +205,15 @@ static int do_sys_od_open(struct file* file, obj_type_t type, int id, mutex_unlock(&inode->i_obj_mutex); - err = fdso_open(entry, config); + /* open only if creation succeeded */ + if (!err) + err = fdso_open(entry, config); if (err < 0) { /* The class rejected the open call. * We need to clean up and tell user space. */ - put_od_entry(entry); + if (obj) + put_od_entry(entry); idx = err; } diff --git a/litmus/locking.c b/litmus/locking.c index d39afae..8ee6a6b 100644 --- a/litmus/locking.c +++ b/litmus/locking.c @@ -5,7 +5,7 @@ #include #include -static void* create_generic_lock(obj_type_t type); +static int create_generic_lock(void** obj_ref, obj_type_t type, void* __user arg); static int open_generic_lock(struct od_table_entry* entry, void* __user arg); static int close_generic_lock(struct od_table_entry* entry); static void destroy_generic_lock(obj_type_t type, void* sem); @@ -28,16 +28,15 @@ static inline struct litmus_lock* get_lock(struct od_table_entry* entry) return (struct litmus_lock*) entry->obj->obj; } -static void* create_generic_lock(obj_type_t type) +static int create_generic_lock(void** obj_ref, obj_type_t type, void* __user arg) { struct litmus_lock* lock; int err; - err = litmus->allocate_lock(&lock, type); + err = litmus->allocate_lock(&lock, type, arg); if (err == 0) - return lock; - else - return NULL; + *obj_ref = lock; + return err; } static int open_generic_lock(struct od_table_entry* entry, void* __user arg) diff --git a/litmus/sched_gsn_edf.c b/litmus/sched_gsn_edf.c index c525d43..c5c9600 100644 --- a/litmus/sched_gsn_edf.c +++ b/litmus/sched_gsn_edf.c @@ -895,7 +895,8 @@ static struct litmus_lock* gsnedf_new_fmlp(void) /* **** lock constructor **** */ -static long gsnedf_allocate_lock(struct litmus_lock **lock, int type) +static long gsnedf_allocate_lock(struct litmus_lock **lock, int type, + void* __user unused) { int err = -ENXIO; diff --git a/litmus/sched_plugin.c b/litmus/sched_plugin.c index 2f8f399..d54886d 100644 --- a/litmus/sched_plugin.c +++ b/litmus/sched_plugin.c @@ -123,7 +123,8 @@ static long litmus_dummy_deactivate_plugin(void) #ifdef CONFIG_LITMUS_LOCKING -static long litmus_dummy_allocate_lock(struct litmus_lock **lock, int type) +static long litmus_dummy_allocate_lock(struct litmus_lock **lock, int type, + void* __user config) { return -ENXIO; } diff --git a/litmus/sched_psn_edf.c b/litmus/sched_psn_edf.c index 801bc92..abb06fa 100644 --- a/litmus/sched_psn_edf.c +++ b/litmus/sched_psn_edf.c @@ -547,7 +547,8 @@ static struct litmus_lock* psnedf_new_fmlp(void) /* **** lock constructor **** */ -static long psnedf_allocate_lock(struct litmus_lock **lock, int type) +static long psnedf_allocate_lock(struct litmus_lock **lock, int type, + void* __user unused) { int err = -ENXIO; struct srp_semaphore* srp; -- 1.7.2.3 From 963fd846e36b48d5338ef2a134d3ee8d208abc07 Mon Sep 17 00:00:00 2001 From: Bjoern B. Brandenburg Date: Sat, 29 Jan 2011 14:45:49 -0500 Subject: [PATCH 16/24] Feather-Trace: rename locking trace points Since we don't expect to trace more than one lock type at a time, having protocol-specific trace points is not required. --- include/litmus/trace.h | 18 ++++-------------- litmus/locking.c | 8 ++++---- 2 files changed, 8 insertions(+), 18 deletions(-) diff --git a/include/litmus/trace.h b/include/litmus/trace.h index b32c711..05f4872 100644 --- a/include/litmus/trace.h +++ b/include/litmus/trace.h @@ -91,20 +91,10 @@ feather_callback void save_timestamp_cpu(unsigned long event, unsigned long cpu) #define TS_EXIT_NP_START TIMESTAMP(150) #define TS_EXIT_NP_END TIMESTAMP(151) -#define TS_SRP_UP_START TIMESTAMP(160) -#define TS_SRP_UP_END TIMESTAMP(161) -#define TS_SRP_DOWN_START TIMESTAMP(162) -#define TS_SRP_DOWN_END TIMESTAMP(163) - -#define TS_PI_UP_START TIMESTAMP(170) -#define TS_PI_UP_END TIMESTAMP(171) -#define TS_PI_DOWN_START TIMESTAMP(172) -#define TS_PI_DOWN_END TIMESTAMP(173) - -#define TS_FIFO_UP_START TIMESTAMP(180) -#define TS_FIFO_UP_END TIMESTAMP(181) -#define TS_FIFO_DOWN_START TIMESTAMP(182) -#define TS_FIFO_DOWN_END TIMESTAMP(183) +#define TS_LOCK_START TIMESTAMP(170) +#define TS_LOCK_END TIMESTAMP(171) +#define TS_UNLOCK_START TIMESTAMP(172) +#define TS_UNLOCK_END TIMESTAMP(173) #define TS_SEND_RESCHED_START(c) CTIMESTAMP(190, c) #define TS_SEND_RESCHED_END DTIMESTAMP(191, TSK_UNKNOWN) diff --git a/litmus/locking.c b/litmus/locking.c index 8ee6a6b..728b568 100644 --- a/litmus/locking.c +++ b/litmus/locking.c @@ -69,7 +69,7 @@ asmlinkage long sys_litmus_lock(int lock_od) struct od_table_entry* entry; struct litmus_lock* l; - TS_PI_DOWN_START; + TS_LOCK_START; entry = get_entry_for_od(lock_od); if (entry && is_lock(entry)) { @@ -80,7 +80,7 @@ asmlinkage long sys_litmus_lock(int lock_od) /* Note: task my have been suspended or preempted in between! Take * this into account when computing overheads. */ - TS_PI_DOWN_END; + TS_UNLOCK_END; return err; } @@ -91,7 +91,7 @@ asmlinkage long sys_litmus_unlock(int lock_od) struct od_table_entry* entry; struct litmus_lock* l; - TS_PI_UP_START; + TS_UNLOCK_START; entry = get_entry_for_od(lock_od); if (entry && is_lock(entry)) { @@ -102,7 +102,7 @@ asmlinkage long sys_litmus_unlock(int lock_od) /* Note: task my have been preempted in between! Take this into * account when computing overheads. */ - TS_PI_UP_END; + TS_UNLOCK_END; return err; } -- 1.7.2.3 From 4ce37704ec0bedb28b5708d32964fca471e793d0 Mon Sep 17 00:00:00 2001 From: Bjoern B. Brandenburg Date: Wed, 26 Jan 2011 20:42:49 -0500 Subject: [PATCH 17/24] Litmus core: extract userspace interface from C-EDF Make the cluster size configuration in C-EDF generic so that it can be used by other clustered schedulers. --- include/litmus/clustered.h | 22 +++++++++++ litmus/litmus_proc.c | 78 +++++++++++++++++++++++++++++++++++++++ litmus/sched_cedf.c | 88 +++----------------------------------------- 3 files changed, 106 insertions(+), 82 deletions(-) create mode 100644 include/litmus/clustered.h diff --git a/include/litmus/clustered.h b/include/litmus/clustered.h new file mode 100644 index 0000000..cad12467 --- /dev/null +++ b/include/litmus/clustered.h @@ -0,0 +1,22 @@ +#ifndef CLUSTERED_H +#define CLUSTERED_H + +/* Which cache level should be used to group CPUs into clusters? + * GLOBAL_CLUSTER means that all CPUs form a single cluster (just like under + * global scheduling). + */ +enum cache_level { + GLOBAL_CLUSTER = 0, + L1_CLUSTER = 1, + L2_CLUSTER = 2, + L3_CLUSTER = 3 +}; + +int parse_cache_level(const char *str, enum cache_level *level); +const char* cache_level_name(enum cache_level level); + +/* expose a cache level in a /proc dir */ +struct proc_dir_entry* create_cluster_file(struct proc_dir_entry* parent, + enum cache_level* level); + +#endif diff --git a/litmus/litmus_proc.c b/litmus/litmus_proc.c index e3f3f11..4bf725a 100644 --- a/litmus/litmus_proc.c +++ b/litmus/litmus_proc.c @@ -8,6 +8,8 @@ #include #include +#include + /* in litmus/litmus.c */ extern atomic_t rt_task_count; @@ -267,3 +269,79 @@ int copy_and_chomp(char *kbuf, unsigned long ksize, return ksize; } + +/* helper functions for clustered plugins */ +static const char* cache_level_names[] = { + "ALL", + "L1", + "L2", + "L3", +}; + +int parse_cache_level(const char *cache_name, enum cache_level *level) +{ + int err = -EINVAL; + int i; + /* do a quick and dirty comparison to find the cluster size */ + for (i = GLOBAL_CLUSTER; i <= L3_CLUSTER; i++) + if (!strcmp(cache_name, cache_level_names[i])) { + *level = (enum cache_level) i; + err = 0; + break; + } + return err; +} + +const char* cache_level_name(enum cache_level level) +{ + int idx = level; + + if (idx >= GLOBAL_CLUSTER && idx <= L3_CLUSTER) + return cache_level_names[idx]; + else + return "INVALID"; +} + + +/* proc file interface to configure the cluster size */ +static int proc_read_cluster_size(char *page, char **start, + off_t off, int count, + int *eof, void *data) +{ + return snprintf(page, PAGE_SIZE, "%s\n", + cache_level_name(*((enum cache_level*) data)));; +} + +static int proc_write_cluster_size(struct file *file, + const char *buffer, + unsigned long count, + void *data) +{ + int len; + char cache_name[8]; + + len = copy_and_chomp(cache_name, sizeof(cache_name), buffer, count); + + if (len > 0 && parse_cache_level(cache_name, (enum cache_level*) data)) + printk(KERN_INFO "Cluster '%s' is unknown.\n", cache_name); + + return len; +} + +struct proc_dir_entry* create_cluster_file(struct proc_dir_entry* parent, + enum cache_level* level) +{ + struct proc_dir_entry* cluster_file; + + cluster_file = create_proc_entry("cluster", 0644, parent); + if (!cluster_file) { + printk(KERN_ERR "Could not allocate %s/cluster " + "procfs entry.\n", parent->name); + } else { + cluster_file->read_proc = proc_read_cluster_size; + cluster_file->write_proc = proc_write_cluster_size; + cluster_file->data = level; + } + return cluster_file; +} + diff --git a/litmus/sched_cedf.c b/litmus/sched_cedf.c index 098a449..73fe1c4 100644 --- a/litmus/sched_cedf.c +++ b/litmus/sched_cedf.c @@ -39,6 +39,8 @@ #include #include +#include + #include /* to configure the cluster size */ @@ -49,12 +51,7 @@ * group CPUs into clusters. GLOBAL_CLUSTER, which is the default, means that * all CPUs form a single cluster (just like GSN-EDF). */ -static enum { - GLOBAL_CLUSTER = 0, - L1_CLUSTER = 1, - L2_CLUSTER = 2, - L3_CLUSTER = 3 -} cluster_config = GLOBAL_CLUSTER; +static enum cache_level cluster_config = GLOBAL_CLUSTER; struct clusterdomain; @@ -770,73 +767,8 @@ static struct sched_plugin cedf_plugin __cacheline_aligned_in_smp = { .activate_plugin = cedf_activate_plugin, }; - -/* proc file interface to configure the cluster size */ - -static int proc_read_cluster_size(char *page, char **start, - off_t off, int count, - int *eof, void *data) -{ - int len; - switch (cluster_config) { - case GLOBAL_CLUSTER: - len = snprintf(page, PAGE_SIZE, "ALL\n"); - break; - case L1_CLUSTER: - case L2_CLUSTER: - case L3_CLUSTER: - len = snprintf(page, PAGE_SIZE, "L%d\n", cluster_config); - break; - default: - /* This should be impossible, but let's be paranoid. */ - len = snprintf(page, PAGE_SIZE, "INVALID (%d)\n", - cluster_config); - break; - } - return len; -} - -static int proc_write_cluster_size(struct file *file, - const char *buffer, - unsigned long count, - void *data) -{ - int len; - /* L2, L3 */ - char cache_name[33]; - - if(count > 32) - len = 32; - else - len = count; - - if(copy_from_user(cache_name, buffer, len)) - return -EFAULT; - - cache_name[len] = '\0'; - /* chomp name */ - if (len > 1 && cache_name[len - 1] == '\n') - cache_name[len - 1] = '\0'; - - /* do a quick and dirty comparison to find the cluster size */ - if (!strcmp(cache_name, "L2")) - cluster_config = L2_CLUSTER; - else if (!strcmp(cache_name, "L3")) - cluster_config = L3_CLUSTER; - else if (!strcmp(cache_name, "L1")) - cluster_config = L1_CLUSTER; - else if (!strcmp(cache_name, "ALL")) - cluster_config = GLOBAL_CLUSTER; - else - printk(KERN_INFO "Cluster '%s' is unknown.\n", cache_name); - - return len; -} - - static struct proc_dir_entry *cluster_file = NULL, *cedf_dir = NULL; - static int __init init_cedf(void) { int err, fs; @@ -844,18 +776,10 @@ static int __init init_cedf(void) err = register_sched_plugin(&cedf_plugin); if (!err) { fs = make_plugin_proc_dir(&cedf_plugin, &cedf_dir); - if (!fs) { - cluster_file = create_proc_entry("cluster", 0644, cedf_dir); - if (!cluster_file) { - printk(KERN_ERR "Could not allocate C-EDF/cluster " - "procfs entry.\n"); - } else { - cluster_file->read_proc = proc_read_cluster_size; - cluster_file->write_proc = proc_write_cluster_size; - } - } else { + if (!fs) + cluster_file = create_cluster_file(cedf_dir, &cluster_config); + else printk(KERN_ERR "Could not allocate C-EDF procfs dir.\n"); - } } return err; } -- 1.7.2.3 From 343d4ead3b12992f494134114cf50e4f37c656c5 Mon Sep 17 00:00:00 2001 From: Bjoern B. Brandenburg Date: Thu, 27 Jan 2011 16:23:46 -0500 Subject: [PATCH 18/24] Litmus core: add generic clustering support Inspired by the existing C-EDF code, this generic version will build clusters of CPUs based on a given cache level. --- include/litmus/clustered.h | 22 +++++++++ litmus/Makefile | 1 + litmus/clustered.c | 111 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 134 insertions(+), 0 deletions(-) create mode 100644 litmus/clustered.c diff --git a/include/litmus/clustered.h b/include/litmus/clustered.h index cad12467..0c18dcb 100644 --- a/include/litmus/clustered.h +++ b/include/litmus/clustered.h @@ -19,4 +19,26 @@ const char* cache_level_name(enum cache_level level); struct proc_dir_entry* create_cluster_file(struct proc_dir_entry* parent, enum cache_level* level); + + +struct scheduling_cluster { + unsigned int id; + /* list of CPUs that are part of this cluster */ + struct list_head cpus; +}; + +struct cluster_cpu { + unsigned int id; /* which CPU is this? */ + struct list_head cluster_list; /* List of the CPUs in this cluster. */ + struct scheduling_cluster* cluster; /* The cluster that this CPU belongs to. */ +}; + +int get_cluster_size(enum cache_level level); + +int assign_cpus_to_clusters(enum cache_level level, + struct scheduling_cluster* clusters[], + unsigned int num_clusters, + struct cluster_cpu* cpus[], + unsigned int num_cpus); + #endif diff --git a/litmus/Makefile b/litmus/Makefile index 62a20e2..ad9936e 100644 --- a/litmus/Makefile +++ b/litmus/Makefile @@ -6,6 +6,7 @@ obj-y = sched_plugin.o litmus.o \ preempt.o \ litmus_proc.o \ budget.o \ + clustered.o \ jobs.o \ sync.o \ rt_domain.o \ diff --git a/litmus/clustered.c b/litmus/clustered.c new file mode 100644 index 0000000..04450a8 --- /dev/null +++ b/litmus/clustered.c @@ -0,0 +1,111 @@ +#include +#include +#include + +#include + +#ifndef CONFIG_X86 +/* fake get_shared_cpu_map() on non-x86 architectures */ + +int get_shared_cpu_map(cpumask_var_t mask, unsigned int cpu, int index) +{ + if (index != 1) + return 1; + else { + /* Fake L1: CPU is all by itself. */ + cpumask_clear(mask); + cpumask_set_cpu(cpu, mask); + return 0; + } +} + +#endif + +int get_cluster_size(enum cache_level level) +{ + cpumask_var_t mask; + int ok; + int num_cpus; + + if (level == GLOBAL_CLUSTER) + return num_online_cpus(); + else { + if (!zalloc_cpumask_var(&mask, GFP_ATOMIC)) + return -ENOMEM; + /* assumes CPU 0 is representative of all CPUs */ + ok = get_shared_cpu_map(mask, 0, level); + /* ok == 0 means we got the map; otherwise it's an invalid cache level */ + if (ok == 0) + num_cpus = cpumask_weight(mask); + free_cpumask_var(mask); + + if (ok == 0) + return num_cpus; + else + return -EINVAL; + } +} + +int assign_cpus_to_clusters(enum cache_level level, + struct scheduling_cluster* clusters[], + unsigned int num_clusters, + struct cluster_cpu* cpus[], + unsigned int num_cpus) +{ + cpumask_var_t mask; + unsigned int i, free_cluster = 0, low_cpu; + int err = 0; + + if (!zalloc_cpumask_var(&mask, GFP_ATOMIC)) + return -ENOMEM; + + /* clear cluster pointers */ + for (i = 0; i < num_cpus; i++) { + cpus[i]->id = i; + cpus[i]->cluster = NULL; + } + + /* initialize clusters */ + for (i = 0; i < num_clusters; i++) { + clusters[i]->id = i; + INIT_LIST_HEAD(&clusters[i]->cpus); + } + + /* Assign each CPU. Two assumtions are made: + * 1) The index of a cpu in cpus corresponds to its processor id (i.e., the index in a cpu mask). + * 2) All cpus that belong to some cluster are online. + */ + for_each_online_cpu(i) { + /* get lowest-id CPU in cluster */ + if (level != GLOBAL_CLUSTER) { + err = get_shared_cpu_map(mask, cpus[i]->id, level); + if (err != 0) { + /* ugh... wrong cache level? Either caller screwed up + * or the CPU topology is weird. */ + printk(KERN_ERR "Could not set up clusters for L%d sharing (max: L%d).\n", + level, err); + err = -EINVAL; + goto out; + } + low_cpu = cpumask_first(mask); + } else + low_cpu = 0; + if (low_cpu == i) { + /* caller must provide an appropriate number of clusters */ + BUG_ON(free_cluster >= num_clusters); + + /* create new cluster */ + cpus[i]->cluster = clusters[free_cluster++]; + } else { + /* low_cpu points to the right cluster + * Assumption: low_cpu is actually online and was processed earlier. */ + cpus[i]->cluster = cpus[low_cpu]->cluster; + } + /* enqueue in cpus list */ + list_add(&cpus[i]->cluster_list, &cpus[i]->cluster->cpus); + printk(KERN_INFO "Assigning CPU%u to cluster %u\n.", i, cpus[i]->cluster->id); + } +out: + free_cpumask_var(mask); + return err; +} -- 1.7.2.3 From 71efbc5459ef95ed902a6980eae646197529364e Mon Sep 17 00:00:00 2001 From: Bjoern B. Brandenburg Date: Fri, 7 Jan 2011 17:37:01 -0500 Subject: [PATCH 19/24] Pfair: support clustered scheduling Just like C-EDF is a global scheduler that is split across several clusters, Pfair can be applied on a per-cluster basis. This patch changes the Pfair implementation to enable clustering based on the recently added generic clustering support. --- litmus/sched_pfair.c | 392 +++++++++++++++++++++++++++++++++++--------------- 1 files changed, 274 insertions(+), 118 deletions(-) diff --git a/litmus/sched_pfair.c b/litmus/sched_pfair.c index c7d5cf7..71ce993 100644 --- a/litmus/sched_pfair.c +++ b/litmus/sched_pfair.c @@ -23,6 +23,13 @@ #include +/* to configure the cluster size */ +#include + +#include + +static enum cache_level pfair_cluster_level = GLOBAL_CLUSTER; + struct subtask { /* measured in quanta relative to job release */ quanta_t release; @@ -43,25 +50,28 @@ struct pfair_param { unsigned int sporadic_release; /* On wakeup, new sporadic release? */ + struct pfair_cluster* cluster; /* where this task is scheduled */ + struct subtask subtasks[0]; /* allocate together with pfair_param */ }; #define tsk_pfair(tsk) ((tsk)->rt_param.pfair) struct pfair_state { - int cpu; + struct cluster_cpu topology; + volatile quanta_t cur_tick; /* updated by the CPU that is advancing * the time */ volatile quanta_t local_tick; /* What tick is the local CPU currently * executing? Updated only by the local * CPU. In QEMU, this may lag behind the - * current tick. In a real system, with - * proper timers and aligned quanta, - * that should only be the - * case for a very short time after the - * time advanced. With staggered quanta, - * it will lag for the duration of the - * offset. + * current tick. In a real system, with + * proper timers and aligned quanta, + * that should only be the case for a + * very short time after the time + * advanced. With staggered quanta, it + * will lag for the duration of the + * offset. */ struct task_struct* linked; /* the task that should be executing */ @@ -79,25 +89,56 @@ struct pfair_state { */ #define PFAIR_MAX_PERIOD 2000 -/* This is the release queue wheel. It is indexed by pfair_time % - * PFAIR_MAX_PERIOD. Each heap is ordered by PFAIR priority, so that it can be - * merged with the ready queue. - */ -static struct bheap release_queue[PFAIR_MAX_PERIOD]; +struct pfair_cluster { + struct scheduling_cluster topology; -DEFINE_PER_CPU(struct pfair_state, pfair_state); -struct pfair_state* *pstate; /* short cut */ + /* The "global" time in this cluster. */ + quanta_t pfair_time; /* the "official" PFAIR clock */ + quanta_t merge_time; /* Updated after the release queue has been + * merged. Used by drop_all_references(). + */ -static quanta_t pfair_time = 0; /* the "official" PFAIR clock */ -static quanta_t merge_time = 0; /* Updated after the release queue has been - * merged. Used by drop_all_references(). - */ + /* The ready queue for this cluster. */ + rt_domain_t pfair; -static rt_domain_t pfair; + /* This is the release queue wheel for this cluster. It is indexed by + * pfair_time % PFAIR_MAX_PERIOD. Each heap is ordered by PFAIR + * priority, so that it can be merged with the ready queue. + */ + struct bheap release_queue[PFAIR_MAX_PERIOD]; +}; -/* The pfair_lock is used to serialize all scheduling events. - */ -#define pfair_lock pfair.ready_lock +static inline struct pfair_cluster* cpu_cluster(struct pfair_state* state) +{ + return container_of(state->topology.cluster, struct pfair_cluster, topology); +} + +static inline int cpu_id(struct pfair_state* state) +{ + return state->topology.id; +} + +static inline struct pfair_state* from_cluster_list(struct list_head* pos) +{ + return list_entry(pos, struct pfair_state, topology.cluster_list); +} + +static inline raw_spinlock_t* cluster_lock(struct pfair_cluster* cluster) +{ + /* The ready_lock is used to serialize all scheduling events. */ + return &cluster->pfair.ready_lock; +} + +static inline raw_spinlock_t* cpu_lock(struct pfair_state* state) +{ + return cluster_lock(cpu_cluster(state)); +} + +DEFINE_PER_CPU(struct pfair_state, pfair_state); +struct pfair_state* *pstate; /* short cut */ + +static struct pfair_cluster* pfair_clusters; +static int num_pfair_clusters; /* Enable for lots of trace info. * #define PFAIR_DEBUG @@ -197,9 +238,9 @@ int pfair_ready_order(struct bheap_node* a, struct bheap_node* b) } /* return the proper release queue for time t */ -static struct bheap* relq(quanta_t t) +static struct bheap* relq(struct pfair_cluster* cluster, quanta_t t) { - struct bheap* rq = &release_queue[t % PFAIR_MAX_PERIOD]; + struct bheap* rq = cluster->release_queue + (t % PFAIR_MAX_PERIOD); return rq; } @@ -215,17 +256,19 @@ static void __pfair_add_release(struct task_struct* t, struct bheap* queue) tsk_rt(t)->heap_node); } -static void pfair_add_release(struct task_struct* t) +static void pfair_add_release(struct pfair_cluster* cluster, + struct task_struct* t) { BUG_ON(bheap_node_in_heap(tsk_rt(t)->heap_node)); - __pfair_add_release(t, relq(cur_release(t))); + __pfair_add_release(t, relq(cluster, cur_release(t))); } /* pull released tasks from the release queue */ -static void poll_releases(quanta_t time) +static void poll_releases(struct pfair_cluster* cluster, + quanta_t time) { - __merge_ready(&pfair, relq(time)); - merge_time = time; + __merge_ready(&cluster->pfair, relq(cluster, time)); + cluster->merge_time = time; } static void check_preempt(struct task_struct* t) @@ -246,18 +289,20 @@ static void check_preempt(struct task_struct* t) } } -/* caller must hold pfair_lock */ +/* caller must hold pfair.ready_lock */ static void drop_all_references(struct task_struct *t) { int cpu; struct pfair_state* s; struct bheap* q; + struct pfair_cluster* cluster; if (bheap_node_in_heap(tsk_rt(t)->heap_node)) { /* figure out what queue the node is in */ - if (time_before_eq(cur_release(t), merge_time)) - q = &pfair.ready_queue; + cluster = tsk_pfair(t)->cluster; + if (time_before_eq(cur_release(t), cluster->merge_time)) + q = &cluster->pfair.ready_queue; else - q = relq(cur_release(t)); + q = relq(cluster, cur_release(t)); bheap_delete(pfair_ready_order, q, tsk_rt(t)->heap_node); } @@ -301,22 +346,25 @@ static int advance_subtask(quanta_t time, struct task_struct* t, int cpu) return to_relq; } -static void advance_subtasks(quanta_t time) +static void advance_subtasks(struct pfair_cluster *cluster, quanta_t time) { - int cpu, missed; + int missed; struct task_struct* l; struct pfair_param* p; + struct list_head* pos; + struct pfair_state* cpu; - for_each_online_cpu(cpu) { - l = pstate[cpu]->linked; - missed = pstate[cpu]->linked != pstate[cpu]->local; + list_for_each(pos, &cluster->topology.cpus) { + cpu = from_cluster_list(pos); + l = cpu->linked; + missed = cpu->linked != cpu->local; if (l) { p = tsk_pfair(l); p->last_quantum = time; - p->last_cpu = cpu; - if (advance_subtask(time, l, cpu)) { - pstate[cpu]->linked = NULL; - pfair_add_release(l); + p->last_cpu = cpu_id(cpu); + if (advance_subtask(time, l, cpu_id(cpu))) { + cpu->linked = NULL; + pfair_add_release(cluster, l); } } } @@ -350,8 +398,10 @@ static int pfair_link(quanta_t time, int cpu, int target = target_cpu(time, t, cpu); struct task_struct* prev = pstate[cpu]->linked; struct task_struct* other; + struct pfair_cluster* cluster = cpu_cluster(pstate[cpu]); if (target != cpu) { + BUG_ON(pstate[target]->topology.cluster != pstate[cpu]->topology.cluster); other = pstate[target]->linked; pstate[target]->linked = t; tsk_rt(t)->linked_on = target; @@ -365,14 +415,14 @@ static int pfair_link(quanta_t time, int cpu, if (prev) { /* prev got pushed back into the ready queue */ tsk_rt(prev)->linked_on = NO_CPU; - __add_ready(&pfair, prev); + __add_ready(&cluster->pfair, prev); } /* we are done with this cpu */ return 0; } else { /* re-add other, it's original CPU was not considered yet */ tsk_rt(other)->linked_on = NO_CPU; - __add_ready(&pfair, other); + __add_ready(&cluster->pfair, other); /* reschedule this CPU */ return 1; } @@ -382,71 +432,77 @@ static int pfair_link(quanta_t time, int cpu, if (prev) { /* prev got pushed back into the ready queue */ tsk_rt(prev)->linked_on = NO_CPU; - __add_ready(&pfair, prev); + __add_ready(&cluster->pfair, prev); } /* we are done with this CPU */ return 0; } } -static void schedule_subtasks(quanta_t time) +static void schedule_subtasks(struct pfair_cluster *cluster, quanta_t time) { - int cpu, retry; + int retry; + struct list_head *pos; + struct pfair_state *cpu_state; - for_each_online_cpu(cpu) { + list_for_each(pos, &cluster->topology.cpus) { + cpu_state = from_cluster_list(pos); retry = 1; while (retry) { - if (pfair_higher_prio(__peek_ready(&pfair), - pstate[cpu]->linked)) - retry = pfair_link(time, cpu, - __take_ready(&pfair)); + if (pfair_higher_prio(__peek_ready(&cluster->pfair), + cpu_state->linked)) + retry = pfair_link(time, cpu_id(cpu_state), + __take_ready(&cluster->pfair)); else retry = 0; } } } -static void schedule_next_quantum(quanta_t time) +static void schedule_next_quantum(struct pfair_cluster *cluster, quanta_t time) { - int cpu; + struct pfair_state *cpu; + struct list_head* pos; /* called with interrupts disabled */ PTRACE("--- Q %lu at %llu PRE-SPIN\n", time, litmus_clock()); - raw_spin_lock(&pfair_lock); + raw_spin_lock(cluster_lock(cluster)); PTRACE("<<< Q %lu at %llu\n", time, litmus_clock()); sched_trace_quantum_boundary(); - advance_subtasks(time); - poll_releases(time); - schedule_subtasks(time); + advance_subtasks(cluster, time); + poll_releases(cluster, time); + schedule_subtasks(cluster, time); - for (cpu = 0; cpu < num_online_cpus(); cpu++) - if (pstate[cpu]->linked) + list_for_each(pos, &cluster->topology.cpus) { + cpu = from_cluster_list(pos); + if (cpu->linked) PTRACE_TASK(pstate[cpu]->linked, - " linked on %d.\n", cpu); + " linked on %d.\n", cpu_id(cpu)); else - PTRACE("(null) linked on %d.\n", cpu); - + PTRACE("(null) linked on %d.\n", cpu_id(cpu)); + } /* We are done. Advance time. */ mb(); - for (cpu = 0; cpu < num_online_cpus(); cpu++) { - if (pstate[cpu]->local_tick != pstate[cpu]->cur_tick) { + list_for_each(pos, &cluster->topology.cpus) { + cpu = from_cluster_list(pos); + if (cpu->local_tick != cpu->cur_tick) { TRACE("BAD Quantum not acked on %d " "(l:%lu c:%lu p:%lu)\n", - cpu, - pstate[cpu]->local_tick, - pstate[cpu]->cur_tick, - pfair_time); - pstate[cpu]->missed_quanta++; + cpu_id(cpu), + cpu->local_tick, + cpu->cur_tick, + cluster->pfair_time); + cpu->missed_quanta++; } - pstate[cpu]->cur_tick = time; + cpu->cur_tick = time; } PTRACE(">>> Q %lu at %llu\n", time, litmus_clock()); - raw_spin_unlock(&pfair_lock); + raw_spin_unlock(cluster_lock(cluster)); } static noinline void wait_for_quantum(quanta_t q, struct pfair_state* state) @@ -479,12 +535,12 @@ static void catchup_quanta(quanta_t from, quanta_t target, while (time_before(cur, target)) { wait_for_quantum(cur, state); cur++; - time = cmpxchg(&pfair_time, + time = cmpxchg(&cpu_cluster(state)->pfair_time, cur - 1, /* expected */ cur /* next */ ); if (time == cur - 1) - schedule_next_quantum(cur); + schedule_next_quantum(cpu_cluster(state), cur); } TRACE("+++> catching up done\n"); } @@ -505,14 +561,14 @@ static void pfair_tick(struct task_struct* t) /* Attempt to advance time. First CPU to get here * will prepare the next quantum. */ - time = cmpxchg(&pfair_time, + time = cmpxchg(&cpu_cluster(state)->pfair_time, cur - 1, /* expected */ cur /* next */ ); if (time == cur - 1) { /* exchange succeeded */ wait_for_quantum(cur - 1, state); - schedule_next_quantum(cur); + schedule_next_quantum(cpu_cluster(state), cur); retry = 0; } else if (time_before(time, cur - 1)) { /* the whole system missed a tick !? */ @@ -562,59 +618,65 @@ static struct task_struct* pfair_schedule(struct task_struct * prev) int blocks; struct task_struct* next = NULL; - raw_spin_lock(&pfair_lock); + raw_spin_lock(cpu_lock(state)); blocks = is_realtime(prev) && !is_running(prev); - if (state->local && safe_to_schedule(state->local, state->cpu)) + if (state->local && safe_to_schedule(state->local, cpu_id(state))) next = state->local; if (prev != next) { tsk_rt(prev)->scheduled_on = NO_CPU; if (next) - tsk_rt(next)->scheduled_on = state->cpu; + tsk_rt(next)->scheduled_on = cpu_id(state); } sched_state_task_picked(); - raw_spin_unlock(&pfair_lock); + raw_spin_unlock(cpu_lock(state)); if (next) TRACE_TASK(next, "scheduled rel=%lu at %lu (%llu)\n", - tsk_pfair(next)->release, pfair_time, litmus_clock()); + tsk_pfair(next)->release, cpu_cluster(state)->pfair_time, litmus_clock()); else if (is_realtime(prev)) - TRACE("Becomes idle at %lu (%llu)\n", pfair_time, litmus_clock()); + TRACE("Becomes idle at %lu (%llu)\n", cpu_cluster(state)->pfair_time, litmus_clock()); return next; } static void pfair_task_new(struct task_struct * t, int on_rq, int running) { - unsigned long flags; + unsigned long flags; + struct pfair_cluster* cluster; TRACE("pfair: task new %d state:%d\n", t->pid, t->state); - raw_spin_lock_irqsave(&pfair_lock, flags); + cluster = tsk_pfair(t)->cluster; + + raw_spin_lock_irqsave(cluster_lock(cluster), flags); if (running) t->rt_param.scheduled_on = task_cpu(t); else t->rt_param.scheduled_on = NO_CPU; - prepare_release(t, pfair_time + 1); + prepare_release(t, cluster->pfair_time + 1); tsk_pfair(t)->sporadic_release = 0; - pfair_add_release(t); + pfair_add_release(cluster, t); check_preempt(t); - raw_spin_unlock_irqrestore(&pfair_lock, flags); + raw_spin_unlock_irqrestore(cluster_lock(cluster), flags); } static void pfair_task_wake_up(struct task_struct *t) { unsigned long flags; lt_t now; + struct pfair_cluster* cluster; + + cluster = tsk_pfair(t)->cluster; TRACE_TASK(t, "wakes at %llu, release=%lu, pfair_time:%lu\n", - litmus_clock(), cur_release(t), pfair_time); + litmus_clock(), cur_release(t), cluster->pfair_time); - raw_spin_lock_irqsave(&pfair_lock, flags); + raw_spin_lock_irqsave(cluster_lock(cluster), flags); /* It is a little unclear how to deal with Pfair * tasks that block for a while and then wake. For now, @@ -629,13 +691,13 @@ static void pfair_task_wake_up(struct task_struct *t) prepare_release(t, time2quanta(now, CEIL)); sched_trace_task_release(t); /* FIXME: race with pfair_time advancing */ - pfair_add_release(t); + pfair_add_release(cluster, t); tsk_pfair(t)->sporadic_release = 0; } check_preempt(t); - raw_spin_unlock_irqrestore(&pfair_lock, flags); + raw_spin_unlock_irqrestore(cluster_lock(cluster), flags); TRACE_TASK(t, "wake up done at %llu\n", litmus_clock()); } @@ -649,9 +711,12 @@ static void pfair_task_block(struct task_struct *t) static void pfair_task_exit(struct task_struct * t) { unsigned long flags; + struct pfair_cluster *cluster; BUG_ON(!is_realtime(t)); + cluster = tsk_pfair(t)->cluster; + /* Remote task from release or ready queue, and ensure * that it is not the scheduled task for ANY CPU. We * do this blanket check because occassionally when @@ -659,12 +724,12 @@ static void pfair_task_exit(struct task_struct * t) * might not be the same as the CPU that the PFAIR scheduler * has chosen for it. */ - raw_spin_lock_irqsave(&pfair_lock, flags); + raw_spin_lock_irqsave(cluster_lock(cluster), flags); TRACE_TASK(t, "RIP, state:%d\n", t->state); drop_all_references(t); - raw_spin_unlock_irqrestore(&pfair_lock, flags); + raw_spin_unlock_irqrestore(cluster_lock(cluster), flags); kfree(t->rt_param.pfair); t->rt_param.pfair = NULL; @@ -676,27 +741,32 @@ static void pfair_release_at(struct task_struct* task, lt_t start) unsigned long flags; quanta_t release; + struct pfair_cluster *cluster; + + cluster = tsk_pfair(task)->cluster; + BUG_ON(!is_realtime(task)); - raw_spin_lock_irqsave(&pfair_lock, flags); + raw_spin_lock_irqsave(cluster_lock(cluster), flags); release_at(task, start); release = time2quanta(start, CEIL); - if (release - pfair_time >= PFAIR_MAX_PERIOD) - release = pfair_time + PFAIR_MAX_PERIOD; + /* FIXME: support arbitrary offsets. */ + if (release - cluster->pfair_time >= PFAIR_MAX_PERIOD) + release = cluster->pfair_time + PFAIR_MAX_PERIOD; TRACE_TASK(task, "sys release at %lu\n", release); drop_all_references(task); prepare_release(task, release); - pfair_add_release(task); + pfair_add_release(cluster, task); /* Clear sporadic release flag, since this release subsumes any * sporadic release on wake. */ tsk_pfair(task)->sporadic_release = 0; - raw_spin_unlock_irqrestore(&pfair_lock, flags); + raw_spin_unlock_irqrestore(cluster_lock(cluster), flags); } static void init_subtask(struct subtask* sub, unsigned long i, @@ -755,6 +825,11 @@ static long pfair_admit_task(struct task_struct* t) struct pfair_param* param; unsigned long i; + /* first check that the task is in the right cluster */ + if (cpu_cluster(pstate[tsk_rt(t)->task_params.cpu]) != + cpu_cluster(pstate[task_cpu(t)])) + return -EINVAL; + /* Pfair is a tick-based method, so the time * of interest is jiffies. Calculate tick-based * times for everything. @@ -798,6 +873,8 @@ static long pfair_admit_task(struct task_struct* t) param->release = 0; param->period = period; + param->cluster = cpu_cluster(pstate[tsk_rt(t)->task_params.cpu]); + for (i = 0; i < quanta; i++) init_subtask(param->subtasks + i, i, quanta, period); @@ -813,24 +890,88 @@ static long pfair_admit_task(struct task_struct* t) return 0; } +static void pfair_init_cluster(struct pfair_cluster* cluster) +{ + int i; + + /* initialize release queue */ + for (i = 0; i < PFAIR_MAX_PERIOD; i++) + bheap_init(&cluster->release_queue[i]); + rt_domain_init(&cluster->pfair, pfair_ready_order, NULL, NULL); + INIT_LIST_HEAD(&cluster->topology.cpus); +} + +static void cleanup_clusters(void) +{ + int i; + + if (num_pfair_clusters) + kfree(pfair_clusters); + pfair_clusters = NULL; + num_pfair_clusters = 0; + + /* avoid stale pointers */ + for (i = 0; i < NR_CPUS; i++) + pstate[i]->topology.cluster = NULL; +} + static long pfair_activate_plugin(void) { - int cpu; + int err, i; struct pfair_state* state; + struct pfair_cluster* cluster ; + quanta_t now; + int cluster_size; + struct cluster_cpu* cpus[NR_CPUS]; + struct scheduling_cluster* clust[NR_CPUS]; - state = &__get_cpu_var(pfair_state); - pfair_time = current_quantum(state); + cluster_size = get_cluster_size(pfair_cluster_level); - TRACE("Activating PFAIR at q=%lu\n", pfair_time); + if (cluster_size <= 0 || num_online_cpus() % cluster_size != 0) + return -EINVAL; - for (cpu = 0; cpu < num_online_cpus(); cpu++) { - state = &per_cpu(pfair_state, cpu); - state->cur_tick = pfair_time; - state->local_tick = pfair_time; + num_pfair_clusters = num_online_cpus() / cluster_size; + + pfair_clusters = kzalloc(num_pfair_clusters * sizeof(struct pfair_cluster), GFP_ATOMIC); + if (!pfair_clusters) { + num_pfair_clusters = 0; + printk(KERN_ERR "Could not allocate Pfair clusters!\n"); + return -ENOMEM; + } + + state = &__get_cpu_var(pfair_state); + now = current_quantum(state); + TRACE("Activating PFAIR at q=%lu\n", now); + + for (i = 0; i < num_pfair_clusters; i++) { + cluster = &pfair_clusters[i]; + pfair_init_cluster(cluster); + cluster->pfair_time = now; + clust[i] = &cluster->topology; + } + + for (i = 0; i < num_online_cpus(); i++) { + state = &per_cpu(pfair_state, i); + state->cur_tick = now; + state->local_tick = now; state->missed_quanta = 0; - state->offset = cpu_stagger_offset(cpu); + state->offset = cpu_stagger_offset(i); + printk(KERN_ERR "cpus[%d] set; %d\n", i, num_online_cpus()); + cpus[i] = &state->topology; } + err = assign_cpus_to_clusters(pfair_cluster_level, clust, num_pfair_clusters, + cpus, num_online_cpus()); + + if (err < 0) + cleanup_clusters(); + + return err; +} + +static long pfair_deactivate_plugin(void) +{ + cleanup_clusters(); return 0; } @@ -847,30 +988,29 @@ static struct sched_plugin pfair_plugin __cacheline_aligned_in_smp = { .release_at = pfair_release_at, .complete_job = complete_job, .activate_plugin = pfair_activate_plugin, + .deactivate_plugin = pfair_deactivate_plugin, }; + +static struct proc_dir_entry *cluster_file = NULL, *pfair_dir = NULL; + static int __init init_pfair(void) { - int cpu, i; + int cpu, err, fs; struct pfair_state *state; - /* * initialize short_cut for per-cpu pfair state; * there may be a problem here if someone removes a cpu * while we are doing this initialization... and if cpus - * are added / removed later... is it a _real_ problem? + * are added / removed later... but we don't support CPU hotplug atm anyway. */ pstate = kmalloc(sizeof(struct pfair_state*) * num_online_cpus(), GFP_KERNEL); - /* initialize release queue */ - for (i = 0; i < PFAIR_MAX_PERIOD; i++) - bheap_init(&release_queue[i]); - /* initialize CPU state */ for (cpu = 0; cpu < num_online_cpus(); cpu++) { state = &per_cpu(pfair_state, cpu); - state->cpu = cpu; + state->topology.id = cpu; state->cur_tick = 0; state->local_tick = 0; state->linked = NULL; @@ -881,13 +1021,29 @@ static int __init init_pfair(void) pstate[cpu] = state; } - rt_domain_init(&pfair, pfair_ready_order, NULL, NULL); - return register_sched_plugin(&pfair_plugin); + pfair_clusters = NULL; + num_pfair_clusters = 0; + + err = register_sched_plugin(&pfair_plugin); + if (!err) { + fs = make_plugin_proc_dir(&pfair_plugin, &pfair_dir); + if (!fs) + cluster_file = create_cluster_file(pfair_dir, &pfair_cluster_level); + else + printk(KERN_ERR "Could not allocate PFAIR procfs dir.\n"); + } + + return err; } static void __exit clean_pfair(void) { kfree(pstate); + + if (cluster_file) + remove_proc_entry("cluster", pfair_dir); + if (pfair_dir) + remove_plugin_proc_dir(&pfair_plugin); } module_init(init_pfair); -- 1.7.2.3 From c05eaa8091d2cadc20363d44a85ee454262f4bc2 Mon Sep 17 00:00:00 2001 From: Bjoern B. Brandenburg Date: Thu, 27 Jan 2011 20:11:59 -0500 Subject: [PATCH 20/24] Pfair: remove sporadic_release flag Instead of having an extra flag, Pfair should just infer sporadic release based on deadlines like other plugins, too. --- litmus/sched_pfair.c | 18 +++--------------- 1 files changed, 3 insertions(+), 15 deletions(-) diff --git a/litmus/sched_pfair.c b/litmus/sched_pfair.c index 71ce993..0a64273 100644 --- a/litmus/sched_pfair.c +++ b/litmus/sched_pfair.c @@ -48,8 +48,6 @@ struct pfair_param { quanta_t last_quantum; /* when scheduled last */ int last_cpu; /* where scheduled last */ - unsigned int sporadic_release; /* On wakeup, new sporadic release? */ - struct pfair_cluster* cluster; /* where this task is scheduled */ struct subtask subtasks[0]; /* allocate together with pfair_param */ @@ -334,7 +332,6 @@ static int advance_subtask(quanta_t time, struct task_struct* t, int cpu) } else { /* remove task from system until it wakes */ drop_all_references(t); - tsk_pfair(t)->sporadic_release = 1; TRACE_TASK(t, "on %d advanced to subtask %lu (not present)\n", cpu, p->cur); return 0; @@ -658,7 +655,6 @@ static void pfair_task_new(struct task_struct * t, int on_rq, int running) t->rt_param.scheduled_on = NO_CPU; prepare_release(t, cluster->pfair_time + 1); - tsk_pfair(t)->sporadic_release = 0; pfair_add_release(cluster, t); check_preempt(t); @@ -678,21 +674,18 @@ static void pfair_task_wake_up(struct task_struct *t) raw_spin_lock_irqsave(cluster_lock(cluster), flags); - /* It is a little unclear how to deal with Pfair - * tasks that block for a while and then wake. For now, - * if a task blocks and wakes before its next job release, + /* If a task blocks and wakes before its next job release, * then it may resume if it is currently linked somewhere * (as if it never blocked at all). Otherwise, we have a * new sporadic job release. */ - if (tsk_pfair(t)->sporadic_release) { - now = litmus_clock(); + now = litmus_clock(); + if (lt_before(get_deadline(t), now)) { release_at(t, now); prepare_release(t, time2quanta(now, CEIL)); sched_trace_task_release(t); /* FIXME: race with pfair_time advancing */ pfair_add_release(cluster, t); - tsk_pfair(t)->sporadic_release = 0; } check_preempt(t); @@ -761,11 +754,6 @@ static void pfair_release_at(struct task_struct* task, lt_t start) prepare_release(task, release); pfair_add_release(cluster, task); - /* Clear sporadic release flag, since this release subsumes any - * sporadic release on wake. - */ - tsk_pfair(task)->sporadic_release = 0; - raw_spin_unlock_irqrestore(cluster_lock(cluster), flags); } -- 1.7.2.3 From 0f6a8e02773f8c23b5b6a3dbfa044e50c9d7d811 Mon Sep 17 00:00:00 2001 From: Glenn Elliott Date: Thu, 31 Mar 2011 10:47:01 -0400 Subject: [PATCH 21/24] Improve FMLP queue management. The next owner of a FMLP-protected resource is dequeued from the FMLP FIFO queue by unlock() (when the resource is freed by the previous owner) instead of performing the dequeue by the next owner immediately after it has been woken up. This simplifies the code a little bit and also reduces potential spinlock contention. --- include/litmus/litmus.h | 2 +- litmus/locking.c | 12 +++++++----- litmus/sched_gsn_edf.c | 4 +--- litmus/sched_psn_edf.c | 6 +----- 4 files changed, 10 insertions(+), 14 deletions(-) diff --git a/include/litmus/litmus.h b/include/litmus/litmus.h index 94086e2..e7769ca 100644 --- a/include/litmus/litmus.h +++ b/include/litmus/litmus.h @@ -26,7 +26,7 @@ static inline int in_list(struct list_head* list) ); } -struct task_struct* waitqueue_first(wait_queue_head_t *wq); +struct task_struct* __waitqueue_remove_first(wait_queue_head_t *wq); #define NO_CPU 0xffffffff diff --git a/litmus/locking.c b/litmus/locking.c index 728b568..2693f1a 100644 --- a/litmus/locking.c +++ b/litmus/locking.c @@ -107,16 +107,18 @@ asmlinkage long sys_litmus_unlock(int lock_od) return err; } -struct task_struct* waitqueue_first(wait_queue_head_t *wq) +struct task_struct* __waitqueue_remove_first(wait_queue_head_t *wq) { - wait_queue_t *q; + wait_queue_t* q; + struct task_struct* t = NULL; if (waitqueue_active(wq)) { q = list_entry(wq->task_list.next, wait_queue_t, task_list); - return (struct task_struct*) q->private; - } else - return NULL; + t = (struct task_struct*) q->private; + __remove_wait_queue(wq, q); + } + return(t); } diff --git a/litmus/sched_gsn_edf.c b/litmus/sched_gsn_edf.c index c5c9600..08b8847 100644 --- a/litmus/sched_gsn_edf.c +++ b/litmus/sched_gsn_edf.c @@ -776,8 +776,6 @@ int gsnedf_fmlp_lock(struct litmus_lock* l) * ->owner. We can thus check it without acquiring the spin * lock. */ BUG_ON(sem->owner != t); - - remove_wait_queue(&sem->wait, &wait); } else { /* it's ours now */ sem->owner = t; @@ -803,7 +801,7 @@ int gsnedf_fmlp_unlock(struct litmus_lock* l) } /* check if there are jobs waiting for this resource */ - next = waitqueue_first(&sem->wait); + next = __waitqueue_remove_first(&sem->wait); if (next) { /* next becomes the resouce holder */ sem->owner = next; diff --git a/litmus/sched_psn_edf.c b/litmus/sched_psn_edf.c index abb06fa..71c0240 100644 --- a/litmus/sched_psn_edf.c +++ b/litmus/sched_psn_edf.c @@ -442,10 +442,6 @@ int psnedf_fmlp_lock(struct litmus_lock* l) * ->owner. We can thus check it without acquiring the spin * lock. */ BUG_ON(sem->owner != t); - - /* FIXME: could we punt the dequeuing to the previous job, - * which is holding the spinlock anyway? */ - remove_wait_queue(&sem->wait, &wait); } else { /* it's ours now */ sem->owner = t; @@ -478,7 +474,7 @@ int psnedf_fmlp_unlock(struct litmus_lock* l) unboost_priority(t); /* check if there are jobs waiting for this resource */ - next = waitqueue_first(&sem->wait); + next = __waitqueue_remove_first(&sem->wait); if (next) { /* boost next job */ boost_priority(next); -- 1.7.2.3 From 6d4cc883ec2470500be6c95fd2e7c6944e89c3e8 Mon Sep 17 00:00:00 2001 From: Bjoern B. Brandenburg Date: Sat, 12 Feb 2011 16:40:43 -0500 Subject: [PATCH 22/24] bugfix: release master CPU must signal task was picked Otherwise, the release master CPU may try to reschedule in an infinite loop. --- litmus/sched_gsn_edf.c | 4 +++- 1 files changed, 3 insertions(+), 1 deletions(-) diff --git a/litmus/sched_gsn_edf.c b/litmus/sched_gsn_edf.c index 08b8847..3092797 100644 --- a/litmus/sched_gsn_edf.c +++ b/litmus/sched_gsn_edf.c @@ -374,8 +374,10 @@ static struct task_struct* gsnedf_schedule(struct task_struct * prev) /* Bail out early if we are the release master. * The release master never schedules any real-time tasks. */ - if (gsnedf.release_master == entry->cpu) + if (gsnedf.release_master == entry->cpu) { + sched_state_task_picked(); return NULL; + } #endif raw_spin_lock(&gsnedf_lock); -- 1.7.2.3 From 7d754596756240fa918b94cd0c3011c77a638987 Mon Sep 17 00:00:00 2001 From: Christopher Kenna Date: Sat, 16 Apr 2011 20:12:00 -0400 Subject: [PATCH 23/24] LITMUS Core: Check for valid class in RT-param syscall. --- litmus/litmus.c | 8 ++++++++ 1 files changed, 8 insertions(+), 0 deletions(-) diff --git a/litmus/litmus.c b/litmus/litmus.c index 11ccaaf..26938ac 100644 --- a/litmus/litmus.c +++ b/litmus/litmus.c @@ -110,6 +110,14 @@ asmlinkage long sys_set_rt_task_param(pid_t pid, struct rt_task __user * param) "because wcet > period\n", pid); goto out_unlock; } + if ( tp.cls != RT_CLASS_HARD && + tp.cls != RT_CLASS_SOFT && + tp.cls != RT_CLASS_BEST_EFFORT) + { + printk(KERN_INFO "litmus: real-time task %d rejected " + "because its class is invalid\n"); + goto out_unlock; + } if (tp.budget_policy != NO_ENFORCEMENT && tp.budget_policy != QUANTUM_ENFORCEMENT && tp.budget_policy != PRECISE_ENFORCEMENT) -- 1.7.2.3 From 6ba981624ea7195b01a56577f1fbef4531cacd83 Mon Sep 17 00:00:00 2001 From: Jonathan Herman Date: Thu, 8 Sep 2011 15:59:37 -0400 Subject: [PATCH 24/24] tmp-branch --- include/litmus/rt_domain.h | 23 +- include/litmus/rt_param.h | 7 +- include/litmus/sched_trace.h | 4 +- include/litmus/servers.h | 221 +++++ litmus/Makefile | 3 +- litmus/bheap.c | 3 + litmus/edf_common.c | 4 +- litmus/litmus.c | 13 +- litmus/rt_domain.c | 59 +- litmus/sched_edf_hsb.c | 2204 ++++++++++++++++++++++++++++++++++++++++++ litmus/servers.c | 767 +++++++++++++++ 11 files changed, 3273 insertions(+), 35 deletions(-) create mode 100644 include/litmus/servers.h create mode 100644 litmus/sched_edf_hsb.c create mode 100644 litmus/servers.c diff --git a/include/litmus/rt_domain.h b/include/litmus/rt_domain.h index ac24929..0756f30 100644 --- a/include/litmus/rt_domain.h +++ b/include/litmus/rt_domain.h @@ -80,7 +80,7 @@ void rt_domain_init(rt_domain_t *rt, bheap_prio_t order, void __add_ready(rt_domain_t* rt, struct task_struct *new); void __merge_ready(rt_domain_t* rt, struct bheap *tasks); -void __add_release(rt_domain_t* rt, struct task_struct *task); +int __add_release(rt_domain_t* rt, struct task_struct *task); static inline struct task_struct* __take_ready(rt_domain_t* rt) { @@ -140,26 +140,31 @@ static inline struct task_struct* take_ready(rt_domain_t* rt) } -static inline void add_release(rt_domain_t* rt, struct task_struct *task) +static inline int add_release(rt_domain_t* rt, struct task_struct *task) { + int rv; unsigned long flags; raw_spin_lock_irqsave(&rt->tobe_lock, flags); - __add_release(rt, task); + rv = __add_release(rt, task); raw_spin_unlock_irqrestore(&rt->tobe_lock, flags); + return rv; } #ifdef CONFIG_RELEASE_MASTER -void __add_release_on(rt_domain_t* rt, struct task_struct *task, - int target_cpu); +int __add_release_on(rt_domain_t* rt, struct task_struct *task, + int target_cpu); -static inline void add_release_on(rt_domain_t* rt, - struct task_struct *task, - int target_cpu) +static inline int add_release_on(rt_domain_t* rt, + struct task_struct *task, + int target_cpu) { + int rv; unsigned long flags; raw_spin_lock_irqsave(&rt->tobe_lock, flags); - __add_release_on(rt, task, target_cpu); + rv = __add_release_on(rt, task, target_cpu); raw_spin_unlock_irqrestore(&rt->tobe_lock, flags); + + return rv; } #endif diff --git a/include/litmus/rt_param.h b/include/litmus/rt_param.h index 5de422c..d40fdad 100644 --- a/include/litmus/rt_param.h +++ b/include/litmus/rt_param.h @@ -5,6 +5,7 @@ #ifndef _LINUX_RT_PARAM_H_ #define _LINUX_RT_PARAM_H_ +struct server; /* Litmus time type. */ typedef unsigned long long lt_t; @@ -22,7 +23,7 @@ static inline int lt_after_eq(lt_t a, lt_t b) /* different types of clients */ typedef enum { - RT_CLASS_HARD, + RT_CLASS_HARD = 0, RT_CLASS_SOFT, RT_CLASS_BEST_EFFORT } task_class_t; @@ -191,12 +192,16 @@ struct rt_param { /* Pointer to the page shared between userspace and kernel. */ struct control_page * ctrl_page; + + /* Used for plugin-specific information */ + void* plugin_data; }; /* Possible RT flags */ #define RT_F_RUNNING 0x00000000 #define RT_F_SLEEP 0x00000001 #define RT_F_EXIT_SEM 0x00000008 +#define RT_F_BLOCK 0x00000010 #endif diff --git a/include/litmus/sched_trace.h b/include/litmus/sched_trace.h index 7ca34cb..e12a993 100644 --- a/include/litmus/sched_trace.h +++ b/include/litmus/sched_trace.h @@ -71,8 +71,8 @@ struct st_resume_data { /* A task resumes. */ struct st_action_data { u64 when; - u8 action; - u8 __unused[7]; + u16 action; + u8 __unused[6]; }; struct st_sys_release_data { diff --git a/include/litmus/servers.h b/include/litmus/servers.h new file mode 100644 index 0000000..b907fec --- /dev/null +++ b/include/litmus/servers.h @@ -0,0 +1,221 @@ +#ifndef _LINUX_SERVERS_H_ +#define _LINUX_SERVERS_H_ + +struct server; +struct periodic_server; +struct proc_dir_entry; +struct server_domain; +struct server_release_heap; +struct completion_timer; +struct server_proc; + +#define SERVER_RELEASE_QUEUE_SLOTS 127 + +#define SERVER_FMT "{server/%d:%d}" +#define SERVER_ARGS(s) (s)->id, (s)->job_no +#define TASK_FMT "(%s/%d:%d)" +#define TASK_ARGS(t) (t)->comm, (t)->pid, (t)->rt_param.job_params.job_no + +#define server_task(s) (((s)->cpu != NO_CPU)?s->domain->linked_tasks[(s)->cpu]:NULL) +#define is_server_linked(s) ((s)->cpu != NO_CPU) + +/* + * A single schedulable server. + */ +typedef struct server { + /* Specified by the user */ + int id; + lt_t wcet; + lt_t period; + + /* Optional */ + int type; + void* data; + + /* Managed internally */ + lt_t deadline; + lt_t release; + lt_t budget; /* The remaining budget for current period */ + int job_no; /* Current job of server */ + int cpu; /* CPU the server is running on or NO_CPU */ + + struct server_domain *domain; + + /* For membership in collections */ + struct bheap_node *hn; + struct list_head list; + + /* Used for grouped releases */ + struct server_release_heap *release_heap; + struct list_head release_list; +} server_t; + +/* + * Called when a server exhausts its budget. + */ +typedef void (*server_completed_t)(struct server *server, + struct task_struct *was_running); +/* + * Called when a group of servers release + */ +typedef void (*servers_released_t)(struct list_head *servers); +/* + * Used to read server entries. + */ +typedef int (*admit_server_t)(unsigned long long wcet, + unsigned long long period, int cpu); +/* + * Lists all servers for a proc entry by calling list_server on each. + */ +typedef void (*list_servers_t)(struct server_proc *proc); +/* + * Stop all servers. Used to destroy servers on a proc entry rewrite. + */ +typedef void (*stop_servers_t)(void); + +/* + * Useful tools for scheduling servers. + */ +typedef struct server_domain { + /* Collection of grouped releases */ + raw_spinlock_t release_lock; + struct list_head release_queue[SERVER_RELEASE_QUEUE_SLOTS]; + + /* List of tasks to be added to the grouped releases */ + raw_spinlock_t tobe_lock; + struct list_head tobe_released; + + /* CPU on which to release servers */ + int release_master; + + /* Per CPU information for running servers */ + struct completion_timer* completion_timers; + server_t** linked_servers; + struct task_struct** linked_tasks; + lt_t* start_times; + + /* Used to lock firing of the completion timer. + * This is needed here and not for the release timer because + * the completion timer actually modifies the state of the + * server itself. + */ + raw_spinlock_t* completion_lock; + + /* Event callbacks */ + server_completed_t server_completed; + servers_released_t servers_released; + + /* Proc entries for controlling groups of servers */ + struct list_head server_procs; +} server_domain_t; + +/* + * A group of servers releasing simultaneously. + */ +typedef struct server_release_heap { + /* Servers to be released */ + struct list_head servers; + lt_t release_time; + + /* For membership in the domain */ + struct list_head list; + + /* For callbacks */ + server_domain_t *domain; + + struct hrtimer timer; + struct hrtimer_start_on_info info; +} server_release_heap_t; + +/* + * A timer for managing server completions. Can be managed concurrently. + */ +typedef struct completion_timer { + int armed; /* Is the timer armed or not? Seperate from the timer + * so that it can be used to disarm a timer which + * is already firing. + */ + int cpu; /* CPU where the server is running. This is not the + * cpu on which the timer will fire. + */ + struct hrtimer timer; + struct hrtimer_start_on_info info; + struct server_domain *domain; /* For callbacks */ +} completion_timer_t; + +/* + * A proc directory entry which controls a group of servers. + */ +typedef struct server_proc { + struct proc_dir_entry *entry; + struct list_head list; + admit_server_t admit_server; /* Add a server from the entry */ + list_servers_t list_servers; /* List each server in the entry */ + stop_servers_t stop_servers; /* Disables all servers in the entry */ + char* page; /* Used internally by proc */ + int length; /* Used internally by proc */ +} server_proc_t; + +/* + * Initialize and exit servers + */ +void server_init(server_t *server, server_domain_t *domain, int id, + lt_t wcet, lt_t period, int grouped); +void server_destroy(server_t *server); + +/* + * Memory manage servers on the module slabs. + */ +server_t* server_alloc(int gfp_flags); +void server_free(server_t *server); + +/* + * Initialize and exit the server domain. + */ +void server_domain_init(server_domain_t *domain, + servers_released_t servers_released, + server_completed_t server_completed, + int release_master, raw_spinlock_t* completion_lock); +void server_domain_destroy(server_domain_t *domain); + +/* + * Adds the next release of the server to the domain's timer. + */ +int add_server_release(server_t *server, server_domain_t *server_domain); + +/* + * Runs a task on the server. + */ +void server_run(server_t *server, struct task_struct *task); + +/* + * Stops server execution. + */ +void server_stop(server_t *server); + +/* + * Begins a server's next period. + */ +void server_release(server_t *server); + +/* + * Set the next period to begin at the given time. + */ +void server_release_at(server_t *server, lt_t time); + +/* + * Call once for every server which should be printed by list_servers. + */ +void list_server(server_t *server, int cpu, server_proc_t *proc); + +/* + * Create and destroy a proc dir entry with the given file name. + */ +server_proc_t* server_proc_init(server_domain_t *domain, + struct proc_dir_entry *proc_dir, char *file, + admit_server_t admit_server, + list_servers_t list_servers, + stop_servers_t stop_servers); +void server_proc_exit(server_proc_t *proc); + +#endif diff --git a/litmus/Makefile b/litmus/Makefile index ad9936e..62c2bb0 100644 --- a/litmus/Makefile +++ b/litmus/Makefile @@ -16,8 +16,9 @@ obj-y = sched_plugin.o litmus.o \ srp.o \ bheap.o \ ctrldev.o \ + servers.o \ sched_gsn_edf.o \ - sched_psn_edf.o + sched_edf_hsb.o obj-$(CONFIG_PLUGIN_CEDF) += sched_cedf.o obj-$(CONFIG_PLUGIN_PFAIR) += sched_pfair.o diff --git a/litmus/bheap.c b/litmus/bheap.c index 528af97..cf9a0f8 100644 --- a/litmus/bheap.c +++ b/litmus/bheap.c @@ -276,6 +276,9 @@ void bheap_delete(bheap_prio_t higher_prio, struct bheap* heap, pos = heap->head; while (pos != node) { prev = pos; + /* a dereferencing error here means that + * the node was not in this heap + */ pos = pos->next; } /* we have prev, now remove node */ diff --git a/litmus/edf_common.c b/litmus/edf_common.c index 9b44dc2..1bb3452 100644 --- a/litmus/edf_common.c +++ b/litmus/edf_common.c @@ -114,5 +114,7 @@ int edf_preemption_needed(rt_domain_t* rt, struct task_struct *t) */ /* make sure to get non-rt stuff out of the way */ - return !is_realtime(t) || edf_higher_prio(__next_ready(rt), t); + return !is_realtime(t) || + (get_deadline(__next_ready(rt)) != get_deadline(t) && + edf_higher_prio(__next_ready(rt), t)); } diff --git a/litmus/litmus.c b/litmus/litmus.c index 26938ac..64f82aa 100644 --- a/litmus/litmus.c +++ b/litmus/litmus.c @@ -16,6 +16,7 @@ #include #include #include +#include /* Number of RT tasks that exist in the system */ atomic_t rt_task_count = ATOMIC_INIT(0); @@ -375,12 +376,12 @@ void litmus_exit_task(struct task_struct* tsk) litmus->task_exit(tsk); - BUG_ON(bheap_node_in_heap(tsk_rt(tsk)->heap_node)); - bheap_node_free(tsk_rt(tsk)->heap_node); - release_heap_free(tsk_rt(tsk)->rel_heap); - + if (!bheap_node_in_heap(tsk_rt(tsk)->heap_node)) { + bheap_node_free(tsk_rt(tsk)->heap_node); + release_heap_free(tsk_rt(tsk)->rel_heap); + reinit_litmus_state(tsk, 1); + } atomic_dec(&rt_task_count); - reinit_litmus_state(tsk, 1); } } @@ -527,7 +528,7 @@ static int __init _init_litmus(void) register_sched_plugin(&linux_sched_plugin); - bheap_node_cache = KMEM_CACHE(bheap_node, SLAB_PANIC); + bheap_node_cache = KMEM_CACHE(bheap_node, SLAB_PANIC); release_heap_cache = KMEM_CACHE(release_heap, SLAB_PANIC); #ifdef CONFIG_MAGIC_SYSRQ diff --git a/litmus/rt_domain.c b/litmus/rt_domain.c index 81a5ac1..011a381 100644 --- a/litmus/rt_domain.c +++ b/litmus/rt_domain.c @@ -147,29 +147,41 @@ static struct release_heap* get_release_heap(rt_domain_t *rt, return heap; } -static void reinit_release_heap(struct task_struct* t) +static int reinit_release_heap(struct task_struct* t) { + int rv = 0; struct release_heap* rh; /* use pre-allocated release heap */ rh = tsk_rt(t)->rel_heap; - /* Make sure it is safe to use. The timer callback could still - * be executing on another CPU; hrtimer_cancel() will wait - * until the timer callback has completed. However, under no - * circumstances should the timer be active (= yet to be - * triggered). - * + /* * WARNING: If the CPU still holds the release_lock at this point, * deadlock may occur! */ - BUG_ON(hrtimer_cancel(&rh->timer)); + rv = hrtimer_try_to_cancel(&rh->timer); + + /* The timer callback is running, it is useless to add + * to the release heap now. + */ + if (rv == -1) { + rv = 0; + goto out; + } + + /* Under no cirumstances should the timer have been active + * but not running. + */ + BUG_ON(rv == 1); + rv = 1; /* initialize */ bheap_init(&rh->heap); #ifdef CONFIG_RELEASE_MASTER atomic_set(&rh->info.state, HRTIMER_START_ON_INACTIVE); #endif + out: + return rv; } /* arm_release_timer() - start local release timer or trigger * remote timer (pull timer) @@ -180,11 +192,12 @@ static void reinit_release_heap(struct task_struct* t) */ #ifdef CONFIG_RELEASE_MASTER #define arm_release_timer(t) arm_release_timer_on((t), NO_CPU) -static void arm_release_timer_on(rt_domain_t *_rt , int target_cpu) +static int arm_release_timer_on(rt_domain_t *_rt , int target_cpu) #else -static void arm_release_timer(rt_domain_t *_rt) +static int arm_release_timer(rt_domain_t *_rt) #endif { + int rv = 1; rt_domain_t *rt = _rt; struct list_head list; struct list_head *pos, *safe; @@ -211,9 +224,15 @@ static void arm_release_timer(rt_domain_t *_rt) VTRACE_TASK(t, "Dropped release_lock 0x%p\n", &rt->release_lock); - reinit_release_heap(t); + rv = reinit_release_heap(t); VTRACE_TASK(t, "release_heap ready\n"); + /* Bail! The heap we should be using just released right + * before we added ourselves to it. + */ + if (!rv) + goto out; + raw_spin_lock(&rt->release_lock); VTRACE_TASK(t, "Re-acquired release_lock 0x%p\n", &rt->release_lock); @@ -257,6 +276,9 @@ static void arm_release_timer(rt_domain_t *_rt) } else VTRACE_TASK(t, "0x%p is not my timer\n", &rh->timer); } + + out: + return rv; } void rt_domain_init(rt_domain_t *rt, @@ -319,9 +341,11 @@ void __merge_ready(rt_domain_t* rt, struct bheap* tasks) #ifdef CONFIG_RELEASE_MASTER -void __add_release_on(rt_domain_t* rt, struct task_struct *task, +int __add_release_on(rt_domain_t* rt, struct task_struct *task, int target_cpu) { + int rv; + TRACE_TASK(task, "add_release_on(), rel=%llu, target=%d\n", get_release(task), target_cpu); list_add(&tsk_rt(task)->list, &rt->tobe_released); @@ -330,17 +354,20 @@ void __add_release_on(rt_domain_t* rt, struct task_struct *task, /* start release timer */ TS_SCHED2_START(task); - arm_release_timer_on(rt, target_cpu); + rv = arm_release_timer_on(rt, target_cpu); TS_SCHED2_END(task); + + return rv; } #endif /* add_release - add a real-time task to the rt release queue. * @task: the sleeping task */ -void __add_release(rt_domain_t* rt, struct task_struct *task) +int __add_release(rt_domain_t* rt, struct task_struct *task) { + int rv; TRACE_TASK(task, "add_release(), rel=%llu\n", get_release(task)); list_add(&tsk_rt(task)->list, &rt->tobe_released); task->rt_param.domain = rt; @@ -348,8 +375,10 @@ void __add_release(rt_domain_t* rt, struct task_struct *task) /* start release timer */ TS_SCHED2_START(task); - arm_release_timer(rt); + rv = arm_release_timer(rt); TS_SCHED2_END(task); + + return rv; } diff --git a/litmus/sched_edf_hsb.c b/litmus/sched_edf_hsb.c new file mode 100644 index 0000000..4bc6c13 --- /dev/null +++ b/litmus/sched_edf_hsb.c @@ -0,0 +1,2204 @@ +/* + * litmus/sched_edf_hsb.c + * + * Implementation of the EDF-HSB scheduling algorithm. + */ +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#define DEBUG_EDF_HSB + +#define BE_PROC_NAME "be_servers" +#define HRT_PROC_NAME "hrt_servers" +#define BE_SERVER_BASE 100 +#define IDLE_SLACK_BASE 1000 +#define SLACK_MIN NSEC_PER_MSEC + +/* Uncomment for human readable time */ +#define TIME(x) \ +({lt_t y = x; \ +do_div(y, NSEC_PER_MSEC); \ +y;}) +#define TRACE_TIMER(fmt, args...) \ + sched_trace_log_message("%d P%d*[%s@%s:%d]: " fmt " at %d\n", \ + TRACE_ARGS, ## args, TIME(litmus_clock())) +#define TRACE_TASK_TIMER(t, fmt, args...) \ + TRACE_TIMER("(%s/%d:%d) " fmt, (t)->comm, (t)->pid, \ + (t)->rt_param.job_params.job_no, ## args) + +/* + * Different types of servers + */ +typedef enum { + S_HRT, + S_SRT, + S_BE, + S_SLACK +} server_type_t; + +/* + * A server running HRT tasks + */ +typedef struct { + server_t server; + rt_domain_t hrt_domain; /* EDF for HRT tasks assigned here */ + int ready; /* False if waiting for next release */ + int no_slack; + struct hrtimer slack_timer; /* Server has no slack when: + * (deadline - budget) <= current_time. + */ + struct hrtimer_start_on_info slack_timer_info; +} hrt_server_t; + +/* + * State of a single CPU + */ +typedef struct { + int cpu; + struct task_struct* scheduled; /* Task that should be running */ + struct task_struct* linked; /* Task that actually is running */ + server_t *scheduled_server; + server_t *linked_server; /* The server running on this cpu. + * Note that what it is 'running' is + * linked, not scheduled. + */ + hrt_server_t hrt_server; /* One HRT server per CPU */ + struct bheap_node* hn; /* For the cpu_heap */ +} cpu_entry_t; + +/* + * Data assigned to each task + */ +typedef struct task_data { + server_t *srt_server; /* If the task is SRT, its server */ + struct list_head candidate_list; /* List of slack canditates */ + struct task_struct *owner; +} task_data_t; + +/* CPU state */ +DEFINE_PER_CPU_SHARED_ALIGNED(cpu_entry_t, cpu_entries); +static struct bheap cpu_heap; +static struct bheap_node cpu_heap_node[NR_CPUS]; +/* Task domains */ +static rt_domain_t srt_domain; +static rt_domain_t be_domain; +/* Useful tools for server scheduling */ +static server_domain_t server_domain; +/* BE server support */ +static struct list_head be_servers; +static struct bheap be_ready_servers; +/* Slack support */ +static struct list_head slack_queue; +static struct list_head slack_candidates; +/* CPU which will release tasks and global servers */ +static int edf_hsb_release_master; +/* Cache to store task_data structs */ +static struct kmem_cache *task_data_cache; + +static struct proc_dir_entry *edf_hsb_proc_dir = NULL; +static struct sched_plugin edf_hsb_plugin __cacheline_aligned_in_smp; + +#define task_sched_entry(task) (&per_cpu(cpu_entries, task_cpu(task))) +#define task_linked_entry(task) (&per_cpu(cpu_entries, task->rt_param.linked_on)) +#define task_job_no(task) (tsk_rt(task)->job_params.job_no) +#define task_data(task) ((task_data_t*)tsk_rt(task)->plugin_data) +#define task_srt_server(task) ((server_t*)task_data(task)->srt_server) +#define server_slack(s) ((server_t*)(s)->data) +#define server_has_slack(s) (server_slack(s)->deadline != 0) +#define local_cpu_entry (&__get_cpu_var(cpu_entries)) +#define global_lock (&srt_domain.ready_lock) +#define is_active_plugin (litmus == &edf_hsb_plugin) + +/* + * This only works if items are deleted with list_del_init. + */ +static inline int head_in_list(struct list_head *head) +{ + return !(head->next == head->prev && head->prev == head); +} + +/* + * Returns slack server running the task or NULL if N/A. + */ +static inline server_t* task_slack_server(struct task_struct *task) +{ + server_t *slack_server = NULL; + if (task->rt_param.linked_on != NO_CPU) { + slack_server = task_linked_entry(task)->linked_server; + if (slack_server->type != S_SLACK) + slack_server = NULL; + } + return slack_server; +} + +static task_data_t* task_data_alloc(int gfp_flags) +{ + return kmem_cache_alloc(task_data_cache, gfp_flags); +} + +static void task_data_free(task_data_t* data) +{ + kmem_cache_free(task_data_cache, data); +} + +/* + * Donating servers pre-allocate a server for slack to avoid runtime + * calls to kmalloc. + */ +static void server_slack_create(server_t *donator) +{ + server_t *slack = server_alloc(GFP_ATOMIC); + + server_init(slack, &server_domain, -donator->id, 0, 0, 1); + slack->type = S_SLACK; + slack->data = donator; + donator->data = slack; +} + + +static void server_slack_destroy(server_t *donator) +{ + server_t *slack = (server_t*)donator->data; + + donator->data = NULL; + server_destroy(slack); + server_free(slack); +} + +static void remove_slack(server_t *slack) +{ + if (!slack) + return; + + if (head_in_list(&slack->list)) + list_del_init(&slack->list); + slack->deadline = 0; + slack->budget = 0; + slack->wcet = 0; +} + +/* + * Slack queue is EDF. + */ +static void add_slack(server_t *slack) +{ + struct list_head *pos; + server_t *queued; + + if (head_in_list(&slack->list)) + return; + + list_for_each_prev(pos, &slack_queue) { + queued = list_entry(pos, server_t, list); + if (lt_before_eq(queued->deadline, slack->deadline)) { + __list_add(&slack->list, pos, pos->next); + return; + } + } + list_add(&slack->list, &slack_queue); +} + +static inline struct task_struct* get_candidate(struct list_head *pos) +{ + struct task_struct *task = NULL; + task_data_t *data; + if (!list_empty(pos)) { + data = list_entry(pos, task_data_t, candidate_list); + task = data->owner; + } + return task; +} + +static inline lt_t real_deadline(struct task_struct *task) +{ + server_t *server = task_srt_server(task); + int job_diff = server->job_no - task_job_no(task); + return get_deadline(task) - job_diff * get_rt_period(task); +} + +/* + * Candidate queue is EDF. + */ +static void add_slack_candidate(struct task_struct *task) +{ + struct list_head *pos; + struct task_struct *queued; + + list_for_each_prev(pos, &slack_candidates) { + queued = get_candidate(pos); + if (lt_before_eq(real_deadline(queued), real_deadline(task))) { + __list_add(&task_data(task)->candidate_list, + pos, pos->next); + return; + } + } + list_add(&task_data(task)->candidate_list, &slack_candidates); +} + +static void donate_slack(server_t *donator) +{ + server_t *slack = (server_t*)donator->data; + hrt_server_t *hrt_server; + + if (donator->type == S_HRT) { + hrt_server = container_of(donator, hrt_server_t, server); + } + + slack->wcet = donator->budget; + slack->budget = donator->budget; + slack->deadline = donator->deadline; + + add_slack(slack); +} + +#ifdef CONFIG_EDF_HSB_SLACK_STEALING +/* + * Donate any available slack from a server. + */ +static noinline void check_donate_slack(server_t *donator, struct task_struct *was_scheduled) +{ + server_t *slack = server_slack(donator); + hrt_server_t *hrt_server; + int donate = 0; + + if (!slack) + return; + + /* Donating small amounts of slack will result in excess migrations */ + if (donator->budget < SLACK_MIN || server_has_slack(donator)) + return; + + if (server_has_slack(donator)) + return; + + if (donator->type == S_HRT) { + hrt_server = container_of(donator, hrt_server_t, server); + } + + /* Donate if the server is waiting for a task release */ + if ((donator->type == S_SRT && + donator->job_no <= task_job_no(was_scheduled)) || + (donator->type == S_HRT && + hrt_server->no_slack && hrt_server->ready && + !__jobs_pending(&hrt_server->hrt_domain)) || + (donator->type == S_BE && + !__jobs_pending(&be_domain))) { + donate = 1; + } + + if (!donate) + return; + + donate_slack(donator); +} + +#else +#define check_donate_slack(a, b) +#endif + +/* + * Adds the task to the candidate queue if it is eligible for slack stealing. + */ +static void check_slack_candidate(struct task_struct *task) +{ + if (is_srt(task) && + /* The task has been synchronously released */ + task_job_no(task) > 2 && + /* The SRT task is behind its server */ + task_srt_server(task)->job_no > task_job_no(task) && + /* The task hasn't already been added to the list */ + !head_in_list(&task_data(task)->candidate_list)) { + + add_slack_candidate(task); + } else if (is_srt(task) && + is_released(task, litmus_clock()) && + !is_queued(task)) { + __add_ready(&srt_domain, task); + } +} + +/* + * Returns the next eligible slack server. This will remove any expired + * slack servers still present in the list. + */ +static noinline server_t* next_eligible_slack_server(void) +{ + server_t *next_slack = NULL; + lt_t now = litmus_clock(); + + while (!list_empty(&slack_queue)) { + next_slack = list_entry(slack_queue.next, server_t, list); + + if (lt_after(next_slack->deadline, now) && + lt_after(next_slack->budget, SLACK_MIN) && + !is_server_linked(next_slack)) { + break; + } else { + /* Slack has expired or has too little time */ + remove_slack(next_slack); + next_slack = NULL; + } + } + + return next_slack; +} + +/* + * Returns the next SRT task that is tardy or will be tardy. If none + * are available, will return a tardy BE task if present. + */ +static noinline struct task_struct* next_eligible_slack(void) +{ + struct task_struct *next = get_candidate(slack_candidates.next); + + while (next && task_srt_server(next)->job_no <= task_job_no(next)) { + list_del_init(&task_data(next)->candidate_list); + next = get_candidate(slack_candidates.next); + } + + /* We couldn't find an SRT to schedule. Find a BE which is + * either tardy or cannot run due to a lack of servers. + */ + if (!next) { + next = __peek_ready(&be_domain); + } + + return next; +} + +/* + * Order BE tasks FIFO. + */ +static inline int be_higher_prio(struct task_struct *first, struct task_struct *second) +{ + return lt_before(get_release(first), get_release(second)) || + + /* Break by PID */ + (get_release(first) == get_release(second) && + (first->pid < second->pid)); +} + +static int be_ready_order(struct bheap_node *a, struct bheap_node *b) +{ + struct task_struct *first, *second; + first = bheap2task(a); + second = bheap2task(b); + if (!first || !second) + return first && !second; + return be_higher_prio(first, second); +} + +/* + * Order servers by EDF. + */ +static inline int server_higher_prio(server_t *first, server_t *second) +{ + return lt_before(first->deadline, second->deadline) || + /* Break by id */ + (first->deadline == second->deadline && + first->id < second->id); +} + +static int server_order(struct bheap_node *a, struct bheap_node *b) +{ + server_t *first, *second; + first = a->value; + second = b->value; + return server_higher_prio(first, second); +} + +/* + * Order CPU's by deadlines of their servers. + */ +static int cpu_lower_prio(struct bheap_node *a, struct bheap_node *b) +{ + cpu_entry_t *first, *second; + first = a->value; + second = b->value; + if (first->linked && second->linked) { + return !server_higher_prio(first->linked_server, + second->linked_server); + } + return second->linked && !first->linked; +} + +/* + * Move the CPU entry to the correct position in the queue. + */ +static inline void update_cpu_position(cpu_entry_t *entry) +{ + if (likely(bheap_node_in_heap(entry->hn))) + bheap_delete(server_order, &cpu_heap, entry->hn); + /* Don't leave HRT CPUs in the heap as its order only matters + * for global preempts. + */ + if (!entry->linked || !is_hrt(entry->linked)) + bheap_insert(cpu_lower_prio, &cpu_heap, entry->hn); +} + +static inline cpu_entry_t* lowest_prio_cpu(void) +{ + struct bheap_node *hn = bheap_peek(cpu_lower_prio, &cpu_heap); + return (hn) ? hn->value : NULL; +} + +static inline int check_hrt_server_initialized(hrt_server_t *hrt_server) +{ + return hrt_server->server.wcet && hrt_server->server.period; +} + +/* + * Arms the slack timer for the server, if necessary. + */ +static void slack_timer_arm(hrt_server_t *hrt_server) +{ + int cpu, err; + cpu_entry_t *entry; + struct hrtimer *timer; + lt_t now = litmus_clock(), when_to_fire; + + if (!check_hrt_server_initialized(hrt_server)) + return; + + timer = &hrt_server->slack_timer; + entry = container_of(hrt_server, cpu_entry_t, hrt_server); + +#ifdef SLACK_ON_MASTER + if (edf_hsb_release_master != NO_CPU) + cpu = edf_hsb_release_master; + else +#endif + cpu = entry->cpu; + + when_to_fire = hrt_server->server.deadline - hrt_server->server.budget; + + /* Ensure the timer is needed */ + if (hrtimer_active(timer) || hrt_server->server.deadline == 0 || + hrt_server->no_slack || hrt_server->server.budget == 0 || + !hrt_server->ready) { + return; + } + + /* Arm timer */ + if (lt_after_eq(now, when_to_fire)) { + /* 'Fire' immediately */ + hrt_server->no_slack = 1; + } else if (cpu != smp_processor_id()) { + err = hrtimer_start_on(cpu, + &hrt_server->slack_timer_info, + &hrt_server->slack_timer, + ns_to_ktime(when_to_fire), + HRTIMER_MODE_ABS_PINNED); + } else { + __hrtimer_start_range_ns(timer, ns_to_ktime(when_to_fire), + 0, HRTIMER_MODE_ABS_PINNED, 0); + } +} + +/* + * Does nothing if the slack timer is not armed. + */ +static inline void slack_timer_cancel(hrt_server_t *hrt_server) +{ + int ret; + if (hrtimer_active(&hrt_server->slack_timer)) { + ret = hrtimer_try_to_cancel(&hrt_server->slack_timer); + } +} + +/* + * Handles subtraction of lt_t without underflows. + */ +static inline lt_t lt_subtract(lt_t a, lt_t b) +{ + long long sub = (long long)a - (long long)b; + if (sub >= 0) + return sub; + else + return 0; +} + +static void requeue_server(server_t *server, lt_t now) +{ + int added = 0; + hrt_server_t *hrt_server; + + if (server->type == S_SRT) + return; + + if (server->type == S_SLACK) { + add_slack(server); + return; + } + + if (lt_before(now, server->release)) { + added = add_server_release(server, &server_domain); + } + + if (!added) { + /* Mark servers as released */ + if (server->type == S_HRT) { + hrt_server = container_of(server, hrt_server_t, server); + hrt_server->ready = 1; + remove_slack(server_slack(server)); + hrt_server->no_slack = 0; + } else if (server->type == S_BE) { + bheap_insert(server_order, &be_ready_servers, server->hn); + } + } +} + +/* + * Absorbs a task's execution time into its donator. + */ +static void reclaim_slack(server_t *slack) +{ + lt_t exec; + server_t *donator = server_slack(slack); + + if (!donator || lt_before_eq(slack->deadline, litmus_clock())) + return; + + /* SRT servers do not ever reclaim slack */ + exec = slack->wcet - slack->budget; + donator->budget = lt_subtract(donator->budget, exec); + slack->wcet = slack->budget; +} + +/* + * Begins server execution and arms any timers necessary. + */ +static noinline void link_server(cpu_entry_t *entry, + server_t *next_server) +{ + + if (entry->linked) { + if (next_server->type != S_SLACK && + (head_in_list(&server_slack(next_server)->list))) { + remove_slack(server_slack(next_server)); + } + + entry->linked_server = next_server; + server_run(entry->linked_server, entry->linked); + } + + /* Timer necessary whenever an HRT is not running */ + if (!entry->linked || !is_hrt(entry->linked)) + slack_timer_arm(&entry->hrt_server); + else + slack_timer_cancel(&entry->hrt_server); +} + +/* + * Stops server execution and timers. This will also re-add servers + * to any collections they should be members of. + */ +static noinline void unlink_server(cpu_entry_t *entry, int requeue) +{ + server_t *server = entry->linked_server; + + server_stop(entry->linked_server); + server = entry->linked_server; + entry->linked_server = NULL; + + if (!requeue) + return; + + if (server->type == S_SLACK && server->deadline) { + add_slack(server); + + /* Donator needs to absorb slack execution time */ + reclaim_slack(server); + } else if (server->type != S_SRT) { + requeue_server(server, litmus_clock()); + } +} + +static void requeue(struct task_struct *task, rt_domain_t *domain); +static inline rt_domain_t* get_rt_domain(cpu_entry_t *entry, struct task_struct *task); + +/* Update the link of a CPU. + * Handles the case where the to-be-linked task is already + * scheduled on a different CPU. The last argument is only needed + * for BE tasks as their servers can't be determined here. + */ +static noinline void link_to_cpu(cpu_entry_t *entry, + struct task_struct* linked, + server_t* next_server) +{ + cpu_entry_t *sched; + server_t *tmp_server; + struct task_struct *tmp_task; + int on_cpu; + + /* Currently linked task is set to be unlinked. */ + if (entry->linked) { + unlink_server(entry, 1); + entry->linked->rt_param.linked_on = NO_CPU; + entry->linked = NULL; + } + + /* Link new task to CPU. */ + if (linked) { + set_rt_flags(linked, RT_F_RUNNING); + /* Handle task is already scheduled somewhere! */ + on_cpu = linked->rt_param.scheduled_on; + if (on_cpu != NO_CPU) { + sched = &per_cpu(cpu_entries, on_cpu); + /* This should only happen if not linked already */ + BUG_ON(sched->linked == linked); + + if (entry != sched && + sched->linked && is_hrt(sched->linked)) { + /* We are already scheduled on a CPU with an HRT */ + requeue_server(next_server, litmus_clock()); + requeue(linked, get_rt_domain(entry, linked)); + + linked = NULL; + next_server = NULL; + } else if (entry != sched) { + /* Link to the CPU we are scheduled on by swapping + * with that CPU's linked task. + */ + tmp_task = sched->linked; + tmp_server = sched->linked_server; + + if (tmp_task) + unlink_server(sched, 0); + + linked->rt_param.linked_on = sched->cpu; + sched->linked = linked; + link_server(sched, next_server); + + update_cpu_position(sched); + + linked = tmp_task; + next_server = tmp_server; + } + } + if (linked) /* Might be NULL due to swap */ + linked->rt_param.linked_on = entry->cpu; + } + entry->linked = linked; + link_server(entry, next_server); + update_cpu_position(entry); +} + +/* + * Grab the local HRT or global SRT or BE domain for the task. + */ +static inline rt_domain_t* get_rt_domain(cpu_entry_t *entry, + struct task_struct *task) +{ + if (is_hrt(task)) + return &entry->hrt_server.hrt_domain; + else if (is_srt(task)) + return &srt_domain; + else /* BE */ + return &be_domain; +} + +/* + * Ensures the task is not linked anywhere nor present in any ready queues. + */ +static noinline void unlink(struct task_struct* t) +{ + cpu_entry_t *entry; + + if (t->rt_param.linked_on != NO_CPU) { + /* Unlink */ + entry = task_linked_entry(t); + link_to_cpu(entry, NULL, NULL); + } else if (is_queued(t)) { + entry = task_sched_entry(t); + + /* A task that is unlinked due to a slack server must be treated + * differently. It is probably queued in a release_queue, but + * a race condition could allow is_released() to return true + * even when the task has not yet been released. Attempting + * to remove the task in this case would be disastrous. + */ + if (entry->scheduled == t && + entry->scheduled_server && /* Can be NULL on task_new */ + entry->scheduled_server->type == S_SLACK) { + + TRACE_TASK(t, "unlinked on slack server\n"); + + } else if (is_released(t, litmus_clock())) { + /* This is an interesting situation: t is scheduled, + * but has already been unlinked. It was re-added to + * a ready queue of some sort but now needs to + * be removed. This usually happens when a job has + * been preempted but completes before it is + * descheduled. + */ + remove(get_rt_domain(entry, t), t); + BUG_ON(is_queued(t)); + } + } + + if (head_in_list(&task_data(t)->candidate_list)) { + list_del_init(&task_data(t)->candidate_list); + } +} + +/* + * A job generated by a HRT task is eligible if either the job's deadline + * is earlier than the server's next deadline, or the server has zero slack + * time in its current period. + */ +static inline int is_eligible(struct task_struct *task, + hrt_server_t *hrt_server) +{ + return hrt_server->ready && !is_server_linked(&hrt_server->server) && + (hrt_server->no_slack || + lt_after_eq(hrt_server->server.deadline, get_deadline(task))); +} + +/* + * Set the server to release at the closest preceding deadline to time. + */ +static inline void catchup_server(server_t *server, lt_t time) +{ + lt_t diff, sub; + diff = time - server->deadline; + sub = diff % server->period; + server_release_at(server, time - sub); +} + +static noinline int catchup_srt_server(struct task_struct *task) +{ + int jobs, rv = 0; + lt_t release; + lt_t now = litmus_clock(); + server_t *srt_server = task_srt_server(task); + + if (lt_before(srt_server->deadline, now) && + srt_server->job_no > 1) { + /* Calculate the number of jobs behind the server is */ + jobs = lt_subtract(now, srt_server->deadline) / + srt_server->period + 1; + + /* Get the new release */ + release = srt_server->release + jobs * srt_server->period; + + /* Update server state */ + server_release_at(srt_server, release); + srt_server->job_no += jobs - 1; + + /* Force task to take characteristics of server */ + tsk_rt(task)->job_params.release = srt_server->release; + tsk_rt(task)->job_params.deadline = srt_server->deadline; + + rv = 1; + } else if (lt_before(srt_server->deadline, now) && + srt_server->job_no <= 1) { + + server_release_at(srt_server, get_release(task)); + srt_server->job_no = task_job_no(task); + } + + return rv; +} + +/* + * If the server is eligible, return the next eligible job. If the server is + * ineligible or there are no eligible jobs, returns NULL. This will re-release + * any servers that are behind. + */ +static noinline struct task_struct* next_eligible_hrt(hrt_server_t *hrt_server) +{ + lt_t now = litmus_clock(); + lt_t dead, slack, budget; + struct task_struct *task = __peek_ready(&hrt_server->hrt_domain); + + /* Catch up server if it is initialized, not running, and late */ + if (check_hrt_server_initialized(hrt_server) && + !is_server_linked(&hrt_server->server)) { + + dead = hrt_server->server.deadline; + budget = hrt_server->server.budget; + slack = lt_subtract(dead, budget); + + if (!head_in_list(&hrt_server->server.release_list) && + lt_before_eq(dead, now)) { + /* The server missed a release */ + catchup_server(&hrt_server->server, now); + hrt_server->ready = 1; + remove_slack(server_slack(&hrt_server->server)); + hrt_server->no_slack = 0; + + slack = lt_subtract(hrt_server->server.deadline, + hrt_server->server.budget); + } + + /* If the slack timer is active, this is not necessary */ + if (!hrtimer_active(&hrt_server->slack_timer) && hrt_server->ready) { + if (lt_before_eq(slack, now) && !hrt_server->no_slack) { + /* The server missed the shift to no slack */ + hrt_server->no_slack = 1; + } else { + slack_timer_arm(hrt_server); + } + } + + } + + if (!hrt_server->server.budget || + (task && !is_eligible(task, hrt_server))) { + + if (!hrt_server->server.budget && + !head_in_list(&hrt_server->server.release_list)) { + catchup_server(&hrt_server->server, now); + requeue_server(&hrt_server->server, now); + slack_timer_arm(hrt_server); + } + + task = NULL; + + /* Donate slack if we have nothing to schedule */ + if (hrt_server->ready && hrt_server->no_slack) { + check_donate_slack(&hrt_server->server, NULL); + } + } + + return task; +} + +/* + * This will catch up the SRT's server if it is behind. + */ +static noinline struct task_struct* next_eligible_srt(void) +{ + int done = 0; + struct task_struct *next_srt; + + while (!done) { + next_srt = __peek_ready(&srt_domain); + + /* A blocking task might pollute the SRT domain if the + * task blocked while it was being run by a slack server. + * Remove and ignore this task. + */ + while (next_srt && (get_rt_flags(next_srt) == RT_F_BLOCK || + unlikely(!is_realtime(next_srt)) || + tsk_rt(next_srt)->linked_on != NO_CPU)) { + remove(&srt_domain, next_srt); + next_srt = __peek_ready(&srt_domain); + } + + /* If the task blocked for awhile or has otherwise not been + * accessed, its server could have fallen behind. + */ + if (next_srt) { + done = !catchup_srt_server(next_srt); + + /* The parameters were modified. Re-insert the task. */ + if (!done) { + remove(&srt_domain, next_srt); + __add_ready(&srt_domain, next_srt); + } else if (is_server_linked(task_srt_server(next_srt))){ + remove(&srt_domain, next_srt); + done = 0; + } + } else { + done = 1; + } + } + + return next_srt; +} + +static inline server_t* next_be_server(void) +{ + struct bheap_node *hn = bheap_peek(server_order, &be_ready_servers); + return (hn) ? hn->value : NULL; +} + +static noinline server_t* next_eligible_be_server(void) +{ + server_t *be_server = next_be_server(); + lt_t now = litmus_clock(); + + /* Catch up any late be servers. This happens when the servers could + * not find tasks to schedule or if the system is overutilized. + */ + while (be_server && (lt_before_eq(be_server->deadline, now) || + is_server_linked(be_server))) { + if (!be_server->deadline) { + return NULL; + } + bheap_delete(server_order, &be_ready_servers, + be_server->hn); + + if (is_server_linked(be_server)) { + be_server = next_be_server(); + return NULL; + } + + catchup_server(be_server, now); + check_donate_slack(be_server, NULL); + bheap_insert(server_order, &be_ready_servers, + be_server->hn); + be_server = next_be_server(); + } + + if (be_server && lt_before(now, be_server->release)) { + be_server = NULL; + } + + return be_server; +} + +/* + * Adds a task to the appropriate queue (ready / release) in a domain. + */ +static noinline void requeue(struct task_struct *task, rt_domain_t *domain) +{ + lt_t now = litmus_clock(); + int was_added; + + if (head_in_list(&task_data(task)->candidate_list)) { + list_del_init(&task_data(task)->candidate_list); + } + + check_slack_candidate(task); + + if (is_queued(task)) { + TRACE_TASK(task, "not requeueing, already queued\n"); + } else if (is_released(task, now)) { + __add_ready(domain, task); + } else { + /* Task needs to wait until it is released */ + was_added = add_release(domain, task); + + /* The release time happened before we added ourselves + * to the heap. We can now add to ready. + */ + if (!was_added) { + __add_ready(domain, task); + } + } +} + +static inline void earlier_server_task(server_t *first, + struct task_struct *first_task, + server_t *second, + struct task_struct *second_task, + server_t **server, + struct task_struct **task) +{ + if (!first || + (second && lt_before_eq(second->deadline, first->deadline))) { + *server = second; + *task = second_task; + } else { + *server = first; + *task = first_task; + } +} + +/* + * Set server and task to the next server and task respectively. + * If entry is not null, the next server will see if it can schedule + * entry's linked task. + */ +static void next_global_task(cpu_entry_t *entry, + server_t **next_server, + struct task_struct **next_task) +{ + struct task_struct *next_srt, *next_be, *next_slack; + server_t *be_server, *slack_server, *srt_server; + + *next_server = NULL; + *next_task = NULL; + + next_srt = next_eligible_srt(); + srt_server = (next_srt) ? task_srt_server(next_srt) : NULL; + + next_be = __peek_ready(&be_domain); + be_server = next_eligible_be_server(); + + next_slack = next_eligible_slack(); + slack_server = next_eligible_slack_server(); + + /* Check if the servers can schedule the task linked to entry */ + if (entry && entry->linked) { + + if (entry->linked_server->type == S_BE && + (!next_be || + lt_before(get_release(entry->linked), + get_release(next_be)))) { + + next_be = entry->linked; + } else if (entry->linked_server->type == S_SLACK && + (!next_slack || + lt_before(get_deadline(entry->linked), + get_deadline(next_slack)))) { + + next_slack = entry->linked; + } + } + + /* Remove tasks without servers and vice versa from contention */ + if (!next_be || !be_server) { + next_be = NULL; + be_server = NULL; + } + if (!next_slack || !slack_server) { + next_slack = NULL; + slack_server = NULL; + } + + /* Favor BE servers. If we don't, then a BE server might lose + * out to its own slack. + */ + if (slack_server && be_server && + be_server->deadline == slack_server->deadline) { + next_slack = NULL; + slack_server = NULL; + } + + /* There is probably a better way to do this */ + earlier_server_task(srt_server, next_srt, + be_server, next_be, + next_server, next_task); + earlier_server_task(*next_server, *next_task, + slack_server, next_slack, + next_server, next_task); +} + +/* + * Remove the task and server from any ready queues. + */ +static void remove_from_ready(server_t *server, struct task_struct *task, + cpu_entry_t *entry) +{ + server_t *slack; + rt_domain_t *domain; + + if (server->type == S_SLACK) { + list_del_init(&server->list); + + /* Remove from consideration of BE servers */ + if (is_be(task) && is_queued(task)) { + remove(&be_domain, task); + } + + /* Remove from consideration of slack servers */ + if (head_in_list(&task_data(task)->candidate_list)) { + list_del_init(&task_data(task)->candidate_list); + } + } else { + slack = server_slack(server); + if (slack && head_in_list(&slack->list)) { + remove_slack(slack); + } + if (server->type == S_BE) { + bheap_delete(server_order, &be_ready_servers, + server->hn); + } + if (is_queued(task)) { + domain = get_rt_domain(entry, task); + remove(domain, task); + } + } +} + +static void check_for_slack_preempt(struct task_struct*,server_t*,cpu_entry_t*, int); + +/* + * Finds and links the next server and task to an entry with no linked task. + */ +static void edf_hsb_pick_next(cpu_entry_t *entry) +{ + struct task_struct *next_task, *linked; + server_t *next_server; + + BUG_ON(entry->linked); + + next_task = next_eligible_hrt(&entry->hrt_server); + if (next_task) + next_server = &entry->hrt_server.server; + else + next_global_task(NULL, &next_server, &next_task); + + + if (next_task) { + remove_from_ready(next_server, next_task, entry); + check_for_slack_preempt(next_task, next_server, entry, 1); + + /* A slack preemption could cause something that was already + * running to be 'swapped' to this CPU in link_to_cpu. + */ + if (entry->linked) { + linked = entry->linked; + unlink(entry->linked); + requeue(linked, get_rt_domain(entry, linked)); + } + link_to_cpu(entry, next_task, next_server); + } +} + +/* + * Preempt the currently running server and task with new ones. + * It is possible that either only the server or the task is different here. + */ +static void preempt(cpu_entry_t *entry, struct task_struct *next, + server_t *next_server, int slack_resched) +{ + struct task_struct *linked; + rt_domain_t *domain; + + remove_from_ready(next_server, next, entry); + + check_for_slack_preempt(next, next_server, entry, slack_resched); + linked = entry->linked; + link_to_cpu(entry, next, next_server); + + /* No need for this if only the server was preempted */ + if (!linked || linked != entry->linked) { + if (linked) { + domain = get_rt_domain(entry, linked); + requeue(linked, domain); + } + preempt_if_preemptable(entry->scheduled, entry->cpu); + } +} + +/* + * Causes a preemption if: + * 1. task is being run by a slack server on a different CPU + * 2. slack donated by server is running a task on a different CPU + */ +static noinline void check_for_slack_preempt(struct task_struct *task, + server_t *server, + cpu_entry_t *next_entry, + int resched) +{ + cpu_entry_t *entry = NULL; + server_t *slack = server_slack(server); + struct task_struct *slack_task; + + /* The task is currently being run by another server */ + if (tsk_rt(task)->linked_on != NO_CPU) { + entry = task_linked_entry(task); + + if (entry != next_entry) { + unlink(task); + } + } + + /* The server's slack is currently being run */ + if (slack && is_server_linked(slack)) { + entry = &per_cpu(cpu_entries, slack->cpu); + slack_task = server_task(slack); + + unlink(slack_task); + remove_slack(slack); + requeue(slack_task, get_rt_domain(entry, slack_task)); + + if (entry != next_entry && resched) { + /* Force a reschedule */ + edf_hsb_pick_next(entry); + preempt_if_preemptable(entry->scheduled, entry->cpu); + } + } +} + +/* + * Check for any necessary non-hrt preemptions. + */ +static void check_for_global_preempt(void) +{ + cpu_entry_t *entry, *sched; + server_t *next_server; + int on_cpu; + struct task_struct *next_task = (struct task_struct*)1; /* Not NULL */ + + for (entry = lowest_prio_cpu(); entry; entry = lowest_prio_cpu()) { + + next_global_task(entry, &next_server, &next_task); + + if (!next_server) + break; + + /* Preempt only if we have an earlier deadline */ + if (entry->linked && + !lt_before(next_server->deadline, + entry->linked_server->deadline)) { + break; + } + + /* If we are scheduled on another CPU, the link code + * will force us to link to that CPU and try and link + * that CPU's task to this CPU. This is impossible + * if that CPU has linked HRT tasks which cannot + * migrate. + */ + on_cpu = next_task->rt_param.scheduled_on; + if (on_cpu != NO_CPU) { + sched = &per_cpu(cpu_entries, on_cpu); + + if (sched != entry && sched->linked && + is_hrt(sched->linked)) { + break; + } + } + + /* We do not reschedule if this causes a slack preemption + * because we will detect if we should reschedule on the + * next iteration of the loop. + */ + preempt(entry, next_task, next_server, + 0 /* Don't reschedule on a slack preemption */); + } +} + +/* + * Correct local link after a change to the local HRT domain. + */ +static void check_for_hrt_preempt(cpu_entry_t *entry) +{ + hrt_server_t *hrt_server = &entry->hrt_server; + struct task_struct *next_hrt = next_eligible_hrt(hrt_server); + + if (next_hrt && + (!entry->linked || !is_hrt(entry->linked) || + !is_eligible(entry->linked, hrt_server) || + edf_preemption_needed(&hrt_server->hrt_domain, entry->linked))) { + + preempt(entry, next_hrt, &hrt_server->server, 1); + + } +} + +/* + * Assumes called with local irqs disabled. + */ +static void job_arrival(struct task_struct *task, cpu_entry_t *entry) +{ + int was_empty; + + BUG_ON(task_cpu(task) == NO_CPU); + + if (is_hrt(task)) { + requeue(task, &entry->hrt_server.hrt_domain); + check_for_hrt_preempt(entry); + } else if (is_srt(task)) { + requeue(task, &srt_domain); + check_for_global_preempt(); + } else /* BE */ { + was_empty = !__jobs_pending(&be_domain); + requeue(task, &be_domain); + + /* Only way this could cause a preemption is if an eligible + * BE server could not queue up a task. + */ + if (was_empty && __jobs_pending(&be_domain)) + check_for_global_preempt(); + } +} + +/****************************************************************************** + * Timer methods + ******************************************************************************/ + +/* + * Merges a group of released HRT tasks into a ready queue and checks + * for preeemptions. + */ +static void release_hrt_jobs(rt_domain_t *domain, struct bheap *tasks) +{ + unsigned long flags; + struct task_struct *first; + cpu_entry_t *entry; + + raw_spin_lock_irqsave(global_lock, flags); + + first = (struct task_struct*)bheap_peek(edf_ready_order, tasks)->value; + entry = task_sched_entry(first); + + TRACE_TASK(first, "HRT tasks released at %llu on P%d\n", + TIME(litmus_clock()), task_cpu(first)); + + __merge_ready(domain, tasks); + check_for_hrt_preempt(entry); + + raw_spin_unlock_irqrestore(global_lock, flags); +} + +/* + * Merges a group of released tasks into a ready queue and checks to see + * if scheduled needs to be called. + */ +static void release_srt_jobs(rt_domain_t *domain, struct bheap *tasks) +{ + unsigned long flags; + struct task_struct *first = (bheap_peek(edf_ready_order, tasks)->value); + + raw_spin_lock_irqsave(global_lock, flags); + + TRACE_TASK(first, "SRT tasks released at %llu\n", TIME(litmus_clock())); + + __merge_ready(domain, tasks); + check_for_global_preempt(); + + raw_spin_unlock_irqrestore(global_lock, flags); +} + +/* + * Merges a group of released tasks into a ready queue and checks to see + * if scheduled needs to be called. + */ +static void release_be_jobs(rt_domain_t *domain, struct bheap *tasks) +{ + unsigned long flags; + int was_empty; + struct task_struct *first = (bheap_peek(edf_ready_order, tasks)->value); + + TRACE_TASK(first, "BE tasks released at %llu\n", TIME(litmus_clock()));; + + raw_spin_lock_irqsave(global_lock, flags); + + was_empty = !__jobs_pending(domain); + __merge_ready(domain, tasks); + if (was_empty) { + /* Only way this could cause a preemption is if an BE server + * could not find a task to run. + */ + check_for_global_preempt(); + } + + raw_spin_unlock_irqrestore(global_lock, flags); +} + +static enum hrtimer_restart slack_timer_fire(struct hrtimer *timer) +{ + unsigned long flags; + hrt_server_t *server = container_of(timer, hrt_server_t, slack_timer); + cpu_entry_t *entry = container_of(server, cpu_entry_t, hrt_server); + + raw_spin_lock_irqsave(global_lock, flags); + + TRACE_TIMER("slack timer fired for P%d", entry->cpu); + BUG_ON(!server->ready); + + /* Set new state of entry */ + server->no_slack = 1; + check_for_hrt_preempt(entry); + + /* Donate slack if the HRT server cannot run anything */ + if (!entry->linked || !is_hrt(entry->linked)) { + check_donate_slack(&server->server, NULL); + check_for_global_preempt(); + } + + raw_spin_unlock_irqrestore(global_lock, flags); + + return HRTIMER_NORESTART; +} + +static void job_completion(cpu_entry_t *entry, struct task_struct* task) +{ + server_t *server = entry->linked_server; + set_rt_flags(task, RT_F_SLEEP); + + unlink(task); + check_donate_slack(server, task); + + /* If a slack server completed an SRT task, the work for the + * next job arrival has already been done. + */ + if (server->type == S_SLACK && is_srt(task)) { + tsk_rt(task)->job_params.job_no++; + sched_trace_task_release(task); + } else if (server->type == S_SRT) { + /* If the task is behind the server it must release immediately, + * leaving its release time and deadline unchanged. + */ + if (server->job_no > tsk_rt(task)->job_params.job_no) { + tsk_rt(task)->job_params.job_no++; + } else { + /* Otherwise release them both */ + prepare_for_next_period(task); + server_release(server); + } + } else { + prepare_for_next_period(task); + } + + if (is_released(task, litmus_clock())) + sched_trace_task_release(task); + + /* Don't requeue a blocking task */ + if (is_running(task)) + job_arrival(task, entry); + + sched_trace_task_completion(task, 1); +} + +/* + * Assumes called with local irqs disabled. + */ +static void server_completed(server_t *server, struct task_struct *task) +{ + hrt_server_t *hrt_server; + cpu_entry_t *entry = task_linked_entry(task); + + BUG_ON(entry->linked != task); + + if (server->type == S_SRT) { + /* The job must now take the priority and release time + * of the next server. We do this so that we can still + * use rt_domain and other handy methods to still work + * with SRT jobs. Because this can ONLY happen if the + * task's job number gets behind the server's, we can + * easily detect the job catching up later. + */ + tsk_rt(task)->job_params.release = server->deadline; + tsk_rt(task)->job_params.deadline = server->deadline + + get_rt_period(task); + } else if (server->type == S_HRT) { + /* Update state of HRT server */ + hrt_server = container_of(server, hrt_server_t, server); + hrt_server->ready = 0; + + if (hrtimer_active(&hrt_server->slack_timer)) + slack_timer_cancel(hrt_server); + } + + if (server->type != S_SLACK) { + server_release(server); + } + + unlink(task); + requeue(task, get_rt_domain(entry, task)); + + /* We know this CPU needs to pick its next task */ + edf_hsb_pick_next(entry); + + /* Only cause a reschedule if something new was scheduled. A task + * could merely have swapped servers. + */ + if (entry->linked != task) + preempt_if_preemptable(entry->scheduled, entry->cpu); + else + entry->scheduled_server = entry->linked_server; +} + +static void hrt_server_released(server_t *server) +{ + hrt_server_t *hrt_server = container_of(server, hrt_server_t, server); + cpu_entry_t *entry = container_of(hrt_server, cpu_entry_t, hrt_server); + + BUG_ON(hrtimer_active(&hrt_server->slack_timer)); + + hrt_server->no_slack = 0; + hrt_server->ready = 1; + remove_slack(server_slack(&hrt_server->server)); + + check_for_hrt_preempt(entry); + + /* Ensure slack timer is only running if the current + * job is not HRT. + */ + if (entry->linked && is_hrt(entry->linked)) + slack_timer_cancel(hrt_server); + else + slack_timer_arm(hrt_server); +} + +static void servers_released(struct list_head *servers) +{ + int was_be = 0; + unsigned long flags; + struct list_head *pos, *safe; + server_t *server; + + raw_spin_lock_irqsave(global_lock, flags); + + TRACE_TIMER("Servers released"); + + list_for_each_safe(pos, safe, servers) { + server = list_entry(pos, server_t, release_list); + + list_del_init(pos); + + if (server->type == S_BE) { + check_donate_slack(server, NULL); + was_be = 1; + BUG_ON(bheap_node_in_heap(server->hn)); + bheap_insert(server_order, &be_ready_servers, + server->hn); + check_donate_slack(server, NULL); + } else { /* HRT server */ + hrt_server_released(server); + } + } + + if (was_be) + check_for_global_preempt(); + + raw_spin_unlock_irqrestore(global_lock, flags); +} + +/****************************************************************************** + * Server management methods + ******************************************************************************/ + +static int curr_be = 0; + +/* + * A BE server has been added in a proc entry. + */ +static int admit_be_server(unsigned long long wcet, + unsigned long long period, + int cpu) +{ + int rv = 0; + server_t *be_server; + + if (cpu != NO_CPU) { + rv = -EINVAL; + goto out; + } + + be_server = server_alloc(GFP_ATOMIC); + server_init(be_server, &server_domain, + BE_SERVER_BASE + ++curr_be, + wcet, period, 1); + be_server->type = S_BE; + server_slack_create(be_server); + + list_add(&be_server->list, &be_servers); + bheap_insert(server_order, &be_ready_servers, be_server->hn); + + out: + return rv; +} + +/* + * Output all BE servers to a proc entry. + */ +static void list_be_servers(server_proc_t *proc) +{ + struct list_head *pos; + server_t *be_server; + + list_for_each(pos, &be_servers) { + be_server = list_entry(pos, server_t, list); + list_server(be_server, NO_CPU, proc); + } +} + +/* + * Halts and destroys all BE servers. + */ +static void stop_be_servers(void) +{ + server_t *be_server; + struct list_head *pos, *safe; + + list_for_each_safe(pos, safe, &be_servers) { + be_server = list_entry(pos, server_t, list); + + list_del_init(pos); + if (bheap_node_in_heap(be_server->hn)) + bheap_delete(server_order, &be_ready_servers, + be_server->hn); + server_slack_destroy(be_server); + server_destroy(be_server); + server_free(be_server); + } +} + +/* + * An HRT server has been added in a proc entry. + */ +static int admit_hrt_server(unsigned long long wcet, + unsigned long long period, + int cpu) +{ + cpu_entry_t *entry = &per_cpu(cpu_entries, cpu); + hrt_server_t *hrt_server = &entry->hrt_server; + struct hrtimer *slack_timer = &hrt_server->slack_timer; + + server_init(&hrt_server->server, &server_domain, + cpu, wcet, period, 1); + server_slack_create(&hrt_server->server); + hrt_server->no_slack = 0; + hrt_server->ready = 1; + hrt_server->server.type = S_HRT; + + edf_domain_init(&hrt_server->hrt_domain, NULL, + release_hrt_jobs); + + hrtimer_init(slack_timer, + CLOCK_MONOTONIC, + HRTIMER_MODE_ABS); + slack_timer->function = slack_timer_fire; + + return 0; +} + +/* + * Print all HRT servers to a proc entry. + */ +static void list_hrt_servers(server_proc_t *proc) +{ + cpu_entry_t *entry; + hrt_server_t *hrt_server; + int cpu; + + for_each_online_cpu(cpu) { + entry = &per_cpu(cpu_entries, cpu); + hrt_server = &entry->hrt_server; + list_server(&hrt_server->server, cpu, proc); + } +} + +/* + * Stops all hrt server timers and resets all fields to 0. + */ +static void stop_hrt_servers(void) +{ + int cpu; + cpu_entry_t *entry; + hrt_server_t *hrt_server; + + for_each_online_cpu(cpu) { + entry = &per_cpu(cpu_entries, cpu); + hrt_server = &entry->hrt_server; + + if (hrt_server->server.data) + server_slack_destroy(&hrt_server->server); + slack_timer_cancel(hrt_server); + + hrt_server->no_slack = 0; + hrt_server->ready = 0; + hrt_server->server.period = 0; + hrt_server->server.wcet = 0; + } +} + +/* + * Starts timers used to manage servers. + */ +static void start_servers(lt_t time) +{ + int cpu; + cpu_entry_t *entry; + server_t *server; + server_t *be_server; + struct list_head *pos; + + /* Start HRT servers */ + for_each_online_cpu(cpu) { + entry = &per_cpu(cpu_entries, cpu); + server = &entry->hrt_server.server; + + if (!check_hrt_server_initialized(&entry->hrt_server)) + goto loop_end; + + /* Cause a catchup later */ + server_release_at(server, time - server->period); + entry->hrt_server.ready = 1; + + TRACE("Setting up cpu %d to have timer deadline %llu\n", + cpu, TIME(server->deadline)); + loop_end: + cpu = cpu; + } + + /* Start BE servers */ + list_for_each(pos, &be_servers) { + be_server = list_entry(pos, server_t, list); + + if (!bheap_node_in_heap(be_server->hn)) + bheap_insert(server_order, &be_ready_servers, be_server->hn); + + /* Cause a catchup later */ + server_release_at(be_server, time - be_server->period); + + TRACE("Releasing BE server %d\n", be_server->id); + } +} + +/****************************************************************************** + * Plugin methods + ******************************************************************************/ + +static long edf_hsb_activate_plugin(void) +{ + int cpu; + cpu_entry_t *entry; +#ifdef CONFIG_RELEASE_MASTER + edf_hsb_release_master = atomic_read(&release_master_cpu); +#else + edf_hsb_release_master = NO_CPU; +#endif + server_domain.release_master = edf_hsb_release_master; + + for_each_online_cpu(cpu) { + entry = &per_cpu(cpu_entries, cpu); +#ifdef CONFIG_RELEASE_MASTER + if (cpu != edf_hsb_release_master) +#endif + update_cpu_position(entry); + } + + start_servers(litmus_clock()); + + TRACE("activating EDF-HSB plugin.\n"); + return 0; +} + +/* + * Requires a processor be specified for any task run on the system. + */ +static long edf_hsb_admit_task(struct task_struct *task) +{ + cpu_entry_t *entry = task_sched_entry(task); + + TRACE_TASK(task, "Admitting\n"); + + if (is_hrt(task)) { + return check_hrt_server_initialized(&entry->hrt_server) && + ((task_cpu(task) == task->rt_param.task_params.cpu) && + (task_cpu(task) == entry->cpu)) ? 0 : -EINVAL; + } else { + /* If the task is not HRT, we don't want to force the user + * to specify a CPU. + */ + return 0; + } +} + +/* + * Stops all servers from running. + */ +static long edf_hsb_deactivate_plugin(void) +{ + cpu_entry_t *cpu_entry; + hrt_server_t *hrt_server; + unsigned long flags; + int cpu; + + local_irq_save(flags); + + for_each_online_cpu(cpu) { + cpu_entry = &per_cpu(cpu_entries, cpu); + hrt_server = &cpu_entry->hrt_server; + + slack_timer_cancel(hrt_server); + + if (likely(bheap_node_in_heap(cpu_entry->hn))) + bheap_delete(server_order, &cpu_heap, cpu_entry->hn); + } + + local_irq_restore(flags); + + return 0; +} + +static void edf_hsb_task_block(struct task_struct *task) +{ + unsigned long flags; + cpu_entry_t *entry = task_sched_entry(task); + struct task_struct *linked; + server_t *linked_server; + + TRACE_TASK(task, "block at %llu in state %llu\n", + litmus_clock(), task->state); + set_rt_flags(task, RT_F_BLOCK); + + raw_spin_lock_irqsave(global_lock, flags); + + linked = entry->linked; + linked_server = entry->linked_server; + + unlink(task); + + if (task == linked) { + check_donate_slack(linked_server, task); + } + + raw_spin_unlock_irqrestore(global_lock, flags); +} + +/* + * A task leaves the system. + */ +static void edf_hsb_task_exit(struct task_struct *task) +{ + unsigned long flags; + cpu_entry_t *entry = task_sched_entry(task); + + BUG_ON(!is_realtime(task)); + TRACE_TASK(task, "RIP at %llu on P%d\n", + TIME(litmus_clock()), tsk_rt(task)->scheduled_on); + + raw_spin_lock_irqsave(global_lock, flags); + + unlink(task); + if (tsk_rt(task)->scheduled_on != NO_CPU) { + entry->scheduled = NULL; + tsk_rt(task)->scheduled_on = NO_CPU; + } + if (is_srt(task)) { + server_slack_destroy(task_srt_server(task)); + server_destroy(task_srt_server(task)); + server_free(task_srt_server(task)); + task_data_free(tsk_rt(task)->plugin_data); + } + + raw_spin_unlock_irqrestore(global_lock, flags); +} + +/* + * Attempts to determine the current scheduler state, then selects the + * next task and updates the scheduler state. + */ +static struct task_struct* edf_hsb_schedule(struct task_struct *prev) +{ + unsigned long flags; + int blocks, preempted, sleep, was_slack, np, hrt_preempt, donated; + struct task_struct *curr; + cpu_entry_t *entry = local_cpu_entry; + +#ifdef CONFIG_RELEASE_MASTER + /* Bail out early if we are the release master. + * The release master never schedules any real-time tasks. + */ + if (edf_hsb_release_master == entry->cpu) { + sched_state_task_picked(); + return NULL; + } +#endif + + raw_spin_lock_irqsave(global_lock, flags); + + curr = entry->scheduled; + + /* Determine state */ + blocks = curr && !is_running(curr); + preempted = entry->scheduled != entry->linked; + sleep = curr && get_rt_flags(curr) == RT_F_SLEEP; + was_slack = !list_empty(&slack_queue); + np = curr && is_np(curr); + + TRACE("blocks: %d, preempted: %d, sleep: %d, np: %d\n", + blocks, preempted, sleep, np); + if (blocks) + unlink(entry->scheduled); + + /* If the task has gone to sleep or exhausted its budget, it + * must complete its current job. + */ + if (sleep && !blocks && !preempted) + job_completion(entry, entry->scheduled); + + /* Pick the next task if there isn't one currently */ + if (!entry->linked) + edf_hsb_pick_next(entry); + + /* Set task states */ + if (entry->linked != entry->scheduled) { + if (entry->linked) + entry->linked->rt_param.scheduled_on = entry->cpu; + if (entry->scheduled) + entry->scheduled->rt_param.scheduled_on = NO_CPU; + } + + entry->scheduled = entry->linked; + entry->scheduled_server = entry->linked_server; + sched_state_task_picked(); + + /* An non-HRT was preempted by an HRT task. Because of the way linking + * works, it cannot link itself to anything else until the non-migratory + * HRT task is scheduled. + */ + hrt_preempt = preempted && entry->linked && curr && + is_hrt(entry->linked) && !is_hrt(curr); + /* A server just donated slack */ + donated = entry->linked && entry->linked_server->type != S_SLACK && + head_in_list(&server_slack(entry->linked_server)->list); + + if (hrt_preempt || donated) + check_for_global_preempt(); + + if (entry->scheduled) + TRACE_TASK(entry->scheduled, "scheduled at %llu\n", + TIME(litmus_clock())); + else + TRACE("NULL scheduled at %llu\n", TIME(litmus_clock())); + + raw_spin_unlock_irqrestore(global_lock, flags); + + return entry->scheduled; +} + +/* + * Prepare a task for running in RT mode + */ +static void edf_hsb_task_new(struct task_struct *task, int on_rq, int running) +{ + unsigned long flags; + task_data_t *data; + server_t *srt_server = NULL; + cpu_entry_t *entry = task_sched_entry(task); + + TRACE_TASK(task, "edf_hsb: task new at %llu\n", TIME(litmus_clock())); + + raw_spin_lock_irqsave(global_lock, flags); + + /* Setup job parameters */ + release_at(task, litmus_clock()); + + /* Create SRT server */ + if (is_srt(task)) { + /* Create SRT server */ + srt_server = server_alloc(GFP_ATOMIC); + server_init(srt_server, &server_domain, + task->pid, get_exec_cost(task), + get_rt_period(task), 0); + srt_server->type = S_SRT; + + server_slack_create(srt_server); + + } + + /* Create task plugin data */ + data = task_data_alloc(GFP_ATOMIC); + data->owner = task; + data->srt_server = srt_server; + INIT_LIST_HEAD(&data->candidate_list); + tsk_rt(task)->plugin_data = data; + + /* Already running, update the cpu entry. + * This tends to happen when the first tasks enter the system. + */ + if (running) { + +#ifdef CONFIG_RELEASE_MASTER + if (entry->cpu != edf_hsb_release_master) { +#endif + entry->scheduled = task; + tsk_rt(task)->scheduled_on = task_cpu(task); +#ifdef CONFIG_RELEASE_MASTER + } else { + /* do not schedule on release master */ + /* Cannot preempt! Causing a preemption with a BE task + * somehow leads to that task never blocking during + * a synchronous release. This is a bug! + */ + preempt_if_preemptable(entry->scheduled, entry->cpu); + tsk_rt(task)->scheduled_on = NO_CPU; + } +#endif + } else { + task->rt_param.scheduled_on = NO_CPU; + } + + task->rt_param.linked_on = NO_CPU; + job_arrival(task, entry); + + raw_spin_unlock_irqrestore(global_lock, flags); +} + +static void edf_hsb_task_wake_up(struct task_struct *task) +{ + lt_t now; + unsigned long flags; + cpu_entry_t *entry = task_sched_entry(task); + + + TRACE_TASK(task, "wake_up at %llu on %d, %d\n", TIME(litmus_clock()), + task_cpu(task), task->rt_param.task_params.cpu); + + raw_spin_lock_irqsave(global_lock, flags); + + if (!is_be(task)) { + if (is_srt(task)) { + catchup_srt_server(task); + } + + /* Non-BE tasks are not sporadic in this model */ + set_rt_flags(task, RT_F_RUNNING); + /* The job blocked while it was being run by a slack server */ + if (is_queued(task)) { + check_slack_candidate(task); + goto out; + } + } else { + /* Re-release all BE tasks on wake-up */ + now = litmus_clock(); + + if (is_tardy(task, now)) { + release_at(task, now); + sched_trace_task_release(task); + } + } + + job_arrival(task, entry); + + out: + raw_spin_unlock_irqrestore(global_lock, flags); +} + +/* + * Unused. + */ +static void edf_hsb_tick(struct task_struct *t) +{ +} + + +/****************************************************************************** + * Plugin + ******************************************************************************/ + +static struct sched_plugin edf_hsb_plugin __cacheline_aligned_in_smp = { + .plugin_name = "EDF-HSB", + + .activate_plugin = edf_hsb_activate_plugin, + .deactivate_plugin = edf_hsb_deactivate_plugin, + + .schedule = edf_hsb_schedule, + .admit_task = edf_hsb_admit_task, + .task_block = edf_hsb_task_block, + .task_exit = edf_hsb_task_exit, + .task_new = edf_hsb_task_new, + .task_wake_up = edf_hsb_task_wake_up, + .tick = edf_hsb_tick, + + /* From jobs.h */ + .complete_job = complete_job, + .release_at = release_at, +}; + +static int __init init_edf_hsb(void) +{ + cpu_entry_t *entry; + hrt_server_t *hrt_server; + server_t *idle_slack; + int rv, cpu; + + rv = register_sched_plugin(&edf_hsb_plugin); + if (rv) { + printk(KERN_ERR "Could not register plugin %s.\n", + edf_hsb_plugin.plugin_name); + goto out; + } + + rv = make_plugin_proc_dir(&edf_hsb_plugin, &edf_hsb_proc_dir); + if (rv) { + printk(KERN_ERR "Could not create %s procfs dir.\n", + edf_hsb_plugin.plugin_name); + goto out; + } + + + task_data_cache = KMEM_CACHE(task_data, SLAB_PANIC); + + /* Global domains */ + edf_domain_init(&srt_domain, NULL, release_srt_jobs); + rt_domain_init(&be_domain, be_ready_order, + NULL, release_be_jobs); + server_domain_init(&server_domain, servers_released, + server_completed, NO_CPU, global_lock); + + /* Server proc interfaces */ + server_proc_init(&server_domain, + edf_hsb_proc_dir, BE_PROC_NAME, + admit_be_server, list_be_servers, + stop_be_servers); + server_proc_init(&server_domain, + edf_hsb_proc_dir, HRT_PROC_NAME, + admit_hrt_server, list_hrt_servers, + stop_hrt_servers); + + + /* Global collections */ + bheap_init(&cpu_heap); + bheap_init(&be_ready_servers); + INIT_LIST_HEAD(&be_servers); + INIT_LIST_HEAD(&slack_queue); + INIT_LIST_HEAD(&slack_candidates); + + for_each_online_cpu(cpu) { + entry = &per_cpu(cpu_entries, cpu); + hrt_server = &entry->hrt_server; + + idle_slack = server_alloc(GFP_ATOMIC); + server_init(idle_slack, &server_domain, + IDLE_SLACK_BASE + cpu, + LLONG_MAX, LLONG_MAX, 1); + idle_slack->deadline = LLONG_MAX; + idle_slack->budget = LLONG_MAX; + idle_slack->job_no = 1; + idle_slack->release = 1; + idle_slack->type = S_SLACK; + add_slack(idle_slack); + + entry->cpu = cpu; + entry->linked = NULL; + entry->scheduled = NULL; + entry->linked_server = NULL; + + /* HRT server */ + hrt_server->server.id = cpu; + hrt_server->server.deadline = 0; + hrt_server->server.period = 0; + hrt_server->server.wcet = 0; + hrt_server->ready = 0; + + hrtimer_start_on_info_init(&hrt_server->slack_timer_info); + + /* CPU entry bheap nodes */ + entry->hn = &cpu_heap_node[cpu]; + bheap_node_init(&entry->hn, entry); + } + + out: + return rv; +} + +static void exit_edf_hsb(void) +{ + int cpu; + cpu_entry_t *entry; + + stop_be_servers(); + stop_hrt_servers(); + + server_domain_destroy(&server_domain); + + for_each_online_cpu(cpu) { + entry = &per_cpu(cpu_entries, cpu); + server_slack_destroy(&entry->hrt_server.server); + server_destroy(&entry->hrt_server.server); + } + + if (edf_hsb_proc_dir) { + remove_plugin_proc_dir(&edf_hsb_plugin); + edf_hsb_proc_dir = NULL; + } +} + +module_init(init_edf_hsb); +module_exit(exit_edf_hsb); diff --git a/litmus/servers.c b/litmus/servers.c new file mode 100644 index 0000000..37af270 --- /dev/null +++ b/litmus/servers.c @@ -0,0 +1,767 @@ +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#define DEBUG_SERVERS + +#define TIME(x) \ + ({lt_t y = x; \ + do_div(y, NSEC_PER_MSEC); \ + y;}) +#ifdef DEBUG_SERVERS +#define _TRACE_TIMER(fmt, args...) \ + sched_trace_log_message("%d P%d*[%s@%s:%d]: " fmt " at %d\n", \ + TRACE_ARGS, ## args, TIME(litmus_clock())) +#define TRACE_TIMER(s, fmt, args...) \ + do { \ + if (is_server_linked(s)) \ + _TRACE_TIMER(TASK_FMT " " SERVER_FMT " " fmt, \ + TASK_ARGS(server_task(s)), \ + SERVER_ARGS(s), ##args); \ + else \ + _TRACE_TIMER("(NULL) " SERVER_FMT " " fmt, \ + SERVER_ARGS(s), ##args); \ + } while(0) +#else +#define TRACE_TIMER(s, fmt, args...) +#define _TRACE_TIMER(fmt, args...) +#endif + +/* Used to run a server on a remote CPU */ +DEFINE_PER_CPU(struct hrtimer_start_on_info, server_cpu_infos); + +/* Memory slabs for servers */ +struct kmem_cache *server_release_cache; +struct kmem_cache *server_cache; + +/* + * Okay to call if the timer is not armed. + */ +static inline int timer_cancel(struct hrtimer *timer) +{ + if (hrtimer_active(timer)) + return hrtimer_try_to_cancel(timer); + else + return 0; +} + +static int completion_timer_arm(server_domain_t* domain, int cpu) +{ + int err = 0, on_cpu; + lt_t now = domain->start_times[cpu]; + server_t *server = domain->linked_servers[cpu]; + lt_t budget_exhausted = now + server->budget; + completion_timer_t *timer = &domain->completion_timers[cpu]; + + /* This happens when someone attempts to call server_run when + * the server completes. When this happens, we can ignore the request + * here because completion_timer_fire will re-arm the timer if + * the server is still running / was run again. + */ + if (hrtimer_active(&timer->timer)) { + return 0; + } + if (timer->armed) { + return 0; + } + + if (lt_after(budget_exhausted, server->deadline)) + budget_exhausted = server->deadline; + +#ifdef COMPLETION_ON_MASTER + if (domain->release_master != NO_CPU) + on_cpu = domain->release_master; + else +#endif + on_cpu = cpu; + + err = 1; + if (cpu != smp_processor_id()) { + err = hrtimer_start_on(on_cpu, &timer->info, &timer->timer, + ns_to_ktime(budget_exhausted), + HRTIMER_MODE_ABS_PINNED); + } else if (atomic_read(&timer->info.state)== HRTIMER_START_ON_INACTIVE){ + err = __hrtimer_start_range_ns(&timer->timer, + ns_to_ktime(budget_exhausted), + 0 /* delta */, + HRTIMER_MODE_ABS_PINNED, + 0 /* no wakeup */); + } + + timer->armed = (err) ? 0 : 1; + + return !err; +} + +static enum hrtimer_restart completion_timer_fire(struct hrtimer *timer) +{ + int cpu; + unsigned long flags; + enum hrtimer_restart rv; + struct task_struct *was_running; + completion_timer_t *completion_timer; + server_domain_t *domain; + server_t *server; + lt_t budget_exhausted; + + rv = HRTIMER_NORESTART; + + completion_timer = container_of(timer, completion_timer_t, timer); + domain = completion_timer->domain; + cpu = completion_timer->cpu; + + raw_spin_lock_irqsave(domain->completion_lock, flags); + + _TRACE_TIMER("completion timer firing on P%d", cpu); + + /* We got the lock before someone tried to re-arm. Proceed. */ + if (completion_timer->armed) { + server = domain->linked_servers[cpu]; + was_running = server_task(server); + + server->budget = 0; + server->cpu = NO_CPU; + domain->start_times[cpu] = 0; + domain->linked_servers[cpu] = NULL; + domain->linked_tasks[cpu] = NULL; + + domain->server_completed(server, was_running); + } + + /* Someone either beat us to the lock or hooked up a new server + * when we called server_completed. Rearm the timer. + */ + if (domain->linked_servers[cpu] && !completion_timer->armed) { + server = domain->linked_servers[cpu]; + budget_exhausted = domain->start_times[cpu] + server->budget; + if (lt_after(budget_exhausted, server->deadline)) + budget_exhausted = server->deadline; + hrtimer_set_expires(timer, ns_to_ktime(budget_exhausted)); + completion_timer->armed = 1; + + rv = HRTIMER_RESTART; + } else { + completion_timer->armed = 0; + } + + raw_spin_unlock_irqrestore(domain->completion_lock, flags); + + return rv; +} + +struct kmem_cache *server_release_cache; /* In litmus.c */ +static enum hrtimer_restart release_servers_fire(struct hrtimer *timer); + +/* + * Initialize heap. + */ +static server_release_heap_t* release_heap_alloc(int gfp_flags) +{ + server_release_heap_t *rh; + rh = kmem_cache_alloc(server_release_cache, gfp_flags); + if (rh) { + hrtimer_init(&rh->timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS); + rh->timer.function = release_servers_fire; + } + return rh; +} + +static void release_heap_free(server_release_heap_t* rh) +{ + kmem_cache_free(server_release_cache, rh); +} + +void server_init(server_t *server, server_domain_t *domain, + int id, lt_t wcet, lt_t period, int grouped) +{ + server->id = id; + server->wcet = wcet; + server->period = period; + + server->deadline = 0; + server->release = 0; + server->budget = 0; + server->job_no = 0; + server->cpu = NO_CPU; + + server->domain = domain; + + server->data = NULL; + + server->hn = bheap_node_alloc(GFP_ATOMIC); + bheap_node_init(&server->hn, server); + INIT_LIST_HEAD(&server->list); + + server->release_heap = NULL; + if (grouped) { + server->release_heap = release_heap_alloc(GFP_ATOMIC); + INIT_LIST_HEAD(&server->release_list); + } +} + +void server_destroy(server_t *server) +{ + bheap_node_free(server->hn); + if (server->release_heap) { + release_heap_free(server->release_heap); + } +} + +server_t* server_alloc(int gfp_flags) +{ + return kmem_cache_alloc(server_cache, gfp_flags); +} + +void server_free(server_t *server) +{ + kmem_cache_free(server_cache, server); +} + +/* + * Handles subtraction of lt_t without underflows. + */ +static inline lt_t lt_subtract(lt_t a, lt_t b) +{ + long long sub = (long long)a - (long long)b; + if (sub >= 0) + return sub; + else + return 0; +} + +void server_run(server_t *server, struct task_struct *task) +{ + int armed, cpu = task->rt_param.linked_on; + server_domain_t *domain = server->domain; + + server->cpu = cpu; + + domain->linked_servers[cpu] = server; + domain->linked_tasks[cpu] = task; + domain->start_times[cpu] = litmus_clock(); + + /* Arm completion timer */ + armed = completion_timer_arm(domain, cpu); + domain->completion_timers[cpu].armed = armed; +} + +void server_stop(server_t *server) +{ + int cpu; + lt_t elapsed_time, now = litmus_clock(); + server_domain_t *domain = server->domain; + + if (!is_server_linked(server)) { + return; + } + + cpu = server->cpu; + BUG_ON(cpu == NO_CPU); + + /* Calculate remaining budget */ + elapsed_time = lt_subtract(now, domain->start_times[cpu]); + server->budget = lt_subtract(server->budget, elapsed_time); + + server->cpu = NO_CPU; + + /* Set domain state */ + domain->completion_timers[cpu].armed = 0; + domain->linked_servers[cpu] = NULL; + domain->linked_tasks[cpu] = NULL; + timer_cancel(&domain->completion_timers[cpu].timer); +} + +void server_release(server_t *server) +{ + server->budget = server->wcet; + server->release = server->deadline; + server->deadline += server->period; + ++server->job_no; + + /* Need to reset for budget calculations */ + if (is_server_linked(server)) + server->domain->start_times[server->cpu] = litmus_clock(); +} + +void server_release_at(server_t *server, lt_t time) +{ + server->deadline = time; + server_release(server); +} + +/****************************************************************************** + * Proc methods + ******************************************************************************/ + +static int server_proc_read(char* page, char **start, off_t off, + int count, int *eof, void *data) +{ + int length; + server_proc_t *proc = (server_proc_t*)data; + + proc->page = page; + proc->length = 0; + proc->list_servers(proc); + + length = proc->length; + *eof = 1; + + proc->length = 0; + proc->page = NULL; + + return length; +} + +void list_server(server_t *server, int cpu, server_proc_t *proc) +{ + if (cpu == NO_CPU) { + proc->length += + snprintf(proc->page + proc->length, + PAGE_SIZE - proc->length, + "%8llu %8llu\n", + server->wcet, server->period); + } else { + proc->length += + snprintf(proc->page + proc->length, + PAGE_SIZE - proc->length, + "%8llu %8llu %3d\n", + server->wcet, server->period, cpu); + } +} + +/* + * Validate server parameters. + */ +static inline int server_param_check(unsigned long long wcet, + unsigned long long period, + int cpu) +{ + int rv = 0; + + if (wcet <= 0) { + printk(KERN_WARNING "Invalid WCET '%llu'\n", wcet); + rv = -EINVAL; + goto out; + } + + if (period < wcet) { + printk(KERN_WARNING "Invalid period '%llu'\n", period); + rv = -EINVAL; + goto out; + } + + if (cpu != NO_CPU && (cpu < 0 || cpu >= nr_cpu_ids)) { + printk(KERN_WARNING "Invalid CPU '%d'\n", cpu); + rv = -EINVAL; + goto out; + } + out: + return rv; +} + +/* Macro to see if we are in the buffer's range and not at the null byte */ +#define buf_in_range(buf, pos, max) (buf <= pos && pos < (buf + max) && *pos) + +#define find_newline(buf, pos, max) \ + do { \ + while (buf_in_range(buf, pos, max) && \ + *pos != '\n') \ + ++pos; \ + } while (0) + +static int server_proc_write(struct file *file, const char __user *input, + unsigned long count, void *data) +{ + server_proc_t *proc = (server_proc_t*)data; +#define SERVER_PROC_BUF 512 + char buffer[SERVER_PROC_BUF]; + unsigned long long wcet, period; + char *pos, *newline, *space_check; + int nums_converted, chars_seen, ret, cpu; + + /* Allow plugin to stop any running servers */ + proc->stop_servers(); + + if (count >= SERVER_PROC_BUF){ + printk(KERN_WARNING "proc buffer possibly too small in %s.\n", + __func__); + return -ENOSPC; + } + + memset(buffer, 0, SERVER_PROC_BUF); + + /* Input is definitely < SERVER_PROC_BUF (see above check) */ + if (copy_from_user(buffer, input, count)) + return -EFAULT; + + buffer[SERVER_PROC_BUF-1] = '\0'; + pos = buffer; + + while (buf_in_range(buffer, pos, SERVER_PROC_BUF)) { + newline = pos; + find_newline(buffer, newline, SERVER_PROC_BUF); + if (buf_in_range(buffer, newline, SERVER_PROC_BUF)) { + /* If there was a newline character */ + *newline = '\0'; + } + nums_converted = sscanf(pos, "%llu %llu %d%n", &wcet, + &period, &cpu, &chars_seen); + if (nums_converted == 2) + cpu = NO_CPU; + if (nums_converted != 2 && nums_converted != 3) { + printk(KERN_WARNING "Didn't see 2-3 integers for " + "server config: %s\n", pos); + goto loop_end; + } + /* space_check = pos + chars_seen; */ + /* if (space_check != newline) { */ + /* /\* If the newline was not right after the numbers */ + /* * converted, ensure extra characters are just space */ + /* *\/ */ + /* for (; *space_check; space_check++) { */ + /* if (!isspace(*space_check)) { */ + /* printk(KERN_WARNING "Extra characters " */ + /* "in line: %s\n", pos); */ + /* goto loop_end; */ + /* } */ + /* } */ + /* } */ + + ret = server_param_check(wcet, period, cpu); + if (ret) goto loop_end; + + ret = proc->admit_server(wcet, period, cpu); + if (ret) { + printk(KERN_WARNING "Litmus plugin rejects server with " + "period: %llu, wcet: %llu, cpu: %d\n", + period, wcet, cpu); + goto loop_end; /* Currently does nothing */ + } +loop_end: + pos = newline + 1; /* Consider next line */ + } + + return count; +} + +server_proc_t* server_proc_init(server_domain_t *domain, + struct proc_dir_entry *proc_dir, char *file, + admit_server_t admit_server, + list_servers_t list_servers, + stop_servers_t stop_servers) +{ + server_proc_t *server_proc = NULL; + struct proc_dir_entry *entry; + + entry = create_proc_entry(file, 0644, proc_dir); + if (!entry) { + printk(KERN_ERR "Could not create proc entry: %s.\n", file); + goto out; + } + + server_proc = kmalloc(sizeof(server_proc_t), GFP_ATOMIC); + + entry->data = server_proc; + entry->read_proc = server_proc_read; + entry->write_proc = server_proc_write; + + server_proc->entry = entry; + server_proc->admit_server = admit_server; + server_proc->list_servers = list_servers; + server_proc->stop_servers = stop_servers; + server_proc->length = 0; + server_proc->page = NULL; + + INIT_LIST_HEAD(&server_proc->list); + list_add(&server_proc->list, &domain->server_procs); + + out: + return server_proc; +} + +void server_proc_exit(server_proc_t *proc) +{ + remove_proc_entry(proc->entry->name, proc->entry->parent); + list_del(&proc->list); + kfree(proc); +} + +/****************************************************************************** + * Domain methods + ******************************************************************************/ + +void server_domain_init(server_domain_t *domain, + servers_released_t servers_released, + server_completed_t server_completed, + int release_master, raw_spinlock_t *completion_lock) +{ + int i; + BUG_ON(!servers_released || !server_completed); + + INIT_LIST_HEAD(&domain->tobe_released); + for (i = 0; i < SERVER_RELEASE_QUEUE_SLOTS; i++) + INIT_LIST_HEAD(&domain->release_queue[i]); + + raw_spin_lock_init(&domain->release_lock); + raw_spin_lock_init(&domain->tobe_lock); + + + domain->release_master = release_master; + domain->completion_lock = completion_lock; + domain->server_completed = server_completed; + domain->servers_released = servers_released; + + INIT_LIST_HEAD(&domain->server_procs); + + domain->completion_timers = + kmalloc(NR_CPUS*sizeof(completion_timer_t), GFP_ATOMIC); + domain->linked_servers = + kmalloc(NR_CPUS*sizeof(server_t*), GFP_ATOMIC); + domain->linked_tasks = + kmalloc(NR_CPUS*sizeof(struct task_struct*), GFP_ATOMIC); + domain->start_times = + kmalloc(NR_CPUS*sizeof(lt_t), GFP_ATOMIC); + + for_each_online_cpu(i) { + domain->linked_tasks[i] = NULL; + domain->linked_servers[i] = NULL; + domain->start_times[i] = 0; + + /* Initialize the completion timer info */ + domain->completion_timers[i].armed = 0; + domain->completion_timers[i].cpu = i; + hrtimer_init(&domain->completion_timers[i].timer, + CLOCK_MONOTONIC, + HRTIMER_MODE_ABS); + domain->completion_timers[i].timer.function = + completion_timer_fire; + hrtimer_start_on_info_init(&domain->completion_timers[i].info); + domain->completion_timers[i].domain = domain; + } +} + +void server_domain_destroy(server_domain_t *domain) +{ + struct list_head *pos, *safe; + server_proc_t *proc; + + kfree(domain->completion_timers); + kfree(domain->linked_tasks); + kfree(domain->linked_servers); + kfree(domain->start_times); + + list_for_each_safe(pos, safe, &domain->server_procs) { + proc = list_entry(pos, server_proc_t, list); + server_proc_exit(proc); + } +} + +static unsigned int time2slot(lt_t time) +{ + return (unsigned int) time2quanta(time, FLOOR) % + SERVER_RELEASE_QUEUE_SLOTS; +} + +/* + * Send a list of servers to a client callback. + */ +static enum hrtimer_restart release_servers_fire(struct hrtimer *timer) +{ + unsigned long flags; + server_release_heap_t *rh; + + rh = container_of(timer, server_release_heap_t, timer); + + raw_spin_lock_irqsave(&rh->domain->release_lock, flags); + + /* Remove from release queue */ + list_del(&rh->list); + + raw_spin_unlock_irqrestore(&rh->domain->release_lock, flags); + + /* Call release callback */ + rh->domain->servers_released(&rh->servers); + + return HRTIMER_NORESTART; +} + +/* + * Caller must hold release lock. + * Will return heap for given time. If no such heap exists prior to + * the invocation it will be created. + */ +static server_release_heap_t* get_release_heap(server_domain_t *rt, + server_t *server, + int use_server_heap) +{ + struct list_head *pos; + server_release_heap_t *heap = NULL; + server_release_heap_t *rh; + lt_t release_time = server->release; + unsigned int slot = time2slot(release_time); + + /* Initialize pos for the case that the list is empty */ + pos = rt->release_queue[slot].next; + list_for_each(pos, &rt->release_queue[slot]) { + rh = list_entry(pos, server_release_heap_t, list); + if (release_time == rh->release_time) { + /* Perfect match -- this happens on hyperperiod + * boundaries + */ + heap = rh; + break; + } else if (lt_before(release_time, rh->release_time)) { + /* We need to insert a new node since rh is + * already in the future + */ + break; + } + } + if (!heap && use_server_heap) { + /* Use pre-allocated release heap */ + rh = server->release_heap; + rh->domain = rt; + rh->release_time = release_time; + + /* Add to release queue */ + list_add(&rh->list, pos->prev); + heap = rh; + } + return heap; +} + +/* + * Prepare a server's release_heap for use. + */ +static int reinit_release_heap(server_t *server) +{ + int rv = 0; + server_release_heap_t* rh; + + /* Use pre-allocated release heap */ + rh = server->release_heap; + + /* WARNING: If the CPU still holds the release_lock at this point, + * deadlock may occur! + */ + rv = hrtimer_try_to_cancel(&rh->timer); + + /* The timer callback is running, it is useless to add + * to the release heap now. + */ + if (rv == -1) { + rv = 0; + goto out; + } + + /* Under no cirumstances should the timer have been active + * but not running. + */ + rv = 1; + + /* initialize */ + INIT_LIST_HEAD(&rh->servers); + atomic_set(&rh->info.state, HRTIMER_START_ON_INACTIVE); + out: + return rv; +} + +/* + * Arm the release timer for the next set of servers. + */ +static int arm_release_timer(server_domain_t *domain) +{ + int rv = 1; + struct list_head list; + struct list_head *pos, *safe; + server_t *server; + server_release_heap_t *rh; + + list_replace_init(&domain->tobe_released, &list); + + list_for_each_safe(pos, safe, &list) { + /* Pick server from work list */ + server = list_entry(pos, server_t, release_list); + list_del(pos); + + /* Put into release heap while holding release_lock */ + raw_spin_lock(&domain->release_lock); + + rh = get_release_heap(domain, server, 0); + if (!rh) { + /* Need to use our own, but drop lock first */ + raw_spin_unlock(&domain->release_lock); + + rv = reinit_release_heap(server); + + /* Bail! We missed the release time */ + if (!rv) { + rv = 0; + goto out; + } + + raw_spin_lock(&domain->release_lock); + + rh = get_release_heap(domain, server, 1); + } + + list_add(&server->release_list, &rh->servers); + + raw_spin_unlock(&domain->release_lock); + + /* To avoid arming the timer multiple times, we only let the + * owner do the arming (which is the "first" task to reference + * this release_heap anyway). + */ + if (rh == server->release_heap) { + /* We cannot arm the timer using hrtimer_start() + * as it may deadlock on rq->lock + * + * PINNED mode is ok on both local and remote CPU + */ + if (domain->release_master == NO_CPU) { + __hrtimer_start_range_ns(&rh->timer, + ns_to_ktime(rh->release_time), + 0, HRTIMER_MODE_ABS_PINNED, 0); + } else { + hrtimer_start_on(domain->release_master, + &rh->info, &rh->timer, + ns_to_ktime(rh->release_time), + HRTIMER_MODE_ABS_PINNED); + } + } + } + out: + return rv; +} + +int add_server_release(server_t *server, server_domain_t *domain) +{ + list_add(&server->release_list, &domain->tobe_released); + return arm_release_timer(domain); +} + +static int __init init_servers(void) +{ + server_cache = KMEM_CACHE(server, SLAB_PANIC); + server_release_cache = KMEM_CACHE(server_release_heap, SLAB_PANIC); + return 1; +} + +static void exit_servers(void) +{ + kmem_cache_destroy(server_cache); + kmem_cache_destroy(server_release_cache); +} + + +module_init(init_servers); +module_exit(exit_servers); -- 1.7.2.3