diff --git a/include/asm-i386/rwsem.h b/include/asm-i386/rwsem.h index 041906f..6983c28 100644 --- a/include/asm-i386/rwsem.h +++ b/include/asm-i386/rwsem.h @@ -42,6 +42,8 @@ #include #include +#include + struct rwsem_waiter; extern struct rw_semaphore *FASTCALL(rwsem_down_read_failed(struct rw_semaphore *sem)); @@ -106,6 +108,7 @@ LOCK_PREFIX " incl (%%eax)\n\t" /* adds 0x00000001, returns the old value : "+m" (sem->count) : "a" (sem) : "memory", "cc"); + TS_SEM_LOCK(sem); } /* @@ -128,7 +131,12 @@ LOCK_PREFIX " cmpxchgl %2,%0\n\t" : "+m" (sem->count), "=&a" (result), "=&r" (tmp) : "i" (RWSEM_ACTIVE_READ_BIAS) : "memory", "cc"); - return result>=0 ? 1 : 0; + if (result >= 0) { + TS_SEM_LOCK(sem); + return 1; + } + else + return 0; } /* @@ -155,6 +163,7 @@ LOCK_PREFIX " xadd %%edx,(%%eax)\n\t" /* subtract 0x0000ffff, returns the static inline void __down_write(struct rw_semaphore *sem) { __down_write_nested(sem, 0); + TS_SEM_LOCK(sem); } /* @@ -165,8 +174,10 @@ static inline int __down_write_trylock(struct rw_semaphore *sem) signed long ret = cmpxchg(&sem->count, RWSEM_UNLOCKED_VALUE, RWSEM_ACTIVE_WRITE_BIAS); - if (ret == RWSEM_UNLOCKED_VALUE) + if (ret == RWSEM_UNLOCKED_VALUE) { + TS_SEM_LOCK(sem); return 1; + } return 0; } @@ -176,6 +187,7 @@ static inline int __down_write_trylock(struct rw_semaphore *sem) static inline void __up_read(struct rw_semaphore *sem) { __s32 tmp = -RWSEM_ACTIVE_READ_BIAS; + TS_SEM_UNLOCK(sem); __asm__ __volatile__( "# beginning __up_read\n\t" LOCK_PREFIX " xadd %%edx,(%%eax)\n\t" /* subtracts 1, returns the old value */ @@ -193,6 +205,7 @@ LOCK_PREFIX " xadd %%edx,(%%eax)\n\t" /* subtracts 1, returns the old valu */ static inline void __up_write(struct rw_semaphore *sem) { + TS_SEM_UNLOCK(sem); __asm__ __volatile__( "# beginning __up_write\n\t" " movl %2,%%edx\n\t" @@ -211,6 +224,7 @@ LOCK_PREFIX " xaddl %%edx,(%%eax)\n\t" /* tries to transition 0xffff0001 -> */ static inline void __downgrade_write(struct rw_semaphore *sem) { + TS_SEM_UNLOCK(sem); __asm__ __volatile__( "# beginning __downgrade_write\n\t" LOCK_PREFIX " addl %2,(%%eax)\n\t" /* transitions 0xZZZZ0001 -> 0xYYYY0001 */ @@ -221,6 +235,7 @@ LOCK_PREFIX " addl %2,(%%eax)\n\t" /* transitions 0xZZZZ0001 -> 0xYYYY0001 : "+m" (sem->count) : "a" (sem), "i" (-RWSEM_WAITING_BIAS) : "memory", "cc"); + TS_SEM_LOCK(sem); } /* diff --git a/include/asm-i386/semaphore.h b/include/asm-i386/semaphore.h index 4e34a46..6ad71c3 100644 --- a/include/asm-i386/semaphore.h +++ b/include/asm-i386/semaphore.h @@ -41,6 +41,8 @@ #include #include +#include + struct semaphore { atomic_t count; int sleepers; @@ -107,6 +109,7 @@ static inline void down(struct semaphore * sem) :"+m" (sem->count) : :"memory","ax"); + TS_SEM_LOCK(sem); } /* @@ -129,6 +132,8 @@ static inline int down_interruptible(struct semaphore * sem) :"=&a" (result), "+m" (sem->count) : :"memory"); + if (!result) + TS_SEM_LOCK(sem); return result; } @@ -151,6 +156,8 @@ static inline int down_trylock(struct semaphore * sem) :"=&a" (result), "+m" (sem->count) : :"memory"); + if (!result) + TS_SEM_LOCK(sem); return result; } @@ -160,6 +167,7 @@ static inline int down_trylock(struct semaphore * sem) */ static inline void up(struct semaphore * sem) { + TS_SEM_UNLOCK(sem); __asm__ __volatile__( "# atomic up operation\n\t" LOCK_PREFIX "incl %0\n\t" /* ++sem->count */ diff --git a/include/linux/feather_buffer.h b/include/linux/feather_buffer.h new file mode 100644 index 0000000..ca49bf2 --- /dev/null +++ b/include/linux/feather_buffer.h @@ -0,0 +1,110 @@ +#ifndef _FEATHER_BUFFER_H_ +#define _FEATHER_BUFFER_H_ + +/* requires UINT_MAX and memcpy */ + +static inline int fetch_and_inc(int *val) +{ + int ret = 1; + __asm__ __volatile__("lock; xaddl %0, %1" : "+r" (ret), "+m" (*val) : : "memory" ); + return ret; +} + +static inline int fetch_and_dec(int *val) +{ + int ret = -1; + __asm__ __volatile__("lock; xaddl %0, %1" : "+r" (ret), "+m" (*val) : : "memory" ); + return ret; +} + +#define SLOT_FREE 0 +#define SLOT_BUSY 1 +#define SLOT_READY 2 + +struct ft_buffer { + unsigned int slot_count; + unsigned int slot_size; + + int free_count; + unsigned int write_idx; + unsigned int read_idx; + + char* slots; + void* buffer_mem; + unsigned int failed_writes; +}; + +static inline int init_ft_buffer(struct ft_buffer* buf, + unsigned int slot_count, + unsigned int slot_size, + char* slots, + void* buffer_mem) +{ + int i = 0; + if (!slot_count || UINT_MAX % slot_count != slot_count - 1) { + /* The slot count must divide UNIT_MAX + 1 so that when it + * wraps around the index correctly points to 0. + */ + return 0; + } else { + buf->slot_count = slot_count; + buf->slot_size = slot_size; + buf->slots = slots; + buf->buffer_mem = buffer_mem; + buf->free_count = slot_count; + buf->write_idx = 0; + buf->read_idx = 0; + buf->failed_writes = 0; + for (i = 0; i < slot_count; i++) + buf->slots[i] = SLOT_FREE; + return 1; + } +} + +static inline int ft_buffer_start_write(struct ft_buffer* buf, void **ptr) +{ + int free = fetch_and_dec(&buf->free_count); + unsigned int idx; + if (free <= 0) { + fetch_and_inc(&buf->free_count); + *ptr = 0; + fetch_and_inc(&buf->failed_writes); + return 0; + } else { + idx = fetch_and_inc((int*) &buf->write_idx) % buf->slot_count; + buf->slots[idx] = SLOT_BUSY; + *ptr = ((char*) buf->buffer_mem) + idx * buf->slot_size; + return 1; + } +} + +static inline void ft_buffer_finish_write(struct ft_buffer* buf, void *ptr) +{ + unsigned int idx; + + idx = ((char*) ptr - (char*) buf->buffer_mem) / buf->slot_size; + buf->slots[idx] = SLOT_READY; +} + + +/* exclusive reader access is assumed */ +static inline int ft_buffer_read(struct ft_buffer* buf, void* dest) +{ + unsigned int idx; + if (buf->free_count == buf->slot_count) + /* nothing available */ + return 0; + idx = buf->read_idx % buf->slot_count; + if (buf->slots[idx] == SLOT_READY) { + memcpy(dest, ((char*) buf->buffer_mem) + idx * buf->slot_size, + buf->slot_size); + buf->slots[idx] = SLOT_FREE; + buf->read_idx++; + fetch_and_inc(&buf->free_count); + return 1; + } else + return 0; +} + + +#endif diff --git a/include/linux/feather_trace.h b/include/linux/feather_trace.h new file mode 100644 index 0000000..1b576a8 --- /dev/null +++ b/include/linux/feather_trace.h @@ -0,0 +1,106 @@ +#ifndef _FEATHER_TRACE_H_ +#define _FEATHER_TRACE_H_ + +#define feather_callback __attribute__((regparm(0))) + +/* make the compiler reload any register that is not saved in + * a cdecl function call + */ +#define __FT_CLOBBER "memory", "cc", "eax", "ecx", "edx" +#define __FT_PAR "ri" + +#define ft_event(id, callback) \ + __asm__ __volatile__( \ + "1: jmp 2f \n\t" \ + " call " #callback " \n\t" \ + ".section __event_table, \"aw\" \n\t" \ + ".long " #id ", 0, 1b, 2f \n\t" \ + ".previous \n\t" \ + "2: \n\t" \ + : : : __FT_CLOBBER) + +#define ft_event0(id, callback) \ + __asm__ __volatile__( \ + "1: jmp 2f \n\t" \ + " pushl $" #id " \n\t" \ + " call " #callback " \n\t" \ + " addl $4, %%esp \n\t" \ + ".section __event_table, \"aw\" \n\t" \ + ".long " #id ", 0, 1b, 2f \n\t" \ + ".previous \n\t" \ + "2: \n\t" \ + : : : __FT_CLOBBER) + +#define ft_event1(id, callback, param) \ + __asm__ __volatile__( \ + "1: jmp 2f \n\t" \ + " pushl %0 \n\t" \ + " pushl $" #id " \n\t" \ + " call " #callback " \n\t" \ + " addl $8, %%esp \n\t" \ + ".section __event_table, \"aw\" \n\t" \ + ".long " #id ", 0, 1b, 2f \n\t" \ + ".previous \n\t" \ + "2: \n\t" \ + : : __FT_PAR (param) : __FT_CLOBBER) + +#define ft_event2(id, callback, param, param2) \ + __asm__ __volatile__( \ + "1: jmp 2f \n\t" \ + " pushl %1 \n\t" \ + " pushl %0 \n\t" \ + " pushl $" #id " \n\t" \ + " call " #callback " \n\t" \ + " addl $12, %%esp \n\t" \ + ".section __event_table, \"aw\" \n\t" \ + ".long " #id ", 0, 1b, 2f \n\t" \ + ".previous \n\t" \ + "2: \n\t" \ + : : __FT_PAR (param), __FT_PAR (param2) : __FT_CLOBBER) + + +#define ft_event3(id, callback, p, p2, p3) \ + __asm__ __volatile__( \ + "1: jmp 2f \n\t" \ + " pushl %2 \n\t" \ + " pushl %1 \n\t" \ + " pushl %0 \n\t" \ + " pushl $" #id " \n\t" \ + " call " #callback " \n\t" \ + " addl $16, %%esp \n\t" \ + ".section __event_table, \"aw\" \n\t" \ + ".long " #id ", 0, 1b, 2f \n\t" \ + ".previous \n\t" \ + "2: \n\t" \ + : : __FT_PAR (p), __FT_PAR (p2), __FT_PAR (p3) : __FT_CLOBBER) + +#define ft_event4(id, callback, p, p2, p3, p4) \ + __asm__ __volatile__( \ + "1: jmp 2f \n\t" \ + " pushl %3 \n\t" \ + " pushl %2 \n\t" \ + " pushl %1 \n\t" \ + " pushl %0 \n\t" \ + " pushl $" #id " \n\t" \ + " call " #callback " \n\t" \ + " addl $20, %%esp \n\t" \ + ".section __event_table, \"aw\" \n\t" \ + ".long " #id ", 0, 1b, 2f \n\t" \ + ".previous \n\t" \ + "2: \n\t" \ + : : __FT_PAR (p), __FT_PAR (p2), __FT_PAR (p3), \ + __FT_PAR (p4) : __FT_CLOBBER) + + +static inline unsigned long long ft_read_tsc(void) +{ + unsigned long long ret; + __asm__ __volatile__("rdtsc" : "=A" (ret)); + return ret; +} + +int ft_enable_event(unsigned long id); +int ft_disable_event(unsigned long id); +int ft_is_event_enabled(unsigned long id); +int ft_disable_all_events(void); +#endif diff --git a/include/linux/spinlock.h b/include/linux/spinlock.h index 94b767d..54f5485 100644 --- a/include/linux/spinlock.h +++ b/include/linux/spinlock.h @@ -56,6 +56,8 @@ #include +#include + /* * Must define these before including other files, inline functions need them */ @@ -168,91 +170,194 @@ do { \ * regardless of whether CONFIG_SMP or CONFIG_PREEMPT are set. The various * methods are defined as nops in the case they are not required. */ -#define spin_trylock(lock) __cond_lock(lock, _spin_trylock(lock)) -#define read_trylock(lock) __cond_lock(lock, _read_trylock(lock)) -#define write_trylock(lock) __cond_lock(lock, _write_trylock(lock)) - -#define spin_lock(lock) _spin_lock(lock) +//#define spin_trylock(lock) __cond_lock(lock, _spin_trylock(lock)) +//#define read_trylock(lock) __cond_lock(lock, _read_trylock(lock)) +//#define write_trylock(lock) __cond_lock(lock, _write_trylock(lock)) + +static inline int spin_trylock(spinlock_t *lock) +{ + if (_spin_trylock(lock)) { + TS_SPIN_START(lock); + TS_SPIN_LOCK(lock); + return 1; + } + return 0; +} + +static inline int read_trylock(rwlock_t *lock) +{ + if (_read_trylock(lock)) { + TS_SPIN_START(lock); + TS_SPIN_LOCK(lock); + return 1; + } + return 0; +} + +static inline int write_trylock(rwlock_t *lock) +{ + if (_write_trylock(lock)) { + TS_SPIN_START(lock); + TS_SPIN_LOCK(lock); + return 1; + } + return 0; +} + + +#define spin_lock(lock) \ +do {TS_SPIN_START(lock); _spin_lock(lock); TS_SPIN_LOCK(lock); } while (0) #ifdef CONFIG_DEBUG_LOCK_ALLOC # define spin_lock_nested(lock, subclass) _spin_lock_nested(lock, subclass) +#error "wrong config for tracing" #else -# define spin_lock_nested(lock, subclass) _spin_lock(lock) +# define spin_lock_nested(lock, subclass) \ +do {TS_SPIN_START(lock); _spin_lock(lock); TS_SPIN_LOCK(lock); } while (0) #endif -#define write_lock(lock) _write_lock(lock) -#define read_lock(lock) _read_lock(lock) +#define write_lock(lock) \ +do {TS_SPIN_START(lock); _write_lock(lock); TS_SPIN_LOCK(lock); } while (0) + +#define read_lock(lock) \ +do {TS_SPIN_START(lock); _read_lock(lock); TS_SPIN_LOCK(lock); } while (0) #if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK) -#define spin_lock_irqsave(lock, flags) flags = _spin_lock_irqsave(lock) -#define read_lock_irqsave(lock, flags) flags = _read_lock_irqsave(lock) -#define write_lock_irqsave(lock, flags) flags = _write_lock_irqsave(lock) +#define spin_lock_irqsave(lock, flags) \ +do {TS_SPIN_START(lock); flags = _spin_lock_irqsave(lock); TS_SPIN_LOCK(lock); } while (0) + +#define read_lock_irqsave(lock, flags) \ +do {TS_SPIN_START(lock); flags = _read_lock_irqsave(lock); TS_SPIN_LOCK(lock); } while (0) + +#define write_lock_irqsave(lock, flags) \ +do {TS_SPIN_START(lock); flags = _write_lock_irqsave(lock); TS_SPIN_LOCK(lock); } while (0) + #ifdef CONFIG_DEBUG_LOCK_ALLOC #define spin_lock_irqsave_nested(lock, flags, subclass) \ flags = _spin_lock_irqsave_nested(lock, subclass) +#error "wrong config" #else #define spin_lock_irqsave_nested(lock, flags, subclass) \ - flags = _spin_lock_irqsave(lock) +do {TS_SPIN_START(lock); flags = _spin_lock_irqsave(lock); TS_SPIN_LOCK(lock); } while (0) + #endif #else -#define spin_lock_irqsave(lock, flags) _spin_lock_irqsave(lock, flags) -#define read_lock_irqsave(lock, flags) _read_lock_irqsave(lock, flags) -#define write_lock_irqsave(lock, flags) _write_lock_irqsave(lock, flags) +#define spin_lock_irqsave(lock, flags) \ +do {TS_SPIN_START(lock); flags = _spin_lock_irqsave(lock, flags); TS_SPIN_LOCK(lock); } while (0) + +#define read_lock_irqsave(lock, flags) \ +do {TS_SPIN_START(lock); flags = _read_lock_irqsave(lock, flags); TS_SPIN_LOCK(lock); } while (0) + +#define write_lock_irqsave(lock, flags) \ +do {TS_SPIN_START(lock); flags = _write_lock_irqsave(lock, flags); TS_SPIN_LOCK(lock); } while (0) + #define spin_lock_irqsave_nested(lock, flags, subclass) \ - spin_lock_irqsave(lock, flags) +do {TS_SPIN_START(lock); flags = _spin_lock_irqsave(lock, flags); TS_SPIN_LOCK(lock); } while (0) #endif -#define spin_lock_irq(lock) _spin_lock_irq(lock) -#define spin_lock_bh(lock) _spin_lock_bh(lock) +#define spin_lock_irq(lock) \ +do {TS_SPIN_START(lock); _spin_lock_irq(lock); TS_SPIN_LOCK(lock); } while (0) + +#define spin_lock_bh(lock) \ +do {TS_SPIN_START(lock); _spin_lock_bh(lock); TS_SPIN_LOCK(lock); } while (0) + +#define read_lock_irq(lock) \ +do {TS_SPIN_START(lock); _read_lock_irq(lock); TS_SPIN_LOCK(lock); } while (0) -#define read_lock_irq(lock) _read_lock_irq(lock) -#define read_lock_bh(lock) _read_lock_bh(lock) +#define read_lock_bh(lock) \ +do {TS_SPIN_START(lock); _read_lock_bh(lock); TS_SPIN_LOCK(lock); } while (0) -#define write_lock_irq(lock) _write_lock_irq(lock) -#define write_lock_bh(lock) _write_lock_bh(lock) +#define write_lock_irq(lock) \ +do {TS_SPIN_START(lock); _write_lock_irq(lock); TS_SPIN_LOCK(lock); } while (0) + +#define write_lock_bh(lock) \ +do {TS_SPIN_START(lock); _write_lock_bh(lock); TS_SPIN_LOCK(lock); } while (0) /* * We inline the unlock functions in the nondebug case: */ #if defined(CONFIG_DEBUG_SPINLOCK) || defined(CONFIG_PREEMPT) || \ !defined(CONFIG_SMP) -# define spin_unlock(lock) _spin_unlock(lock) -# define read_unlock(lock) _read_unlock(lock) -# define write_unlock(lock) _write_unlock(lock) -# define spin_unlock_irq(lock) _spin_unlock_irq(lock) -# define read_unlock_irq(lock) _read_unlock_irq(lock) -# define write_unlock_irq(lock) _write_unlock_irq(lock) + +# define spin_unlock(lock) \ +do { TS_SPIN_UNLOCK(lock); _spin_unlock(lock); } while (0) + +# define read_unlock(lock) \ +do { TS_SPIN_UNLOCK(lock); _read_unlock(lock); } while (0) + +# define write_unlock(lock) \ +do { TS_SPIN_UNLOCK(lock); _write_unlock(lock); } while (0) + +# define spin_unlock_irq(lock) \ +do { TS_SPIN_UNLOCK(lock); _spin_unlock_irq(lock); } while (0) + +# define read_unlock_irq(lock) \ +do { TS_SPIN_UNLOCK(lock); _read_unlock_irq(lock); } while (0) + +# define write_unlock_irq(lock) \ +do { TS_SPIN_UNLOCK(lock); _write_unlock_irq(lock); } while (0) + #else -# define spin_unlock(lock) __raw_spin_unlock(&(lock)->raw_lock) -# define read_unlock(lock) __raw_read_unlock(&(lock)->raw_lock) -# define write_unlock(lock) __raw_write_unlock(&(lock)->raw_lock) + + +# define spin_unlock(lock) \ +do { TS_SPIN_UNLOCK(lock);__raw_spin_unlock(&(lock)->raw_lock); } while (0) + +# define read_unlock(lock) \ +do { TS_SPIN_UNLOCK(lock);__raw_read_unlock(&(lock)->raw_lock); } while (0) + +# define write_unlock(lock) \ +do { TS_SPIN_UNLOCK(lock);__raw_write_unlock(&(lock)->raw_lock); } while (0) + # define spin_unlock_irq(lock) \ - do { __raw_spin_unlock(&(lock)->raw_lock); local_irq_enable(); } while (0) + do { TS_SPIN_UNLOCK(lock); __raw_spin_unlock(&(lock)->raw_lock); local_irq_enable(); } while (0) + # define read_unlock_irq(lock) \ - do { __raw_read_unlock(&(lock)->raw_lock); local_irq_enable(); } while (0) + do { TS_SPIN_UNLOCK(lock); __raw_read_unlock(&(lock)->raw_lock); local_irq_enable(); } while (0) + # define write_unlock_irq(lock) \ - do { __raw_write_unlock(&(lock)->raw_lock); local_irq_enable(); } while (0) + do { TS_SPIN_UNLOCK(lock); __raw_write_unlock(&(lock)->raw_lock); local_irq_enable(); } while (0) + #endif + + + + + + + + #define spin_unlock_irqrestore(lock, flags) \ - _spin_unlock_irqrestore(lock, flags) -#define spin_unlock_bh(lock) _spin_unlock_bh(lock) +do { TS_SPIN_UNLOCK(lock); _spin_unlock_irqrestore(lock, flags); } while (0) + + +#define spin_unlock_bh(lock) \ +do { TS_SPIN_UNLOCK(lock); _spin_unlock_bh(lock); } while (0) #define read_unlock_irqrestore(lock, flags) \ - _read_unlock_irqrestore(lock, flags) -#define read_unlock_bh(lock) _read_unlock_bh(lock) +do { TS_SPIN_UNLOCK(lock); _read_unlock_irqrestore(lock, flags); } while (0) + +#define read_unlock_bh(lock) \ +do { TS_SPIN_UNLOCK(lock); _read_unlock_bh(lock); } while (0) + #define write_unlock_irqrestore(lock, flags) \ - _write_unlock_irqrestore(lock, flags) -#define write_unlock_bh(lock) _write_unlock_bh(lock) +do { TS_SPIN_UNLOCK(lock); _write_unlock_irqrestore(lock, flags); } while (0) + +#define write_unlock_bh(lock) \ +do { TS_SPIN_UNLOCK(lock); _write_unlock_bh(lock); } while (0) + +/* TODO: */ #define spin_trylock_bh(lock) __cond_lock(lock, _spin_trylock_bh(lock)) + #define spin_trylock_irq(lock) \ ({ \ local_irq_disable(); \ diff --git a/include/linux/trace.h b/include/linux/trace.h new file mode 100644 index 0000000..cec85ef --- /dev/null +++ b/include/linux/trace.h @@ -0,0 +1,65 @@ + +#ifndef _SYS_TRACE_H_ +#define _SYS_TRACE_H_ + +#include +#include + +/* + * #define CONFIG_TRACE_LOCK_LOC + * + * Uncomment this to trace the location of lock calls. + * Works best for spinlocks. + */ + +/*********************** TIMESTAMPS ************************/ + +struct ft_event_data { + unsigned long event; + unsigned long long timestamp; + union { + int cpu; + pid_t pid; + } ctx; + unsigned long lock; +#ifdef CONFIG_TRACE_LOCK_LOC + char file[16]; + int line; +#endif +}; + + +/* buffer holding time stamps - will be provided by driver */ +extern struct ft_buffer* trace_ts_buf; + + +#define TS_SPIN_START(l) /* no tracing of spin start, not needed atm */ + + +#ifdef CONFIG_TRACE_LOCK_LOC +/* pass information on location of lock requests */ + +/* spin locks */ +#define TS_SPIN_LOCK(l) ft_event3(1, save_cpu_timestamp, l, \ + __FILE__, __LINE__) +#define TS_SPIN_UNLOCK(l) ft_event3(2, save_cpu_timestamp, l, \ + __FILE__, __LINE__) + + +/* semaphores */ +#define TS_SEM_LOCK(l) ft_event3(10, save_task_timestamp, l, \ + __FILE__, __LINE__) +#define TS_SEM_UNLOCK(l) ft_event3(11, save_task_timestamp, l, \ + __FILE__, __LINE__) + +#else /* !CONFIG_TRACE_LOCK_LOC */ + +#define TS_SPIN_LOCK(l) ft_event1(1, save_cpu_timestamp, l) +#define TS_SPIN_UNLOCK(l) ft_event1(2, save_cpu_timestamp, l) + +#define TS_SEM_LOCK(l) ft_event1(10, save_task_timestamp, l) +#define TS_SEM_UNLOCK(l) ft_event1(11, save_task_timestamp, l) + +#endif + +#endif /* !_SYS_TRACE_H_ */ diff --git a/kernel/Makefile b/kernel/Makefile index 14f4d45..438547a 100644 --- a/kernel/Makefile +++ b/kernel/Makefile @@ -8,7 +8,8 @@ obj-y = sched.o fork.o exec_domain.o panic.o printk.o profile.o \ signal.o sys.o kmod.o workqueue.o pid.o \ rcupdate.o extable.o params.o posix-timers.o \ kthread.o wait.o kfifo.o sys_ni.o posix-cpu-timers.o mutex.o \ - hrtimer.o rwsem.o latency.o nsproxy.o srcu.o + hrtimer.o rwsem.o latency.o nsproxy.o srcu.o \ + trace.o ft_event.o obj-$(CONFIG_STACKTRACE) += stacktrace.o obj-y += time/ diff --git a/kernel/ft_event.c b/kernel/ft_event.c new file mode 100644 index 0000000..7f0ba9a --- /dev/null +++ b/kernel/ft_event.c @@ -0,0 +1,105 @@ +#include + +#include + +/* the feather trace management functions assume + * exclusive access to the event table + */ + + +#define BYTE_JUMP 0xeb +#define BYTE_JUMP_LEN 0x02 + +/* for each event, there is an entry in the event table */ +struct trace_event { + long id; + long count; + long start_addr; + long end_addr; +}; + +extern struct trace_event __start___event_table[]; +extern struct trace_event __stop___event_table[]; + +int ft_enable_event(unsigned long id) +{ + struct trace_event* te = __start___event_table; + int count = 0; + char* delta; + unsigned char* instr; + + while (te < __stop___event_table) { + if (te->id == id && ++te->count == 1) { + instr = (unsigned char*) te->start_addr; + /* make sure we don't clobber something wrong */ + if (*instr == BYTE_JUMP) { + delta = (((unsigned char*) te->start_addr) + 1); + *delta = 0; + } + } + if (te->id == id) + count++; + te++; + } + return count; +} + +int ft_disable_all_events(void) +{ + struct trace_event* te = __start___event_table; + int count = 0; + char* delta; + unsigned char* instr; + + while (te < __stop___event_table) { + if (te->count) { + instr = (unsigned char*) te->start_addr; + if (*instr == BYTE_JUMP) { + delta = (((unsigned char*) te->start_addr) + + 1); + *delta = te->end_addr - te->start_addr - + BYTE_JUMP_LEN; + te->count = 0; + count++; + } + } + te++; + } + return count; +} + + +int ft_disable_event(unsigned long id) +{ + struct trace_event* te = __start___event_table; + int count = 0; + char* delta; + unsigned char* instr; + + while (te < __stop___event_table) { + if (te->id == id && --te->count == 0) { + instr = (unsigned char*) te->start_addr; + if (*instr == BYTE_JUMP) { + delta = (((unsigned char*) te->start_addr) + 1); + *delta = te->end_addr - te->start_addr - + BYTE_JUMP_LEN; + } + } + if (te->id == id) + count++; + te++; + } + return count; +} + +int ft_is_event_enabled(unsigned long id) +{ + struct trace_event* te = __start___event_table; + + while (te < __stop___event_table) { + if (te->id == id) + return te->count; + te++; + } + return 0; +} diff --git a/kernel/mutex.c b/kernel/mutex.c index e7cbbb8..637659f 100644 --- a/kernel/mutex.c +++ b/kernel/mutex.c @@ -19,6 +19,8 @@ #include #include +#include + /* * In the DEBUG case we are using the "NULL fastpath" for mutexes, * which forces all calls into the slowpath: @@ -89,6 +91,7 @@ void inline fastcall __sched mutex_lock(struct mutex *lock) * 'unlocked' into 'locked' state. */ __mutex_fastpath_lock(&lock->count, __mutex_lock_slowpath); + TS_SEM_LOCK(lock); } EXPORT_SYMBOL(mutex_lock); @@ -113,6 +116,7 @@ void fastcall __sched mutex_unlock(struct mutex *lock) * The unlocking fastpath is the 0->1 transition from 'locked' * into 'unlocked' state: */ + TS_SEM_UNLOCK(lock); __mutex_fastpath_unlock(&lock->count, __mutex_unlock_slowpath); } @@ -283,9 +287,13 @@ __mutex_lock_interruptible_slowpath(atomic_t *lock_count); */ int fastcall __sched mutex_lock_interruptible(struct mutex *lock) { + int ret; might_sleep(); - return __mutex_fastpath_lock_retval + ret = __mutex_fastpath_lock_retval (&lock->count, __mutex_lock_interruptible_slowpath); + if (!ret) + TS_SEM_LOCK(lock); + return ret; } EXPORT_SYMBOL(mutex_lock_interruptible); @@ -340,8 +348,12 @@ static inline int __mutex_trylock_slowpath(atomic_t *lock_count) */ int fastcall __sched mutex_trylock(struct mutex *lock) { - return __mutex_fastpath_trylock(&lock->count, + int ret; + ret = __mutex_fastpath_trylock(&lock->count, __mutex_trylock_slowpath); + if (ret) + TS_SEM_LOCK(lock); + return ret; } EXPORT_SYMBOL(mutex_trylock); diff --git a/kernel/trace.c b/kernel/trace.c new file mode 100644 index 0000000..e9d536c --- /dev/null +++ b/kernel/trace.c @@ -0,0 +1,294 @@ +#include +#include +#include +#include +#include +#include +#include + +#include + +/******************************************************************************/ +/* Allocation */ +/******************************************************************************/ + +struct ft_buffer* trace_ts_buf = NULL; + +atomic_t cpu_ctr = ATOMIC_INIT(0); +atomic_t tsk_ctr = ATOMIC_INIT(0); + +feather_callback void save_cpu_timestamp(unsigned long event, void* lock_addr +#ifdef CONFIG_TRACE_LOCK_LOC + , const char* file, int line +#endif + ) +{ + struct ft_event_data *ts; + if (event == 1) + atomic_inc(&cpu_ctr); + if (ft_buffer_start_write(trace_ts_buf, (void**) &ts)) { + ts->event = event; + ts->timestamp = ft_read_tsc(); + ts->ctx.cpu = raw_smp_processor_id(); + ts->lock = (unsigned long) lock_addr; +#ifdef CONFIG_TRACE_LOCK_LOC + ts->line = line; + strncpy(ts->file, file, 15); +#endif + ft_buffer_finish_write(trace_ts_buf, ts); + } +} + +feather_callback void save_task_timestamp(unsigned long event, void* lock_addr +#ifdef CONFIG_TRACE_LOCK_LOC + , const char* file, int line +#endif + ) +{ + struct ft_event_data *ts; + if (event == 10) + atomic_inc(&tsk_ctr); + if (ft_buffer_start_write(trace_ts_buf, (void**) &ts)) { + ts->event = event; + ts->timestamp = ft_read_tsc(); + ts->ctx.pid = current->pid; + ts->lock = (unsigned long) lock_addr; +#ifdef CONFIG_TRACE_LOCK_LOC + ts->line = line; + strncpy(ts->file, file, 15); +#endif + ft_buffer_finish_write(trace_ts_buf, ts); + } +} + + +static struct ft_buffer* alloc_ft_buffer(unsigned int count, size_t size) +{ + struct ft_buffer* buf; + size_t total = (size + 1) * count; + char* mem; + int order = 0, pages = 1; + + buf = kmalloc(sizeof(struct ft_buffer), GFP_KERNEL); + if (!buf) + return NULL; + + total = (total / PAGE_SIZE) + (total % PAGE_SIZE != 0); + while (pages < total) { + order++; + pages *= 2; + } + + mem = (char*) __get_free_pages(GFP_KERNEL, order); + if (!mem) { + kfree(buf); + return NULL; + } + + if (!init_ft_buffer(buf, count, size, + mem + (count * size), /* markers at the end */ + mem)) { /* buffer objects */ + free_pages((unsigned long) mem, order); + kfree(buf); + return NULL; + } + return buf; +} + +static void free_ft_buffer(struct ft_buffer* buf) +{ + int order = 0, pages = 1; + size_t total; + + if (buf) { + total = (buf->slot_size + 1) * buf->slot_count; + total = (total / PAGE_SIZE) + (total % PAGE_SIZE != 0); + while (pages < total) { + order++; + pages *= 2; + } + free_pages((unsigned long) buf->buffer_mem, order); + kfree(buf); + } +} + + +/******************************************************************************/ +/* DEVICE FILE DRIVER */ +/******************************************************************************/ + +#define NO_TIMESTAMPS 262144 + +static int trace_release(struct inode *in, struct file *filp) +{ + int error = -EINVAL; + + /* disable events */ + ft_disable_all_events(); + set_current_state(TASK_UNINTERRUPTIBLE); + /* wait for any pending events to complete */ + schedule_timeout(HZ); + printk(KERN_ALERT "Failed trace writes: %u\nCPU ctr: %u\nTSK ctr: %u\n", + trace_ts_buf->failed_writes, atomic_read(&cpu_ctr), atomic_read(&tsk_ctr)); + free_ft_buffer(trace_ts_buf); + trace_ts_buf = NULL; + return error; +} + +static ssize_t trace_read(struct file *filp, char __user *to, size_t len, + loff_t *f_pos) +{ + /* we ignore f_pos, this is strictly sequential */ + ssize_t error = 0; + struct ft_event_data ts; + + while (len >= sizeof(struct ft_event_data)) { + if (ft_buffer_read(trace_ts_buf, &ts)) { + if (copy_to_user(to, &ts, sizeof(struct ft_event_data))) { + error = -EFAULT; + break; + } else { + len -= sizeof(struct ft_event_data); + to += sizeof(struct ft_event_data); + error += sizeof(struct ft_event_data); + } + } else { + set_current_state(TASK_INTERRUPTIBLE); + schedule_timeout(50); + if (signal_pending(current)) { + error = -ERESTARTSYS; + break; + } + } + } + return error; +} + +#define ENABLE_CMD 0 +#define DISABLE_CMD 1 + +static ssize_t trace_write(struct file *filp, const char __user *from, + size_t len, loff_t *f_pos) +{ + ssize_t error = -EINVAL; + unsigned long cmd; + unsigned long id; + + if (len % sizeof(long) || len < 2 * sizeof(long)) + goto out; + + if (copy_from_user(&cmd, from, sizeof(long))) { + error = -EFAULT; + goto out; + } + len -= sizeof(long); + from += sizeof(long); + + if (cmd != ENABLE_CMD && cmd != DISABLE_CMD) + goto out; + + error = sizeof(long); + while (len) { + if (copy_from_user(&id, from, sizeof(long))) { + error = -EFAULT; + goto out; + } + len -= sizeof(long); + from += sizeof(long); + if (cmd) { + printk(KERN_INFO + "Disabling feather-trace event %lu.\n", id); + ft_disable_event(id); + } else { + printk(KERN_INFO + "Enabling feather-trace event %lu.\n", id); + ft_enable_event(id); + } + error += sizeof(long); + } + + + out: + return error; +} + +static int trace_open(struct inode *in, struct file *filp) +{ + int err = 0; + unsigned int count = NO_TIMESTAMPS; + while (count && !trace_ts_buf) { + printk("trace: trying to allocate %u time stamps.\n", count); + trace_ts_buf = alloc_ft_buffer(count, sizeof(struct ft_event_data)); + count /= 2; + } + atomic_set(&tsk_ctr, 0); + atomic_set(&cpu_ctr, 0); + if (!trace_ts_buf) + err = -ENOMEM; + + return err; +} + +/******************************************************************************/ +/* Device Registration */ +/******************************************************************************/ + +#define FT_TRACE_MAJOR 252 + +struct file_operations ft_trace_fops = { + .owner = THIS_MODULE, + .open = trace_open, + .release = trace_release, + .write = trace_write, + .read = trace_read, +}; + + +static int __init register_buffer_dev(const char* name, + struct file_operations* fops, + int major, int count) +{ + dev_t trace_dev; + struct cdev *cdev; + int error = 0; + + trace_dev = MKDEV(major, 0); + error = register_chrdev_region(trace_dev, count, name); + if (error) + { + printk(KERN_WARNING "trace: " + "Could not register major/minor number %d\n", major); + return error; + } + cdev = cdev_alloc(); + if (!cdev) { + printk(KERN_WARNING "trace: " + "Could not get a cdev for %s.\n", name); + return -ENOMEM; + } + cdev->owner = THIS_MODULE; + cdev->ops = fops; + error = cdev_add(cdev, trace_dev, count); + if (error) { + printk(KERN_WARNING "trace: " + "add_cdev failed for %s.\n", name); + return -ENOMEM; + } + return error; + +} + +static int __init init_feather_trace(void) +{ + int error = 0; + + printk("Initializing Feather-Trace device\n"); + /* dummy entry to make linker happy */ + ft_event0(666, save_cpu_timestamp); + + error = register_buffer_dev("ft_trace", &ft_trace_fops, + FT_TRACE_MAJOR, 1); + return error; +} + +module_init(init_feather_trace);