Gecko [ dataharbor.africa ]

Name	Size	Permission
auditbt.meta	429 B	-rw-r--r--
auditbt.stp	622 B	-rwxr-xr-x
auditbt.tcl	79 B	-rw-r--r--
chng_cpu.meta	775 B	-rw-r--r--
chng_cpu.stp	1.02 KB	-rwxr-xr-x
chng_cpu.tcl	140 B	-rw-r--r--
cycle_thief.meta	1.06 KB	-rw-r--r--
cycle_thief.stp	2.83 KB	-rwxr-xr-x
cycle_thief.txt	5.83 KB	-rw-r--r--
errsnoop.meta	575 B	-rw-r--r--
errsnoop.stp	1015 B	-rwxr-xr-x
forktracker.meta	525 B	-rw-r--r--
forktracker.stp	601 B	-rwxr-xr-x
futexes.meta	520 B	-rw-r--r--
futexes.stp	1.09 KB	-rwxr-xr-x
futexes.txt	944 B	-rw-r--r--
futexes2.meta	593 B	-rw-r--r--
futexes2.stp	1.97 KB	-rw-r--r--
ltrace.meta	314 B	-rw-r--r--
ltrace.stp	2.94 KB	-rwxr-xr-x
ltrace.tcl	122 B	-rw-r--r--
migrate.meta	746 B	-rw-r--r--
migrate.stp	1.01 KB	-rwxr-xr-x
mutex-contention.meta	479 B	-rw-r--r--
mutex-contention.stp	5.43 KB	-rwxr-xr-x
mutex-contention.tcl	79 B	-rw-r--r--
noptrace.meta	512 B	-rw-r--r--
noptrace.stp	1.98 KB	-rwxr-xr-x
noptrace.txt	1.18 KB	-rw-r--r--
pfiles.meta	450 B	-rw-r--r--
pfiles.stp	23.14 KB	-rwxr-xr-x
plimit.meta	387 B	-rw-r--r--
plimit.stp	2.96 KB	-rwxr-xr-x
proc_snoop.stp	1.11 KB	-rwxr-xr-x
proc_snoop_parser.xml	818 B	-rw-r--r--
proc_snoop_parser_instructions...	2.13 KB	-rw-r--r--
procmod_watcher.meta	657 B	-rw-r--r--
procmod_watcher.stp	2.35 KB	-rw-r--r--
proctop.meta	601 B	-rw-r--r--
proctop.stp	4.15 KB	-rwxr-xr-x
proctop.txt	8.45 KB	-rw-r--r--
psig.meta	354 B	-rw-r--r--
psig.stp	5.48 KB	-rwxr-xr-x
pstrace_exec.meta	545 B	-rw-r--r--
pstrace_exec.stp	465 B	-rwxr-xr-x
pstree.meta	455 B	-rw-r--r--
pstree.stp	1.91 KB	-rwxr-xr-x
rlimit_nofile.meta	319 B	-rw-r--r--
rlimit_nofile.stp	1.12 KB	-rwxr-xr-x
sched-latency.meta	470 B	-rw-r--r--
sched-latency.stp	508 B	-rw-r--r--
sched-latency.txt	1.53 KB	-rw-r--r--
schedtimes.meta	814 B	-rw-r--r--
schedtimes.stp	3.84 KB	-rwxr-xr-x
schedtimes.txt	3.2 KB	-rw-r--r--
semop-watch.meta	267 B	-rw-r--r--
semop-watch.stp	795 B	-rwxr-xr-x
sig_by_pid.meta	357 B	-rw-r--r--
sig_by_pid.stp	1.04 KB	-rwxr-xr-x
sig_by_pid.txt	1.48 KB	-rw-r--r--
sig_by_proc.meta	364 B	-rw-r--r--
sig_by_proc.stp	839 B	-rwxr-xr-x
sig_by_proc.txt	951 B	-rw-r--r--
sigkill.meta	560 B	-rw-r--r--
sigkill.stp	656 B	-rwxr-xr-x
sigmon.meta	669 B	-rw-r--r--
sigmon.stp	932 B	-rwxr-xr-x
sleepingBeauties.meta	581 B	-rw-r--r--
sleepingBeauties.stp	1.33 KB	-rwxr-xr-x
sleepingBeauties.tcl	140 B	-rw-r--r--
sleeptime.meta	601 B	-rw-r--r--
sleeptime.stp	1.2 KB	-rwxr-xr-x
spawn_seeker.meta	854 B	-rw-r--r--
spawn_seeker.stp	1.49 KB	-rwxr-xr-x
spawn_seeker.txt	1.29 KB	-rw-r--r--
strace.meta	385 B	-rw-r--r--
strace.stp	2.03 KB	-rwxr-xr-x
strace.txt	3.59 KB	-rw-r--r--
syscalls_by_pid.meta	514 B	-rw-r--r--
syscalls_by_pid.stp	623 B	-rwxr-xr-x
syscalls_by_pid.txt	597 B	-rw-r--r--
syscalls_by_proc.meta	531 B	-rw-r--r--
syscalls_by_proc.stp	677 B	-rwxr-xr-x
syscalls_by_proc.txt	1.1 KB	-rw-r--r--
syscalltimes	6.14 KB	-rwxr-xr-x
syscalltimes.meta	454 B	-rw-r--r--
syscalltimes.txt	11.19 KB	-rw-r--r--
thread-business.meta	346 B	-rw-r--r--
thread-business.stp	866 B	-rwxr-xr-x
thread-business.txt	2.76 KB	-rw-r--r--
threadstacks.meta	585 B	-rw-r--r--
threadstacks.stp	1.43 KB	-rwxr-xr-x
threadstacks.tcl	79 B	-rw-r--r--
wait4time.meta	658 B	-rw-r--r--
wait4time.stp	1.32 KB	-rwxr-xr-x

Code Editor : mutex-contention.stp

#!/usr/bin/stap

// NOTE: a popup mutex is one that is seen in contention but the init
// of it was not detected not sure why there are popups - static mutex
// init?

// stack trace where mutex was initialized, if known
global mutex_init_stack
// stack trace where mutex was first referenced, if init stack not known
global mutex_uninit
// stats about the contention on this mutex
global mutex_contention
// usymdata for this mutex
global mutex_names
// stack trace of last contention for this mutex
global mutex_last_cont_stack

// NOTE: the way 389 works is that many common mutexes are initialized
// in the same place.  For example - each connection object has its
// own mutex - there may be thousands of these.  They are all
// initialized in the same place - same with entry objects, where
// there may be millions.  So, if we want to get aggregate contention
// statistics for _all_ connection objects, not just each individual
// one, we have to keep track of each unique call stack.  This is what
// mutex_cont_stack is used for.

// stats about unique stacks
global mutex_cont_stack
// report mutexes that had more than this many contentions
global cont_count_threshold = 1
// usec - report mutexes that had a max time more than this
global cont_max_threshold = 100000
global verbose = 0
global keep_stats = 1

/* ... and a variant of the futexes.stp script: */
global FUTEX_WAIT = 0 /*, FUTEX_WAKE = 1 */
global FUTEX_PRIVATE_FLAG = 128 /* linux 2.6.22+ */
global FUTEX_CLOCK_REALTIME = 256 /* linux 2.6.29+ */

function process_mutex_init(mutex, ppfunc)
{
  if (verbose && (mutex in mutex_init_stack)) {
    printf("error: %s: mutex %p is already initialized at\n%s\n",
	   ppfunc, mutex, mutex_init_stack[mutex])
  }
  if (keep_stats) {
    mutex_init_stack[mutex] = sprint_ubacktrace()
    mutex_names[mutex] = usymdata (mutex)
  }
  else {
    printf("init %p at\n%s\n", mutex, sprint_ubacktrace())
  }
  if (verbose) {
    printf("%s: mutex %p %s\n", ppfunc, mutex, mutex_names[mutex])
  }
}

function show_contention(mutex, stack, type)
{
  count = @count(mutex_contention[mutex])
  max = @max(mutex_contention[mutex])
  if ((count > cont_count_threshold) || (max > cont_max_threshold)) {
    printf("=======================================\nmutex %p (%s) contended %d times, %d avg usec, %d max usec, %d total usec, %s at\n%s\n",
	   mutex, mutex_names[mutex], count, @avg(mutex_contention[mutex]),
	   max, @sum(mutex_contention[mutex]), type, stack)
    if (mutex in mutex_last_cont_stack) {
      printf("\nmutex was last contended at\n%s\n",
	     mutex_last_cont_stack[mutex])
    }
  }
}

// we use @defined($muex) and @defined($rwlock) here because
// on 64 bit systems, the wildcard search takes us through
// both 64 bit and 32 bit libc (which doesn't have debuginfo),
// this means the probe point resolved from the 32 bit library
// has no context info
probe process("/lib*/libc.so*").function("pthread_mutex_init")
{
  if(@defined($mutex))
    process_mutex_init($mutex, ppfunc())
}
probe process("/lib*/libpthread.so*").function("__pthread_mutex_init")
{
  if(@defined($mutex))
    process_mutex_init($mutex, ppfunc())
}
probe process("/lib*/libpthread.so*").function("__pthread_rwlock_init")
{
  if(@defined($rwlock))
    process_mutex_init($rwlock, ppfunc())
}

global entry_times%, uaddrs%

probe syscall.futex {
  if (pid() != target()) next		// skip irrelevant processes
  if ((op & ~(FUTEX_PRIVATE_FLAG|FUTEX_CLOCK_REALTIME)) != FUTEX_WAIT) next
  entry_times[tid()] = gettimeofday_us()
  uaddrs[tid()] = futex_uaddr
}

probe syscall.futex.return
{
  if (!(tid() in entry_times)) next
  entry = entry_times[tid()]
  delete entry_times[tid()]

uaddr = uaddrs[tid()]
  delete uaddrs[tid()]

elapsed = gettimeofday_us() - entry
  if (keep_stats) {
    mutex_contention[uaddr] <<< elapsed
    stack = sprint_ubacktrace()
    mutex_last_cont_stack[uaddr] = stack
    mutex_cont_stack[stack] <<< elapsed
    if (uaddr in mutex_init_stack) {
      if (verbose) {
        printf("contention time %d on mutex %p initialized at\n%s\n",
	       elapsed, uaddr, mutex_init_stack[uaddr])
      }
    }
    else if (uaddr in mutex_uninit) {
      if (verbose) {
        printf("contention time %d on popup mutex %p at\n%s\n",
	       elapsed, uaddr, stack)
      }
    }
    else {
      if (verbose) {
        printf("contention time %d on popup mutex %p at\n%s\n",
	       elapsed, uaddr, stack)
      }
      mutex_uninit[uaddr] = stack
      mutex_names[uaddr] = usymdata (uaddr)
    }
  }
  else {
    printf("contention %p elapsed %d at\n%s\n", uaddr, elapsed,
	   sprint_ubacktrace())
  }
}

probe end
{
  if (!keep_stats) {
    printf("======== END\n")
    next
  }
  printf("<<<<<<<< aggregate stats\n")
  foreach (stack in mutex_cont_stack-) {
    count = @count(mutex_cont_stack[stack])
    max = @max(mutex_cont_stack[stack])
    if ((count > cont_count_threshold) || (max > cont_max_threshold)) {
      printf("=======================================\nstack contended %d times, %d avg usec, %d max usec, %d total usec, at\n%s\n",
	     @count(mutex_cont_stack[stack]), @avg(mutex_cont_stack[stack]),
	     @max(mutex_cont_stack[stack]), @sum(mutex_cont_stack[stack]),
	     stack)
    }
  }
  printf(">>>>>>>> aggregate stats\n")
  foreach (mutex in mutex_contention-) {
    if (mutex in mutex_init_stack) {
      stack = mutex_init_stack[mutex]
      type = "init"
    }
    else if (mutex in mutex_uninit) {
      stack = mutex_uninit[mutex]
      type = "popup"
    }
    show_contention(mutex, stack, type)
  }
}

${this.title}

Code Editor : mutex-contention.stp