diff options
| -rw-r--r-- | ChangeLog | 16 | ||||
| -rw-r--r-- | Makefile.am | 2 | ||||
| -rw-r--r-- | Makefile.in | 2 | ||||
| -rw-r--r-- | buildrun.cxx | 8 | ||||
| -rw-r--r-- | runtime/uprobes/Makefile | 6 | ||||
| -rw-r--r-- | runtime/uprobes/uprobes.c | 13 | ||||
| -rw-r--r-- | runtime/uprobes/uprobes.h | 14 | ||||
| -rw-r--r-- | runtime/uprobes2/uprobes.c | 2797 | ||||
| -rw-r--r-- | runtime/uprobes2/uprobes.h | 406 | ||||
| -rw-r--r-- | runtime/uprobes2/uprobes_arch.c | 11 | ||||
| -rw-r--r-- | runtime/uprobes2/uprobes_arch.h | 11 | ||||
| -rw-r--r-- | runtime/uprobes2/uprobes_x86.c | 722 | ||||
| -rw-r--r-- | runtime/uprobes2/uprobes_x86.h | 107 |
13 files changed, 4109 insertions, 6 deletions
@@ -1,3 +1,19 @@ +2008-11-03 Jim Keniston <jkenisto@us.ibm.com> + + Add a version of uprobes that works with the 2.6.27 version of utrace. + * runtime/uprobes2/uprobes.[ch]: uprobes2 + * runtime/uprobes2/uprobes_x86.[ch]: Simplified to use post-2.6.24 + unified pt_regs names ("ax" rather than "eax" or "rax"). + * runtime/uprobes2/uprobes_arch.[ch]: Pull in the arch-specific + file from the appropriate directory. + * runtime/uprobes/uprobes.[ch]: #ifdef added to pull in file from + ../uprobes2 if we're running the new utrace. + * runtime/uprobes/Makefile: Updated + * Makefile.am: Install runtime/uprobes2/*. + * Makefile.in: Regenerated + * buildrun.cxx: Always copy uprobes's Module.symvers into the + directory where stap builds the script .ko. + 2008-11-03 Mark Wielaard <mjw@redhat.com> * tapsets.cxx (literal_stmt_for_return): Add function name and file diff --git a/Makefile.am b/Makefile.am index b3a4801e..0b21231a 100644 --- a/Makefile.am +++ b/Makefile.am @@ -163,6 +163,8 @@ install-data-local: do $(INSTALL_DATA) -D $$f $(DESTDIR)$(pkgdatadir)/runtime/transport/$$f; done) (cd $(srcdir)/runtime/uprobes; for f in Makefile *.[ch]; \ do $(INSTALL_DATA) -D $$f $(DESTDIR)$(pkgdatadir)/runtime/uprobes/$$f; done) + (cd $(srcdir)/runtime/uprobes2; for f in *.[ch]; \ + do $(INSTALL_DATA) -D $$f $(DESTDIR)$(pkgdatadir)/runtime/uprobes2/$$f; done) (cd $(srcdir)/tapset; find . \( -name '*.stp' -o -name README \) -print \ | while read f; do $(INSTALL_DATA) -D $$f $(DESTDIR)$(pkgdatadir)/tapset/$$f; done) (cd $(srcdir)/testsuite/systemtap.examples; find . -type f -print \ diff --git a/Makefile.in b/Makefile.in index d43fde6f..d802d954 100644 --- a/Makefile.in +++ b/Makefile.in @@ -1377,6 +1377,8 @@ install-data-local: do $(INSTALL_DATA) -D $$f $(DESTDIR)$(pkgdatadir)/runtime/transport/$$f; done) (cd $(srcdir)/runtime/uprobes; for f in Makefile *.[ch]; \ do $(INSTALL_DATA) -D $$f $(DESTDIR)$(pkgdatadir)/runtime/uprobes/$$f; done) + (cd $(srcdir)/runtime/uprobes2; for f in *.[ch]; \ + do $(INSTALL_DATA) -D $$f $(DESTDIR)$(pkgdatadir)/runtime/uprobes2/$$f; done) (cd $(srcdir)/tapset; find . \( -name '*.stp' -o -name README \) -print \ | while read f; do $(INSTALL_DATA) -D $$f $(DESTDIR)$(pkgdatadir)/tapset/$$f; done) (cd $(srcdir)/testsuite/systemtap.examples; find . -type f -print \ diff --git a/buildrun.cxx b/buildrun.cxx index 5f31303b..c23a4747 100644 --- a/buildrun.cxx +++ b/buildrun.cxx @@ -253,12 +253,12 @@ uprobes_pass (systemtap_session& s) * stap must fail. */ int rc; - if (geteuid() == 0) { + if (geteuid() == 0) rc = make_uprobes(s); - if (rc == 0) - rc = copy_uprobes_symbols(s); - } else + else rc = verify_uprobes_uptodate(s); + if (rc == 0) + rc = copy_uprobes_symbols(s); return rc; } diff --git a/runtime/uprobes/Makefile b/runtime/uprobes/Makefile index 40af7aa2..a9630e5a 100644 --- a/runtime/uprobes/Makefile +++ b/runtime/uprobes/Makefile @@ -1,8 +1,9 @@ obj-m := uprobes.o KDIR := /lib/modules/$(shell uname -r)/build PWD := $(shell pwd) -DEPENDENCIES := $(shell echo uprobes_arch.[ch] uprobes.[ch] uprobes_*.[ch]) -DEPENDENCIES += Makefile $(KDIR)/Module.symvers +DEPENDENCIES := $(shell echo uprobes.[ch] uprobes_*.[ch]) +DEPENDENCIES += $(shell echo ../uprobes2/uprobes.[ch] ../uprobes2/uprobes_*.[ch]) +DEPENDENCIES += Makefile default: $(MAKE) -C $(KDIR) SUBDIRS=$(PWD) modules @@ -13,4 +14,5 @@ uprobes.ko: $(DEPENDENCIES) clean: rm -f *.mod.c *.ko *.o .*.cmd *~ + rm -f Module.markers modules.order Module.symvers rm -rf .tmp_versions diff --git a/runtime/uprobes/uprobes.c b/runtime/uprobes/uprobes.c index 8c187bd1..22d62ecc 100644 --- a/runtime/uprobes/uprobes.c +++ b/runtime/uprobes/uprobes.c @@ -1,3 +1,14 @@ +#include <linux/utrace.h> +#ifndef UTRACE_ACTION_RESUME + +/* + * Assume the kernel is running the 2008 version of utrace. + * Skip the code in this file and instead use uprobes 2. + */ +#include "../uprobes2/uprobes.c" + +#else /* uprobes 1 (based on original utrace) */ + /* * Userspace Probes (UProbes) * kernel/uprobes_core.c @@ -2575,3 +2586,5 @@ static int __access_process_vm(struct task_struct *tsk, unsigned long addr, void #endif #include "uprobes_arch.c" MODULE_LICENSE("GPL"); + +#endif /* uprobes 1 (based on original utrace) */ diff --git a/runtime/uprobes/uprobes.h b/runtime/uprobes/uprobes.h index dc970bbf..0266cb7d 100644 --- a/runtime/uprobes/uprobes.h +++ b/runtime/uprobes/uprobes.h @@ -1,4 +1,16 @@ #ifndef _LINUX_UPROBES_H + +#include <linux/utrace.h> +#ifndef UTRACE_ACTION_RESUME + +/* + * Assume the kernel is running the 2008 version of utrace. + * Skip the code in this file and instead use uprobes 2. + */ +#include "../uprobes2/uprobes.h" + +#else /* uprobes 1 (based on original utrace) */ + #define _LINUX_UPROBES_H /* * Userspace Probes (UProbes) @@ -400,4 +412,6 @@ static void uprobe_post_ssout(struct uprobe_task*, struct uprobe_probept*, #endif /* UPROBES_IMPLEMENTATION */ +#endif /* uprobes 1 (based on original utrace) */ + #endif /* _LINUX_UPROBES_H */ diff --git a/runtime/uprobes2/uprobes.c b/runtime/uprobes2/uprobes.c new file mode 100644 index 00000000..02496a4e --- /dev/null +++ b/runtime/uprobes2/uprobes.c @@ -0,0 +1,2797 @@ +/* + * Userspace Probes (UProbes) + * kernel/uprobes_core.c + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + * + * Copyright (C) IBM Corporation, 2006 + */ +#include <linux/types.h> +#include <linux/hash.h> +#include <linux/init.h> +#include <linux/module.h> +#include <linux/sched.h> +#include <linux/rcupdate.h> +#include <linux/err.h> +#include <linux/kref.h> +#include <linux/utrace.h> +#include <linux/regset.h> +#define UPROBES_IMPLEMENTATION 1 +#include "uprobes.h" +#include <linux/tracehook.h> +#include <linux/mm.h> +#include <linux/string.h> +#include <linux/uaccess.h> +// #include <asm/tracehook.h> +#include <asm/errno.h> +#include <asm/mman.h> + +#define UPROBE_SET_FLAGS 1 +#define UPROBE_CLEAR_FLAGS 0 + +#define MAX_SSOL_SLOTS 1024 +#define SLOT_SIZE MAX_UINSN_BYTES + +#define NO_ACCESS_PROCESS_VM_EXPORT +#ifdef NO_ACCESS_PROCESS_VM_EXPORT +static int __access_process_vm(struct task_struct *tsk, unsigned long addr, + void *buf, int len, int write); +#define access_process_vm __access_process_vm +#else +extern int access_process_vm(struct task_struct *tsk, unsigned long addr, + void *buf, int len, int write); +#endif +static int utask_fake_quiesce(struct uprobe_task *utask); + +static void uretprobe_handle_entry(struct uprobe *u, struct pt_regs *regs, + struct uprobe_task *utask); +static void uretprobe_handle_return(struct pt_regs *regs, + struct uprobe_task *utask); +static void uretprobe_set_trampoline(struct uprobe_process *uproc, + struct task_struct *tsk); +static void zap_uretprobe_instances(struct uprobe *u, + struct uprobe_process *uproc); + +typedef void (*uprobe_handler_t)(struct uprobe*, struct pt_regs*); +#define URETPROBE_HANDLE_ENTRY ((uprobe_handler_t)-1L) +#define is_uretprobe(u) (u->handler == URETPROBE_HANDLE_ENTRY) +/* Point utask->active_probe at this while running uretprobe handler. */ +static struct uprobe_probept uretprobe_trampoline_dummy_probe; + +/* Table of currently probed processes, hashed by tgid. */ +static struct hlist_head uproc_table[UPROBE_TABLE_SIZE]; + +/* Protects uproc_table during uprobe (un)registration */ +static DEFINE_MUTEX(uproc_mutex); + +/* Table of uprobe_tasks, hashed by task_struct pointer. */ +static struct hlist_head utask_table[UPROBE_TABLE_SIZE]; +static DEFINE_SPINLOCK(utask_table_lock); + +#define lock_uproc_table() mutex_lock(&uproc_mutex) +#define unlock_uproc_table() mutex_unlock(&uproc_mutex) + +#define lock_utask_table(flags) spin_lock_irqsave(&utask_table_lock, (flags)) +#define unlock_utask_table(flags) \ + spin_unlock_irqrestore(&utask_table_lock, (flags)) + +/* p_uprobe_utrace_ops = &uprobe_utrace_ops. Fwd refs are a pain w/o this. */ +static const struct utrace_engine_ops *p_uprobe_utrace_ops; + +struct deferred_registration { + struct list_head list; + struct uprobe *uprobe; + int regflag; /* 0 - unregister, 1 - register */ + enum uprobe_type type; +}; + +/* + * Calling a signal handler cancels single-stepping, so uprobes delays + * calling the handler, as necessary, until after single-stepping is completed. + */ +struct delayed_signal { + struct list_head list; + siginfo_t info; +}; + +static struct uprobe_task *uprobe_find_utask(struct task_struct *tsk) +{ + struct hlist_head *head; + struct hlist_node *node; + struct uprobe_task *utask; + unsigned long flags; + + head = &utask_table[hash_ptr(tsk, UPROBE_HASH_BITS)]; + lock_utask_table(flags); + hlist_for_each_entry(utask, node, head, hlist) { + if (utask->tsk == tsk) { + unlock_utask_table(flags); + return utask; + } + } + unlock_utask_table(flags); + return NULL; +} + +static void uprobe_hash_utask(struct uprobe_task *utask) +{ + struct hlist_head *head; + unsigned long flags; + + INIT_HLIST_NODE(&utask->hlist); + head = &utask_table[hash_ptr(utask->tsk, UPROBE_HASH_BITS)]; + lock_utask_table(flags); + hlist_add_head(&utask->hlist, head); + unlock_utask_table(flags); +} + +static void uprobe_unhash_utask(struct uprobe_task *utask) +{ + unsigned long flags; + + lock_utask_table(flags); + hlist_del(&utask->hlist); + unlock_utask_table(flags); +} + +static inline void uprobe_get_process(struct uprobe_process *uproc) +{ + atomic_inc(&uproc->refcount); +} + +/* + * Decrement uproc's refcount in a situation where we "know" it can't + * reach zero. It's OK to call this with uproc locked. Compare with + * uprobe_put_process(). + */ +static inline void uprobe_decref_process(struct uprobe_process *uproc) +{ + if (atomic_dec_and_test(&uproc->refcount)) + BUG(); +} + +/* + * Runs with the uproc_mutex held. Returns with uproc ref-counted and + * write-locked. + * + * Around exec time, briefly, it's possible to have one (finished) uproc + * for the old image and one for the new image. We find the latter. + */ +static struct uprobe_process *uprobe_find_process(struct pid *tg_leader) +{ + struct hlist_head *head; + struct hlist_node *node; + struct uprobe_process *uproc; + + head = &uproc_table[hash_ptr(tg_leader, UPROBE_HASH_BITS)]; + hlist_for_each_entry(uproc, node, head, hlist) { + if (uproc->tg_leader == tg_leader && !uproc->finished) { + uprobe_get_process(uproc); + down_write(&uproc->rwsem); + return uproc; + } + } + return NULL; +} + +/* + * In the given uproc's hash table of probepoints, find the one with the + * specified virtual address. Runs with uproc->rwsem locked. + */ +static struct uprobe_probept *uprobe_find_probept(struct uprobe_process *uproc, + unsigned long vaddr) +{ + struct uprobe_probept *ppt; + struct hlist_node *node; + struct hlist_head *head = &uproc->uprobe_table[hash_long(vaddr, + UPROBE_HASH_BITS)]; + + hlist_for_each_entry(ppt, node, head, ut_node) { + if (ppt->vaddr == vaddr && ppt->state != UPROBE_DISABLED) + return ppt; + } + return NULL; +} + +/* + * set_bp: Store a breakpoint instruction at ppt->vaddr. + * Returns BP_INSN_SIZE on success. + * + * NOTE: BREAKPOINT_INSTRUCTION on all archs is the same size as + * uprobe_opcode_t. + */ +static int set_bp(struct uprobe_probept *ppt, struct task_struct *tsk) +{ + uprobe_opcode_t bp_insn = BREAKPOINT_INSTRUCTION; + return access_process_vm(tsk, ppt->vaddr, &bp_insn, BP_INSN_SIZE, 1); +} + +/* + * set_orig_insn: For probepoint ppt, replace the breakpoint instruction + * with the original opcode. Returns BP_INSN_SIZE on success. + */ +static int set_orig_insn(struct uprobe_probept *ppt, struct task_struct *tsk) +{ + return access_process_vm(tsk, ppt->vaddr, &ppt->opcode, BP_INSN_SIZE, + 1); +} + +static void bkpt_insertion_failed(struct uprobe_probept *ppt, const char *why) +{ + printk(KERN_ERR "Can't place uprobe at pid %d vaddr %#lx: %s\n", + pid_nr(ppt->uproc->tg_leader), ppt->vaddr, why); +} + +/* + * Save a copy of the original instruction (so it can be single-stepped + * out of line), insert the breakpoint instruction, and awake + * register_uprobe(). + */ +static void insert_bkpt(struct uprobe_probept *ppt, struct task_struct *tsk) +{ + struct uprobe_kimg *uk; + long result = 0; + int len; + + if (!tsk) { + /* No surviving tasks associated with ppt->uproc */ + result = -ESRCH; + goto out; + } + + /* + * If access_process_vm() transfers fewer bytes than the maximum + * instruction size, assume that the probed instruction is smaller + * than the max and near the end of the last page of instructions. + * But there must be room at least for a breakpoint-size instruction. + */ + len = access_process_vm(tsk, ppt->vaddr, ppt->insn, MAX_UINSN_BYTES, 0); + if (len < BP_INSN_SIZE) { + bkpt_insertion_failed(ppt, + "error reading original instruction"); + result = -EIO; + goto out; + } + memcpy(&ppt->opcode, ppt->insn, BP_INSN_SIZE); + if (ppt->opcode == BREAKPOINT_INSTRUCTION) { + bkpt_insertion_failed(ppt, "bkpt already exists at that addr"); + result = -EEXIST; + goto out; + } + + if ((result = arch_validate_probed_insn(ppt, tsk)) < 0) { + bkpt_insertion_failed(ppt, "instruction type cannot be probed"); + goto out; + } + + len = set_bp(ppt, tsk); + if (len < BP_INSN_SIZE) { + bkpt_insertion_failed(ppt, "failed to insert bkpt instruction"); + result = -EIO; + goto out; + } +out: + ppt->state = (result ? UPROBE_DISABLED : UPROBE_BP_SET); + list_for_each_entry(uk, &ppt->uprobe_list, list) + uk->status = result; + wake_up_all(&ppt->waitq); +} + +static void remove_bkpt(struct uprobe_probept *ppt, struct task_struct *tsk) +{ + int len; + + if (tsk) { + len = set_orig_insn(ppt, tsk); + if (len < BP_INSN_SIZE) { + printk(KERN_ERR + "Error removing uprobe at pid %d vaddr %#lx:" + " can't restore original instruction\n", + tsk->tgid, ppt->vaddr); + /* + * This shouldn't happen, since we were previously + * able to write the breakpoint at that address. + * There's not much we can do besides let the + * process die with a SIGTRAP the next time the + * breakpoint is hit. + */ + } + } + /* Wake up unregister_uprobe(). */ + ppt->state = UPROBE_DISABLED; + wake_up_all(&ppt->waitq); +} + +/* + * Runs with all of uproc's threads quiesced and uproc->rwsem write-locked. + * As specified, insert or remove the breakpoint instruction for each + * uprobe_probept on uproc's pending list. + * tsk = one of the tasks associated with uproc -- NULL if there are + * no surviving threads. + * It's OK for uproc->pending_uprobes to be empty here. It can happen + * if a register and an unregister are requested (by different probers) + * simultaneously for the same pid/vaddr. + */ +static void handle_pending_uprobes(struct uprobe_process *uproc, + struct task_struct *tsk) +{ + struct uprobe_probept *ppt, *tmp; + + list_for_each_entry_safe(ppt, tmp, &uproc->pending_uprobes, pd_node) { + switch (ppt->state) { + case UPROBE_INSERTING: + insert_bkpt(ppt, tsk); + break; + case UPROBE_REMOVING: + remove_bkpt(ppt, tsk); + break; + default: + BUG(); + } + list_del(&ppt->pd_node); + } +} + +static void utask_adjust_flags(struct uprobe_task *utask, int set, + unsigned long flags) +{ + unsigned long newflags, oldflags; + + newflags = oldflags = utask->engine->flags; + if (set) + newflags |= flags; + else + newflags &= ~flags; + /* + * utrace_barrier[_pid] is not appropriate here. If we're + * adjusting current, it's not needed. And if we're adjusting + * some other task, we're holding utask->uproc->rwsem, which + * could prevent that task from completing the callback we'd + * be waiting on. + */ + if (newflags != oldflags) { + if (utrace_set_events_pid(utask->pid, utask->engine, + newflags) != 0) + /* We don't care. */ + ; + } +} + +static inline void clear_utrace_quiesce(struct uprobe_task *utask, bool resume) +{ + utask_adjust_flags(utask, UPROBE_CLEAR_FLAGS, UTRACE_EVENT(QUIESCE)); + if (resume) { + if (utrace_control_pid(utask->pid, utask->engine, + UTRACE_RESUME) != 0) + /* We don't care. */ + ; + } +} + +/* Opposite of quiesce_all_threads(). Same locking applies. */ +static void rouse_all_threads(struct uprobe_process *uproc) +{ + struct uprobe_task *utask; + + list_for_each_entry(utask, &uproc->thread_list, list) { + if (utask->quiescing) { + utask->quiescing = 0; + if (utask->state == UPTASK_QUIESCENT) { + utask->state = UPTASK_RUNNING; + uproc->n_quiescent_threads--; + clear_utrace_quiesce(utask, true); + } + } + } + /* Wake any threads that decided to sleep rather than quiesce. */ + wake_up_all(&uproc->waitq); +} + +/* + * If all of uproc's surviving threads have quiesced, do the necessary + * breakpoint insertions or removals, un-quiesce everybody, and return 1. + * tsk is a surviving thread, or NULL if there is none. Runs with + * uproc->rwsem write-locked. + */ +static int check_uproc_quiesced(struct uprobe_process *uproc, + struct task_struct *tsk) +{ + if (uproc->n_quiescent_threads >= uproc->nthreads) { + handle_pending_uprobes(uproc, tsk); + rouse_all_threads(uproc); + return 1; + } + return 0; +} + +/* Direct the indicated thread to quiesce. */ +static void uprobe_stop_thread(struct uprobe_task *utask) +{ + int result; + /* + * As with utask_adjust_flags, calling utrace_barrier_pid below + * could deadlock. + */ + BUG_ON(utask->tsk == current); + result = utrace_control_pid(utask->pid, utask->engine, UTRACE_STOP); + if (result == 0) { + /* Already stopped. */ + utask->state = UPTASK_QUIESCENT; + utask->uproc->n_quiescent_threads++; + } else if (result == -EINPROGRESS) { + if (utask->tsk->state & TASK_INTERRUPTIBLE) { + /* + * Task could be in interruptible wait for a long + * time -- e.g., if stopped for I/O. But we know + * it's not going to run user code before all + * threads quiesce, so pretend it's quiesced. + * This avoids terminating a system call via + * UTRACE_INTERRUPT. + */ + utask->state = UPTASK_QUIESCENT; + utask->uproc->n_quiescent_threads++; + } else { + /* + * Task will eventually stop, but it may be a long time. + * Don't wait. + */ + result = utrace_control_pid(utask->pid, utask->engine, + UTRACE_INTERRUPT); + if (result != 0) + /* We don't care. */ + ; + } + } +} + +/* + * Quiesce all threads in the specified process -- e.g., prior to + * breakpoint insertion. Runs with uproc->rwsem write-locked. + * Returns false if all threads have died. + */ +static bool quiesce_all_threads(struct uprobe_process *uproc, + struct uprobe_task **cur_utask_quiescing) +{ + struct uprobe_task *utask; + struct task_struct *survivor = NULL; // any survivor + bool survivors = false; + + *cur_utask_quiescing = NULL; + list_for_each_entry(utask, &uproc->thread_list, list) { + if (!survivors) { + survivor = pid_task(utask->pid, PIDTYPE_PID); + if (survivor) + survivors = true; + } + if (!utask->quiescing) { + /* + * If utask is currently handling a probepoint, it'll + * check utask->quiescing and quiesce when it's done. + */ + utask->quiescing = 1; + if (utask->tsk == current) + *cur_utask_quiescing = utask; + else if (utask->state == UPTASK_RUNNING) { + utask_adjust_flags(utask, UPROBE_SET_FLAGS, + UTRACE_EVENT(QUIESCE)); + uprobe_stop_thread(utask); + } + } + } + /* + * If all the (other) threads are already quiesced, it's up to the + * current thread to do the necessary work. + */ + check_uproc_quiesced(uproc, survivor); + return survivors; +} + +/* Called with utask->uproc write-locked. */ +static void uprobe_free_task(struct uprobe_task *utask, bool in_callback) +{ + struct deferred_registration *dr, *d; + struct delayed_signal *ds, *ds2; + struct uretprobe_instance *ri; + struct hlist_node *r1, *r2; + + if (utask->engine && (utask->tsk != current || !in_callback)) { + /* + * No other tasks in this process should be running + * uprobe_report_* callbacks. (If they are, utrace_barrier() + * here could deadlock.) + */ + int result = utrace_control_pid(utask->pid, utask->engine, + UTRACE_DETACH); + BUG_ON(result == -EINPROGRESS); + } + put_pid(utask->pid); /* null pid OK */ + + uprobe_unhash_utask(utask); + list_del(&utask->list); + list_for_each_entry_safe(dr, d, &utask->deferred_registrations, list) { + list_del(&dr->list); + kfree(dr); + } + + list_for_each_entry_safe(ds, ds2, &utask->delayed_signals, list) { + list_del(&ds->list); + kfree(ds); + } + + hlist_for_each_entry_safe(ri, r1, r2, &utask->uretprobe_instances, + hlist) { + hlist_del(&ri->hlist); + kfree(ri); + uprobe_decref_process(utask->uproc); + } + kfree(utask); +} + +/* + * Dismantle uproc and all its remaining uprobe_tasks. + * in_callback = 1 if the caller is a uprobe_report_* callback who will + * handle the UTRACE_DETACH operation. + * Runs with uproc_mutex held; called with uproc->rwsem write-locked. + */ +static void uprobe_free_process(struct uprobe_process *uproc, int in_callback) +{ + struct uprobe_task *utask, *tmp; + struct uprobe_ssol_area *area = &uproc->ssol_area; + + if (area->slots) + kfree(area->slots); + if (!hlist_unhashed(&uproc->hlist)) + hlist_del(&uproc->hlist); + list_for_each_entry_safe(utask, tmp, &uproc->thread_list, list) + uprobe_free_task(utask, in_callback); + put_pid(uproc->tg_leader); + up_write(&uproc->rwsem); // So kfree doesn't complain + kfree(uproc); +} + +/* + * Decrement uproc's ref count. If it's zero, free uproc and return 1. + * Else return 0. If uproc is locked, don't call this; use + * uprobe_decref_process(). + * + * If we free uproc, we also decrement the ref-count on the uprobes + * module, if any. If somebody is doing "rmmod --wait uprobes", this + * function could schedule removal of the module. Therefore, don't call + * this function and then sleep in uprobes code, unless you know you'll + * return with the module ref-count > 0. + */ +static int uprobe_put_process(struct uprobe_process *uproc, bool in_callback) +{ + int freed = 0; + if (atomic_dec_and_test(&uproc->refcount)) { + lock_uproc_table(); + down_write(&uproc->rwsem); + if (unlikely(atomic_read(&uproc->refcount) != 0)) { + /* + * The works because uproc_mutex is held any + * time the ref count can go from 0 to 1 -- e.g., + * register_uprobe() sneaks in with a new probe. + */ + up_write(&uproc->rwsem); + } else { + uprobe_free_process(uproc, in_callback); + freed = 1; + } + unlock_uproc_table(); + } + if (freed) + module_put(THIS_MODULE); + return freed; +} + +static struct uprobe_kimg *uprobe_mk_kimg(struct uprobe *u) +{ + struct uprobe_kimg *uk = (struct uprobe_kimg*)kzalloc(sizeof *uk, + GFP_USER); + if (unlikely(!uk)) + return ERR_PTR(-ENOMEM); + u->kdata = uk; + uk->uprobe = u; + uk->ppt = NULL; + INIT_LIST_HEAD(&uk->list); + uk->status = -EBUSY; + return uk; +} + +/* + * Allocate a uprobe_task object for p and add it to uproc's list. + * Called with p "got" and uproc->rwsem write-locked. Called in one of + * the following cases: + * - before setting the first uprobe in p's process + * - we're in uprobe_report_clone() and p is the newly added thread + * Returns: + * - pointer to new uprobe_task on success + * - NULL if t dies before we can utrace_attach it + * - negative errno otherwise + */ +static struct uprobe_task *uprobe_add_task(struct pid *p, + struct uprobe_process *uproc) +{ + struct uprobe_task *utask; + struct utrace_attached_engine *engine; + struct task_struct *t = pid_task(p, PIDTYPE_PID); + + if (!t) + return NULL; + utask = (struct uprobe_task *)kzalloc(sizeof *utask, GFP_USER); + if (unlikely(utask == NULL)) + return ERR_PTR(-ENOMEM); + + utask->pid = p; + utask->tsk = t; + utask->state = UPTASK_RUNNING; + utask->quiescing = 0; + utask->uproc = uproc; + utask->active_probe = NULL; + utask->doomed = 0; + INIT_HLIST_HEAD(&utask->uretprobe_instances); + INIT_LIST_HEAD(&utask->deferred_registrations); + INIT_LIST_HEAD(&utask->delayed_signals); + INIT_LIST_HEAD(&utask->list); + list_add_tail(&utask->list, &uproc->thread_list); + uprobe_hash_utask(utask); + + engine = utrace_attach_pid(p, UTRACE_ATTACH_CREATE, + p_uprobe_utrace_ops, utask); + if (IS_ERR(engine)) { + long err = PTR_ERR(engine); + printk("uprobes: utrace_attach_task failed, returned %ld\n", + err); + uprobe_free_task(utask, 0); + if (err == -ESRCH) + return NULL; + return ERR_PTR(err); + } + utask->engine = engine; + /* + * Always watch for traps, clones, execs and exits. Caller must + * set any other engine flags. + */ + utask_adjust_flags(utask, UPROBE_SET_FLAGS, + UTRACE_EVENT(SIGNAL) | UTRACE_EVENT(SIGNAL_IGN) | + UTRACE_EVENT(SIGNAL_CORE) | UTRACE_EVENT(EXEC) | + UTRACE_EVENT(CLONE) | UTRACE_EVENT(EXIT)); + /* + * Note that it's OK if t dies just after utrace_attach, because + * with the engine in place, the appropriate report_* callback + * should handle it after we release uproc->rwsem. + */ + return utask; +} + +/* + * start_pid is the pid for a thread in the probed process. Find the + * next thread that doesn't have a corresponding uprobe_task yet. Return + * a ref-counted pid for that task, if any, else NULL. + */ +static struct pid *find_next_thread_to_add(struct uprobe_process *uproc, + struct pid *start_pid) +{ + struct task_struct *t, *start; + struct uprobe_task *utask; + struct pid *pid = NULL; + + rcu_read_lock(); + t = start = pid_task(start_pid, PIDTYPE_PID); + if (t) { + do { + if (unlikely(t->flags & PF_EXITING)) + goto dont_add; + list_for_each_entry(utask, &uproc->thread_list, list) { + if (utask->tsk == t) + /* Already added */ + goto dont_add; + } + /* Found thread/task to add. */ + pid = get_pid(task_pid(t)); + break; + dont_add: + t = next_thread(t); + } while (t != start); + } + rcu_read_unlock(); + return pid; +} + +/* Runs with uproc_mutex held; returns with uproc->rwsem write-locked. */ +static struct uprobe_process *uprobe_mk_process(struct pid *tg_leader) +{ + struct uprobe_process *uproc; + struct uprobe_task *utask; + struct pid *add_me; + int i; + long err; + + uproc = (struct uprobe_process *)kzalloc(sizeof *uproc, GFP_USER); + if (unlikely(uproc == NULL)) + return ERR_PTR(-ENOMEM); + + /* Initialize fields */ + atomic_set(&uproc->refcount, 1); + init_rwsem(&uproc->rwsem); + down_write(&uproc->rwsem); + init_waitqueue_head(&uproc->waitq); + for (i = 0; i < UPROBE_TABLE_SIZE; i++) + INIT_HLIST_HEAD(&uproc->uprobe_table[i]); + uproc->nppt = 0; + INIT_LIST_HEAD(&uproc->pending_uprobes); + INIT_LIST_HEAD(&uproc->thread_list); + uproc->nthreads = 0; + uproc->n_quiescent_threads = 0; + INIT_HLIST_NODE(&uproc->hlist); + uproc->tg_leader = get_pid(tg_leader); + uproc->tgid = pid_task(tg_leader, PIDTYPE_PID)->tgid; + uproc->finished = 0; + uproc->uretprobe_trampoline_addr = NULL; + + uproc->ssol_area.insn_area = NULL; + uproc->ssol_area.initialized = 0; + mutex_init(&uproc->ssol_area.setup_mutex); + /* Initialize rest of area in uprobe_init_ssol(). */ +#ifdef CONFIG_UPROBES_SSOL + uproc->sstep_out_of_line = 1; +#else + uproc->sstep_out_of_line = 0; +#endif + + /* + * Create and populate one utask per thread in this process. We + * can't call uprobe_add_task() while holding RCU lock, so we: + * 1. rcu_read_lock() + * 2. Find the next thread, add_me, in this process that's not + * already on uproc's thread_list. + * 3. rcu_read_unlock() + * 4. uprobe_add_task(add_me, uproc) + * Repeat 1-4 'til we have utasks for all threads. + */ + add_me = tg_leader; + while ((add_me = find_next_thread_to_add(uproc, add_me)) != NULL) { + utask = uprobe_add_task(add_me, uproc); + if (IS_ERR(utask)) { + err = PTR_ERR(utask); + goto fail; + } + if (utask) + uproc->nthreads++; + } + + if (uproc->nthreads == 0) { + /* All threads -- even p -- are dead. */ + err = -ESRCH; + goto fail; + } + return uproc; + +fail: + uprobe_free_process(uproc, 0); + return ERR_PTR(err); +} + +/* + * Creates a uprobe_probept and connects it to uk and uproc. Runs with + * uproc->rwsem write-locked. + */ +static struct uprobe_probept *uprobe_add_probept(struct uprobe_kimg *uk, + struct uprobe_process *uproc) +{ + struct uprobe_probept *ppt; + + ppt = (struct uprobe_probept *)kzalloc(sizeof *ppt, GFP_USER); + if (unlikely(ppt == NULL)) + return ERR_PTR(-ENOMEM); + init_waitqueue_head(&ppt->waitq); + mutex_init(&ppt->ssil_mutex); + mutex_init(&ppt->slot_mutex); + ppt->slot = NULL; + + /* Connect to uk. */ + INIT_LIST_HEAD(&ppt->uprobe_list); + list_add_tail(&uk->list, &ppt->uprobe_list); + uk->ppt = ppt; + uk->status = -EBUSY; + ppt->vaddr = uk->uprobe->vaddr; + + /* Connect to uproc. */ + ppt->state = UPROBE_INSERTING; + ppt->uproc = uproc; + INIT_LIST_HEAD(&ppt->pd_node); + list_add_tail(&ppt->pd_node, &uproc->pending_uprobes); + INIT_HLIST_NODE(&ppt->ut_node); + hlist_add_head(&ppt->ut_node, + &uproc->uprobe_table[hash_long(ppt->vaddr, UPROBE_HASH_BITS)]); + uproc->nppt++; + uprobe_get_process(uproc); + return ppt; +} + +/* ppt is going away. Free its slot (if it owns one) in the SSOL area. */ +static void uprobe_free_slot(struct uprobe_probept *ppt) +{ + struct uprobe_ssol_slot *slot = ppt->slot; + if (slot) { + down_write(&slot->rwsem); + if (slot->owner == ppt) { + unsigned long flags; + struct uprobe_ssol_area *area = &ppt->uproc->ssol_area; + spin_lock_irqsave(&area->lock, flags); + slot->state = SSOL_FREE; + slot->owner = NULL; + area->nfree++; + spin_unlock_irqrestore(&area->lock, flags); + } + up_write(&slot->rwsem); + } +} + +/* + * Runs with ppt->uproc write-locked. Frees ppt and decrements the ref count + * on ppt->uproc (but ref count shouldn't hit 0). + */ +static void uprobe_free_probept(struct uprobe_probept *ppt) +{ + struct uprobe_process *uproc = ppt->uproc; + uprobe_free_slot(ppt); + hlist_del(&ppt->ut_node); + uproc->nppt--; + kfree(ppt); + uprobe_decref_process(uproc); +} + +static void uprobe_free_kimg(struct uprobe_kimg *uk) +{ + uk->uprobe->kdata = NULL; + kfree(uk); +} + +/* + * Runs with uprobe_process write-locked. + * Note that we never free uk->uprobe, because the user owns that. + */ +static void purge_uprobe(struct uprobe_kimg *uk) +{ + struct uprobe_probept *ppt = uk->ppt; + list_del(&uk->list); + uprobe_free_kimg(uk); + if (list_empty(&ppt->uprobe_list)) + uprobe_free_probept(ppt); +} + +/* Probed address must be in an executable VM area, outside the SSOL area. */ +static int uprobe_validate_vaddr(struct pid *p, unsigned long vaddr, + struct uprobe_process *uproc) +{ + struct task_struct *t; + struct vm_area_struct *vma; + struct mm_struct *mm = NULL; + int ret = -EINVAL; + + rcu_read_lock(); + t = pid_task(p, PIDTYPE_PID); + if (t) + mm = get_task_mm(t); + rcu_read_unlock(); + if (!mm) + return -EINVAL; + down_read(&mm->mmap_sem); + vma = find_vma(mm, vaddr); + if (vma && vaddr >= vma->vm_start && (vma->vm_flags & VM_EXEC) && + vma->vm_start != (unsigned long) uproc->ssol_area.insn_area) + ret = 0; + up_read(&mm->mmap_sem); + mmput(mm); + return ret; +} + +/* Runs with utask->uproc read-locked. Returns -EINPROGRESS on success. */ +static int defer_registration(struct uprobe *u, int regflag, + struct uprobe_task *utask) +{ + struct deferred_registration *dr = + kmalloc(sizeof(struct deferred_registration), GFP_USER); + if (!dr) + return -ENOMEM; + dr->type = (is_uretprobe(u) ? UPTY_URETPROBE : UPTY_UPROBE); + dr->uprobe = u; + dr->regflag = regflag; + INIT_LIST_HEAD(&dr->list); + list_add_tail(&dr->list, &utask->deferred_registrations); + return -EINPROGRESS; +} + +/* + * Given a numeric thread ID, return a ref-counted struct pid for the + * task-group-leader thread. + */ +static struct pid *uprobe_get_tg_leader(pid_t p) +{ + struct pid *pid; + + rcu_read_lock(); + pid = find_vpid(p); + if (pid) { + struct task_struct *t = pid_task(pid, PIDTYPE_PID); + if (t) + pid = task_tgid(t); + else + pid = NULL; + } + rcu_read_unlock(); + return get_pid(pid); /* null pid OK here */ +} + +/* See Documentation/uprobes.txt. */ +int register_uprobe(struct uprobe *u) +{ + struct pid *p; + struct uprobe_process *uproc; + struct uprobe_kimg *uk; + struct uprobe_probept *ppt; + struct uprobe_task *cur_utask, *cur_utask_quiescing = NULL; + int ret = 0, uproc_is_new = 0; + bool survivors; + if (!u || !u->handler) + return -EINVAL; + + p = uprobe_get_tg_leader(u->pid); + if (!p) + return -ESRCH; + + cur_utask = uprobe_find_utask(current); + if (cur_utask && cur_utask->active_probe) { + /* + * Called from handler; cur_utask->uproc is read-locked. + * Do this registration later. + */ + put_pid(p); + return defer_registration(u, 1, cur_utask); + } + + /* Get the uprobe_process for this pid, or make a new one. */ + lock_uproc_table(); + uproc = uprobe_find_process(p); + + if (uproc) + unlock_uproc_table(); + else { + /* Creating a new uprobe_process. Ref-count the module. */ + if (!try_module_get(THIS_MODULE)) { + /* uprobes.ko is being removed. */ + ret = -ENOSYS; + unlock_uproc_table(); + goto fail_tsk; + } + uproc = uprobe_mk_process(p); + if (IS_ERR(uproc)) { + ret = (int) PTR_ERR(uproc); + unlock_uproc_table(); + module_put(THIS_MODULE); + goto fail_tsk; + } + /* Hold uproc_mutex until we've added uproc to uproc_table. */ + uproc_is_new = 1; + } + + if (is_uretprobe(u) && IS_ERR(uproc->uretprobe_trampoline_addr)) { + /* Previously failed to set up trampoline. */ + ret = -ENOMEM; + goto fail_uproc; + } + + if ((ret = uprobe_validate_vaddr(p, u->vaddr, uproc)) < 0) + goto fail_uproc; + + if (u->kdata) { + /* + * Probe is already/still registered. This is the only + * place we return -EBUSY to the user. + */ + ret = -EBUSY; + goto fail_uproc; + } + + uk = uprobe_mk_kimg(u); + if (IS_ERR(uk)) { + ret = (int) PTR_ERR(uk); + goto fail_uproc; + } + + /* See if we already have a probepoint at the vaddr. */ + ppt = (uproc_is_new ? NULL : uprobe_find_probept(uproc, u->vaddr)); + if (ppt) { + /* Breakpoint is already in place, or soon will be. */ + uk->ppt = ppt; + list_add_tail(&uk->list, &ppt->uprobe_list); + switch (ppt->state) { + case UPROBE_INSERTING: + uk->status = -EBUSY; // in progress + if (uproc->tg_leader == task_tgid(current)) { + cur_utask_quiescing = cur_utask; + BUG_ON(!cur_utask_quiescing); + } + break; + case UPROBE_REMOVING: + /* Wait! Don't remove that bkpt after all! */ + ppt->state = UPROBE_BP_SET; + list_del(&ppt->pd_node); // Remove from pending list. + wake_up_all(&ppt->waitq); // Wake unregister_uprobe(). + /*FALLTHROUGH*/ + case UPROBE_BP_SET: + uk->status = 0; + break; + default: + BUG(); + } + up_write(&uproc->rwsem); + put_pid(p); + if (uk->status == 0) { + uprobe_decref_process(uproc); + return 0; + } + goto await_bkpt_insertion; + } else { + ppt = uprobe_add_probept(uk, uproc); + if (IS_ERR(ppt)) { + ret = (int) PTR_ERR(ppt); + goto fail_uk; + } + } + + if (uproc_is_new) { + hlist_add_head(&uproc->hlist, + &uproc_table[hash_ptr(uproc->tg_leader, + UPROBE_HASH_BITS)]); + unlock_uproc_table(); + } + put_pid(p); + survivors = quiesce_all_threads(uproc, &cur_utask_quiescing); + + if (!survivors) { + purge_uprobe(uk); + up_write(&uproc->rwsem); + uprobe_put_process(uproc, false); + return -ESRCH; + } + up_write(&uproc->rwsem); + +await_bkpt_insertion: + if (cur_utask_quiescing) + /* Current task is probing its own process. */ + (void) utask_fake_quiesce(cur_utask_quiescing); + else + wait_event(ppt->waitq, ppt->state != UPROBE_INSERTING); + ret = uk->status; + if (ret != 0) { + down_write(&uproc->rwsem); + purge_uprobe(uk); + up_write(&uproc->rwsem); + } + uprobe_put_process(uproc, false); + return ret; + +fail_uk: + uprobe_free_kimg(uk); + +fail_uproc: + if (uproc_is_new) { + uprobe_free_process(uproc, 0); + unlock_uproc_table(); + module_put(THIS_MODULE); + } else { + up_write(&uproc->rwsem); + uprobe_put_process(uproc, false); + } + +fail_tsk: + put_pid(p); + return ret; +} +EXPORT_SYMBOL_GPL(register_uprobe); + +/* See Documentation/uprobes.txt. */ +void unregister_uprobe(struct uprobe *u) +{ + struct pid *p; + struct uprobe_process *uproc; + struct uprobe_kimg *uk; + struct uprobe_probept *ppt; + struct uprobe_task *cur_utask, *cur_utask_quiescing = NULL; + + if (!u) + return; + p = uprobe_get_tg_leader(u->pid); + if (!p) + return; + + cur_utask = uprobe_find_utask(current); + if (cur_utask && cur_utask->active_probe) { + /* Called from handler; uproc is read-locked; do this later */ + put_pid(p); + (void) defer_registration(u, 0, cur_utask); + return; + } + + /* + * Lock uproc before walking the graph, in case the process we're + * probing is exiting. + */ + lock_uproc_table(); + uproc = uprobe_find_process(p); + unlock_uproc_table(); + put_pid(p); + if (!uproc) + return; + + uk = (struct uprobe_kimg *)u->kdata; + if (!uk) + /* + * This probe was never successfully registered, or + * has already been unregistered. + */ + goto done; + if (uk->status == -EBUSY) + /* Looks like register or unregister is already in progress. */ + goto done; + ppt = uk->ppt; + + list_del(&uk->list); + uprobe_free_kimg(uk); + + if (is_uretprobe(u)) + zap_uretprobe_instances(u, uproc); + + if (!list_empty(&ppt->uprobe_list)) + goto done; + + /* + * The last uprobe at ppt's probepoint is being unregistered. + * Queue the breakpoint for removal. + */ + ppt->state = UPROBE_REMOVING; + list_add_tail(&ppt->pd_node, &uproc->pending_uprobes); + + (void) quiesce_all_threads(uproc, &cur_utask_quiescing); + up_write(&uproc->rwsem); + if (cur_utask_quiescing) + /* Current task is probing its own process. */ + (void) utask_fake_quiesce(cur_utask_quiescing); + else + wait_event(ppt->waitq, ppt->state != UPROBE_REMOVING); + + if (likely(ppt->state == UPROBE_DISABLED)) { + down_write(&uproc->rwsem); + uprobe_free_probept(ppt); + /* else somebody else's register_uprobe() resurrected ppt. */ + up_write(&uproc->rwsem); + } + uprobe_put_process(uproc, false); + return; + +done: + up_write(&uproc->rwsem); + uprobe_put_process(uproc, false); +} +EXPORT_SYMBOL_GPL(unregister_uprobe); + +/* Find a surviving thread in uproc. Runs with uproc->rwsem locked. */ +static struct task_struct *find_surviving_thread(struct uprobe_process *uproc) +{ + struct uprobe_task *utask; + + list_for_each_entry(utask, &uproc->thread_list, list) { + if (!(utask->tsk->flags & PF_EXITING)) + return utask->tsk; + } + return NULL; +} + +/* + * Run all the deferred_registrations previously queued by the current utask. + * Runs with no locks or mutexes held. The current utask's uprobe_process + * is ref-counted, so it won't disappear as the result of unregister_u*probe() + * called here. + */ +static void uprobe_run_def_regs(struct list_head *drlist) +{ + struct deferred_registration *dr, *d; + + list_for_each_entry_safe(dr, d, drlist, list) { + int result = 0; + struct uprobe *u = dr->uprobe; + + if (dr->type == UPTY_URETPROBE) { + struct uretprobe *rp = + container_of(u, struct uretprobe, u); + if (dr->regflag) + result = register_uretprobe(rp); + else + unregister_uretprobe(rp); + } else { + if (dr->regflag) + result = register_uprobe(u); + else + unregister_uprobe(u); + } + if (u && u->registration_callback) + u->registration_callback(u, dr->regflag, dr->type, + result); + list_del(&dr->list); + kfree(dr); + } +} + +/* + * Functions for allocation of the SSOL area, and the instruction slots + * therein + */ + +/* + * We leave the SSOL vma in place even after all the probes are gone. + * We used to remember its address in current->mm->context.uprobes_ssol_area, + * but adding that field to mm_context broke KAPI compatibility. + * Instead, when we create the SSOL area, we "tag" the vma for later + * use by a new uproc. This is not particularly robust, but it's + * no more vulnerable to ptrace or mprotect mischief than any other part + * of the address space. We keep the tag small to avoid wasting slots. + */ +#define UPROBES_SSOL_VMA_TAG "uprobes vma" +#define UPROBES_SSOL_TAGSZ ((int)sizeof(UPROBES_SSOL_VMA_TAG)) + +static void uprobe_tag_vma(struct uprobe_ssol_area *area) +{ + static const char *buf = UPROBES_SSOL_VMA_TAG; + struct uprobe_ssol_slot *slot = &area->slots[area->next_slot]; + + if (access_process_vm(current, (unsigned long) slot->insn, (void*)buf, + UPROBES_SSOL_TAGSZ, 1) == UPROBES_SSOL_TAGSZ) { + int nb; + for (nb = 0; nb < UPROBES_SSOL_TAGSZ; nb += SLOT_SIZE) { + slot->state = SSOL_RESERVED; + slot++; + area->next_slot++; + area->nfree--; + } + } else { + printk(KERN_ERR "Failed to tag uprobes SSOL vma: " + "pid/tgid=%d/%d, vaddr=%p\n", + current->pid, current->tgid, slot->insn); + } +} + +/* + * Searching downward from ceiling address (0 signifies top of memory), + * find the next vma whose flags indicate it could be an SSOL area. + * Return its address, or 0 for no match. + */ +static unsigned long find_next_possible_ssol_vma(unsigned long ceiling) +{ + struct mm_struct *mm; + struct rb_node *rb_node; + struct vm_area_struct *vma; + unsigned long good_flags = VM_EXEC | VM_DONTEXPAND; + unsigned long bad_flags = VM_WRITE | VM_GROWSDOWN | VM_GROWSUP; + unsigned long addr = 0; + + mm = get_task_mm(current); + if (!mm) + return 0; + down_read(&mm->mmap_sem); + for (rb_node=rb_last(&mm->mm_rb); rb_node; rb_node=rb_prev(rb_node)) { + vma = rb_entry(rb_node, struct vm_area_struct, vm_rb); + if (ceiling && vma->vm_start >= ceiling) + continue; + if ((vma->vm_flags & good_flags) != good_flags) + continue; + if ((vma->vm_flags & bad_flags) != 0) + continue; + addr = vma->vm_start; + break; + } + up_read(&mm->mmap_sem); + mmput(mm); + return addr; +} + +static noinline unsigned long find_old_ssol_vma(void) +{ + unsigned long addr; + unsigned long ceiling = 0; // top of memory + char buf[UPROBES_SSOL_TAGSZ]; + while ((addr = find_next_possible_ssol_vma(ceiling)) != 0) { + ceiling = addr; + if (copy_from_user(buf, (const void __user*)addr, + UPROBES_SSOL_TAGSZ)) + continue; + if (!strcmp(buf, UPROBES_SSOL_VMA_TAG)) + return addr; + } + return 0; +} + +/* + * Mmap nbytes bytes for the uprobes SSOL area for the current process. + * Returns the address of the page, or a negative errno. + * This approach was suggested by Roland McGrath. + */ +static noinline unsigned long uprobe_setup_ssol_vma(unsigned long nbytes) +{ + unsigned long addr; + struct mm_struct *mm; + struct vm_area_struct *vma; + + BUG_ON(nbytes & ~PAGE_MASK); + if ((addr = find_old_ssol_vma()) != 0) + return addr; + + mm = get_task_mm(current); + if (!mm) + return (unsigned long) (-ESRCH); + down_write(&mm->mmap_sem); + /* + * Find the end of the top mapping and skip a page. + * If there is no space for PAGE_SIZE above + * that, mmap will ignore our address hint. + */ + vma = rb_entry(rb_last(&mm->mm_rb), struct vm_area_struct, vm_rb); + addr = vma->vm_end + PAGE_SIZE; + addr = do_mmap_pgoff(NULL, addr, nbytes, PROT_EXEC, + MAP_PRIVATE|MAP_ANONYMOUS, 0); + if (addr & ~PAGE_MASK) { + up_write(&mm->mmap_sem); + mmput(mm); + printk(KERN_ERR "Uprobes failed to allocate a vma for" + " pid/tgid %d/%d for single-stepping out of line.\n", + current->pid, current->tgid); + return addr; + } + + vma = find_vma(mm, addr); + BUG_ON(!vma); + /* + * Don't expand vma on mremap(). Allow vma to be copied on + * fork() -- see uprobe_fork_uproc(). + */ + vma->vm_flags |= VM_DONTEXPAND; + + up_write(&mm->mmap_sem); + mmput(mm); + return addr; +} + +/** + * uprobe_init_ssol -- initialize per-process area for single stepping + * out-of-line. + * @uproc: probed process + * @tsk: probed task: must be current if @insn_area is %NULL + * @insn_area: virtual address of the already-established SSOL vma -- + * see uprobe_fork_uproc(). + * + * Returns with @uproc->ssol_area.insn_area pointing to the initialized + * area, or set to a negative errno. + */ +static void uprobe_init_ssol(struct uprobe_process *uproc, + struct task_struct *tsk, __user uprobe_opcode_t *insn_area) +{ + struct uprobe_ssol_area *area = &uproc->ssol_area; + struct uprobe_ssol_slot *slot; + int i; + char *slot_addr; // Simplify pointer arithmetic + + /* Trampoline setup will either fail or succeed here. */ + uproc->uretprobe_trampoline_addr = ERR_PTR(-ENOMEM); + + if (insn_area) { + BUG_ON(IS_ERR(insn_area)); + area->insn_area = insn_area; + } else { + BUG_ON(tsk != current); + area->insn_area = + (uprobe_opcode_t *) uprobe_setup_ssol_vma(PAGE_SIZE); + if (IS_ERR(area->insn_area)) + return; + } + + area->nfree = area->nslots = PAGE_SIZE / SLOT_SIZE; + if (area->nslots > MAX_SSOL_SLOTS) + area->nfree = area->nslots = MAX_SSOL_SLOTS; + area->slots = (struct uprobe_ssol_slot *) + kzalloc(sizeof(struct uprobe_ssol_slot) * area->nslots, + GFP_USER); + if (!area->slots) { + area->insn_area = ERR_PTR(-ENOMEM); + return; + } + mutex_init(&area->populate_mutex); + spin_lock_init(&area->lock); + area->next_slot = 0; + slot_addr = (char*) area->insn_area; + for (i = 0; i < area->nslots; i++) { + slot = &area->slots[i]; + init_rwsem(&slot->rwsem); + slot->state = SSOL_FREE; + slot->owner = NULL; + slot->last_used = 0; + slot->insn = (__user uprobe_opcode_t *) slot_addr; + slot_addr += SLOT_SIZE; + } + uprobe_tag_vma(area); + uretprobe_set_trampoline(uproc, tsk); + area->first_ssol_slot = area->next_slot; +} + +/* + * Verify that the SSOL area has been set up for uproc. Returns a + * pointer to the SSOL area, or a negative erro if we couldn't set it up. + */ +static __user uprobe_opcode_t + *uprobe_verify_ssol(struct uprobe_process *uproc) +{ + struct uprobe_ssol_area *area = &uproc->ssol_area; + + if (unlikely(!area->initialized)) { + /* First time through for this probed process */ + mutex_lock(&uproc->ssol_area.setup_mutex); + if (likely(!area->initialized)) { + /* Nobody snuck in and set things up ahead of us. */ + uprobe_init_ssol(uproc, current, NULL); + area->initialized = 1; + } + mutex_unlock(&uproc->ssol_area.setup_mutex); + } + return area->insn_area; +} + +static inline int advance_slot(int slot, struct uprobe_ssol_area *area) +{ + /* First few slots are reserved for vma tag, uretprobe trampoline. */ + slot++; + if (unlikely(slot >= area->nslots)) + slot = area->first_ssol_slot; + return slot; +} + +/* + * Return the slot number of the least-recently-used slot in the + * neighborhood of area->next_slot. Limit the number of slots we test + * to keep it fast. Nobody dies if this isn't the best choice. + */ +static int uprobe_lru_insn_slot(struct uprobe_ssol_area *area) +{ +#define MAX_LRU_TESTS 10 + struct uprobe_ssol_slot *s; + int lru_slot = -1; + unsigned long lru_time = ULONG_MAX; + int nr_lru_tests = 0; + int slot = area->next_slot; + do { + s = &area->slots[slot]; + if (likely(s->state == SSOL_ASSIGNED)) { + if( lru_time > s->last_used) { + lru_time = s->last_used; + lru_slot = slot; + } + if (++nr_lru_tests >= MAX_LRU_TESTS) + break; + } + slot = advance_slot(slot, area); + } while (slot != area->next_slot); + + if (unlikely(lru_slot < 0)) + /* All slots are in the act of being stolen. Join the melee. */ + return area->next_slot; + else + return lru_slot; +} + +/* + * Choose an instruction slot and take it. Choose a free slot if there is one. + * Otherwise choose the least-recently-used slot. Returns with slot + * read-locked and containing the desired instruction. Runs with + * ppt->slot_mutex locked. + */ +static struct uprobe_ssol_slot + *uprobe_take_insn_slot(struct uprobe_probept *ppt) +{ + struct uprobe_process *uproc = ppt->uproc; + struct uprobe_ssol_area *area = &uproc->ssol_area; + struct uprobe_ssol_slot *s; + int len, slot; + unsigned long flags; + + spin_lock_irqsave(&area->lock, flags); + + if (area->nfree) { + for (slot = 0; slot < area->nslots; slot++) { + if (area->slots[slot].state == SSOL_FREE) { + area->nfree--; + goto found_slot; + } + } + /* Shouldn't get here. Fix nfree and get on with life. */ + area->nfree = 0; + } + slot = uprobe_lru_insn_slot(area); + +found_slot: + area->next_slot = advance_slot(slot, area); + s = &area->slots[slot]; + s->state = SSOL_BEING_STOLEN; + + spin_unlock_irqrestore(&area->lock, flags); + + /* Wait for current users of slot to finish. */ + down_write(&s->rwsem); + ppt->slot = s; + s->owner = ppt; + s->last_used = jiffies; + s->state = SSOL_ASSIGNED; + /* Copy the original instruction to the chosen slot. */ + mutex_lock(&area->populate_mutex); + len = access_process_vm(current, (unsigned long)s->insn, + ppt->insn, MAX_UINSN_BYTES, 1); + mutex_unlock(&area->populate_mutex); + if (unlikely(len < MAX_UINSN_BYTES)) { + up_write(&s->rwsem); + printk(KERN_ERR "Failed to copy instruction at %#lx" + " to SSOL area (%#lx)\n", ppt->vaddr, + (unsigned long) area->slots); + return NULL; + } + /* Let other threads single-step in this slot. */ + downgrade_write(&s->rwsem); + return s; +} + +/* ppt doesn't own a slot. Get one for ppt, and return it read-locked. */ +static struct uprobe_ssol_slot + *uprobe_find_insn_slot(struct uprobe_probept *ppt) +{ + struct uprobe_ssol_slot *slot; + + mutex_lock(&ppt->slot_mutex); + slot = ppt->slot; + if (unlikely(slot && slot->owner == ppt)) { + /* Looks like another thread snuck in and got a slot for us. */ + down_read(&slot->rwsem); + if (likely(slot->owner == ppt)) { + slot->last_used = jiffies; + mutex_unlock(&ppt->slot_mutex); + return slot; + } + /* ... but then somebody stole it. */ + up_read(&slot->rwsem); + } + slot = uprobe_take_insn_slot(ppt); + mutex_unlock(&ppt->slot_mutex); + return slot; +} + +/* + * Ensure that ppt owns an instruction slot for single-stepping. + * Returns with the slot read-locked and ppt->slot pointing at it. + */ +static +struct uprobe_ssol_slot *uprobe_get_insn_slot(struct uprobe_probept *ppt) +{ + struct uprobe_ssol_slot *slot; + +retry: + slot = ppt->slot; + if (unlikely(!slot)) + return uprobe_find_insn_slot(ppt); + + down_read(&slot->rwsem); + if (unlikely(slot != ppt->slot)) { + up_read(&slot->rwsem); + goto retry; + } + if (unlikely(slot->owner != ppt)) { + up_read(&slot->rwsem); + return uprobe_find_insn_slot(ppt); + } + slot->last_used = jiffies; + return slot; +} + +/* + * utrace engine report callbacks + */ + +/* + * We've been asked to quiesce, but aren't in a position to do so. + * This could happen in either of the following cases: + * + * 1) Our own thread is doing a register or unregister operation -- + * e.g., as called from a u[ret]probe handler or a non-uprobes utrace + * callback. We can't wait_event() for ourselves in [un]register_uprobe(). + * + * 2) We've been asked to quiesce, but we hit a probepoint first. Now + * we're in the report_signal callback, having handled the probepoint. + * We'd like to just turn on UTRACE_EVENT(QUIESCE) and coast into + * quiescence. Unfortunately, it's possible to hit a probepoint again + * before we quiesce. When processing the SIGTRAP, utrace would call + * uprobe_report_quiesce(), which must decline to take any action so + * as to avoid removing the uprobe just hit. As a result, we could + * keep hitting breakpoints and never quiescing. + * + * So here we do essentially what we'd prefer to do in uprobe_report_quiesce(). + * If we're the last thread to quiesce, handle_pending_uprobes() and + * rouse_all_threads(). Otherwise, pretend we're quiescent and sleep until + * the last quiescent thread handles that stuff and then wakes us. + * + * Called and returns with no mutexes held. Returns 1 if we free utask->uproc, + * else 0. + */ +static int utask_fake_quiesce(struct uprobe_task *utask) +{ + struct uprobe_process *uproc = utask->uproc; + enum uprobe_task_state prev_state = utask->state; + + down_write(&uproc->rwsem); + + /* In case we're somehow set to quiesce for real... */ + clear_utrace_quiesce(utask, false); + + if (uproc->n_quiescent_threads == uproc->nthreads-1) { + /* We're the last thread to "quiesce." */ + handle_pending_uprobes(uproc, utask->tsk); + rouse_all_threads(uproc); + up_write(&uproc->rwsem); + return 0; + } else { + utask->state = UPTASK_SLEEPING; + uproc->n_quiescent_threads++; + up_write(&uproc->rwsem); + /* We ref-count sleepers. */ + uprobe_get_process(uproc); + + wait_event(uproc->waitq, !utask->quiescing); + + down_write(&uproc->rwsem); + utask->state = prev_state; + uproc->n_quiescent_threads--; + up_write(&uproc->rwsem); + + /* + * If uproc's last uprobe has been unregistered, and + * unregister_uprobe() woke up before we did, it's up + * to us to free uproc. + */ + return uprobe_put_process(uproc, false); + } +} + +/* Prepare to single-step ppt's probed instruction inline. */ +static inline void uprobe_pre_ssin(struct uprobe_task *utask, + struct uprobe_probept *ppt, struct pt_regs *regs) +{ + int len; + arch_reset_ip_for_sstep(regs); + mutex_lock(&ppt->ssil_mutex); + len = set_orig_insn(ppt, utask->tsk); + if (unlikely(len != BP_INSN_SIZE)) { + printk("Failed to temporarily restore original " + "instruction for single-stepping: " + "pid/tgid=%d/%d, vaddr=%#lx\n", + utask->tsk->pid, utask->tsk->tgid, ppt->vaddr); + utask->doomed = 1; + } +} + +/* Prepare to continue execution after single-stepping inline. */ +static inline void uprobe_post_ssin(struct uprobe_task *utask, + struct uprobe_probept *ppt) +{ + + int len = set_bp(ppt, utask->tsk); + if (unlikely(len != BP_INSN_SIZE)) { + printk("Couldn't restore bp: pid/tgid=%d/%d, addr=%#lx\n", + utask->tsk->pid, utask->tsk->tgid, ppt->vaddr); + ppt->state = UPROBE_DISABLED; + } + mutex_unlock(&ppt->ssil_mutex); +} + +/* uprobe_pre_ssout() and uprobe_post_ssout() are architecture-specific. */ + +/* + * If this thread is supposed to be quiescing, mark it quiescent; and + * if it was the last thread to quiesce, do the work we quiesced for. + * Runs with utask->uproc->rwsem write-locked. Returns true if we can + * let this thread resume. + */ +static bool utask_quiesce(struct uprobe_task *utask) +{ + if (utask->quiescing) { + if (utask->state != UPTASK_QUIESCENT) { + utask->state = UPTASK_QUIESCENT; + utask->uproc->n_quiescent_threads++; + } + return check_uproc_quiesced(utask->uproc, current); + } else { + clear_utrace_quiesce(utask, false); + return true; + } +} + +/* + * Delay delivery of the indicated signal until after single-step. + * Otherwise single-stepping will be cancelled as part of calling + * the signal handler. + */ +static void uprobe_delay_signal(struct uprobe_task *utask, siginfo_t *info) +{ + struct delayed_signal *ds = kmalloc(sizeof(*ds), GFP_USER); + if (ds) { + ds->info = *info; + INIT_LIST_HEAD(&ds->list); + list_add_tail(&ds->list, &utask->delayed_signals); + } +} + +static void uprobe_inject_delayed_signals(struct list_head *delayed_signals) +{ + struct delayed_signal *ds, *tmp; + list_for_each_entry_safe(ds, tmp, delayed_signals, list) { + send_sig_info(ds->info.si_signo, &ds->info, current); + list_del(&ds->list); + kfree(ds); + } +} + +/* + * Signal callback: + * + * We get called here with: + * state = UPTASK_RUNNING => we are here due to a breakpoint hit + * - Read-lock the process + * - Figure out which probepoint, based on regs->IP + * - Set state = UPTASK_BP_HIT + * - Reset regs->IP to beginning of the insn, if necessary + * - Invoke handler for each uprobe at this probepoint + * - Start watching for quiesce events, in case another + * engine cancels our UTRACE_SINGLESTEP with a + * UTRACE_STOP. + * - Set singlestep in motion (UTRACE_SINGLESTEP), + * with state = UPTASK_SSTEP + * + * state = UPTASK_SSTEP => here after single-stepping + * - Validate we are here per the state machine + * - Clean up after single-stepping + * - Set state = UPTASK_RUNNING + * - Read-unlock the process + * - If it's time to quiesce, take appropriate action. + * - If the handler(s) we ran called [un]register_uprobe(), + * complete those via uprobe_run_def_regs(). + * + * state = ANY OTHER STATE + * - Not our signal, pass it on (UTRACE_RESUME) + * Note: Intermediate states such as UPTASK_POST_SSTEP help + * uprobe_report_exit() decide what to unlock if we die. + */ +static u32 uprobe_report_signal(u32 action, + struct utrace_attached_engine *engine, + struct task_struct *tsk, + struct pt_regs *regs, + siginfo_t *info, + const struct k_sigaction *orig_ka, + struct k_sigaction *return_ka) +{ + struct uprobe_task *utask; + struct uprobe_probept *ppt; + struct uprobe_process *uproc; + struct uprobe_kimg *uk; + unsigned long probept; + enum utrace_signal_action signal_action = utrace_signal_action(action); + enum utrace_resume_action resume_action; + int hit_uretprobe_trampoline = 0; + + utask = (struct uprobe_task *)rcu_dereference(engine->data); + BUG_ON(!utask); + uproc = utask->uproc; + + /* + * We may need to re-assert UTRACE_SINGLESTEP if this signal + * is not associated with the breakpoint. + */ + if (utask->state == UPTASK_SSTEP) + resume_action = UTRACE_SINGLESTEP; + else + resume_action = UTRACE_RESUME; + + /* Keep uproc intact until just before we return. */ + uprobe_get_process(uproc); + + if (unlikely(signal_action == UTRACE_SIGNAL_REPORT)) { + /* This thread was quiesced using UTRACE_INTERRUPT. */ + bool done_quiescing; + if (utask->active_probe) { + /* + * We already hold uproc->rwsem read-locked. + * We'll fake quiescence after we're done + * processing the probepoint. + */ + uprobe_decref_process(uproc); + return UTRACE_SIGNAL_IGN | resume_action; + } + down_write(&uproc->rwsem); + done_quiescing = utask_quiesce(utask); + up_write(&uproc->rwsem); + if (uprobe_put_process(uproc, true)) + resume_action = UTRACE_DETACH; + else if (done_quiescing) + resume_action = UTRACE_RESUME; + else + resume_action = UTRACE_STOP; + return UTRACE_SIGNAL_IGN | resume_action; + } + + /* + * info will be null if we're called with action=UTRACE_SIGNAL_HANDLER, + * which means that single-stepping has been disabled so a signal + * handler can be called in the probed process. That should never + * happen because we intercept and delay handled signals (action = + * UTRACE_RESUME) until after we're done single-stepping. + * TODO: Verify that this is still the case in utrace 2008. + * UTRACE_SIGNAL_HANDLER seems to be defined, but not used anywhere. + */ + BUG_ON(!info); + if (signal_action == UTRACE_SIGNAL_DELIVER && utask->active_probe && + info->si_signo != SSTEP_SIGNAL) { + uprobe_delay_signal(utask, info); + uprobe_decref_process(uproc); + return UTRACE_SIGNAL_IGN | UTRACE_SINGLESTEP; + } + + if (info->si_signo != BREAKPOINT_SIGNAL && + info->si_signo != SSTEP_SIGNAL) + goto no_interest; + + /* + * Set up the SSOL area if it's not already there. We do this + * here because we have to do it before handling the first + * probepoint hit, the probed process has to do it, and this may + * be the first time our probed process runs uprobes code. + * + * We need the SSOL area for the uretprobe trampoline even if + * this architectures doesn't single-step out of line. + */ +#ifdef CONFIG_UPROBES_SSOL + if (uproc->sstep_out_of_line && + unlikely(IS_ERR(uprobe_verify_ssol(uproc)))) + uproc->sstep_out_of_line = 0; +#elif defined(CONFIG_URETPROBES) + (void) uprobe_verify_ssol(uproc); +#endif + + switch (utask->state) { + case UPTASK_RUNNING: + if (info->si_signo != BREAKPOINT_SIGNAL) + goto no_interest; + down_read(&uproc->rwsem); + /* Don't quiesce while running handlers. */ + clear_utrace_quiesce(utask, false); + probept = arch_get_probept(regs); + + hit_uretprobe_trampoline = (probept == (unsigned long) + uproc->uretprobe_trampoline_addr); + if (hit_uretprobe_trampoline) { + uretprobe_handle_return(regs, utask); + goto bkpt_done; + } + + ppt = uprobe_find_probept(uproc, probept); + if (!ppt) { + up_read(&uproc->rwsem); + goto no_interest; + } + utask->active_probe = ppt; + utask->state = UPTASK_BP_HIT; + + if (likely(ppt->state == UPROBE_BP_SET)) { + list_for_each_entry(uk, &ppt->uprobe_list, list) { + struct uprobe *u = uk->uprobe; + if (is_uretprobe(u)) + uretprobe_handle_entry(u, regs, utask); + else if (u->handler) + u->handler(u, regs); + } + } + + utask->state = UPTASK_PRE_SSTEP; +#ifdef CONFIG_UPROBES_SSOL + if (uproc->sstep_out_of_line) + uprobe_pre_ssout(utask, ppt, regs); + else +#endif + uprobe_pre_ssin(utask, ppt, regs); + if (unlikely(utask->doomed)) { + uprobe_decref_process(uproc); + do_exit(SIGSEGV); + } + utask->state = UPTASK_SSTEP; + /* In case another engine cancels our UTRACE_SINGLESTEP... */ + utask_adjust_flags(utask, UPROBE_SET_FLAGS, + UTRACE_EVENT(QUIESCE)); + /* Don't deliver this signal to the process. */ + resume_action = UTRACE_SINGLESTEP; + signal_action = UTRACE_SIGNAL_IGN; + break; + case UPTASK_SSTEP: + if (info->si_signo != SSTEP_SIGNAL) + goto no_interest; + /* No further need to re-assert UTRACE_SINGLESTEP. */ + clear_utrace_quiesce(utask, false); + + ppt = utask->active_probe; + BUG_ON(!ppt); + utask->state = UPTASK_POST_SSTEP; +#ifdef CONFIG_UPROBES_SSOL + if (uproc->sstep_out_of_line) + uprobe_post_ssout(utask, ppt, regs); + else +#endif + uprobe_post_ssin(utask, ppt); +bkpt_done: + /* Note: Can come here after running uretprobe handlers */ + if (unlikely(utask->doomed)) { + uprobe_decref_process(uproc); + do_exit(SIGSEGV); + } + + utask->active_probe = NULL; + + utask->state = UPTASK_RUNNING; + if (utask->quiescing) { + int uproc_freed; + up_read(&uproc->rwsem); + uproc_freed = utask_fake_quiesce(utask); + BUG_ON(uproc_freed); + } else + up_read(&uproc->rwsem); + + if (hit_uretprobe_trampoline) + /* + * It's possible that the uretprobe_instance + * we just recycled was the last reason for + * keeping uproc around. + */ + uprobe_decref_process(uproc); + + /* + * We hold a ref count on uproc, so this should never + * make utask or uproc disappear. + */ + uprobe_run_def_regs(&utask->deferred_registrations); + + uprobe_inject_delayed_signals(&utask->delayed_signals); + + resume_action = UTRACE_RESUME; + signal_action = UTRACE_SIGNAL_IGN; + break; + default: + goto no_interest; + } + +no_interest: + if (uprobe_put_process(uproc, true)) + resume_action = UTRACE_DETACH; + return (signal_action | resume_action); +} + +#if 0 +/* + * utask_quiesce_pending_sigtrap: The utask entered the quiesce callback + * through the signal delivery path, apparently. Check if the associated + * signal happened due to a uprobe hit. + * + * Called with utask->uproc write-locked. Returns 1 if quiesce was + * entered with SIGTRAP pending due to a uprobe hit. + */ +static int utask_quiesce_pending_sigtrap(struct uprobe_task *utask) +{ + const struct user_regset_view *view; + const struct user_regset *regset; + struct uprobe_probept *ppt; + unsigned long insn_ptr; + + view = task_user_regset_view(utask->tsk); + regset = &view->regsets[0]; + if (regset->get(utask->tsk, regset, + SLOT_IP(utask->tsk) * regset->size, + regset->size, &insn_ptr, NULL) != 0) + return -EIO; + + if (regset->size != sizeof(insn_ptr)) { + /* Assume 32-bit app and 64-bit kernel. */ + u32 *insn_ptr32 = (u32*) &insn_ptr; + BUG_ON(regset->size != sizeof(u32)); + insn_ptr = *insn_ptr32; + } + + ppt = uprobe_find_probept(utask->uproc, ARCH_BP_INST_PTR(insn_ptr)); + return (ppt != NULL); +} +#endif + +/* + * Quiesce callback: The associated process has one or more breakpoint + * insertions or removals pending. If we're the last thread in this + * process to quiesce, do the insertion(s) and/or removal(s). + */ +static u32 uprobe_report_quiesce(enum utrace_resume_action action, + struct utrace_attached_engine *engine, + struct task_struct *tsk, + unsigned long event) +{ + struct uprobe_task *utask; + struct uprobe_process *uproc; + bool done_quiescing = false; + + utask = (struct uprobe_task *)rcu_dereference(engine->data); + BUG_ON(!utask); + BUG_ON(tsk != current); // guaranteed by utrace 2008 + + if (utask->state == UPTASK_SSTEP) + /* + * We got a breakpoint trap and tried to single-step, + * but somebody else's report_signal callback overrode + * our UTRACE_SINGLESTEP with a UTRACE_STOP. Try again. + */ + return UTRACE_SINGLESTEP; + + BUG_ON(utask->active_probe); + uproc = utask->uproc; + down_write(&uproc->rwsem); +#if 0 + // TODO: Is this a concern any more? + /* + * When a thread hits a breakpoint or single-steps, utrace calls + * this quiesce callback before our signal callback. We must + * let uprobe_report_signal() handle the uprobe hit and THEN + * quiesce, because (a) there's a chance that we're quiescing + * in order to remove that very uprobe, and (b) there's a tiny + * chance that even though that uprobe isn't marked for removal + * now, it may be before all threads manage to quiesce. + */ + if (!utask->quiescing || utask_quiesce_pending_sigtrap(utask) == 1) { + clear_utrace_quiesce(utask, false); + done_quiescing = true; + goto done; + } +#endif + + done_quiescing = utask_quiesce(utask); +// done: + up_write(&uproc->rwsem); + return (done_quiescing ? UTRACE_RESUME : UTRACE_STOP); +} + +/* + * uproc's process is exiting or exec-ing, so zap all the (now irrelevant) + * probepoints. Runs with uproc->rwsem write-locked. Caller must ref-count + * uproc before calling this function, to ensure that uproc doesn't get + * freed in the middle of this. + */ +static void uprobe_cleanup_process(struct uprobe_process *uproc) +{ + int i; + struct uprobe_probept *ppt; + struct hlist_node *pnode1, *pnode2; + struct hlist_head *head; + struct uprobe_kimg *uk, *unode; + + uproc->finished = 1; + + for (i = 0; i < UPROBE_TABLE_SIZE; i++) { + head = &uproc->uprobe_table[i]; + hlist_for_each_entry_safe(ppt, pnode1, pnode2, head, ut_node) { + if (ppt->state == UPROBE_INSERTING || + ppt->state == UPROBE_REMOVING) { + /* + * This task is (exec/exit)ing with + * a [un]register_uprobe pending. + * [un]register_uprobe will free ppt. + */ + ppt->state = UPROBE_DISABLED; + list_del(&ppt->pd_node); + list_for_each_entry_safe(uk, unode, + &ppt->uprobe_list, list) + uk->status = -ESRCH; + wake_up_all(&ppt->waitq); + } else if (ppt->state == UPROBE_BP_SET) { + list_for_each_entry_safe(uk, unode, + &ppt->uprobe_list, list) { + list_del(&uk->list); + uprobe_free_kimg(uk); + } + uprobe_free_probept(ppt); + /* else */ + /* + * If ppt is UPROBE_DISABLED, assume that + * [un]register_uprobe() has been notified + * and will free it soon. + */ + } + } + } +} + +/* + * Exit callback: The associated task/thread is exiting. + */ +static u32 uprobe_report_exit(enum utrace_resume_action action, + struct utrace_attached_engine *engine, + struct task_struct *tsk, long orig_code, long *code) +{ + struct uprobe_task *utask; + struct uprobe_process *uproc; + struct uprobe_probept *ppt; + int utask_quiescing; + + utask = (struct uprobe_task *)rcu_dereference(engine->data); + uproc = utask->uproc; + uprobe_get_process(uproc); + + ppt = utask->active_probe; + if (ppt) { + if (utask->state == UPTASK_TRAMPOLINE_HIT) + printk(KERN_WARNING "Task died during uretprobe return:" + " pid/tgid = %d/%d\n", tsk->pid, tsk->tgid); + else + printk(KERN_WARNING "Task died at uprobe probepoint:" + " pid/tgid = %d/%d, probepoint = %#lx\n", + tsk->pid, tsk->tgid, ppt->vaddr); + /* Mutex cleanup depends on where we died and SSOL vs. SSIL. */ + if (uproc->sstep_out_of_line) { + if (utask->state == UPTASK_SSTEP + && ppt->slot && ppt->slot->owner == ppt) + up_read(&ppt->slot->rwsem); + } else { + switch (utask->state) { + case UPTASK_PRE_SSTEP: + case UPTASK_SSTEP: + case UPTASK_POST_SSTEP: + mutex_unlock(&ppt->ssil_mutex); + break; + default: + break; + } + } + up_read(&uproc->rwsem); + if (utask->state == UPTASK_TRAMPOLINE_HIT) + uprobe_decref_process(uproc); + } + + down_write(&uproc->rwsem); + utask_quiescing = utask->quiescing; + uprobe_free_task(utask, 1); + + uproc->nthreads--; + if (uproc->nthreads) { + if (utask_quiescing) + /* + * In case other threads are waiting for + * us to quiesce... + */ + (void) check_uproc_quiesced(uproc, + find_surviving_thread(uproc)); + } else { + /* + * We were the last remaining thread - clean up the uprobe + * remnants a la unregister_uprobe(). We don't have to + * remove the breakpoints, though. + */ + uprobe_cleanup_process(uproc); + } + up_write(&uproc->rwsem); + uprobe_put_process(uproc, true); + + return UTRACE_DETACH; +} + +/* + * Duplicate the FIFO of uretprobe_instances from parent_utask into + * child_utask. Zap the uretprobe pointer, since all we care about is + * vectoring to the proper return address. Where there are multiple + * uretprobe_instances for the same function instance, copy only the + * one that contains the real return address. + */ +static int uprobe_fork_uretprobe_instances(struct uprobe_task *parent_utask, + struct uprobe_task *child_utask) +{ + struct uprobe_process *parent_uproc = parent_utask->uproc; + struct uprobe_process *child_uproc = child_utask->uproc; + __user uprobe_opcode_t *trampoline_addr = + child_uproc->uretprobe_trampoline_addr; + struct hlist_node *tmp, *tail; + struct uretprobe_instance *pri, *cri; + + BUG_ON(trampoline_addr != parent_uproc->uretprobe_trampoline_addr); + + /* Since there's no hlist_add_tail()... */ + tail = NULL; + hlist_for_each_entry(pri, tmp, &parent_utask->uretprobe_instances, + hlist) { + if (pri->ret_addr == (unsigned long) trampoline_addr) + continue; + cri = kmalloc(sizeof(*cri), GFP_USER); + if (!cri) + return -ENOMEM; + cri->rp = NULL; + cri->ret_addr = pri->ret_addr; + cri->sp = pri->sp; + INIT_HLIST_NODE(&cri->hlist); + if (tail) + hlist_add_after(tail, &cri->hlist); + else + hlist_add_head(&cri->hlist, + &child_utask->uretprobe_instances); + tail = &cri->hlist; + + /* Ref-count uretprobe_instances. */ + uprobe_get_process(child_uproc); + } + BUG_ON(hlist_empty(&child_utask->uretprobe_instances)); + return 0; +} + +/* + * A probed process is forking, and at least one function in the + * call stack has a uretprobe on it. Since the child inherits the + * call stack, it's possible that the child could attempt to return + * through the uretprobe trampoline. Create a uprobe_process for + * the child, initialize its SSOL vma (which has been cloned from + * the parent), and clone the parent's list of uretprobe_instances. + * + * Called with uproc_table locked and parent_uproc->rwsem write-locked. + * + * (On architectures where it's easy to keep track of where in the + * stack the return addresses are stored, we could just poke the real + * return addresses back into the child's stack. We use this more + * general solution.) + */ +static int uprobe_fork_uproc(struct uprobe_process *parent_uproc, + struct uprobe_task *parent_utask, + struct task_struct *child_tsk) +{ + int ret = 0; + struct uprobe_process *child_uproc; + struct uprobe_task *child_utask; + struct pid *child_pid; + + BUG_ON(parent_uproc->tgid == child_tsk->tgid); + BUG_ON(!parent_uproc->uretprobe_trampoline_addr || + IS_ERR(parent_uproc->uretprobe_trampoline_addr)); + + if (!try_module_get(THIS_MODULE)) + return -ENOSYS; + child_pid = get_pid(find_vpid(child_tsk->pid)); + if (!child_pid) { + module_put(THIS_MODULE); + return -ESRCH; + } + child_uproc = uprobe_mk_process(child_pid); + put_pid(child_pid); + if (IS_ERR(child_uproc)) { + ret = (int) PTR_ERR(child_uproc); + module_put(THIS_MODULE); + return ret; + } + /* child_uproc is write-locked and ref-counted at this point. */ + + mutex_lock(&child_uproc->ssol_area.setup_mutex); + uprobe_init_ssol(child_uproc, child_tsk, + parent_uproc->ssol_area.insn_area); + child_uproc->ssol_area.initialized = 1; + mutex_unlock(&child_uproc->ssol_area.setup_mutex); + + child_utask = uprobe_find_utask(child_tsk); + BUG_ON(!child_utask); + ret = uprobe_fork_uretprobe_instances(parent_utask, child_utask); + + hlist_add_head(&child_uproc->hlist, + &uproc_table[hash_ptr(child_pid, UPROBE_HASH_BITS)]); + + up_write(&child_uproc->rwsem); + uprobe_decref_process(child_uproc); + return ret; +} + +/* + * Clone callback: The current task has spawned a thread/process. + * Utrace guarantees that parent and child pointers will be valid + * for the duration of this callback. + * + * NOTE: For now, we don't pass on uprobes from the parent to the + * child. We now do the necessary clearing of breakpoints in the + * child's address space. + * + * TODO: + * - Provide option for child to inherit uprobes. + */ +static u32 uprobe_report_clone(enum utrace_resume_action action, + struct utrace_attached_engine *engine, + struct task_struct *parent, + unsigned long clone_flags, + struct task_struct *child) +{ + int len; + struct uprobe_process *uproc; + struct uprobe_task *ptask, *ctask; + + ptask = (struct uprobe_task *)rcu_dereference(engine->data); + uproc = ptask->uproc; + + /* + * Lock uproc so no new uprobes can be installed 'til all + * report_clone activities are completed. Lock uproc_table + * in case we have to run uprobe_fork_uproc(). + */ + lock_uproc_table(); + down_write(&uproc->rwsem); + + if (clone_flags & CLONE_THREAD) { + /* New thread in the same process. */ + ctask = uprobe_find_utask(child); + if (unlikely(ctask)) { + /* + * uprobe_mk_process() ran just as this clone + * happened, and has already accounted for the + * new child. + */ + } else { + struct pid *child_pid = get_pid(task_pid(child)); + BUG_ON(!child_pid); + ctask = uprobe_add_task(child_pid, uproc); + BUG_ON(!ctask); + if (IS_ERR(ctask)) + goto done; + uproc->nthreads++; + /* + * FIXME: Handle the case where uproc is quiescing + * (assuming it's possible to clone while quiescing). + */ + } + } else { + /* + * New process spawned by parent. Remove the probepoints + * in the child's text. + * + * Its not necessary to quiesce the child as we are assured + * by utrace that this callback happens *before* the child + * gets to run userspace. + * + * We also hold the uproc->rwsem for the parent - so no + * new uprobes will be registered 'til we return. + */ + int i; + struct uprobe_probept *ppt; + struct hlist_node *node; + struct hlist_head *head; + + for (i = 0; i < UPROBE_TABLE_SIZE; i++) { + head = &uproc->uprobe_table[i]; + hlist_for_each_entry(ppt, node, head, ut_node) { + len = set_orig_insn(ppt, child); + if (len != BP_INSN_SIZE) { + /* Ratelimit this? */ + printk(KERN_ERR "Pid %d forked %d;" + " failed to remove probepoint" + " at %#lx in child\n", + parent->pid, child->pid, + ppt->vaddr); + } + } + } + + if (!hlist_empty(&ptask->uretprobe_instances)) { + int result = uprobe_fork_uproc(uproc, ptask, child); + if (result != 0) + printk(KERN_ERR "Failed to create" + " uprobe_process on fork: child=%d," + " parent=%d, error=%d\n", + child->pid, parent->pid, result); + } + } + +done: + up_write(&uproc->rwsem); + unlock_uproc_table(); + return UTRACE_RESUME; +} + +/* + * Exec callback: The associated process called execve() or friends + * + * The new program is about to start running and so there is no + * possibility of a uprobe from the previous user address space + * to be hit. + * + * NOTE: + * Typically, this process would have passed through the clone + * callback, where the necessary action *should* have been + * taken. However, if we still end up at this callback: + * - We don't have to clear the uprobes - memory image + * will be overlaid. + * - We have to free up uprobe resources associated with + * this process. + */ +static u32 uprobe_report_exec(enum utrace_resume_action action, + struct utrace_attached_engine *engine, + struct task_struct *tsk, + const struct linux_binfmt *fmt, + const struct linux_binprm *bprm, + struct pt_regs *regs) +{ + struct uprobe_process *uproc; + struct uprobe_task *utask; + int uproc_freed; + + utask = (struct uprobe_task *)rcu_dereference(engine->data); + uproc = utask->uproc; + uprobe_get_process(uproc); + + down_write(&uproc->rwsem); + uprobe_cleanup_process(uproc); + /* + * TODO: Is this necessary? + * If [un]register_uprobe() is in progress, cancel the quiesce. + * Otherwise, utrace_report_exec() might call uprobe_report_exec() + * while the [un]register_uprobe thread is freeing the uproc. + */ + clear_utrace_quiesce(utask, false); + up_write(&uproc->rwsem); + + /* If any [un]register_uprobe is pending, it'll clean up. */ + uproc_freed = uprobe_put_process(uproc, true); + return (uproc_freed ? UTRACE_DETACH : UTRACE_RESUME); +} + +static const struct utrace_engine_ops uprobe_utrace_ops = +{ + .report_quiesce = uprobe_report_quiesce, + .report_signal = uprobe_report_signal, + .report_exit = uprobe_report_exit, + .report_clone = uprobe_report_clone, + .report_exec = uprobe_report_exec +}; + +static int __init init_uprobes(void) +{ + int i; + + for (i = 0; i < UPROBE_TABLE_SIZE; i++) { + INIT_HLIST_HEAD(&uproc_table[i]); + INIT_HLIST_HEAD(&utask_table[i]); + } + + p_uprobe_utrace_ops = &uprobe_utrace_ops; + return 0; +} + +static void __exit exit_uprobes(void) +{ +} + +module_init(init_uprobes); +module_exit(exit_uprobes); + +#ifdef CONFIG_URETPROBES + +/* Returns true if ri_sp lies outside the stack (beyond cursp). */ +static inline bool compare_stack_ptrs(unsigned long cursp, + unsigned long ri_sp) +{ +#ifdef CONFIG_STACK_GROWSUP + if (cursp < ri_sp) + return true; +#else + if (cursp > ri_sp) + return true; +#endif + return false; +} + +/* + * A longjmp may cause one or more uretprobed functions to terminate without + * returning. Those functions' uretprobe_instances need to be recycled. + * We detect this when any uretprobed function is subsequently called + * or returns. A bypassed uretprobe_instance's stack_ptr is beyond the + * current stack. + */ +static inline void uretprobe_bypass_instances(unsigned long cursp, + struct uprobe_task *utask) +{ + struct hlist_node *r1, *r2; + struct uretprobe_instance *ri; + struct hlist_head *head = &utask->uretprobe_instances; + + hlist_for_each_entry_safe(ri, r1, r2, head, hlist) { + if (compare_stack_ptrs(cursp, ri->sp)) { + hlist_del(&ri->hlist); + kfree(ri); + uprobe_decref_process(utask->uproc); + } else + return; + } +} + +/* Called when the entry-point probe u is hit. */ +static void uretprobe_handle_entry(struct uprobe *u, struct pt_regs *regs, + struct uprobe_task *utask) +{ + struct uretprobe_instance *ri; + unsigned long trampoline_addr; + + if (IS_ERR(utask->uproc->uretprobe_trampoline_addr)) + return; + trampoline_addr = (unsigned long) + utask->uproc->uretprobe_trampoline_addr; + ri = (struct uretprobe_instance *) + kmalloc(sizeof(struct uretprobe_instance), GFP_USER); + if (!ri) + return; + ri->ret_addr = arch_hijack_uret_addr(trampoline_addr, regs, utask); + if (likely(ri->ret_addr)) { + ri->sp = arch_predict_sp_at_ret(regs, utask->tsk); + uretprobe_bypass_instances(ri->sp, utask); + ri->rp = container_of(u, struct uretprobe, u); + INIT_HLIST_NODE(&ri->hlist); + hlist_add_head(&ri->hlist, &utask->uretprobe_instances); + /* We ref-count outstanding uretprobe_instances. */ + uprobe_get_process(utask->uproc); + } else + kfree(ri); +} + +/* + * For each uretprobe_instance pushed onto the LIFO for the function + * instance that's now returning, call the handler, free the ri, and + * decrement the uproc's ref count. Caller ref-counts uproc, so we + * should never hit zero in this function. + * + * Returns the original return address. + */ +static unsigned long uretprobe_run_handlers(struct uprobe_task *utask, + struct pt_regs *regs, unsigned long trampoline_addr) +{ + unsigned long ret_addr, cur_sp; + struct hlist_head *head = &utask->uretprobe_instances; + struct uretprobe_instance *ri; + struct hlist_node *r1, *r2; + + cur_sp = arch_get_cur_sp(regs); + uretprobe_bypass_instances(cur_sp, utask); + hlist_for_each_entry_safe(ri, r1, r2, head, hlist) { + if (ri->rp && ri->rp->handler) + ri->rp->handler(ri, regs); + ret_addr = ri->ret_addr; + hlist_del(&ri->hlist); + kfree(ri); + uprobe_decref_process(utask->uproc); + if (ret_addr != trampoline_addr) + /* + * This is the first ri (chronologically) pushed for + * this particular instance of the probed function. + */ + return ret_addr; + } + printk(KERN_ERR "No uretprobe instance with original return address!" + " pid/tgid=%d/%d", current->pid, current->tgid); + utask->doomed = 1; + return 0; +} + +/* Called when the uretprobe trampoline is hit. */ +static void uretprobe_handle_return(struct pt_regs *regs, + struct uprobe_task *utask) +{ + unsigned long orig_ret_addr; + /* Delay recycling of uproc until end of uprobe_report_signal() */ + uprobe_get_process(utask->uproc); + utask->state = UPTASK_TRAMPOLINE_HIT; + utask->active_probe = &uretprobe_trampoline_dummy_probe; + orig_ret_addr = uretprobe_run_handlers(utask, regs, + (unsigned long) utask->uproc->uretprobe_trampoline_addr); + arch_restore_uret_addr(orig_ret_addr, regs); +} + +int register_uretprobe(struct uretprobe *rp) +{ + if (!rp || !rp->handler) + return -EINVAL; + rp->u.handler = URETPROBE_HANDLE_ENTRY; + return register_uprobe(&rp->u); +} +EXPORT_SYMBOL_GPL(register_uretprobe); + +/* + * The uretprobe containing u is being unregistered. Its uretprobe_instances + * have to hang around 'til their associated instances return (but we can't + * run rp's handler). Zap ri->rp for each one to indicate unregistration. + * + * Runs with uproc write-locked. + */ +static void zap_uretprobe_instances(struct uprobe *u, + struct uprobe_process *uproc) +{ + struct uprobe_task *utask; + struct uretprobe *rp = container_of(u, struct uretprobe, u); + + if (!uproc) + return; + + list_for_each_entry(utask, &uproc->thread_list, list) { + struct hlist_node *r; + struct uretprobe_instance *ri; + + hlist_for_each_entry(ri, r, &utask->uretprobe_instances, hlist) + if (ri->rp == rp) + ri->rp = NULL; + } +} + +void unregister_uretprobe(struct uretprobe *rp) +{ + if (!rp) + return; + unregister_uprobe(&rp->u); +} +EXPORT_SYMBOL_GPL(unregister_uretprobe); + +/* + * uproc->ssol_area has been successfully set up. Establish the + * uretprobe trampoline in the next available slot following the + * vma tag. + */ +static void uretprobe_set_trampoline(struct uprobe_process *uproc, + struct task_struct *tsk) +{ + uprobe_opcode_t bp_insn = BREAKPOINT_INSTRUCTION; + struct uprobe_ssol_area *area = &uproc->ssol_area; + struct uprobe_ssol_slot *slot = &area->slots[area->next_slot]; + + if (access_process_vm(tsk, (unsigned long) slot->insn, + &bp_insn, BP_INSN_SIZE, 1) == BP_INSN_SIZE) { + uproc->uretprobe_trampoline_addr = slot->insn; + slot->state = SSOL_RESERVED; + area->next_slot++; + area->nfree--; + } else { + printk(KERN_ERR "uretprobes disabled for pid %d:" + " cannot set uretprobe trampoline at %p\n", + pid_nr(uproc->tg_leader), slot->insn); + } +} + +#else /* ! CONFIG_URETPROBES */ + +static void uretprobe_handle_entry(struct uprobe *u, struct pt_regs *regs, + struct uprobe_task *utask) +{ +} +static void uretprobe_handle_return(struct pt_regs *regs, + struct uprobe_task *utask) +{ +} +static void uretprobe_set_trampoline(struct uprobe_process *uproc, + struct task_struct *tsk) +{ +} +static void zap_uretprobe_instances(struct uprobe *u, + struct uprobe_process *uproc) +{ +} +#endif /* CONFIG_URETPROBES */ + +#define UPROBES_DEBUG +#ifdef UPROBES_DEBUG +struct uprobe_task *updebug_find_utask(struct task_struct *tsk) +{ + return uprobe_find_utask(tsk); +} +EXPORT_SYMBOL_GPL(updebug_find_utask); + +/* NB: No locking, no ref-counting */ +struct uprobe_process *updebug_find_process(pid_t tgid) +{ + struct hlist_head *head; + struct hlist_node *node; + struct uprobe_process *uproc; + struct pid *p; + + p = uprobe_get_tg_leader(tgid); + head = &uproc_table[hash_ptr(p, UPROBE_HASH_BITS)]; + hlist_for_each_entry(uproc, node, head, hlist) { + if (uproc->tg_leader == p && !uproc->finished) + return uproc; + } + return NULL; +} +EXPORT_SYMBOL_GPL(updebug_find_process); + +struct uprobe_probept *updebug_find_probept(struct uprobe_process *uproc, + unsigned long vaddr) +{ + return uprobe_find_probept(uproc, vaddr); +} +EXPORT_SYMBOL_GPL(updebug_find_probept); +#endif /* UPROBES_DEBUG */ + +#ifdef NO_ACCESS_PROCESS_VM_EXPORT +/* + * Some kernel versions export everything that uprobes.ko needs except + * access_process_vm, so we copied and pasted it here. Fortunately, + * everything it calls is exported. + */ +#include <linux/pagemap.h> +#include <asm/cacheflush.h> +static int __access_process_vm(struct task_struct *tsk, unsigned long addr, void *buf, int len, int write) +{ + struct mm_struct *mm; + struct vm_area_struct *vma; + struct page *page; + void *old_buf = buf; + + mm = get_task_mm(tsk); + if (!mm) + return 0; + + down_read(&mm->mmap_sem); + /* ignore errors, just check how much was sucessfully transfered */ + while (len) { + int bytes, ret, offset; + void *maddr; + + ret = get_user_pages(tsk, mm, addr, 1, + write, 1, &page, &vma); + if (ret <= 0) + break; + + bytes = len; + offset = addr & (PAGE_SIZE-1); + if (bytes > PAGE_SIZE-offset) + bytes = PAGE_SIZE-offset; + + maddr = kmap(page); + if (write) { + copy_to_user_page(vma, page, addr, + maddr + offset, buf, bytes); + set_page_dirty_lock(page); + } else { + copy_from_user_page(vma, page, addr, + buf, maddr + offset, bytes); + } + kunmap(page); + page_cache_release(page); + len -= bytes; + buf += bytes; + addr += bytes; + } + up_read(&mm->mmap_sem); + mmput(mm); + + return buf - old_buf; +} +#endif + +#include "uprobes_arch.c" + +MODULE_LICENSE("GPL"); diff --git a/runtime/uprobes2/uprobes.h b/runtime/uprobes2/uprobes.h new file mode 100644 index 00000000..11d01f5c --- /dev/null +++ b/runtime/uprobes2/uprobes.h @@ -0,0 +1,406 @@ +#ifndef _LINUX_UPROBES_H +#define _LINUX_UPROBES_H +/* + * Userspace Probes (UProbes) + * include/linux/uprobes.h + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + * + * Copyright (C) IBM Corporation, 2006 + */ +#include <linux/types.h> +#include <linux/list.h> + +struct pt_regs; + +enum uprobe_type { + UPTY_UPROBE, + UPTY_URETPROBE +}; + +/* This is what the user supplies us. */ +struct uprobe { + /* + * The pid of the probed process. Currently, this can be the + * thread ID (task->pid) of any active thread in the process. + */ + pid_t pid; + + /* Location of the probepoint */ + unsigned long vaddr; + + /* Handler to run when the probepoint is hit */ + void (*handler)(struct uprobe*, struct pt_regs*); + + /* + * This function, if non-NULL, will be called upon completion of + * an ASYNCHRONOUS registration (i.e., one initiated by a uprobe + * handler). reg = 1 for register, 0 for unregister. type + * specifies the type of [un]register call (uprobe or uretprobe). + */ + void (*registration_callback)(struct uprobe *u, int reg, + enum uprobe_type type, int result); + + /* Reserved for use by uprobes */ + void *kdata; +}; + +struct uretprobe_instance; + +struct uretprobe { + struct uprobe u; + void (*handler)(struct uretprobe_instance*, struct pt_regs*); +}; + +struct uretprobe_instance { + struct uretprobe *rp; + unsigned long ret_addr; + struct hlist_node hlist; + unsigned long sp; // stack pointer value expected on return + unsigned long reserved1; +}; + +extern int register_uprobe(struct uprobe *u); +extern void unregister_uprobe(struct uprobe *u); +/* For runtime, assume uprobes support includes uretprobes. */ +extern int register_uretprobe(struct uretprobe *rp); +extern void unregister_uretprobe(struct uretprobe *rp); + +#ifdef UPROBES_IMPLEMENTATION + +#include <linux/mutex.h> +#include <linux/rwsem.h> +#include <linux/wait.h> +#include <asm/atomic.h> +#include "uprobes_arch.h" + +struct task_struct; +struct utrace_attached_engine; +struct pid; + +enum uprobe_probept_state { + UPROBE_INSERTING, // process quiescing prior to insertion + UPROBE_BP_SET, // breakpoint in place + UPROBE_REMOVING, // process quiescing prior to removal + UPROBE_DISABLED // removal completed +}; + +enum uprobe_task_state { + UPTASK_QUIESCENT, + UPTASK_SLEEPING, // used when task may not be able to quiesce + UPTASK_RUNNING, + UPTASK_BP_HIT, + UPTASK_TRAMPOLINE_HIT, + UPTASK_PRE_SSTEP, + UPTASK_SSTEP, + UPTASK_POST_SSTEP +}; + +#define UPROBE_HASH_BITS 5 +#define UPROBE_TABLE_SIZE (1 << UPROBE_HASH_BITS) + +/* Used when deciding which instruction slot to steal. */ +enum uprobe_slot_state { + SSOL_FREE, + SSOL_ASSIGNED, + SSOL_BEING_STOLEN, + SSOL_RESERVED // e.g., for uretprobe trampoline +}; + +/* + * For a uprobe_process that uses an SSOL area, there's an array of these + * objects matching the array of instruction slots in the SSOL area. + */ +struct uprobe_ssol_slot { + /* The slot in the SSOL area that holds the instruction-copy */ + __user uprobe_opcode_t *insn; + + enum uprobe_slot_state state; + + /* The probepoint that currently owns this slot */ + struct uprobe_probept *owner; + + /* + * Read-locked when slot is in use during single-stepping. + * Write-locked by stealing task. + */ + struct rw_semaphore rwsem; + + /* Used for LRU heuristics. If this overflows, it's OK. */ + unsigned long last_used; +}; + +/* + * The per-process single-stepping out-of-line (SSOL) area + */ +struct uprobe_ssol_area { + /* Array of instruction slots in the vma we allocate */ + __user uprobe_opcode_t *insn_area; + + int nslots; + int nfree; + + /* Array of slot objects, one per instruction slot */ + struct uprobe_ssol_slot *slots; + + /* lock held while finding a free slot */ + spinlock_t lock; + + /* + * We currently use access_process_vm() to populate instruction + * slots. Calls must be serialized because access_process_vm() + * isn't thread-safe. + */ + struct mutex populate_mutex; + + /* Next slot to steal */ + int next_slot; + + /* First slot not reserved for trampoline or some such */ + int first_ssol_slot; + + /* Ensures 2 threads don't try to set up the vma simultaneously. */ + struct mutex setup_mutex; + + /* 1 = we've at least tried. IS_ERR(insn_area) if we failed. */ + int initialized; +}; + +/* + * uprobe_process -- not a user-visible struct. + * A uprobe_process represents a probed process. A process can have + * multiple probepoints (each represented by a uprobe_probept) and + * one or more threads (each represented by a uprobe_task). + */ +struct uprobe_process { + /* + * rwsem is write-locked for any change to the uprobe_process's + * graph (including uprobe_tasks, uprobe_probepts, and uprobe_kimgs) -- + * e.g., due to probe [un]registration or special events like exit. + * It's read-locked during the whole time we process a probepoint hit. + */ + struct rw_semaphore rwsem; + + /* Table of uprobe_probepts registered for this process */ + /* TODO: Switch to list_head[] per Ingo. */ + struct hlist_head uprobe_table[UPROBE_TABLE_SIZE]; + int nppt; /* number of probepoints */ + + /* List of uprobe_probepts awaiting insertion or removal */ + struct list_head pending_uprobes; + + /* List of uprobe_tasks in this task group */ + struct list_head thread_list; + int nthreads; + int n_quiescent_threads; + + /* this goes on the uproc_table */ + struct hlist_node hlist; + + /* + * All threads (tasks) in a process share the same uprobe_process. + */ + struct pid *tg_leader; + pid_t tgid; + + /* Threads in UTASK_SLEEPING state wait here to be roused. */ + wait_queue_head_t waitq; + + /* + * We won't free the uprobe_process while... + * - any register/unregister operations on it are in progress; or + * - any uprobe_report_* callbacks are running; or + * - uprobe_table[] is not empty; or + * - any tasks are UTASK_SLEEPING in the waitq; or + * - any uretprobe_instances are outstanding. + * refcount reflects this. We do NOT ref-count tasks (threads), + * since once the last thread has exited, the rest is academic. + */ + atomic_t refcount; + + /* Return-probed functions return via this trampoline. */ + __user uprobe_opcode_t *uretprobe_trampoline_addr; + + /* + * finished = 1 means the process is execing or the last thread + * is exiting, and we're cleaning up the uproc. If the execed + * process is probed, a new uproc will be created. + */ + int finished; + + /* + * Manages slots for instruction-copies to be single-stepped + * out of line. + */ + struct uprobe_ssol_area ssol_area; + + /* + * 1 to single-step out of line; 0 for inline. This can drop to + * 0 if we can't set up the SSOL area, but never goes from 0 to 1. + */ + int sstep_out_of_line; +}; + +/* + * uprobe_kimg -- not a user-visible struct. + * Holds implementation-only per-uprobe data. + * uprobe->kdata points to this. + */ +struct uprobe_kimg { + struct uprobe *uprobe; + struct uprobe_probept *ppt; + + /* + * -EBUSY while we're waiting for all threads to quiesce so the + * associated breakpoint can be inserted or removed. + * 0 if the the insert/remove operation has succeeded, or -errno + * otherwise. + */ + int status; + + /* on ppt's list */ + struct list_head list; +}; + +/* + * uprobe_probept -- not a user-visible struct. + * A probepoint, at which several uprobes can be registered. + * Guarded by uproc->rwsem. + */ +struct uprobe_probept { + /* vaddr copied from (first) uprobe */ + unsigned long vaddr; + + /* The uprobe_kimg(s) associated with this uprobe_probept */ + struct list_head uprobe_list; + + enum uprobe_probept_state state; + + /* Saved opcode (which has been replaced with breakpoint) */ + uprobe_opcode_t opcode; + + /* + * Saved original instruction. This may be modified by + * architecture-specific code if the original instruction + * can't be single-stepped out of line as-is. + */ + uprobe_opcode_t insn[MAX_UINSN_BYTES / sizeof(uprobe_opcode_t)]; + + struct uprobe_probept_arch_info arch_info; + + /* The parent uprobe_process */ + struct uprobe_process *uproc; + + /* + * ppt goes in the uprobe_process->uprobe_table when registered -- + * even before the breakpoint has been inserted. + */ + struct hlist_node ut_node; + + /* + * ppt sits in the uprobe_process->pending_uprobes queue while + * awaiting insertion or removal of the breakpoint. + */ + struct list_head pd_node; + + /* [un]register_uprobe() waits 'til bkpt inserted/removed. */ + wait_queue_head_t waitq; + + /* + * Serialize single-stepping inline, so threads don't clobber + * each other swapping the breakpoint instruction in and out. + * This helps prevent crashing the probed app, but it does NOT + * prevent probe misses while the breakpoint is swapped out. + */ + struct mutex ssil_mutex; + + /* + * We put the instruction-copy here to single-step it. + * We don't own it unless slot->owner points back to us. + */ + struct uprobe_ssol_slot *slot; + + /* + * Hold this while stealing an insn slot to ensure that no + * other thread, having also hit this probepoint, simultaneously + * steals a slot for it. + */ + struct mutex slot_mutex; +}; + +/* + * uprobe_utask -- not a user-visible struct. + * Corresponds to a thread in a probed process. + * Guarded by uproc->rwsem. + */ +struct uprobe_task { + /* Lives in the global utask_table */ + struct hlist_node hlist; + + /* Lives on the thread_list for the uprobe_process */ + struct list_head list; + + struct task_struct *tsk; + struct pid *pid; + + /* The utrace engine for this task */ + struct utrace_attached_engine *engine; + + /* Back pointer to the associated uprobe_process */ + struct uprobe_process *uproc; + + enum uprobe_task_state state; + + /* + * quiescing = 1 means this task has been asked to quiesce. + * It may not be able to comply immediately if it's hit a bkpt. + */ + int quiescing; + + /* Set before running handlers; cleared after single-stepping. */ + struct uprobe_probept *active_probe; + + /* Saved address of copied original instruction */ + long singlestep_addr; + + struct uprobe_task_arch_info arch_info; + + /* + * Unexpected error in probepoint handling has left task's + * text or stack corrupted. Kill task ASAP. + */ + int doomed; + + /* LIFO -- active instances */ + struct hlist_head uretprobe_instances; + + /* [un]registrations initiated by handlers must be asynchronous. */ + struct list_head deferred_registrations; + + /* Delay handler-destined signals 'til after single-step done. */ + struct list_head delayed_signals; +}; + +#ifdef CONFIG_UPROBES_SSOL +static struct uprobe_ssol_slot *uprobe_get_insn_slot(struct uprobe_probept*); +static void uprobe_pre_ssout(struct uprobe_task*, struct uprobe_probept*, + struct pt_regs*); +static void uprobe_post_ssout(struct uprobe_task*, struct uprobe_probept*, + struct pt_regs*); +#endif + +#endif /* UPROBES_IMPLEMENTATION */ + +#endif /* _LINUX_UPROBES_H */ diff --git a/runtime/uprobes2/uprobes_arch.c b/runtime/uprobes2/uprobes_arch.c new file mode 100644 index 00000000..3c86804e --- /dev/null +++ b/runtime/uprobes2/uprobes_arch.c @@ -0,0 +1,11 @@ +#if defined (__x86_64__) || defined(__i386) +#include "uprobes_x86.c" +#elif defined (__powerpc64__) +#include "../uprobes/uprobes_ppc64.c" +#elif defined (__s390__) || defined (__s390x__) +#include "../uprobes/uprobes_s390.c" +#elif defined (__ia64__) +#include "../uprobes/uprobes_ia64.c" +#else +#error "Unsupported architecture" +#endif diff --git a/runtime/uprobes2/uprobes_arch.h b/runtime/uprobes2/uprobes_arch.h new file mode 100644 index 00000000..87e0cc83 --- /dev/null +++ b/runtime/uprobes2/uprobes_arch.h @@ -0,0 +1,11 @@ +#if defined (__x86_64__) || defined(__i386) +#include "uprobes_x86.h" +#elif defined (__powerpc64__) +#include "../uprobes/uprobes_ppc64.h" +#elif defined (__s390__) || defined (__s390x__) +#include "../uprobes/uprobes_s390.h" +#elif defined (__ia64__) +#include "../uprobes/uprobes_ia64.h" +#else +#error "Unsupported architecture" +#endif diff --git a/runtime/uprobes2/uprobes_x86.c b/runtime/uprobes2/uprobes_x86.c new file mode 100644 index 00000000..38c20ba0 --- /dev/null +++ b/runtime/uprobes2/uprobes_x86.c @@ -0,0 +1,722 @@ +/* + * Userspace Probes (UProbes) + * uprobes.c + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + * + * Copyright (C) IBM Corporation, 2006-2008 + */ +/* + * In versions of uprobes built in the SystemTap runtime, this file + * is #included at the end of uprobes.c. + */ +#include <asm/uaccess.h> + +#ifdef CONFIG_X86_32 +#define is_32bit_app(tsk) 1 +#else +#define is_32bit_app(tsk) (test_tsk_thread_flag(tsk, TIF_IA32)) +#endif + +/* Adapted from arch/x86_64/kprobes.c */ +#undef W +#define W(row,b0,b1,b2,b3,b4,b5,b6,b7,b8,b9,ba,bb,bc,bd,be,bf) \ + (((b0##ULL<< 0x0)|(b1##ULL<< 0x1)|(b2##ULL<< 0x2)|(b3##ULL<< 0x3) | \ + (b4##ULL<< 0x4)|(b5##ULL<< 0x5)|(b6##ULL<< 0x6)|(b7##ULL<< 0x7) | \ + (b8##ULL<< 0x8)|(b9##ULL<< 0x9)|(ba##ULL<< 0xa)|(bb##ULL<< 0xb) | \ + (bc##ULL<< 0xc)|(bd##ULL<< 0xd)|(be##ULL<< 0xe)|(bf##ULL<< 0xf)) \ + << (row % 64)) + +static const u64 good_insns_64[256 / 64] = { + /* 0 1 2 3 4 5 6 7 8 9 a b c d e f */ + /* ------------------------------- */ + W(0x00, 1,1,1,1,1,1,0,0,1,1,1,1,1,1,0,0)| /* 00 */ + W(0x10, 1,1,1,1,1,1,0,0,1,1,1,1,1,1,0,0)| /* 10 */ + W(0x20, 1,1,1,1,1,1,0,0,1,1,1,1,1,1,0,0)| /* 20 */ + W(0x30, 1,1,1,1,1,1,0,0,1,1,1,1,1,1,0,0), /* 30 */ + W(0x40, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0)| /* 40 */ + W(0x50, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1)| /* 50 */ + W(0x60, 0,0,0,1,0,0,0,0,1,1,1,1,0,0,0,0)| /* 60 */ + W(0x70, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1), /* 70 */ + W(0x80, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1)| /* 80 */ + W(0x90, 1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1)| /* 90 */ + W(0xa0, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1)| /* a0 */ + W(0xb0, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1), /* b0 */ + W(0xc0, 1,1,1,1,0,0,1,1,1,1,1,1,0,0,0,0)| /* c0 */ + W(0xd0, 1,1,1,1,0,0,0,1,1,1,1,1,1,1,1,1)| /* d0 */ + W(0xe0, 1,1,1,1,0,0,0,0,1,1,1,1,0,0,0,0)| /* e0 */ + W(0xf0, 0,0,1,1,0,1,1,1,1,1,0,0,1,1,1,1) /* f0 */ + /* ------------------------------- */ + /* 0 1 2 3 4 5 6 7 8 9 a b c d e f */ +}; + +/* Good-instruction tables for 32-bit apps -- copied from i386 uprobes */ + +static const u64 good_insns_32[256 / 64] = { + /* 0 1 2 3 4 5 6 7 8 9 a b c d e f */ + /* ------------------------------- */ + W(0x00, 1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0)| /* 00 */ + W(0x10, 1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0)| /* 10 */ + W(0x20, 1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,1)| /* 20 */ + W(0x30, 1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,1), /* 30 */ + W(0x40, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1)| /* 40 */ + W(0x50, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1)| /* 50 */ + W(0x60, 1,1,1,0,0,0,0,0,1,1,1,1,0,0,0,0)| /* 60 */ + W(0x70, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1), /* 70 */ + W(0x80, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1)| /* 80 */ + W(0x90, 1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1)| /* 90 */ + W(0xa0, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1)| /* a0 */ + W(0xb0, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1), /* b0 */ + W(0xc0, 1,1,1,1,1,1,1,1,1,1,1,1,0,0,0,0)| /* c0 */ + W(0xd0, 1,1,1,1,1,1,0,1,1,1,1,1,1,1,1,1)| /* d0 */ + W(0xe0, 1,1,1,1,0,0,0,0,1,1,1,1,0,0,0,0)| /* e0 */ + W(0xf0, 0,0,1,1,0,1,1,1,1,1,0,0,1,1,1,1) /* f0 */ + /* ------------------------------- */ + /* 0 1 2 3 4 5 6 7 8 9 a b c d e f */ +}; + +/* Using this for both 64-bit and 32-bit apps */ +static const u64 good_2byte_insns[256 / 64] = { + /* 0 1 2 3 4 5 6 7 8 9 a b c d e f */ + /* ------------------------------- */ + W(0x00, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1)| /* 00 */ + W(0x10, 1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1)| /* 10 */ + W(0x20, 1,1,1,1,0,0,0,0,1,1,1,1,1,1,1,1)| /* 20 */ + W(0x30, 0,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0), /* 30 */ + W(0x40, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1)| /* 40 */ + W(0x50, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1)| /* 50 */ + W(0x60, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1)| /* 60 */ + W(0x70, 1,1,1,1,1,1,1,1,0,0,0,0,0,0,1,1), /* 70 */ + W(0x80, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1)| /* 80 */ + W(0x90, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1)| /* 90 */ + W(0xa0, 1,1,1,1,1,1,0,0,1,1,1,1,1,1,0,1)| /* a0 */ + W(0xb0, 1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1), /* b0 */ + W(0xc0, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1)| /* c0 */ + W(0xd0, 0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1)| /* d0 */ + W(0xe0, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1)| /* e0 */ + W(0xf0, 0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0) /* f0 */ + /* ------------------------------- */ + /* 0 1 2 3 4 5 6 7 8 9 a b c d e f */ +}; + +/* + * opcodes we'll probably never support: + * 6c-6d, e4-e5, ec-ed - in + * 6e-6f, e6-e7, ee-ef - out + * cc, cd - int3, int + * cf - iret + * d6 - illegal instruction + * f1 - int1/icebp + * f4 - hlt + * fa, fb - cli, sti + * 0f - lar, lsl, syscall, clts, sysret, sysenter, sysexit, invd, wbinvd, ud2 + * + * invalid opcodes in 64-bit mode: + * 06, 0e, 16, 1e, 27, 2f, 37, 3f, 60-62, c4-c5, d4-d5 + * + * 63 - we support this opcode in x86_64 but not in i386. + * opcodes we may need to refine support for: + * 0f - 2-byte instructions: For many of these instructions, the validity + * depends on the prefix and/or the reg field. On such instructions, we + * just consider the opcode combination valid if it corresponds to any + * valid instruction. + * 8f - Group 1 - only reg = 0 is OK + * c6-c7 - Group 11 - only reg = 0 is OK + * d9-df - fpu insns with some illegal encodings + * f2, f3 - repnz, repz prefixes. These are also the first byte for + * certain floating-point instructions, such as addsd. + * fe - Group 4 - only reg = 0 or 1 is OK + * ff - Group 5 - only reg = 0-6 is OK + * + * others -- Do we need to support these? + * 0f - (floating-point?) prefetch instructions + * 07, 17, 1f - pop es, pop ss, pop ds + * 26, 2e, 36, 3e, 64, 65 - es:, cs:, ss:, ds:, fs:, gs: segment prefixes + * 67 - addr16 prefix + * 9b - wait/fwait + * ce - into + * f0 - lock prefix + */ + +/* + * TODO: + * - Where necessary, examine the modrm byte and allow only valid instructions + * in the different Groups and fpu instructions. + * - Note: If we go past the first byte, do we need to verify that + * subsequent bytes were actually there, rather than off the last page? + * - Be clearer about which instructions we'll never probe. + */ + +/* + * Return 1 if this is a legacy instruction prefix we support, -1 if + * it's one we don't support, or 0 if it's not a prefix at all. + */ +static inline int check_legacy_prefix(u8 byte) +{ + switch (byte) { + case 0x26: + case 0x2e: + case 0x36: + case 0x3e: + case 0x64: + case 0x65: + case 0xf0: + return -1; + case 0x66: + case 0x67: + case 0xf2: + case 0xf3: + return 1; + default: + return 0; + } +} + +static void report_bad_1byte_opcode(int mode, uprobe_opcode_t op) +{ + printk(KERN_ERR "In %d-bit apps, " + "uprobes does not currently support probing " + "instructions whose first byte is 0x%2.2x\n", mode, op); +} + +static void report_bad_2byte_opcode(uprobe_opcode_t op) +{ + printk(KERN_ERR "uprobes does not currently support probing " + "instructions with the 2-byte opcode 0x0f 0x%2.2x\n", op); +} + +static int validate_insn_32bits(struct uprobe_probept *ppt) +{ + uprobe_opcode_t *insn = ppt->insn; + int pfx; + + /* Skip good instruction prefixes; reject "bad" ones. */ + while ((pfx = check_legacy_prefix(insn[0])) == 1) + insn++; + if (pfx < 0) { + report_bad_1byte_opcode(32, insn[0]); + return -EPERM; + } + if (test_bit(insn[0], (unsigned long*)good_insns_32)) + return 0; + if (insn[0] == 0x0f) { + if (test_bit(insn[1], (unsigned long*)good_2byte_insns)) + return 0; + report_bad_2byte_opcode(insn[1]); + } else + report_bad_1byte_opcode(32, insn[0]); + return -EPERM; +} + +static int validate_insn_64bits(struct uprobe_probept *ppt) +{ + uprobe_opcode_t *insn = ppt->insn; + int pfx; + + /* Skip good instruction prefixes; reject "bad" ones. */ + while ((pfx = check_legacy_prefix(insn[0])) == 1) + insn++; + if (pfx < 0) { + report_bad_1byte_opcode(64, insn[0]); + return -EPERM; + } + /* Skip REX prefix. */ + if ((insn[0] & 0xf0) == 0x40) + insn++; + if (test_bit(insn[0], (unsigned long*)good_insns_64)) + return 0; + if (insn[0] == 0x0f) { + if (test_bit(insn[1], (unsigned long*)good_2byte_insns)) + return 0; + report_bad_2byte_opcode(insn[1]); + } else + report_bad_1byte_opcode(64, insn[0]); + return -EPERM; +} + +#ifdef CONFIG_X86_64 +static int handle_riprel_insn(struct uprobe_probept *ppt); +#endif + +static +int arch_validate_probed_insn(struct uprobe_probept *ppt, + struct task_struct *tsk) +{ + int ret; + +#ifdef CONFIG_X86_64 + ppt->arch_info.flags = 0x0; + ppt->arch_info.rip_target_address = 0x0; +#endif + + if (is_32bit_app(tsk)) + return validate_insn_32bits(ppt); + if ((ret = validate_insn_64bits(ppt)) != 0) + return ret; +#ifdef CONFIG_X86_64 + (void) handle_riprel_insn(ppt); +#endif + return 0; +} + +#ifdef CONFIG_X86_64 +/* + * Returns 0 if the indicated instruction has no immediate operand + * and/or can't use rip-relative addressing. Otherwise returns + * the size of the immediate operand in the instruction. (Note that + * for instructions such as "movq $7,xxxx(%rip)" the immediate-operand + * field is 4 bytes, even though 8 bytes are stored.) + */ +static int immediate_operand_size(u8 opcode1, u8 opcode2, u8 reg, + int operand_size_prefix) +{ + switch (opcode1) { + case 0x6b: /* imul immed,mem,reg */ + case 0x80: /* Group 1 */ + case 0x83: /* Group 1 */ + case 0xc0: /* Group 2 */ + case 0xc1: /* Group 2 */ + case 0xc6: /* Group 11 */ + return 1; + case 0x69: /* imul immed,mem,reg */ + case 0x81: /* Group 1 */ + case 0xc7: /* Group 11 */ + return (operand_size_prefix ? 2 : 4); + case 0xf6: /* Group 3, reg field == 0 or 1 */ + return (reg > 1 ? 0 : 1); + case 0xf7: /* Group 3, reg field == 0 or 1 */ + if (reg > 1) + return 0; + return (operand_size_prefix ? 2 : 4); + case 0x0f: + /* 2-byte opcodes */ + switch (opcode2) { + /* + * Note: 0x71-73 (Groups 12-14) have immediate operands, + * but not memory operands. + */ + case 0x70: /* pshuf* immed,mem,reg */ + case 0xa4: /* shld immed,reg,mem */ + case 0xac: /* shrd immed,reg,mem */ + case 0xc2: /* cmpps or cmppd */ + case 0xc4: /* pinsrw */ + case 0xc5: /* pextrw */ + case 0xc6: /* shufps or shufpd */ + case 0x0f: /* 3DNow extensions */ + return 1; + default: + return 0; + } + } + return 0; +} + +/* + * TODO: These tables are common for kprobes and uprobes and can be moved + * to a common place. + */ +static const u64 onebyte_has_modrm[256 / 64] = { + /* 0 1 2 3 4 5 6 7 8 9 a b c d e f */ + /* ------------------------------- */ + W(0x00, 1,1,1,1,0,0,0,0,1,1,1,1,0,0,0,0)| /* 00 */ + W(0x10, 1,1,1,1,0,0,0,0,1,1,1,1,0,0,0,0)| /* 10 */ + W(0x20, 1,1,1,1,0,0,0,0,1,1,1,1,0,0,0,0)| /* 20 */ + W(0x30, 1,1,1,1,0,0,0,0,1,1,1,1,0,0,0,0), /* 30 */ + W(0x40, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0)| /* 40 */ + W(0x50, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0)| /* 50 */ + W(0x60, 0,0,1,1,0,0,0,0,0,1,0,1,0,0,0,0)| /* 60 */ + W(0x70, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0), /* 70 */ + W(0x80, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1)| /* 80 */ + W(0x90, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0)| /* 90 */ + W(0xa0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0)| /* a0 */ + W(0xb0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0), /* b0 */ + W(0xc0, 1,1,0,0,1,1,1,1,0,0,0,0,0,0,0,0)| /* c0 */ + W(0xd0, 1,1,1,1,0,0,0,0,1,1,1,1,1,1,1,1)| /* d0 */ + W(0xe0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0)| /* e0 */ + W(0xf0, 0,0,0,0,0,0,1,1,0,0,0,0,0,0,1,1) /* f0 */ + /* ------------------------------- */ + /* 0 1 2 3 4 5 6 7 8 9 a b c d e f */ +}; +static const u64 twobyte_has_modrm[256 / 64] = { + /* 0 1 2 3 4 5 6 7 8 9 a b c d e f */ + /* ------------------------------- */ + W(0x00, 1,1,1,1,0,0,0,0,0,0,0,0,0,1,0,1)| /* 0f */ + W(0x10, 1,1,1,1,1,1,1,1,1,0,0,0,0,0,0,0)| /* 1f */ + W(0x20, 1,1,1,1,1,0,1,0,1,1,1,1,1,1,1,1)| /* 2f */ + W(0x30, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0), /* 3f */ + W(0x40, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1)| /* 4f */ + W(0x50, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1)| /* 5f */ + W(0x60, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1)| /* 6f */ + W(0x70, 1,1,1,1,1,1,1,0,0,0,0,0,1,1,1,1), /* 7f */ + W(0x80, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0)| /* 8f */ + W(0x90, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1)| /* 9f */ + W(0xa0, 0,0,0,1,1,1,1,1,0,0,0,1,1,1,1,1)| /* af */ + W(0xb0, 1,1,1,1,1,1,1,1,0,0,1,1,1,1,1,1), /* bf */ + W(0xc0, 1,1,1,1,1,1,1,1,0,0,0,0,0,0,0,0)| /* cf */ + W(0xd0, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1)| /* df */ + W(0xe0, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1)| /* ef */ + W(0xf0, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0) /* ff */ + /* ------------------------------- */ + /* 0 1 2 3 4 5 6 7 8 9 a b c d e f */ +}; + +/* + * If pp->insn doesn't use rip-relative addressing, return 0. Otherwise, + * rewrite the instruction so that it accesses its memory operand + * indirectly through a scratch register. Set flags and rip_target_address + * in ppt->arch_info accordingly. (The contents of the scratch register + * will be saved before we single-step the modified instruction, and + * restored afterward.) Return 1. + * + * We do this because a rip-relative instruction can access only a + * relatively small area (+/- 2 GB from the instruction), and the SSOL + * area typically lies beyond that area. At least for instructions + * that store to memory, we can't single-step the original instruction + * and "fix things up" later, because the misdirected store could be + * disastrous. + * + * Some useful facts about rip-relative instructions: + * - There's always a modrm byte. + * - There's never a SIB byte. + * - The offset is always 4 bytes. + */ +static int handle_riprel_insn(struct uprobe_probept *ppt) +{ + u8 *insn = (u8*) ppt->insn; + u8 opcode1, opcode2, modrm, reg; + int need_modrm; + int operand_size_prefix = 0; + int immed_size, instruction_size; + + /* + * Skip legacy instruction prefixes. Some of these we don't + * support (yet), but here we pretend to support all of them. + * Skip the REX prefix, if any. + */ + while (check_legacy_prefix(*insn)) { + if (*insn == 0x66) + operand_size_prefix = 1; + insn++; + } + if ((*insn & 0xf0) == 0x40) + insn++; + + opcode1 = *insn; + if (opcode1 == 0x0f) { /* Two-byte opcode. */ + opcode2 = *++insn; + need_modrm = test_bit(opcode2, twobyte_has_modrm); + } else { /* One-byte opcode. */ + opcode2 = 0x0; + need_modrm = test_bit(opcode1, onebyte_has_modrm); + } + + if (!need_modrm) + return 0; + + modrm = *++insn; + /* + * For rip-relative instructions, the mod field (top 2 bits) + * is zero and the r/m field (bottom 3 bits) is 0x5. + */ + if ((modrm & 0xc7) != 0x5) + return 0; + + /* + * We have a rip-relative instruction. insn points at the + * modrm byte. The next 4 bytes are the offset. Beyond the + * offset, for some instructions, is the immediate operand. + */ + reg = (modrm >> 3) & 0x7; + immed_size = immediate_operand_size(opcode1, opcode2, reg, + operand_size_prefix); + instruction_size = + (insn - (u8*) ppt->insn) /* prefixes + opcodes */ + + 1 /* modrm byte */ + + 4 /* offset */ + + immed_size; /* immediate field */ +#undef DEBUG_UPROBES_RIP +#ifdef DEBUG_UPROBES_RIP +{ + int i; + BUG_ON(instruction_size > 15); + printk(KERN_INFO "Munging rip-relative insn:"); + for (i = 0; i < instruction_size; i++) + printk(" %2.2x", ppt->insn[i]); + printk("\n"); +} +#endif + + /* + * Convert from rip-relative addressing to indirect addressing + * via a scratch register. Change the r/m field from 0x5 (%rip) + * to 0x0 (%rax) or 0x1 (%rcx), and squeeze out the offset field. + */ + if (reg == 0) { + /* + * The register operand (if any) is either the A register + * (%rax, %eax, etc.) or (if the 0x4 bit is set in the + * REX prefix) %r8. In any case, we know the C register + * is NOT the register operand, so we use %rcx (register + * #1) for the scratch register. + */ + ppt->arch_info.flags = UPFIX_RIP_RCX; + /* Change modrm from 00 000 101 to 00 000 001. */ + *insn = 0x1; + } else { + /* Use %rax (register #0) for the scratch register. */ + ppt->arch_info.flags = UPFIX_RIP_RAX; + /* Change modrm from 00 xxx 101 to 00 xxx 000 */ + *insn = (reg << 3); + } + + /* Target address = address of next instruction + (signed) offset */ + insn++; + ppt->arch_info.rip_target_address = + (long) ppt->vaddr + instruction_size + *((s32*)insn); + if (immed_size) + memmove(insn, insn+4, immed_size); +#ifdef DEBUG_UPROBES_RIP +{ + int i; + printk(KERN_INFO "Munged rip-relative insn: "); + for (i = 0; i < instruction_size-4; i++) + printk(" %2.2x", ppt->insn[i]); + printk("\n"); + printk(KERN_INFO "Target address = %#lx\n", + ppt->arch_info.rip_target_address); +} +#endif + return 1; +} +#endif + +/* + * Get an instruction slot from the process's SSOL area, containing the + * instruction at ppt's probepoint. Point the rip at that slot, in + * preparation for single-stepping out of line. + * + * If we're emulating a rip-relative instruction, save the contents + * of the scratch register and store the target address in that register. + */ +static +void uprobe_pre_ssout(struct uprobe_task *utask, struct uprobe_probept *ppt, + struct pt_regs *regs) +{ + struct uprobe_ssol_slot *slot; + + slot = uprobe_get_insn_slot(ppt); + if (!slot) { + utask->doomed = 1; + return; + } + + regs->ip = (long)slot->insn; + utask->singlestep_addr = regs->ip; +#ifdef CONFIG_X86_64 + if (ppt->arch_info.flags == UPFIX_RIP_RAX) { + utask->arch_info.saved_scratch_register = regs->ax; + regs->ax = ppt->arch_info.rip_target_address; + } else if (ppt->arch_info.flags == UPFIX_RIP_RCX) { + utask->arch_info.saved_scratch_register = regs->cx; + regs->cx = ppt->arch_info.rip_target_address; + } +#endif +} + +/* + * Called by uprobe_post_ssout() to adjust the return address + * pushed by a call instruction executed out of line. + */ +static void adjust_ret_addr(unsigned long rsp, long correction, + struct uprobe_task *utask) +{ + unsigned long nleft; + if (is_32bit_app(current)) { + s32 ra; + nleft = copy_from_user(&ra, (const void __user *) rsp, 4); + if (unlikely(nleft != 0)) + goto fail; + ra += (s32) correction; + nleft = copy_to_user((void __user *) rsp, &ra, 4); + if (unlikely(nleft != 0)) + goto fail; + } else { + s64 ra; + nleft = copy_from_user(&ra, (const void __user *) rsp, 8); + if (unlikely(nleft != 0)) + goto fail; + ra += correction; + nleft = copy_to_user((void __user *) rsp, &ra, 8); + if (unlikely(nleft != 0)) + goto fail; + } + return; + +fail: + printk(KERN_ERR + "uprobes: Failed to adjust return address after" + " single-stepping call instruction;" + " pid=%d, rsp=%#lx\n", current->pid, rsp); + utask->doomed = 1; +} + +/* + * Called after single-stepping. ppt->vaddr is the address of the + * instruction whose first byte has been replaced by the "int3" + * instruction. To avoid the SMP problems that can occur when we + * temporarily put back the original opcode to single-step, we + * single-stepped a copy of the instruction. The address of this + * copy is utask->singlestep_addr. + * + * This function prepares to return from the post-single-step + * trap. We have to fix things up as follows: + * + * 0) Typically, the new rip is relative to the copied instruction. We + * need to make it relative to the original instruction. Exceptions are + * return instructions and absolute or indirect jump or call instructions. + * + * 1) If the single-stepped instruction was a call, the return address + * that is atop the stack is the address following the copied instruction. + * We need to make it the address following the original instruction. + * + * 2) If the original instruction was a rip-relative instruction such as + * "movl %edx,0xnnnn(%rip)", we have instead executed an equivalent + * instruction using a scratch register -- e.g., "movl %edx,(%rax)". + * We need to restore the contents of the scratch register and adjust + * the rip, keeping in mind that the instruction we executed is 4 bytes + * shorter than the original instruction (since we squeezed out the offset + * field). + */ +static +void uprobe_post_ssout(struct uprobe_task *utask, struct uprobe_probept *ppt, + struct pt_regs *regs) +{ + unsigned long next_ip = 0; + unsigned long copy_ip = utask->singlestep_addr; + unsigned long orig_ip = ppt->vaddr; + long correction = (long) (orig_ip - copy_ip); + uprobe_opcode_t *insn = ppt->insn; +#ifdef CONFIG_X86_64 + unsigned long flags = ppt->arch_info.flags; +#endif + + up_read(&ppt->slot->rwsem); +#ifdef CONFIG_X86_64 + if (flags & (UPFIX_RIP_RAX | UPFIX_RIP_RCX)) { + if (flags & UPFIX_RIP_RAX) + regs->ax = utask->arch_info.saved_scratch_register; + else + regs->cx = utask->arch_info.saved_scratch_register; + /* + * The original instruction includes a displacement, and so + * is 4 bytes longer than what we've just single-stepped. + * Fall through to handle stuff like "jmpq *...(%rip)" and + * "callq *...(%rip)". + */ + correction += 4; + } +#endif + /* + * TODO: Move all this instruction parsing to + * arch_validate_probed_insn(), and store what we learn in + * ppt->arch_info.flags. + * + * We don't bother skipping prefixes here because none of the + * instructions that require special treatment (other than + * rip-relative instructions, handled above) involve prefixes. + */ + + switch (*insn) { + case 0xc3: /* ret/lret */ + case 0xcb: + case 0xc2: + case 0xca: + /* rip is correct */ + next_ip = regs->ip; + break; + case 0xe8: /* call relative - Fix return addr */ + adjust_ret_addr(regs->sp, correction, utask); + break; + case 0x9a: /* call absolute - Fix return addr */ + adjust_ret_addr(regs->sp, correction, utask); + next_ip = regs->ip; + break; + case 0xff: + if ((insn[1] & 0x30) == 0x10) { + /* call absolute, indirect */ + /* Fix return addr; rip is correct. */ + next_ip = regs->ip; + adjust_ret_addr(regs->sp, correction, utask); + } else if ((insn[1] & 0x31) == 0x20 || /* jmp near, absolute indirect */ + (insn[1] & 0x31) == 0x21) { /* jmp far, absolute indirect */ + /* rip is correct. */ + next_ip = regs->ip; + } + break; + case 0xea: /* jmp absolute -- rip is correct */ + next_ip = regs->ip; + break; + default: + break; + } + + if (next_ip) + regs->ip = next_ip; + else + regs->ip += correction; +} + +/* + * Replace the return address with the trampoline address. Returns + * the original return address. + */ +static +unsigned long arch_hijack_uret_addr(unsigned long trampoline_address, + struct pt_regs *regs, struct uprobe_task *utask) +{ + int nleft; + unsigned long orig_ret_addr = 0; /* clear high bits for 32-bit apps */ + size_t rasize; + + if (is_32bit_app(current)) + rasize = 4; + else + rasize = 8; + nleft = copy_from_user(&orig_ret_addr, (const void __user *) regs->sp, + rasize); + if (unlikely(nleft != 0)) + return 0; + if (orig_ret_addr == trampoline_address) + /* + * There's another uretprobe on this function, and it was + * processed first, so the return address has already + * been hijacked. + */ + return orig_ret_addr; + + nleft = copy_to_user((void __user *) regs->sp, &trampoline_address, + rasize); + if (unlikely(nleft != 0)) { + if (nleft != rasize) { + printk(KERN_ERR "uretprobe_entry_handler: " + "return address partially clobbered -- " + "pid=%d, %%sp=%#lx, %%ip=%#lx\n", + current->pid, regs->sp, regs->ip); + utask->doomed = 1; + } // else nothing written, so no harm + return 0; + } + return orig_ret_addr; +} diff --git a/runtime/uprobes2/uprobes_x86.h b/runtime/uprobes2/uprobes_x86.h new file mode 100644 index 00000000..ca3f4873 --- /dev/null +++ b/runtime/uprobes2/uprobes_x86.h @@ -0,0 +1,107 @@ +#ifndef _ASM_UPROBES_H +#define _ASM_UPROBES_H +/* + * Userspace Probes (UProbes) + * uprobes.h + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + * + * Copyright (C) IBM Corporation, 2008 + */ +#include <linux/types.h> +#include <linux/ptrace.h> +#include <linux/sched.h> +#include <asm/thread_info.h> + +/* Normally defined in Kconfig */ +#define CONFIG_URETPROBES 1 +#define CONFIG_UPROBES_SSOL 1 + +typedef u8 uprobe_opcode_t; +#define BREAKPOINT_INSTRUCTION 0xcc +#define BP_INSN_SIZE 1 +#define MAX_UINSN_BYTES 16 + +// SLOT_IP should be 16 for 64-bit apps (include/asm-x86_64/elf.h) +// but 12 for 32-bit apps (arch/x86_64/ia32/ia32_binfmt.c) +#ifdef CONFIG_X86_32 +#define SLOT_IP(tsk) 12 +#else +#define SLOT_IP(tsk) (test_tsk_thread_flag(tsk, TIF_IA32) ? 12 : 16) +#endif + +#define BREAKPOINT_SIGNAL SIGTRAP +#define SSTEP_SIGNAL SIGTRAP + +/* Architecture specific switch for where the IP points after a bp hit */ +#define ARCH_BP_INST_PTR(inst_ptr) (inst_ptr - BP_INSN_SIZE) + +#define UPFIX_RIP_RAX 0x1 /* (%rip) insn rewritten to use (%rax) */ +#define UPFIX_RIP_RCX 0x2 /* (%rip) insn rewritten to use (%rcx) */ + +#ifdef CONFIG_X86_64 +struct uprobe_probept_arch_info { + unsigned long flags; + unsigned long rip_target_address; +}; + +struct uprobe_task_arch_info { + unsigned long saved_scratch_register; +}; +#else +struct uprobe_probept_arch_info {}; +struct uprobe_task_arch_info {}; +#endif + +struct uprobe_probept; +struct uprobe_task; + +static int arch_validate_probed_insn(struct uprobe_probept *ppt, + struct task_struct *tsk); + +/* On x86, the int3 traps leaves ip pointing past the int3 instruction. */ +static inline unsigned long arch_get_probept(struct pt_regs *regs) +{ + return (unsigned long) (regs->ip - BP_INSN_SIZE); +} + +static inline void arch_reset_ip_for_sstep(struct pt_regs *regs) +{ + regs->ip -= BP_INSN_SIZE; +} + +static inline void arch_restore_uret_addr(unsigned long ret_addr, + struct pt_regs *regs) +{ + regs->ip = ret_addr; +} + +static inline unsigned long arch_get_cur_sp(struct pt_regs *regs) +{ + return (unsigned long) regs->sp; +} + +static inline unsigned long arch_predict_sp_at_ret(struct pt_regs *regs, + struct task_struct *tsk) +{ + if (test_tsk_thread_flag(tsk, TIF_IA32)) + return (unsigned long) (regs->sp + 4); + else + return (unsigned long) (regs->sp + 8); +} + +static unsigned long arch_hijack_uret_addr(unsigned long trampoline_addr, + struct pt_regs*, struct uprobe_task*); +#endif /* _ASM_UPROBES_H */ |