uretprobes: invoke handlers and restore return address when trampoline is hit

The uretprobe handlers are invoked when the trampoline is hit, on
completion the trampoline is replaced with the saved return address and
the uretprobe instance deleted.

Signed-off-by: Anithra P Janakiraman <anithra@linux.vnet.ibm.com>
Signed-off-by: Anton Arapov <anton@redhat.com>

2 files changed, 109 insertions, 8 deletions

diff --git a/include/linux/uprobes.h b/include/linux/uprobes.h
index 460f2f674ee..6c1cf58ae28 100644
--- a/include/linux/uprobes.h
+++ b/include/linux/uprobes.h
@@ -46,6 +46,17 @@ struct uprobe_consumer {
 	struct uprobe_consumer *next;
 };
 
+struct uretprobe_consumer {
+	int (*handler)(struct uretprobe_consumer *self, struct pt_regs *regs);
+	/*
+	 * filter is optional; If a filter exists, handler is run
+	 * if and only if filter returns true.
+	 */
+	bool (*filter)(struct uretprobe_consumer *self, struct task_struct *task);
+
+	struct uretprobe_consumer *next;
+};
+
 #ifdef CONFIG_UPROBES
 enum uprobe_task_state {
 	UTASK_RUNNING,
diff --git a/kernel/events/uprobes.c b/kernel/events/uprobes.c
index 61e5525000c..6b619a3144c 100644
--- a/kernel/events/uprobes.c
+++ b/kernel/events/uprobes.c
@@ -107,8 +107,11 @@ struct uprobe {
 static struct uprobe_consumer *uretprobe_dummy_consumer;
 static void uretprobe_handle_entry(struct uprobe *, struct pt_regs *, struct uprobe_task *);
 static struct xol_area *get_xol_area(struct mm_struct *mm);
+static struct uprobe *uretprobe_trampoline_dummy_probe;
 
 struct uretprobe {
+	struct rw_semaphore       consumer_rwsem;
+	struct uretprobe_consumer *consumers;
 };
 
 struct uretprobe_instance {
@@ -283,6 +286,79 @@ static inline void uretprobe_bypass_instances(unsigned long cursp,
 	}
 }
 
+static void uretprobe_handler_chain(struct uretprobe *rp, struct pt_regs *regs)
+{
+	struct uretprobe_consumer *rpc;
+
+	down_read(&rp->consumer_rwsem);
+	rpc = rp->consumers;
+	while (rpc) {
+		if (!rpc->filter || rpc->filter(rpc, current))
+			rpc->handler(rpc, regs);
+		rpc = rpc->next;
+	}
+	up_read(&rp->consumer_rwsem);
+}
+
+/*
+ * For each uretprobe_instance pushed onto the LIFO for the function instance
+ * that's now returning, call the handler, free the ri, and decrement the
+ * uproc's ref count.  Caller ref-counts uproc, so we should never hit zero in
+ * this function.
+ *
+ * Return the original return address.
+ */
+static unsigned long uretprobe_run_handlers(struct uprobe_task *utask, struct pt_regs *regs)
+{
+	unsigned long ret_addr, cur_sp;
+	struct hlist_head *head = &utask->uretprobe_instances;
+	struct uretprobe_instance *ri;
+	struct hlist_node *r1, *r2;
+	struct xol_area *area;
+	unsigned long trampoline_addr;
+
+	area = get_xol_area(current->mm);
+	trampoline_addr = area->uretprobe_trampoline_addr;
+	cur_sp = kernel_stack_pointer(regs);
+	uretprobe_bypass_instances(cur_sp, utask);
+
+	hlist_for_each_entry_safe(ri, r1, r2, head, hlist) {
+		if (ri->rp && ri->rp->consumers)
+			uretprobe_handler_chain(ri->rp, regs);
+
+		ret_addr = ri->ret_addr;
+		hlist_del(&ri->hlist);
+		kfree(ri);
+		if (ret_addr != trampoline_addr)
+			/*
+			 * This is the first ri (chronologically) pushed for
+			 * this particular instance of the probed function.
+			 */
+			return ret_addr;
+	}
+
+	printk(KERN_ERR "No uretprobe instance with original return address!"
+			" pid/tgid=%d/%d", current->pid, current->tgid);
+	utask->autask.doomed=1;
+
+	return 0;
+}
+
+/* Called when the uretprobe trampoline is hit. */
+static void uretprobe_handle_return(struct pt_regs *regs, struct uprobe_task *utask)
+{
+	unsigned long ret_addr;
+
+	if (!uretprobe_trampoline_dummy_probe)
+		uretprobe_trampoline_dummy_probe = kzalloc(sizeof(struct uprobe), GFP_KERNEL);
+
+	atomic_inc(&uretprobe_trampoline_dummy_probe->ref);
+	// TODO: utask->active_uprobe = uretprobe_trampoline_dummy_probe;
+	ret_addr = uretprobe_run_handlers(utask, regs);
+	/* Restore original return address */
+	regs->ip = ret_addr;
+}
+
 /*
  * write_opcode - write the opcode at a given virtual address.
  * @mm: the probed process address space.
@@ -1550,10 +1626,32 @@ static void handle_swbp(struct pt_regs *regs)
 	struct uprobe_task *utask;
 	struct uprobe *uprobe;
 	unsigned long bp_vaddr;
+	struct xol_area *area;
 	int uninitialized_var(is_swbp);
+	int hit_uretprobe_trampoline = 0;
 
 	bp_vaddr = uprobe_get_swbp_addr(regs);
+	area = get_xol_area(current->mm);
+	if (area)
+		hit_uretprobe_trampoline = (bp_vaddr == area->uretprobe_trampoline_addr);
+
 	uprobe = find_active_uprobe(bp_vaddr, &is_swbp);
+	utask = current->utask;
+	if (!utask) {
+		utask = add_utask();
+		/* Cannot allocate; re-execute the instruction. */
+		if (!utask)
+			goto restart;
+	}
+
+	if (hit_uretprobe_trampoline) {
+		uretprobe_handle_return(regs, utask);
+		if (unlikely(utask->autask.doomed))
+			do_exit(SIGSEGV);
+
+		put_uprobe(uretprobe_trampoline_dummy_probe);
+		return;
+	}
 
 	if (!uprobe) {
 		if (is_swbp > 0) {
@@ -1581,14 +1679,6 @@ static void handle_swbp(struct pt_regs *regs)
 	if (unlikely(!test_bit(UPROBE_COPY_INSN, &uprobe->flags)))
 		goto restart;
 
-	utask = current->utask;
-	if (!utask) {
-		utask = add_utask();
-		/* Cannot allocate; re-execute the instruction. */
-		if (!utask)
-			goto restart;
-	}
-
 	handler_chain(uprobe, regs, utask);
 	if (can_skip_sstep(uprobe, regs))
 		goto out;