uretprobes: handler uretprobe entry

When a uretprobe is registered the corresponding uprobe that is created
is assigned a dummy consumer. When this uprobe is hit, the
uretprobe_handle_entry function is invoked instead of the handler. This
creates a uretprobe_instance, hijacks the return address and replaces it
with the trampoline if it has not already been done so.

Signed-off-by: Anithra P Janakiraman <anithra@linux.vnet.ibm.com>
Signed-off-by: Anton Arapov <anton@redhat.com>

2 files changed, 103 insertions, 4 deletions

diff --git a/include/linux/uprobes.h b/include/linux/uprobes.h
index ffb1984db93..460f2f674ee 100644
--- a/include/linux/uprobes.h
+++ b/include/linux/uprobes.h
@@ -60,6 +60,7 @@ enum uprobe_task_state {
 struct uprobe_task {
 	enum uprobe_task_state		state;
 	struct arch_uprobe_task		autask;
+	struct hlist_head		uretprobe_instances;
 
 	struct uprobe			*active_uprobe;
 
diff --git a/kernel/events/uprobes.c b/kernel/events/uprobes.c
index 29c73d7b1c2..61e5525000c 100644
--- a/kernel/events/uprobes.c
+++ b/kernel/events/uprobes.c
@@ -92,6 +92,7 @@ struct uprobe {
 	struct mutex		copy_mutex;	/* TODO: kill me and UPROBE_COPY_INSN */
 	struct list_head	pending_list;
 	struct uprobe_consumer	*consumers;
+	struct uretprobe	*rp;		/* Has return probe if not NULL */
 	struct inode		*inode;		/* Also hold a ref to inode */
 	loff_t			offset;
 	unsigned long		flags;
@@ -99,7 +100,25 @@ struct uprobe {
 };
 
 /*
+ * When a uretprobe is registered the corresponding uprobe that is created
+ * is assigned a dummy consumer. When this uprobe is hit, the
+ * uretprobe_handle_entry function is invoked instead of the handler.
+ */
+static struct uprobe_consumer *uretprobe_dummy_consumer;
+static void uretprobe_handle_entry(struct uprobe *, struct pt_regs *, struct uprobe_task *);
 static struct xol_area *get_xol_area(struct mm_struct *mm);
+
+struct uretprobe {
+};
+
+struct uretprobe_instance {
+	struct uretprobe	*rp;
+	unsigned long		ret_addr;
+	struct hlist_node	hlist;
+	unsigned long		sp;
+};
+
+/*
  * valid_vma: Verify if the specified vma is an executable vma
  * Relax restrictions while unregistering: vm_flags might have
  * changed after breakpoint was inserted.
@@ -228,6 +247,42 @@ static int verify_opcode(struct page *page, unsigned long vaddr, uprobe_opcode_t
  * have fixed length instructions.
  */
 
+/* Returns true if @ri_sp lies outside the stack (beyond @cursp). */
+static inline bool compare_stack_ptrs(unsigned long cursp, unsigned long ri_sp)
+{
+#ifdef CONFIG_STACK_GROWSUP
+	if (cursp < ri_sp)
+		return true;
+#else
+	if (cursp > ri_sp)
+		return true;
+#endif
+	return false;
+}
+
+/*
+ * A longjmp may cause one or more uretprobed functions to terminate without
+ * returning. Those functions' uretprobe_instances need to be recycled.
+ * We detect this when any uretprobed function is subsequently called
+ * or returns. A bypassed uretprobe_instance's stack_ptr is beyond the
+ * current stack.
+ */
+static inline void uretprobe_bypass_instances(unsigned long cursp,
+					struct uprobe_task *utask)
+{
+	struct hlist_node *r1, *r2;
+	struct uretprobe_instance *ri;
+	struct hlist_head *head = &utask->uretprobe_instances;
+
+	hlist_for_each_entry_safe(ri, r1, r2, head, hlist) {
+		if (compare_stack_ptrs(cursp, ri->sp)) {
+			hlist_del(&ri->hlist);
+			kfree(ri);
+		} else
+			return;
+	}
+}
+
 /*
  * write_opcode - write the opcode at a given virtual address.
  * @mm: the probed process address space.
@@ -445,6 +500,7 @@ static struct uprobe *alloc_uprobe(struct inode *inode, loff_t offset)
 
 	uprobe->inode = igrab(inode);
 	uprobe->offset = offset;
+	uprobe->rp = NULL;
 	init_rwsem(&uprobe->consumer_rwsem);
 	mutex_init(&uprobe->copy_mutex);
 
@@ -463,7 +519,8 @@ static struct uprobe *alloc_uprobe(struct inode *inode, loff_t offset)
 	return uprobe;
 }
 
-static void handler_chain(struct uprobe *uprobe, struct pt_regs *regs)
+static void handler_chain(struct uprobe *uprobe, struct pt_regs *regs,
+			struct uprobe_task *utask)
 {
 	struct uprobe_consumer *uc;
 
@@ -472,8 +529,12 @@ static void handler_chain(struct uprobe *uprobe, struct pt_regs *regs)
 
 	down_read(&uprobe->consumer_rwsem);
 	for (uc = uprobe->consumers; uc; uc = uc->next) {
-		if (!uc->filter || uc->filter(uc, current))
-			uc->handler(uc, regs);
+		if (!uc->filter || uc->filter(uc, current)) {
+			if (uc == uretprobe_dummy_consumer)
+				uretprobe_handle_entry(uprobe, regs, utask);
+			else
+				uc->handler(uc, regs);
+		}
 	}
 	up_read(&uprobe->consumer_rwsem);
 }
@@ -1334,6 +1395,8 @@ static struct uprobe_task *add_utask(void)
 	if (unlikely(!utask))
 		return NULL;
 
+	utask->autask.doomed = 0;
+	INIT_HLIST_HEAD(&utask->uretprobe_instances);
 	current->utask = utask;
 	return utask;
 }
@@ -1526,7 +1589,7 @@ static void handle_swbp(struct pt_regs *regs)
 			goto restart;
 	}
 
-	handler_chain(uprobe, regs);
+	handler_chain(uprobe, regs, utask);
 	if (can_skip_sstep(uprobe, regs))
 		goto out;
 
@@ -1598,6 +1661,41 @@ void uprobe_notify_resume(struct pt_regs *regs)
 		handle_swbp(regs);
 }
 
+/* Called when the entry-point probe @u is hit. */
+static void uretprobe_handle_entry(struct uprobe *uprobe, struct pt_regs *regs,
+				struct uprobe_task *utask)
+{
+	struct uretprobe_instance *ri;
+	unsigned long trampoline_addr = 0;
+	struct xol_area *area;
+
+	area = get_xol_area(current->mm);
+	if (area)
+		trampoline_addr = area->uretprobe_trampoline_addr;
+
+	if (!trampoline_addr) {
+		trampoline_addr = xol_get_trampoline_slot();
+		if (!(trampoline_addr))
+			return;
+	}
+
+	ri = (struct uretprobe_instance *)
+		kzalloc(sizeof(struct uretprobe_instance), GFP_KERNEL);
+	if (!ri)
+		return;
+
+	ri->ret_addr = arch_uretprobe_hijack_uret_addr(trampoline_addr, regs,
+						&utask->autask);
+	if (likely(ri->ret_addr)) {
+		ri->sp = arch_uretprobe_predict_sp_at_ret(regs, current);
+		uretprobe_bypass_instances(ri->sp, utask);
+		ri->rp = uprobe->rp;
+		INIT_HLIST_NODE(&ri->hlist);
+		hlist_add_head(&ri->hlist, &utask->uretprobe_instances);
+	} else
+		kfree(ri);
+}
+
 /*
  * uprobe_pre_sstep_notifier gets called from interrupt context as part of
  * notifier mechanism. Set TIF_UPROBE flag and indicate breakpoint hit.