ftrace: Have the function probes call their own function

Now that the function probes have their own ftrace_ops, there's no reason to
continue using the ftrace_func_hash to find which probe to call in the
function callback. The ops that is passed in to the function callback is
part of the probe_ops to call.

Signed-off-by: Steven Rostedt (VMware) <rostedt@goodmis.org>

2 files changed, 99 insertions, 127 deletions

diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c
index cf6b7263199a..493c7ff7e860 100644
--- a/kernel/trace/ftrace.c
+++ b/kernel/trace/ftrace.c
@@ -1096,14 +1096,7 @@ static bool update_all_ops;
 # error Dynamic ftrace depends on MCOUNT_RECORD
 #endif
 
-static struct hlist_head ftrace_func_hash[FTRACE_FUNC_HASHSIZE] __read_mostly;
-
-struct ftrace_func_probe {
-	struct hlist_node	node;
-	struct ftrace_probe_ops	*ops;
-	unsigned long		ip;
-	struct list_head	free_list;
-};
+static LIST_HEAD(ftrace_func_probes);
 
 struct ftrace_func_entry {
 	struct hlist_node hlist;
@@ -1270,7 +1263,7 @@ static void
 remove_hash_entry(struct ftrace_hash *hash,
 		  struct ftrace_func_entry *entry)
 {
-	hlist_del(&entry->hlist);
+	hlist_del_rcu(&entry->hlist);
 	hash->count--;
 }
 
@@ -3063,35 +3056,58 @@ struct ftrace_iterator {
 	loff_t				func_pos;
 	struct ftrace_page		*pg;
 	struct dyn_ftrace		*func;
-	struct ftrace_func_probe	*probe;
+	struct ftrace_probe_ops		*probe;
+	struct ftrace_func_entry	*probe_entry;
 	struct trace_parser		parser;
 	struct ftrace_hash		*hash;
 	struct ftrace_ops		*ops;
-	int				hidx;
+	int				pidx;
 	int				idx;
 	unsigned			flags;
 };
 
 static void *
-t_hash_next(struct seq_file *m, loff_t *pos)
+t_probe_next(struct seq_file *m, loff_t *pos)
 {
 	struct ftrace_iterator *iter = m->private;
+	struct ftrace_hash *hash;
+	struct list_head *next;
 	struct hlist_node *hnd = NULL;
 	struct hlist_head *hhd;
+	int size;
 
 	(*pos)++;
 	iter->pos = *pos;
 
-	if (iter->probe)
-		hnd = &iter->probe->node;
- retry:
-	if (iter->hidx >= FTRACE_FUNC_HASHSIZE)
+	if (list_empty(&ftrace_func_probes))
 		return NULL;
 
-	hhd = &ftrace_func_hash[iter->hidx];
+	if (!iter->probe) {
+		next = ftrace_func_probes.next;
+		iter->probe = list_entry(next, struct ftrace_probe_ops, list);
+	}
+
+	if (iter->probe_entry)
+		hnd = &iter->probe_entry->hlist;
+
+	hash = iter->probe->ops.func_hash->filter_hash;
+	size = 1 << hash->size_bits;
+
+ retry:
+	if (iter->pidx >= size) {
+		if (iter->probe->list.next == &ftrace_func_probes)
+			return NULL;
+		next = iter->probe->list.next;
+		iter->probe = list_entry(next, struct ftrace_probe_ops, list);
+		hash = iter->probe->ops.func_hash->filter_hash;
+		size = 1 << hash->size_bits;
+		iter->pidx = 0;
+	}
+
+	hhd = &hash->buckets[iter->pidx];
 
 	if (hlist_empty(hhd)) {
-		iter->hidx++;
+		iter->pidx++;
 		hnd = NULL;
 		goto retry;
 	}
@@ -3101,7 +3117,7 @@ t_hash_next(struct seq_file *m, loff_t *pos)
 	else {
 		hnd = hnd->next;
 		if (!hnd) {
-			iter->hidx++;
+			iter->pidx++;
 			goto retry;
 		}
 	}
@@ -3109,26 +3125,28 @@ t_hash_next(struct seq_file *m, loff_t *pos)
 	if (WARN_ON_ONCE(!hnd))
 		return NULL;
 
-	iter->probe = hlist_entry(hnd, struct ftrace_func_probe, node);
+	iter->probe_entry = hlist_entry(hnd, struct ftrace_func_entry, hlist);
 
 	return iter;
 }
 
-static void *t_hash_start(struct seq_file *m, loff_t *pos)
+static void *t_probe_start(struct seq_file *m, loff_t *pos)
 {
 	struct ftrace_iterator *iter = m->private;
 	void *p = NULL;
 	loff_t l;
 
-	if (!(iter->flags & FTRACE_ITER_DO_HASH))
+	if (!(iter->flags & FTRACE_ITER_DO_PROBES))
 		return NULL;
 
 	if (iter->func_pos > *pos)
 		return NULL;
 
-	iter->hidx = 0;
+	iter->probe = NULL;
+	iter->probe_entry = NULL;
+	iter->pidx = 0;
 	for (l = 0; l <= (*pos - iter->func_pos); ) {
-		p = t_hash_next(m, &l);
+		p = t_probe_next(m, &l);
 		if (!p)
 			break;
 	}
@@ -3136,24 +3154,27 @@ static void *t_hash_start(struct seq_file *m, loff_t *pos)
 		return NULL;
 
 	/* Only set this if we have an item */
-	iter->flags |= FTRACE_ITER_HASH;
+	iter->flags |= FTRACE_ITER_PROBE;
 
 	return iter;
 }
 
 static int
-t_hash_show(struct seq_file *m, struct ftrace_iterator *iter)
+t_probe_show(struct seq_file *m, struct ftrace_iterator *iter)
 {
-	struct ftrace_func_probe *rec;
+	struct ftrace_probe_ops *probe;
+	struct ftrace_func_entry *probe_entry;
 
-	rec = iter->probe;
-	if (WARN_ON_ONCE(!rec))
+	probe = iter->probe;
+	probe_entry = iter->probe_entry;
+
+	if (WARN_ON_ONCE(!probe || !probe_entry))
 		return -EIO;
 
-	if (rec->ops->print)
-		return rec->ops->print(m, rec->ip, rec->ops, NULL);
+	if (probe->print)
+		return probe->print(m, probe_entry->ip, probe, NULL);
 
-	seq_printf(m, "%ps:%ps\n", (void *)rec->ip, (void *)rec->ops->func);
+	seq_printf(m, "%ps:%ps\n", (void *)probe_entry->ip, (void *)probe->func);
 
 	return 0;
 }
@@ -3205,19 +3226,19 @@ t_next(struct seq_file *m, void *v, loff_t *pos)
 	if (unlikely(ftrace_disabled))
 		return NULL;
 
-	if (iter->flags & FTRACE_ITER_HASH)
-		return t_hash_next(m, pos);
+	if (iter->flags & FTRACE_ITER_PROBE)
+		return t_probe_next(m, pos);
 
 	if (iter->flags & FTRACE_ITER_PRINTALL) {
-		/* next must increment pos, and t_hash_start does not */
+		/* next must increment pos, and t_probe_start does not */
 		(*pos)++;
-		return t_hash_start(m, &l);
+		return t_probe_start(m, &l);
 	}
 
 	ret = t_func_next(m, pos);
 
 	if (!ret)
-		return t_hash_start(m, &l);
+		return t_probe_start(m, &l);
 
 	return ret;
 }
@@ -3226,7 +3247,7 @@ static void reset_iter_read(struct ftrace_iterator *iter)
 {
 	iter->pos = 0;
 	iter->func_pos = 0;
-	iter->flags &= ~(FTRACE_ITER_PRINTALL | FTRACE_ITER_HASH);
+	iter->flags &= ~(FTRACE_ITER_PRINTALL | FTRACE_ITER_PROBE);
 }
 
 static void *t_start(struct seq_file *m, loff_t *pos)
@@ -3255,15 +3276,15 @@ static void *t_start(struct seq_file *m, loff_t *pos)
 	    ftrace_hash_empty(iter->hash)) {
 		iter->func_pos = 1; /* Account for the message */
 		if (*pos > 0)
-			return t_hash_start(m, pos);
+			return t_probe_start(m, pos);
 		iter->flags |= FTRACE_ITER_PRINTALL;
 		/* reset in case of seek/pread */
-		iter->flags &= ~FTRACE_ITER_HASH;
+		iter->flags &= ~FTRACE_ITER_PROBE;
 		return iter;
 	}
 
-	if (iter->flags & FTRACE_ITER_HASH)
-		return t_hash_start(m, pos);
+	if (iter->flags & FTRACE_ITER_PROBE)
+		return t_probe_start(m, pos);
 
 	/*
 	 * Unfortunately, we need to restart at ftrace_pages_start
@@ -3279,7 +3300,7 @@ static void *t_start(struct seq_file *m, loff_t *pos)
 	}
 
 	if (!p)
-		return t_hash_start(m, pos);
+		return t_probe_start(m, pos);
 
 	return iter;
 }
@@ -3310,8 +3331,8 @@ static int t_show(struct seq_file *m, void *v)
 	struct ftrace_iterator *iter = m->private;
 	struct dyn_ftrace *rec;
 
-	if (iter->flags & FTRACE_ITER_HASH)
-		return t_hash_show(m, iter);
+	if (iter->flags & FTRACE_ITER_PROBE)
+		return t_probe_show(m, iter);
 
 	if (iter->flags & FTRACE_ITER_PRINTALL) {
 		if (iter->flags & FTRACE_ITER_NOTRACE)
@@ -3490,7 +3511,7 @@ ftrace_filter_open(struct inode *inode, struct file *file)
 	struct ftrace_ops *ops = inode->i_private;
 
 	return ftrace_regex_open(ops,
-			FTRACE_ITER_FILTER | FTRACE_ITER_DO_HASH,
+			FTRACE_ITER_FILTER | FTRACE_ITER_DO_PROBES,
 			inode, file);
 }
 
@@ -3765,16 +3786,9 @@ core_initcall(ftrace_mod_cmd_init);
 static void function_trace_probe_call(unsigned long ip, unsigned long parent_ip,
 				      struct ftrace_ops *op, struct pt_regs *pt_regs)
 {
-	struct ftrace_func_probe *entry;
-	struct hlist_head *hhd;
-	unsigned long key;
-
-	key = hash_long(ip, FTRACE_HASH_BITS);
+	struct ftrace_probe_ops *probe_ops;
 
-	hhd = &ftrace_func_hash[key];
-
-	if (hlist_empty(hhd))
-		return;
+	probe_ops = container_of(op, struct ftrace_probe_ops, ops);
 
 	/*
 	 * Disable preemption for these calls to prevent a RCU grace
@@ -3782,20 +3796,10 @@ static void function_trace_probe_call(unsigned long ip, unsigned long parent_ip,
 	 * on the hash. rcu_read_lock is too dangerous here.
 	 */
 	preempt_disable_notrace();
-	hlist_for_each_entry_rcu_notrace(entry, hhd, node) {
-		if (entry->ip == ip)
-			entry->ops->func(ip, parent_ip, entry->ops, NULL);
-	}
+	probe_ops->func(ip, parent_ip, probe_ops, NULL);
 	preempt_enable_notrace();
 }
 
-static void ftrace_free_entry(struct ftrace_func_probe *entry)
-{
-	if (entry->ops->free)
-		entry->ops->free(entry->ops, entry->ip, NULL);
-	kfree(entry);
-}
-
 struct ftrace_func_map {
 	struct ftrace_func_entry	entry;
 	void				*data;
@@ -3942,13 +3946,9 @@ register_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
 			       void *data)
 {
 	struct ftrace_func_entry *entry;
-	struct ftrace_func_probe *probe;
 	struct ftrace_hash **orig_hash;
 	struct ftrace_hash *old_hash;
 	struct ftrace_hash *hash;
-	struct hlist_head hl;
-	struct hlist_node *n;
-	unsigned long key;
 	int count = 0;
 	int size;
 	int ret;
@@ -3961,6 +3961,7 @@ register_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
 	if (!(ops->ops.flags & FTRACE_OPS_FL_INITIALIZED)) {
 		ops->ops.func = function_trace_probe_call;
 		ftrace_ops_init(&ops->ops);
+		INIT_LIST_HEAD(&ops->list);
 	}
 
 	mutex_lock(&ops->ops.func_hash->regex_lock);
@@ -3978,31 +3979,21 @@ register_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
 	if (ret < 0)
 		goto out;
 
-	INIT_HLIST_HEAD(&hl);
-
 	size = 1 << hash->size_bits;
 	for (i = 0; i < size; i++) {
 		hlist_for_each_entry(entry, &hash->buckets[i], hlist) {
 			if (ftrace_lookup_ip(old_hash, entry->ip))
 				continue;
-			probe = kmalloc(sizeof(*probe), GFP_KERNEL);
-			if (!probe) {
-				count = -ENOMEM;
-				goto err_free;
-			}
-			probe->ops = ops;
-			probe->ip = entry->ip;
 			/*
 			 * The caller might want to do something special
 			 * for each function we find. We call the callback
 			 * to give the caller an opportunity to do so.
 			 */
-			if (ops->init && ops->init(ops, entry->ip, data) < 0) {
-				kfree(probe);
-				goto err_free;
+			if (ops->init) {
+				ret = ops->init(ops, entry->ip, data);
+				if (ret < 0)
+					goto out;
 			}
-			hlist_add_head(&probe->node, &hl);
-
 			count++;
 		}
 	}
@@ -4012,17 +4003,15 @@ register_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
 	ret = ftrace_hash_move_and_update_ops(&ops->ops, orig_hash,
 						      hash, 1);
 	if (ret < 0)
-		goto err_free_unlock;
+		goto out_unlock;
 
-	hlist_for_each_entry_safe(probe, n, &hl, node) {
-		hlist_del(&probe->node);
-		key = hash_long(probe->ip, FTRACE_HASH_BITS);
-		hlist_add_head_rcu(&probe->node, &ftrace_func_hash[key]);
-	}
+	if (list_empty(&ops->list))
+		list_add(&ops->list, &ftrace_func_probes);
 
 	if (!(ops->ops.flags & FTRACE_OPS_FL_ENABLED))
 		ret = ftrace_startup(&ops->ops, 0);
 
+ out_unlock:
 	mutex_unlock(&ftrace_lock);
 
 	if (!ret)
@@ -4032,34 +4021,22 @@ register_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
 	free_ftrace_hash(hash);
 
 	return ret;
-
- err_free_unlock:
-	mutex_unlock(&ftrace_lock);
- err_free:
-	hlist_for_each_entry_safe(probe, n, &hl, node) {
-		hlist_del(&probe->node);
-		if (ops->free)
-			ops->free(ops, probe->ip, NULL);
-		kfree(probe);
-	}
-	goto out;
 }
 
 int
 unregister_ftrace_function_probe_func(char *glob, struct ftrace_probe_ops *ops)
 {
 	struct ftrace_ops_hash old_hash_ops;
-	struct ftrace_func_entry *rec_entry;
-	struct ftrace_func_probe *entry;
-	struct ftrace_func_probe *p;
+	struct ftrace_func_entry *entry;
 	struct ftrace_glob func_g;
 	struct ftrace_hash **orig_hash;
 	struct ftrace_hash *old_hash;
-	struct list_head free_list;
 	struct ftrace_hash *hash = NULL;
 	struct hlist_node *tmp;
+	struct hlist_head hhd;
 	char str[KSYM_SYMBOL_LEN];
 	int i, ret;
+	int size;
 
 	if (!(ops->ops.flags & FTRACE_OPS_FL_INITIALIZED))
 		return -EINVAL;
@@ -4097,18 +4074,12 @@ unregister_ftrace_function_probe_func(char *glob, struct ftrace_probe_ops *ops)
 	if (!hash)
 		goto out_unlock;
 
-	INIT_LIST_HEAD(&free_list);
-
-	for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) {
-		struct hlist_head *hhd = &ftrace_func_hash[i];
+	INIT_HLIST_HEAD(&hhd);
 
-		hlist_for_each_entry_safe(entry, tmp, hhd, node) {
-
-			/* break up if statements for readability */
-			if (entry->ops != ops)
-				continue;
+	size = 1 << hash->size_bits;
+	for (i = 0; i < size; i++) {
+		hlist_for_each_entry_safe(entry, tmp, &hash->buckets[i], hlist) {
 
-			/* do this last, since it is the most expensive */
 			if (func_g.search) {
 				kallsyms_lookup(entry->ip, NULL, NULL,
 						NULL, str);
@@ -4116,26 +4087,24 @@ unregister_ftrace_function_probe_func(char *glob, struct ftrace_probe_ops *ops)
 					continue;
 			}
 
-			rec_entry = ftrace_lookup_ip(hash, entry->ip);
-			/* It is possible more than one entry had this ip */
-			if (rec_entry)
-				free_hash_entry(hash, rec_entry);
-
-			hlist_del_rcu(&entry->node);
-			list_add(&entry->free_list, &free_list);
+			remove_hash_entry(hash, entry);
+			hlist_add_head(&entry->hlist, &hhd);
 		}
 	}
 
 	/* Nothing found? */
-	if (list_empty(&free_list)) {
+	if (hlist_empty(&hhd)) {
 		ret = -EINVAL;
 		goto out_unlock;
 	}
 
 	mutex_lock(&ftrace_lock);
 
-	if (ftrace_hash_empty(hash))
+	if (ftrace_hash_empty(hash)) {
 		ftrace_shutdown(&ops->ops, 0);
+		list_del_init(&ops->list);
+	}
+
 
 	ret = ftrace_hash_move_and_update_ops(&ops->ops, orig_hash,
 					      hash, 1);
@@ -4146,9 +4115,11 @@ unregister_ftrace_function_probe_func(char *glob, struct ftrace_probe_ops *ops)
 				       &old_hash_ops);
 	synchronize_sched();
 
-	list_for_each_entry_safe(entry, p, &free_list, free_list) {
-		list_del(&entry->free_list);
-		ftrace_free_entry(entry);
+	hlist_for_each_entry_safe(entry, tmp, &hhd, hlist) {
+		hlist_del(&entry->hlist);
+		if (ops->free)
+			ops->free(ops, entry->ip, NULL);
+		kfree(entry);
 	}
 	mutex_unlock(&ftrace_lock);
 
diff --git a/kernel/trace/trace.h b/kernel/trace/trace.h
index e16c67c49de4..d457addcc224 100644
--- a/kernel/trace/trace.h
+++ b/kernel/trace/trace.h
@@ -933,6 +933,7 @@ static inline void ftrace_pid_follow_fork(struct trace_array *tr, bool enable) {
 
 struct ftrace_probe_ops {
 	struct ftrace_ops	ops;
+	struct list_head	list;
 	void			(*func)(unsigned long ip,
 					unsigned long parent_ip,
 					struct ftrace_probe_ops *ops,