1/*-
2 * Copyright (c) 2017 Mellanox Technologies, Ltd.
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 *    notice unmodified, this list of conditions, and the following
10 *    disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 *    notice, this list of conditions and the following disclaimer in the
13 *    documentation and/or other materials provided with the distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
17 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
18 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
19 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
20 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
21 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25 */
26
27#include <sys/cdefs.h>
28__FBSDID("$FreeBSD$");
29
30#include <linux/slab.h>
31#include <linux/rcupdate.h>
32#include <linux/kernel.h>
33#include <linux/irq_work.h>
34#include <linux/llist.h>
35
36#include <sys/param.h>
37#include <sys/taskqueue.h>
38
39struct linux_kmem_rcu {
40	struct rcu_head rcu_head;
41	struct linux_kmem_cache *cache;
42};
43
44#define	LINUX_KMEM_TO_RCU(c, m)					\
45	((struct linux_kmem_rcu *)((char *)(m) +		\
46	(c)->cache_size - sizeof(struct linux_kmem_rcu)))
47
48#define	LINUX_RCU_TO_KMEM(r)					\
49	((void *)((char *)(r) + sizeof(struct linux_kmem_rcu) - \
50	(r)->cache->cache_size))
51
52static LLIST_HEAD(linux_kfree_async_list);
53
54static int
55linux_kmem_ctor(void *mem, int size, void *arg, int flags)
56{
57	struct linux_kmem_cache *c = arg;
58
59	if (unlikely(c->cache_flags & SLAB_TYPESAFE_BY_RCU)) {
60		struct linux_kmem_rcu *rcu = LINUX_KMEM_TO_RCU(c, mem);
61
62		/* duplicate cache pointer */
63		rcu->cache = c;
64	}
65
66	/* check for constructor */
67	if (likely(c->cache_ctor != NULL))
68		c->cache_ctor(mem);
69
70	return (0);
71}
72
73static void
74linux_kmem_cache_free_rcu_callback(struct rcu_head *head)
75{
76	struct linux_kmem_rcu *rcu =
77	    container_of(head, struct linux_kmem_rcu, rcu_head);
78
79	uma_zfree(rcu->cache->cache_zone, LINUX_RCU_TO_KMEM(rcu));
80}
81
82struct linux_kmem_cache *
83linux_kmem_cache_create(const char *name, size_t size, size_t align,
84    unsigned flags, linux_kmem_ctor_t *ctor)
85{
86	struct linux_kmem_cache *c;
87
88	c = malloc(sizeof(*c), M_KMALLOC, M_WAITOK);
89
90	if (flags & SLAB_HWCACHE_ALIGN)
91		align = UMA_ALIGN_CACHE;
92	else if (align != 0)
93		align--;
94
95	if (flags & SLAB_TYPESAFE_BY_RCU) {
96		/* make room for RCU structure */
97		size = ALIGN(size, sizeof(void *));
98		size += sizeof(struct linux_kmem_rcu);
99
100		/* create cache_zone */
101		c->cache_zone = uma_zcreate(name, size,
102		    linux_kmem_ctor, NULL, NULL, NULL,
103		    align, UMA_ZONE_ZINIT);
104	} else {
105		/* create cache_zone */
106		c->cache_zone = uma_zcreate(name, size,
107		    ctor ? linux_kmem_ctor : NULL, NULL,
108		    NULL, NULL, align, 0);
109	}
110
111	c->cache_flags = flags;
112	c->cache_ctor = ctor;
113	c->cache_size = size;
114	return (c);
115}
116
117void
118linux_kmem_cache_free_rcu(struct linux_kmem_cache *c, void *m)
119{
120	struct linux_kmem_rcu *rcu = LINUX_KMEM_TO_RCU(c, m);
121
122	call_rcu(&rcu->rcu_head, linux_kmem_cache_free_rcu_callback);
123}
124
125void
126linux_kmem_cache_destroy(struct linux_kmem_cache *c)
127{
128	if (unlikely(c->cache_flags & SLAB_TYPESAFE_BY_RCU)) {
129		/* make sure all free callbacks have been called */
130		rcu_barrier();
131	}
132
133	uma_zdestroy(c->cache_zone);
134	free(c, M_KMALLOC);
135}
136
137static void
138linux_kfree_async_fn(void *context, int pending)
139{
140	struct llist_node *freed;
141
142	while((freed = llist_del_first(&linux_kfree_async_list)) != NULL)
143		kfree(freed);
144}
145static struct task linux_kfree_async_task =
146    TASK_INITIALIZER(0, linux_kfree_async_fn, &linux_kfree_async_task);
147
148void
149linux_kfree_async(void *addr)
150{
151	if (addr == NULL)
152		return;
153	llist_add(addr, &linux_kfree_async_list);
154	taskqueue_enqueue(linux_irq_work_tq, &linux_kfree_async_task);
155}
156