1 // SPDX-License-Identifier: (GPL-2.0+ OR MIT)
2 /*
3 * Copyright (c) 2022 Rockchip Electronics Co., Ltd.
4 */
5
6 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
7
8 #include <linux/init.h>
9 #include <linux/ktime.h>
10 #include <linux/module.h>
11 #include <linux/moduleparam.h>
12 #include <linux/err.h>
13 #include <linux/random.h>
14 #include <linux/slab.h>
15 #include <linux/sched.h>
16 #include <linux/kthread.h>
17
18 #include "pcie-designware.h"
19 #include "pcie-dw-dmatest.h"
20 #include "../rockchip-pcie-dma.h"
21
22 static int test_size = 0x20;
23 module_param_named(size, test_size, int, 0644);
24 MODULE_PARM_DESC(size, "each packet size in bytes");
25
26 static unsigned int cycles_count = 1;
27 module_param(cycles_count, uint, 0644);
28 MODULE_PARM_DESC(cycles_count, "how many erase cycles to do (default 1)");
29
30 static unsigned int chn_en = 1;
31 module_param(chn_en, uint, 0644);
32 MODULE_PARM_DESC(chn_en, "Each bits for one dma channel, up to 2 channels, (default enable channel 0)");
33
34 static unsigned int rw_test = 3;
35 module_param(rw_test, uint, 0644);
36 MODULE_PARM_DESC(rw_test, "Read/Write test, 1-read 2-write 3-both(default 3)");
37
38 static unsigned int bus_addr = 0x3c000000;
39 module_param(bus_addr, uint, 0644);
40 MODULE_PARM_DESC(bus_addr, "Dmatest chn0 bus_addr(remote), chn1 add offset 0x100000, (default 0x3c000000)");
41
42 static unsigned int local_addr = 0x3c000000;
43 module_param(local_addr, uint, 0644);
44 MODULE_PARM_DESC(local_addr, "Dmatest chn0 local_addr(local), chn1 add offset 0x100000, (default 0x3c000000)");
45
46 static unsigned int test_dev;
47 module_param(test_dev, uint, 0644);
48 MODULE_PARM_DESC(test_dev, "Choose dma_obj device,(default 0)");
49
50 static bool is_rc = true;
51 module_param_named(is_rc, is_rc, bool, 0644);
52 MODULE_PARM_DESC(is_rc, "Test port is rc(default true)");
53
54 #define PCIE_DW_MISC_DMATEST_DEV_MAX 5
55
56 #define PCIE_DMA_CHANEL_MAX_NUM 2
57
58 struct pcie_dw_dmatest_dev {
59 struct dma_trx_obj *obj;
60
61 bool irq_en;
62 struct completion rd_done[PCIE_DMA_CHANEL_MAX_NUM];
63 struct completion wr_done[PCIE_DMA_CHANEL_MAX_NUM];
64
65 struct mutex rd_lock[PCIE_DMA_CHANEL_MAX_NUM]; /* Corresponding to each read DMA channel */
66 struct mutex wr_lock[PCIE_DMA_CHANEL_MAX_NUM]; /* Corresponding to each write DMA channel */
67 };
68
69 static struct pcie_dw_dmatest_dev s_dmatest_dev[PCIE_DW_MISC_DMATEST_DEV_MAX];
70 static int cur_dmatest_dev;
71
pcie_dw_dmatest_show(void)72 static void pcie_dw_dmatest_show(void)
73 {
74 int i;
75
76 for (i = 0; i < PCIE_DW_MISC_DMATEST_DEV_MAX; i++) {
77 if (s_dmatest_dev[i].obj)
78 dev_info(s_dmatest_dev[i].obj->dev, " test_dev index %d\n", i);
79 else
80 break;
81 }
82
83 dev_info(s_dmatest_dev[test_dev].obj->dev, " is current test_dev\n");
84 }
85
rk_pcie_dma_wait_for_finised(struct dma_trx_obj * obj,struct dma_table * table)86 static int rk_pcie_dma_wait_for_finised(struct dma_trx_obj *obj, struct dma_table *table)
87 {
88 int ret;
89
90 do {
91 ret = obj->get_dma_status(obj, table->chn, table->dir);
92 } while (!ret);
93
94 return ret;
95 }
96
rk_pcie_ep_dma_frombus(struct pcie_dw_dmatest_dev * dmatest_dev,u32 chn,u32 local_paddr,u32 bus_paddr,u32 size)97 static int rk_pcie_ep_dma_frombus(struct pcie_dw_dmatest_dev *dmatest_dev, u32 chn,
98 u32 local_paddr, u32 bus_paddr, u32 size)
99 {
100 struct dma_table *table;
101 struct dma_trx_obj *obj = dmatest_dev->obj;
102 int ret;
103
104 if (chn >= PCIE_DMA_CHANEL_MAX_NUM)
105 return -1;
106
107 table = kzalloc(sizeof(struct dma_table), GFP_KERNEL);
108 if (!table)
109 return -ENOMEM;
110
111 mutex_lock(&dmatest_dev->rd_lock[chn]);
112 if (dmatest_dev->irq_en)
113 reinit_completion(&dmatest_dev->rd_done[chn]);
114
115 table->buf_size = size;
116 table->bus = bus_paddr;
117 table->local = local_paddr;
118 table->chn = chn;
119 table->dir = DMA_FROM_BUS;
120
121 obj->config_dma_func(table);
122 obj->start_dma_func(obj, table);
123
124 if (dmatest_dev->irq_en) {
125 ret = wait_for_completion_interruptible_timeout(&dmatest_dev->rd_done[chn], HZ);
126 if (ret < 0)
127 dev_err(obj->dev, "%s interrupted\n", __func__);
128 else if (ret == 0)
129 dev_err(obj->dev, "%s timed out\n", __func__);
130 } else {
131 ret = rk_pcie_dma_wait_for_finised(obj, table);
132 }
133 mutex_unlock(&dmatest_dev->rd_lock[chn]);
134
135 kfree(table);
136
137 return ret;
138 }
139
rk_pcie_ep_dma_tobus(struct pcie_dw_dmatest_dev * dmatest_dev,u32 chn,u32 bus_paddr,u32 local_paddr,u32 size)140 static int rk_pcie_ep_dma_tobus(struct pcie_dw_dmatest_dev *dmatest_dev, u32 chn,
141 u32 bus_paddr, u32 local_paddr, u32 size)
142 {
143 struct dma_table *table;
144 struct dma_trx_obj *obj = dmatest_dev->obj;
145 int ret;
146
147 if (chn >= PCIE_DMA_CHANEL_MAX_NUM)
148 return -1;
149
150 table = kzalloc(sizeof(struct dma_table), GFP_KERNEL);
151 if (!table)
152 return -ENOMEM;
153
154 mutex_lock(&dmatest_dev->wr_lock[chn]);
155 if (dmatest_dev->irq_en)
156 reinit_completion(&dmatest_dev->wr_done[chn]);
157
158 table->buf_size = size;
159 table->bus = bus_paddr;
160 table->local = local_paddr;
161 table->chn = chn;
162 table->dir = DMA_TO_BUS;
163
164 obj->config_dma_func(table);
165 obj->start_dma_func(obj, table);
166
167 if (dmatest_dev->irq_en) {
168 ret = wait_for_completion_interruptible_timeout(&dmatest_dev->wr_done[chn], HZ);
169 if (ret < 0)
170 dev_err(obj->dev, "%s interrupted\n", __func__);
171 else if (ret == 0)
172 dev_err(obj->dev, "%s timed out\n", __func__);
173 } else {
174 ret = rk_pcie_dma_wait_for_finised(obj, table);
175 }
176 mutex_unlock(&dmatest_dev->wr_lock[chn]);
177
178 kfree(table);
179
180 return ret;
181 }
182
rk_pcie_rc_dma_frombus(struct pcie_dw_dmatest_dev * dmatest_dev,u32 chn,u32 local_paddr,u32 bus_paddr,u32 size)183 static int rk_pcie_rc_dma_frombus(struct pcie_dw_dmatest_dev *dmatest_dev, u32 chn,
184 u32 local_paddr, u32 bus_paddr, u32 size)
185 {
186 return rk_pcie_ep_dma_tobus(dmatest_dev, chn, local_paddr, bus_paddr, size);
187 }
188
rk_pcie_rc_dma_tobus(struct pcie_dw_dmatest_dev * dmatest_dev,u32 chn,u32 bus_paddr,u32 local_paddr,u32 size)189 static int rk_pcie_rc_dma_tobus(struct pcie_dw_dmatest_dev *dmatest_dev, u32 chn,
190 u32 bus_paddr, u32 local_paddr, u32 size)
191 {
192 return rk_pcie_ep_dma_frombus(dmatest_dev, chn, bus_paddr, local_paddr, size);
193 }
194
rk_pcie_dma_interrupt_handler_call_back(struct dma_trx_obj * obj,u32 chn,enum dma_dir dir)195 static int rk_pcie_dma_interrupt_handler_call_back(struct dma_trx_obj *obj, u32 chn, enum dma_dir dir)
196 {
197 struct pcie_dw_dmatest_dev *dmatest_dev = (struct pcie_dw_dmatest_dev *)obj->priv;
198
199 if (chn >= PCIE_DMA_CHANEL_MAX_NUM)
200 return -1;
201
202 if (dir == DMA_FROM_BUS)
203 complete(&dmatest_dev->rd_done[chn]);
204 else
205 complete(&dmatest_dev->wr_done[chn]);
206
207 return 0;
208 }
209
pcie_dw_dmatest_register(struct device * dev,bool irq_en)210 struct dma_trx_obj *pcie_dw_dmatest_register(struct device *dev, bool irq_en)
211 {
212 struct dma_trx_obj *obj;
213 struct pcie_dw_dmatest_dev *dmatest_dev = &s_dmatest_dev[cur_dmatest_dev];
214 int i;
215
216 obj = devm_kzalloc(dev, sizeof(struct dma_trx_obj), GFP_KERNEL);
217 if (!obj)
218 return ERR_PTR(-ENOMEM);
219
220 obj->dev = dev;
221 obj->priv = dmatest_dev;
222 obj->cb = rk_pcie_dma_interrupt_handler_call_back;
223
224 /* Save for dmatest */
225 dmatest_dev->obj = obj;
226 for (i = 0; i < PCIE_DMA_CHANEL_MAX_NUM; i++) {
227 init_completion(&dmatest_dev->rd_done[i]);
228 init_completion(&dmatest_dev->wr_done[i]);
229 mutex_init(&dmatest_dev->rd_lock[i]);
230 mutex_init(&dmatest_dev->wr_lock[i]);
231 }
232
233 /* Enable IRQ transfer as default */
234 dmatest_dev->irq_en = irq_en;
235 cur_dmatest_dev++;
236
237 return obj;
238 }
239
dma_test(struct pcie_dw_dmatest_dev * dmatest_dev,u32 chn,u32 bus_paddr,u32 local_paddr,u32 size,u32 loop,u8 rd_en,u8 wr_en)240 static int dma_test(struct pcie_dw_dmatest_dev *dmatest_dev, u32 chn,
241 u32 bus_paddr, u32 local_paddr, u32 size, u32 loop, u8 rd_en, u8 wr_en)
242 {
243 ktime_t start_time;
244 ktime_t end_time;
245 ktime_t cost_time;
246 u32 i;
247 long long total_byte;
248 long long us = 0;
249 struct dma_trx_obj *obj = dmatest_dev->obj;
250
251 /*
252 * Clean the cache to ensure memory consistency. The CPU writes to the normal memory
253 * cache before the transmission is initiated, which may cause IO consistency problems,
254 * such as IO commands.
255 */
256 if (rd_en)
257 dma_sync_single_for_device(obj->dev, local_paddr, size, DMA_TO_DEVICE);
258
259 start_time = ktime_get();
260 for (i = 0; i < loop; i++) {
261 if (rd_en) {
262 if (is_rc)
263 rk_pcie_rc_dma_frombus(dmatest_dev, chn, local_paddr, bus_paddr, size);
264 else
265 rk_pcie_ep_dma_frombus(dmatest_dev, chn, local_paddr, bus_paddr, size);
266 dma_sync_single_for_cpu(obj->dev, local_paddr, size, DMA_FROM_DEVICE);
267 }
268
269 if (wr_en) {
270 dma_sync_single_for_device(obj->dev, local_paddr, size, DMA_TO_DEVICE);
271 if (is_rc)
272 rk_pcie_rc_dma_tobus(dmatest_dev, chn, bus_paddr, local_paddr, size);
273 else
274 rk_pcie_ep_dma_tobus(dmatest_dev, chn, bus_paddr, local_paddr, size);
275 }
276 }
277 end_time = ktime_get();
278 cost_time = ktime_sub(end_time, start_time);
279 us = ktime_to_us(cost_time);
280
281 total_byte = (wr_en + rd_en) * size * loop; /* 1 rd,1 wr */
282 total_byte = total_byte * (1000000 / 1024) / us;
283 pr_err("pcie dma %s/%s test (%d+%d)*%d*%d cost %lldus speed:%lldKB/S\n",
284 wr_en ? "wr" : "", rd_en ? "rd" : "", wr_en, rd_en, size, loop, us, total_byte);
285
286 return 0;
287 }
288
dma_test_ch0(void * p)289 static int dma_test_ch0(void *p)
290 {
291 dma_test(&s_dmatest_dev[test_dev], 0, bus_addr, local_addr, test_size,
292 cycles_count, rw_test & 0x1, (rw_test & 0x2) >> 1);
293
294 return 0;
295 }
296
dma_test_ch1(void * p)297 static int dma_test_ch1(void *p)
298 {
299 /* Test in different area with ch0 */
300 if (chn_en == 3)
301 dma_test(&s_dmatest_dev[test_dev], 1, bus_addr + test_size, local_addr + test_size, test_size,
302 cycles_count, rw_test & 0x1, (rw_test & 0x2) >> 1);
303 else
304 dma_test(&s_dmatest_dev[test_dev], 1, bus_addr, local_addr, test_size,
305 cycles_count, rw_test & 0x1, (rw_test & 0x2) >> 1);
306
307 return 0;
308 }
309
dma_run(void)310 static int dma_run(void)
311 {
312 if (chn_en == 3) {
313 kthread_run(dma_test_ch0, NULL, "dma_test_ch0");
314 kthread_run(dma_test_ch1, NULL, "dma_test_ch1");
315 } else if (chn_en == 2) {
316 dma_test_ch1(NULL);
317 } else {
318 dma_test_ch0(NULL);
319 }
320
321 return 0;
322 }
323
pcie_dw_dmatest(const char * val,const struct kernel_param * kp)324 static int pcie_dw_dmatest(const char *val, const struct kernel_param *kp)
325 {
326 char tmp[8];
327
328 if (!s_dmatest_dev[0].obj) {
329 pr_err("dmatest dev not exits\n");
330 kfree(tmp);
331
332 return -1;
333 }
334
335 strncpy(tmp, val, 8);
336 if (!strncmp(tmp, "run", 3)) {
337 dma_run();
338 } else if (!strncmp(tmp, "show", 4)) {
339 pcie_dw_dmatest_show();
340 } else {
341 pr_info("input error\n");
342 }
343
344 return 0;
345 }
346
347 static const struct kernel_param_ops pcie_dw_dmatest_ops = {
348 .set = pcie_dw_dmatest,
349 .get = param_get_uint,
350 };
351
352 module_param_cb(dmatest, &pcie_dw_dmatest_ops, &pcie_dw_dmatest, 0644);
353 MODULE_PARM_DESC(dmatest, "test rockchip pcie dma module");
354
355 MODULE_AUTHOR("Jon Lin");
356 MODULE_LICENSE("GPL");
357