1*4882a593Smuzhiyun // SPDX-License-Identifier: GPL-2.0-only
2*4882a593Smuzhiyun /*
3*4882a593Smuzhiyun * PowerPC version derived from arch/arm/mm/consistent.c
4*4882a593Smuzhiyun * Copyright (C) 2001 Dan Malek (dmalek@jlc.net)
5*4882a593Smuzhiyun *
6*4882a593Smuzhiyun * Copyright (C) 2000 Russell King
7*4882a593Smuzhiyun */
8*4882a593Smuzhiyun
9*4882a593Smuzhiyun #include <linux/kernel.h>
10*4882a593Smuzhiyun #include <linux/errno.h>
11*4882a593Smuzhiyun #include <linux/types.h>
12*4882a593Smuzhiyun #include <linux/highmem.h>
13*4882a593Smuzhiyun #include <linux/dma-direct.h>
14*4882a593Smuzhiyun #include <linux/dma-map-ops.h>
15*4882a593Smuzhiyun
16*4882a593Smuzhiyun #include <asm/tlbflush.h>
17*4882a593Smuzhiyun #include <asm/dma.h>
18*4882a593Smuzhiyun
19*4882a593Smuzhiyun /*
20*4882a593Smuzhiyun * make an area consistent.
21*4882a593Smuzhiyun */
__dma_sync(void * vaddr,size_t size,int direction)22*4882a593Smuzhiyun static void __dma_sync(void *vaddr, size_t size, int direction)
23*4882a593Smuzhiyun {
24*4882a593Smuzhiyun unsigned long start = (unsigned long)vaddr;
25*4882a593Smuzhiyun unsigned long end = start + size;
26*4882a593Smuzhiyun
27*4882a593Smuzhiyun switch (direction) {
28*4882a593Smuzhiyun case DMA_NONE:
29*4882a593Smuzhiyun BUG();
30*4882a593Smuzhiyun case DMA_FROM_DEVICE:
31*4882a593Smuzhiyun /*
32*4882a593Smuzhiyun * invalidate only when cache-line aligned otherwise there is
33*4882a593Smuzhiyun * the potential for discarding uncommitted data from the cache
34*4882a593Smuzhiyun */
35*4882a593Smuzhiyun if ((start | end) & (L1_CACHE_BYTES - 1))
36*4882a593Smuzhiyun flush_dcache_range(start, end);
37*4882a593Smuzhiyun else
38*4882a593Smuzhiyun invalidate_dcache_range(start, end);
39*4882a593Smuzhiyun break;
40*4882a593Smuzhiyun case DMA_TO_DEVICE: /* writeback only */
41*4882a593Smuzhiyun clean_dcache_range(start, end);
42*4882a593Smuzhiyun break;
43*4882a593Smuzhiyun case DMA_BIDIRECTIONAL: /* writeback and invalidate */
44*4882a593Smuzhiyun flush_dcache_range(start, end);
45*4882a593Smuzhiyun break;
46*4882a593Smuzhiyun }
47*4882a593Smuzhiyun }
48*4882a593Smuzhiyun
49*4882a593Smuzhiyun #ifdef CONFIG_HIGHMEM
50*4882a593Smuzhiyun /*
51*4882a593Smuzhiyun * __dma_sync_page() implementation for systems using highmem.
52*4882a593Smuzhiyun * In this case, each page of a buffer must be kmapped/kunmapped
53*4882a593Smuzhiyun * in order to have a virtual address for __dma_sync(). This must
54*4882a593Smuzhiyun * not sleep so kmap_atomic()/kunmap_atomic() are used.
55*4882a593Smuzhiyun *
56*4882a593Smuzhiyun * Note: yes, it is possible and correct to have a buffer extend
57*4882a593Smuzhiyun * beyond the first page.
58*4882a593Smuzhiyun */
__dma_sync_page_highmem(struct page * page,unsigned long offset,size_t size,int direction)59*4882a593Smuzhiyun static inline void __dma_sync_page_highmem(struct page *page,
60*4882a593Smuzhiyun unsigned long offset, size_t size, int direction)
61*4882a593Smuzhiyun {
62*4882a593Smuzhiyun size_t seg_size = min((size_t)(PAGE_SIZE - offset), size);
63*4882a593Smuzhiyun size_t cur_size = seg_size;
64*4882a593Smuzhiyun unsigned long flags, start, seg_offset = offset;
65*4882a593Smuzhiyun int nr_segs = 1 + ((size - seg_size) + PAGE_SIZE - 1)/PAGE_SIZE;
66*4882a593Smuzhiyun int seg_nr = 0;
67*4882a593Smuzhiyun
68*4882a593Smuzhiyun local_irq_save(flags);
69*4882a593Smuzhiyun
70*4882a593Smuzhiyun do {
71*4882a593Smuzhiyun start = (unsigned long)kmap_atomic(page + seg_nr) + seg_offset;
72*4882a593Smuzhiyun
73*4882a593Smuzhiyun /* Sync this buffer segment */
74*4882a593Smuzhiyun __dma_sync((void *)start, seg_size, direction);
75*4882a593Smuzhiyun kunmap_atomic((void *)start);
76*4882a593Smuzhiyun seg_nr++;
77*4882a593Smuzhiyun
78*4882a593Smuzhiyun /* Calculate next buffer segment size */
79*4882a593Smuzhiyun seg_size = min((size_t)PAGE_SIZE, size - cur_size);
80*4882a593Smuzhiyun
81*4882a593Smuzhiyun /* Add the segment size to our running total */
82*4882a593Smuzhiyun cur_size += seg_size;
83*4882a593Smuzhiyun seg_offset = 0;
84*4882a593Smuzhiyun } while (seg_nr < nr_segs);
85*4882a593Smuzhiyun
86*4882a593Smuzhiyun local_irq_restore(flags);
87*4882a593Smuzhiyun }
88*4882a593Smuzhiyun #endif /* CONFIG_HIGHMEM */
89*4882a593Smuzhiyun
90*4882a593Smuzhiyun /*
91*4882a593Smuzhiyun * __dma_sync_page makes memory consistent. identical to __dma_sync, but
92*4882a593Smuzhiyun * takes a struct page instead of a virtual address
93*4882a593Smuzhiyun */
__dma_sync_page(phys_addr_t paddr,size_t size,int dir)94*4882a593Smuzhiyun static void __dma_sync_page(phys_addr_t paddr, size_t size, int dir)
95*4882a593Smuzhiyun {
96*4882a593Smuzhiyun struct page *page = pfn_to_page(paddr >> PAGE_SHIFT);
97*4882a593Smuzhiyun unsigned offset = paddr & ~PAGE_MASK;
98*4882a593Smuzhiyun
99*4882a593Smuzhiyun #ifdef CONFIG_HIGHMEM
100*4882a593Smuzhiyun __dma_sync_page_highmem(page, offset, size, dir);
101*4882a593Smuzhiyun #else
102*4882a593Smuzhiyun unsigned long start = (unsigned long)page_address(page) + offset;
103*4882a593Smuzhiyun __dma_sync((void *)start, size, dir);
104*4882a593Smuzhiyun #endif
105*4882a593Smuzhiyun }
106*4882a593Smuzhiyun
arch_sync_dma_for_device(phys_addr_t paddr,size_t size,enum dma_data_direction dir)107*4882a593Smuzhiyun void arch_sync_dma_for_device(phys_addr_t paddr, size_t size,
108*4882a593Smuzhiyun enum dma_data_direction dir)
109*4882a593Smuzhiyun {
110*4882a593Smuzhiyun __dma_sync_page(paddr, size, dir);
111*4882a593Smuzhiyun }
112*4882a593Smuzhiyun
arch_sync_dma_for_cpu(phys_addr_t paddr,size_t size,enum dma_data_direction dir)113*4882a593Smuzhiyun void arch_sync_dma_for_cpu(phys_addr_t paddr, size_t size,
114*4882a593Smuzhiyun enum dma_data_direction dir)
115*4882a593Smuzhiyun {
116*4882a593Smuzhiyun __dma_sync_page(paddr, size, dir);
117*4882a593Smuzhiyun }
118*4882a593Smuzhiyun
arch_dma_prep_coherent(struct page * page,size_t size)119*4882a593Smuzhiyun void arch_dma_prep_coherent(struct page *page, size_t size)
120*4882a593Smuzhiyun {
121*4882a593Smuzhiyun unsigned long kaddr = (unsigned long)page_address(page);
122*4882a593Smuzhiyun
123*4882a593Smuzhiyun flush_dcache_range(kaddr, kaddr + size);
124*4882a593Smuzhiyun }
125