1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
|
// SPDX-License-Identifier: GPL-2.0 OR MIT
/* Copyright 2017-2019 Qiang Yu <yuq825@gmail.com> */
#include <linux/interrupt.h>
#include <linux/iopoll.h>
#include <linux/device.h>
#include "lima_device.h"
#include "lima_mmu.h"
#include "lima_vm.h"
#include "lima_regs.h"
#define mmu_write(reg, data) writel(data, ip->iomem + reg)
#define mmu_read(reg) readl(ip->iomem + reg)
#define lima_mmu_send_command(cmd, addr, val, cond) \
({ \
int __ret; \
\
mmu_write(LIMA_MMU_COMMAND, cmd); \
__ret = readl_poll_timeout(ip->iomem + (addr), val, \
cond, 0, 100); \
if (__ret) \
dev_err(dev->dev, \
"%s command %x timeout\n", \
lima_ip_name(ip), cmd); \
__ret; \
})
static irqreturn_t lima_mmu_irq_handler(int irq, void *data)
{
struct lima_ip *ip = data;
struct lima_device *dev = ip->dev;
u32 status = mmu_read(LIMA_MMU_INT_STATUS);
struct lima_sched_pipe *pipe;
/* for shared irq case */
if (!status)
return IRQ_NONE;
if (status & LIMA_MMU_INT_PAGE_FAULT) {
u32 fault = mmu_read(LIMA_MMU_PAGE_FAULT_ADDR);
dev_err(dev->dev, "%s page fault at 0x%x from bus id %d of type %s\n",
lima_ip_name(ip), fault, LIMA_MMU_STATUS_BUS_ID(status),
status & LIMA_MMU_STATUS_PAGE_FAULT_IS_WRITE ? "write" : "read");
}
if (status & LIMA_MMU_INT_READ_BUS_ERROR)
dev_err(dev->dev, "%s irq bus error\n", lima_ip_name(ip));
/* mask all interrupts before resume */
mmu_write(LIMA_MMU_INT_MASK, 0);
mmu_write(LIMA_MMU_INT_CLEAR, status);
pipe = dev->pipe + (ip->id == lima_ip_gpmmu ? lima_pipe_gp : lima_pipe_pp);
lima_sched_pipe_mmu_error(pipe);
return IRQ_HANDLED;
}
static int lima_mmu_hw_init(struct lima_ip *ip)
{
struct lima_device *dev = ip->dev;
int err;
u32 v;
mmu_write(LIMA_MMU_COMMAND, LIMA_MMU_COMMAND_HARD_RESET);
err = lima_mmu_send_command(LIMA_MMU_COMMAND_HARD_RESET,
LIMA_MMU_DTE_ADDR, v, v == 0);
if (err)
return err;
mmu_write(LIMA_MMU_INT_MASK,
LIMA_MMU_INT_PAGE_FAULT | LIMA_MMU_INT_READ_BUS_ERROR);
mmu_write(LIMA_MMU_DTE_ADDR, dev->empty_vm->pd.dma);
return lima_mmu_send_command(LIMA_MMU_COMMAND_ENABLE_PAGING,
LIMA_MMU_STATUS, v,
v & LIMA_MMU_STATUS_PAGING_ENABLED);
}
int lima_mmu_resume(struct lima_ip *ip)
{
if (ip->id == lima_ip_ppmmu_bcast)
return 0;
return lima_mmu_hw_init(ip);
}
void lima_mmu_suspend(struct lima_ip *ip)
{
}
int lima_mmu_init(struct lima_ip *ip)
{
struct lima_device *dev = ip->dev;
int err;
if (ip->id == lima_ip_ppmmu_bcast)
return 0;
mmu_write(LIMA_MMU_DTE_ADDR, 0xCAFEBABE);
if (mmu_read(LIMA_MMU_DTE_ADDR) != 0xCAFEB000) {
dev_err(dev->dev, "%s dte write test fail\n", lima_ip_name(ip));
return -EIO;
}
err = devm_request_irq(dev->dev, ip->irq, lima_mmu_irq_handler,
IRQF_SHARED, lima_ip_name(ip), ip);
if (err) {
dev_err(dev->dev, "%s fail to request irq\n", lima_ip_name(ip));
return err;
}
return lima_mmu_hw_init(ip);
}
void lima_mmu_fini(struct lima_ip *ip)
{
struct lima_device *dev = ip->dev;
if (ip->id == lima_ip_ppmmu_bcast)
return;
devm_free_irq(dev->dev, ip->irq, ip);
}
void lima_mmu_flush_tlb(struct lima_ip *ip)
{
mmu_write(LIMA_MMU_COMMAND, LIMA_MMU_COMMAND_ZAP_CACHE);
}
void lima_mmu_switch_vm(struct lima_ip *ip, struct lima_vm *vm)
{
struct lima_device *dev = ip->dev;
u32 v;
lima_mmu_send_command(LIMA_MMU_COMMAND_ENABLE_STALL,
LIMA_MMU_STATUS, v,
v & LIMA_MMU_STATUS_STALL_ACTIVE);
mmu_write(LIMA_MMU_DTE_ADDR, vm->pd.dma);
/* flush the TLB */
mmu_write(LIMA_MMU_COMMAND, LIMA_MMU_COMMAND_ZAP_CACHE);
lima_mmu_send_command(LIMA_MMU_COMMAND_DISABLE_STALL,
LIMA_MMU_STATUS, v,
!(v & LIMA_MMU_STATUS_STALL_ACTIVE));
}
void lima_mmu_page_fault_resume(struct lima_ip *ip)
{
struct lima_device *dev = ip->dev;
u32 status = mmu_read(LIMA_MMU_STATUS);
u32 v;
if (status & LIMA_MMU_STATUS_PAGE_FAULT_ACTIVE) {
dev_info(dev->dev, "%s resume\n", lima_ip_name(ip));
mmu_write(LIMA_MMU_INT_MASK, 0);
mmu_write(LIMA_MMU_DTE_ADDR, 0xCAFEBABE);
lima_mmu_send_command(LIMA_MMU_COMMAND_HARD_RESET,
LIMA_MMU_DTE_ADDR, v, v == 0);
mmu_write(LIMA_MMU_INT_MASK, LIMA_MMU_INT_PAGE_FAULT | LIMA_MMU_INT_READ_BUS_ERROR);
mmu_write(LIMA_MMU_DTE_ADDR, dev->empty_vm->pd.dma);
lima_mmu_send_command(LIMA_MMU_COMMAND_ENABLE_PAGING,
LIMA_MMU_STATUS, v,
v & LIMA_MMU_STATUS_PAGING_ENABLED);
}
}
|
