1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
|
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* arch/arm64/include/asm/arch_timer.h
*
* Copyright (C) 2012 ARM Ltd.
* Author: Marc Zyngier <marc.zyngier@arm.com>
*/
#ifndef __ASM_ARCH_TIMER_H
#define __ASM_ARCH_TIMER_H
#include <asm/barrier.h>
#include <asm/hwcap.h>
#include <asm/sysreg.h>
#include <linux/bug.h>
#include <linux/init.h>
#include <linux/jump_label.h>
#include <linux/smp.h>
#include <linux/types.h>
#include <clocksource/arm_arch_timer.h>
#if IS_ENABLED(CONFIG_ARM_ARCH_TIMER_OOL_WORKAROUND)
#define has_erratum_handler(h) \
({ \
const struct arch_timer_erratum_workaround *__wa; \
__wa = __this_cpu_read(timer_unstable_counter_workaround); \
(__wa && __wa->h); \
})
#define erratum_handler(h) \
({ \
const struct arch_timer_erratum_workaround *__wa; \
__wa = __this_cpu_read(timer_unstable_counter_workaround); \
(__wa && __wa->h) ? ({ isb(); __wa->h;}) : arch_timer_##h; \
})
#else
#define has_erratum_handler(h) false
#define erratum_handler(h) (arch_timer_##h)
#endif
enum arch_timer_erratum_match_type {
ate_match_dt,
ate_match_local_cap_id,
ate_match_acpi_oem_info,
};
struct clock_event_device;
struct arch_timer_erratum_workaround {
enum arch_timer_erratum_match_type match_type;
const void *id;
const char *desc;
u64 (*read_cntpct_el0)(void);
u64 (*read_cntvct_el0)(void);
int (*set_next_event_phys)(unsigned long, struct clock_event_device *);
int (*set_next_event_virt)(unsigned long, struct clock_event_device *);
bool disable_compat_vdso;
};
DECLARE_PER_CPU(const struct arch_timer_erratum_workaround *,
timer_unstable_counter_workaround);
static inline notrace u64 arch_timer_read_cntpct_el0(void)
{
u64 cnt;
asm volatile(ALTERNATIVE("isb\n mrs %0, cntpct_el0",
"nop\n" __mrs_s("%0", SYS_CNTPCTSS_EL0),
ARM64_HAS_ECV)
: "=r" (cnt));
return cnt;
}
static inline notrace u64 arch_timer_read_cntvct_el0(void)
{
u64 cnt;
asm volatile(ALTERNATIVE("isb\n mrs %0, cntvct_el0",
"nop\n" __mrs_s("%0", SYS_CNTVCTSS_EL0),
ARM64_HAS_ECV)
: "=r" (cnt));
return cnt;
}
#define arch_timer_reg_read_stable(reg) \
({ \
u64 _val; \
\
preempt_disable_notrace(); \
_val = erratum_handler(read_ ## reg)(); \
preempt_enable_notrace(); \
\
_val; \
})
/*
* These register accessors are marked inline so the compiler can
* nicely work out which register we want, and chuck away the rest of
* the code.
*/
static __always_inline
void arch_timer_reg_write_cp15(int access, enum arch_timer_reg reg, u64 val)
{
if (access == ARCH_TIMER_PHYS_ACCESS) {
switch (reg) {
case ARCH_TIMER_REG_CTRL:
write_sysreg(val, cntp_ctl_el0);
isb();
break;
case ARCH_TIMER_REG_CVAL:
write_sysreg(val, cntp_cval_el0);
break;
default:
BUILD_BUG();
}
} else if (access == ARCH_TIMER_VIRT_ACCESS) {
switch (reg) {
case ARCH_TIMER_REG_CTRL:
write_sysreg(val, cntv_ctl_el0);
isb();
break;
case ARCH_TIMER_REG_CVAL:
write_sysreg(val, cntv_cval_el0);
break;
default:
BUILD_BUG();
}
} else {
BUILD_BUG();
}
}
static __always_inline
u64 arch_timer_reg_read_cp15(int access, enum arch_timer_reg reg)
{
if (access == ARCH_TIMER_PHYS_ACCESS) {
switch (reg) {
case ARCH_TIMER_REG_CTRL:
return read_sysreg(cntp_ctl_el0);
default:
BUILD_BUG();
}
} else if (access == ARCH_TIMER_VIRT_ACCESS) {
switch (reg) {
case ARCH_TIMER_REG_CTRL:
return read_sysreg(cntv_ctl_el0);
default:
BUILD_BUG();
}
}
BUILD_BUG();
unreachable();
}
static inline u32 arch_timer_get_cntfrq(void)
{
return read_sysreg(cntfrq_el0);
}
static inline u32 arch_timer_get_cntkctl(void)
{
return read_sysreg(cntkctl_el1);
}
static inline void arch_timer_set_cntkctl(u32 cntkctl)
{
write_sysreg(cntkctl, cntkctl_el1);
isb();
}
static __always_inline u64 __arch_counter_get_cntpct_stable(void)
{
u64 cnt;
cnt = arch_timer_reg_read_stable(cntpct_el0);
arch_counter_enforce_ordering(cnt);
return cnt;
}
static __always_inline u64 __arch_counter_get_cntpct(void)
{
u64 cnt;
asm volatile(ALTERNATIVE("isb\n mrs %0, cntpct_el0",
"nop\n" __mrs_s("%0", SYS_CNTPCTSS_EL0),
ARM64_HAS_ECV)
: "=r" (cnt));
arch_counter_enforce_ordering(cnt);
return cnt;
}
static __always_inline u64 __arch_counter_get_cntvct_stable(void)
{
u64 cnt;
cnt = arch_timer_reg_read_stable(cntvct_el0);
arch_counter_enforce_ordering(cnt);
return cnt;
}
static __always_inline u64 __arch_counter_get_cntvct(void)
{
u64 cnt;
asm volatile(ALTERNATIVE("isb\n mrs %0, cntvct_el0",
"nop\n" __mrs_s("%0", SYS_CNTVCTSS_EL0),
ARM64_HAS_ECV)
: "=r" (cnt));
arch_counter_enforce_ordering(cnt);
return cnt;
}
static inline int arch_timer_arch_init(void)
{
return 0;
}
static inline void arch_timer_set_evtstrm_feature(void)
{
cpu_set_named_feature(EVTSTRM);
#ifdef CONFIG_COMPAT
compat_elf_hwcap |= COMPAT_HWCAP_EVTSTRM;
#endif
}
static inline bool arch_timer_have_evtstrm_feature(void)
{
return cpu_have_named_feature(EVTSTRM);
}
#endif
|
