1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
|
/* ===-- clear_cache.c - Implement __clear_cache ---------------------------===
*
* The LLVM Compiler Infrastructure
*
* This file is dual licensed under the MIT and the University of Illinois Open
* Source Licenses. See LICENSE.TXT for details.
*
* ===----------------------------------------------------------------------===
*/
#include "int_lib.h"
#include <assert.h>
#include <stddef.h>
#if __APPLE__
#include <libkern/OSCacheControl.h>
#endif
#if defined(_WIN32)
/* Forward declare Win32 APIs since the GCC mode driver does not handle the
newer SDKs as well as needed. */
uint32_t FlushInstructionCache(uintptr_t hProcess, void *lpBaseAddress,
uintptr_t dwSize);
uintptr_t GetCurrentProcess(void);
#endif
#if defined(__FreeBSD__) && defined(__arm__)
#include <sys/types.h>
#include <machine/sysarch.h>
#endif
#if defined(__NetBSD__) && defined(__arm__)
#include <machine/sysarch.h>
#endif
#if defined(__OpenBSD__) && defined(__mips__)
#include <sys/types.h>
#include <machine/sysarch.h>
#endif
#if defined(__linux__) && defined(__mips__)
#include <sys/cachectl.h>
#include <sys/syscall.h>
#include <unistd.h>
#if defined(__ANDROID__) && defined(__LP64__)
/*
* clear_mips_cache - Invalidates instruction cache for Mips.
*/
static void clear_mips_cache(const void* Addr, size_t Size) {
__asm__ volatile (
".set push\n"
".set noreorder\n"
".set noat\n"
"beq %[Size], $zero, 20f\n" /* If size == 0, branch around. */
"nop\n"
"daddu %[Size], %[Addr], %[Size]\n" /* Calculate end address + 1 */
"rdhwr $v0, $1\n" /* Get step size for SYNCI.
$1 is $HW_SYNCI_Step */
"beq $v0, $zero, 20f\n" /* If no caches require
synchronization, branch
around. */
"nop\n"
"10:\n"
"synci 0(%[Addr])\n" /* Synchronize all caches around
address. */
"daddu %[Addr], %[Addr], $v0\n" /* Add step size. */
"sltu $at, %[Addr], %[Size]\n" /* Compare current with end
address. */
"bne $at, $zero, 10b\n" /* Branch if more to do. */
"nop\n"
"sync\n" /* Clear memory hazards. */
"20:\n"
"bal 30f\n"
"nop\n"
"30:\n"
"daddiu $ra, $ra, 12\n" /* $ra has a value of $pc here.
Add offset of 12 to point to the
instruction after the last nop.
*/
"jr.hb $ra\n" /* Return, clearing instruction
hazards. */
"nop\n"
".set pop\n"
: [Addr] "+r"(Addr), [Size] "+r"(Size)
:: "at", "ra", "v0", "memory"
);
}
#endif
#endif
/*
* The compiler generates calls to __clear_cache() when creating
* trampoline functions on the stack for use with nested functions.
* It is expected to invalidate the instruction cache for the
* specified range.
*/
void __clear_cache(void *start, void *end) {
#if __i386__ || __x86_64__ || defined(_M_IX86) || defined(_M_X64)
/*
* Intel processors have a unified instruction and data cache
* so there is nothing to do
*/
#elif defined(__arm__) && !defined(__APPLE__)
#if defined(__FreeBSD__) || defined(__NetBSD__)
struct arm_sync_icache_args arg;
arg.addr = (uintptr_t)start;
arg.len = (uintptr_t)end - (uintptr_t)start;
sysarch(ARM_SYNC_ICACHE, &arg);
#elif defined(__linux__)
/*
* We used to include asm/unistd.h for the __ARM_NR_cacheflush define, but
* it also brought many other unused defines, as well as a dependency on
* kernel headers to be installed.
*
* This value is stable at least since Linux 3.13 and should remain so for
* compatibility reasons, warranting it's re-definition here.
*/
#define __ARM_NR_cacheflush 0x0f0002
register int start_reg __asm("r0") = (int) (intptr_t) start;
const register int end_reg __asm("r1") = (int) (intptr_t) end;
const register int flags __asm("r2") = 0;
const register int syscall_nr __asm("r7") = __ARM_NR_cacheflush;
__asm __volatile("svc 0x0"
: "=r"(start_reg)
: "r"(syscall_nr), "r"(start_reg), "r"(end_reg),
"r"(flags));
assert(start_reg == 0 && "Cache flush syscall failed.");
#elif defined(_WIN32)
FlushInstructionCache(GetCurrentProcess(), start, end - start);
#else
compilerrt_abort();
#endif
#elif defined(__linux__) && defined(__mips__)
const uintptr_t start_int = (uintptr_t) start;
const uintptr_t end_int = (uintptr_t) end;
#if defined(__ANDROID__) && defined(__LP64__)
// Call synci implementation for short address range.
const uintptr_t address_range_limit = 256;
if ((end_int - start_int) <= address_range_limit) {
clear_mips_cache(start, (end_int - start_int));
} else {
syscall(__NR_cacheflush, start, (end_int - start_int), BCACHE);
}
#else
syscall(__NR_cacheflush, start, (end_int - start_int), BCACHE);
#endif
#elif defined(__mips__) && defined(__OpenBSD__)
cacheflush(start, (uintptr_t)end - (uintptr_t)start, BCACHE);
#elif defined(__aarch64__) && !defined(__APPLE__)
uint64_t xstart = (uint64_t)(uintptr_t) start;
uint64_t xend = (uint64_t)(uintptr_t) end;
uint64_t addr;
// Get Cache Type Info
uint64_t ctr_el0;
__asm __volatile("mrs %0, ctr_el0" : "=r"(ctr_el0));
/*
* dc & ic instructions must use 64bit registers so we don't use
* uintptr_t in case this runs in an IPL32 environment.
*/
const size_t dcache_line_size = 4 << ((ctr_el0 >> 16) & 15);
for (addr = xstart & ~(dcache_line_size - 1); addr < xend;
addr += dcache_line_size)
__asm __volatile("dc cvau, %0" :: "r"(addr));
__asm __volatile("dsb ish");
const size_t icache_line_size = 4 << ((ctr_el0 >> 0) & 15);
for (addr = xstart & ~(icache_line_size - 1); addr < xend;
addr += icache_line_size)
__asm __volatile("ic ivau, %0" :: "r"(addr));
__asm __volatile("isb sy");
#elif defined (__powerpc64__)
const size_t line_size = 32;
const size_t len = (uintptr_t)end - (uintptr_t)start;
const uintptr_t mask = ~(line_size - 1);
const uintptr_t start_line = ((uintptr_t)start) & mask;
const uintptr_t end_line = ((uintptr_t)start + len + line_size - 1) & mask;
for (uintptr_t line = start_line; line < end_line; line += line_size)
__asm__ volatile("dcbf 0, %0" : : "r"(line));
__asm__ volatile("sync");
for (uintptr_t line = start_line; line < end_line; line += line_size)
__asm__ volatile("icbi 0, %0" : : "r"(line));
__asm__ volatile("isync");
#else
#if __APPLE__
/* On Darwin, sys_icache_invalidate() provides this functionality */
sys_icache_invalidate(start, end-start);
#else
compilerrt_abort();
#endif
#endif
}
|
