File: | src/gnu/usr.bin/clang/liblldELF/../../../llvm/lld/ELF/AArch64ErrataFix.cpp |
Warning: | line 509, column 27 Assigned value is garbage or undefined |
Press '?' to see keyboard shortcuts
Keyboard shortcuts:
1 | //===- AArch64ErrataFix.cpp -----------------------------------------------===// | ||||
2 | // | ||||
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | ||||
4 | // See https://llvm.org/LICENSE.txt for license information. | ||||
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | ||||
6 | // | ||||
7 | //===----------------------------------------------------------------------===// | ||||
8 | // This file implements Section Patching for the purpose of working around | ||||
9 | // the AArch64 Cortex-53 errata 843419 that affects r0p0, r0p1, r0p2 and r0p4 | ||||
10 | // versions of the core. | ||||
11 | // | ||||
12 | // The general principle is that an erratum sequence of one or | ||||
13 | // more instructions is detected in the instruction stream, one of the | ||||
14 | // instructions in the sequence is replaced with a branch to a patch sequence | ||||
15 | // of replacement instructions. At the end of the replacement sequence the | ||||
16 | // patch branches back to the instruction stream. | ||||
17 | |||||
18 | // This technique is only suitable for fixing an erratum when: | ||||
19 | // - There is a set of necessary conditions required to trigger the erratum that | ||||
20 | // can be detected at static link time. | ||||
21 | // - There is a set of replacement instructions that can be used to remove at | ||||
22 | // least one of the necessary conditions that trigger the erratum. | ||||
23 | // - We can overwrite an instruction in the erratum sequence with a branch to | ||||
24 | // the replacement sequence. | ||||
25 | // - We can place the replacement sequence within range of the branch. | ||||
26 | //===----------------------------------------------------------------------===// | ||||
27 | |||||
28 | #include "AArch64ErrataFix.h" | ||||
29 | #include "Config.h" | ||||
30 | #include "LinkerScript.h" | ||||
31 | #include "OutputSections.h" | ||||
32 | #include "Relocations.h" | ||||
33 | #include "Symbols.h" | ||||
34 | #include "SyntheticSections.h" | ||||
35 | #include "Target.h" | ||||
36 | #include "lld/Common/Memory.h" | ||||
37 | #include "lld/Common/Strings.h" | ||||
38 | #include "llvm/Support/Endian.h" | ||||
39 | #include "llvm/Support/raw_ostream.h" | ||||
40 | #include <algorithm> | ||||
41 | |||||
42 | using namespace llvm; | ||||
43 | using namespace llvm::ELF; | ||||
44 | using namespace llvm::object; | ||||
45 | using namespace llvm::support; | ||||
46 | using namespace llvm::support::endian; | ||||
47 | using namespace lld; | ||||
48 | using namespace lld::elf; | ||||
49 | |||||
50 | // Helper functions to identify instructions and conditions needed to trigger | ||||
51 | // the Cortex-A53-843419 erratum. | ||||
52 | |||||
53 | // ADRP | ||||
54 | // | 1 | immlo (2) | 1 | 0 0 0 0 | immhi (19) | Rd (5) | | ||||
55 | static bool isADRP(uint32_t instr) { | ||||
56 | return (instr & 0x9f000000) == 0x90000000; | ||||
57 | } | ||||
58 | |||||
59 | // Load and store bit patterns from ARMv8-A ARM ARM. | ||||
60 | // Instructions appear in order of appearance starting from table in | ||||
61 | // C4.1.3 Loads and Stores. | ||||
62 | |||||
63 | // All loads and stores have 1 (at bit position 27), (0 at bit position 25). | ||||
64 | // | op0 x op1 (2) | 1 op2 0 op3 (2) | x | op4 (5) | xxxx | op5 (2) | x (10) | | ||||
65 | static bool isLoadStoreClass(uint32_t instr) { | ||||
66 | return (instr & 0x0a000000) == 0x08000000; | ||||
67 | } | ||||
68 | |||||
69 | // LDN/STN multiple no offset | ||||
70 | // | 0 Q 00 | 1100 | 0 L 00 | 0000 | opcode (4) | size (2) | Rn (5) | Rt (5) | | ||||
71 | // LDN/STN multiple post-indexed | ||||
72 | // | 0 Q 00 | 1100 | 1 L 0 | Rm (5)| opcode (4) | size (2) | Rn (5) | Rt (5) | | ||||
73 | // L == 0 for stores. | ||||
74 | |||||
75 | // Utility routine to decode opcode field of LDN/STN multiple structure | ||||
76 | // instructions to find the ST1 instructions. | ||||
77 | // opcode == 0010 ST1 4 registers. | ||||
78 | // opcode == 0110 ST1 3 registers. | ||||
79 | // opcode == 0111 ST1 1 register. | ||||
80 | // opcode == 1010 ST1 2 registers. | ||||
81 | static bool isST1MultipleOpcode(uint32_t instr) { | ||||
82 | return (instr & 0x0000f000) == 0x00002000 || | ||||
83 | (instr & 0x0000f000) == 0x00006000 || | ||||
84 | (instr & 0x0000f000) == 0x00007000 || | ||||
85 | (instr & 0x0000f000) == 0x0000a000; | ||||
86 | } | ||||
87 | |||||
88 | static bool isST1Multiple(uint32_t instr) { | ||||
89 | return (instr & 0xbfff0000) == 0x0c000000 && isST1MultipleOpcode(instr); | ||||
90 | } | ||||
91 | |||||
92 | // Writes to Rn (writeback). | ||||
93 | static bool isST1MultiplePost(uint32_t instr) { | ||||
94 | return (instr & 0xbfe00000) == 0x0c800000 && isST1MultipleOpcode(instr); | ||||
95 | } | ||||
96 | |||||
97 | // LDN/STN single no offset | ||||
98 | // | 0 Q 00 | 1101 | 0 L R 0 | 0000 | opc (3) S | size (2) | Rn (5) | Rt (5)| | ||||
99 | // LDN/STN single post-indexed | ||||
100 | // | 0 Q 00 | 1101 | 1 L R | Rm (5) | opc (3) S | size (2) | Rn (5) | Rt (5)| | ||||
101 | // L == 0 for stores | ||||
102 | |||||
103 | // Utility routine to decode opcode field of LDN/STN single structure | ||||
104 | // instructions to find the ST1 instructions. | ||||
105 | // R == 0 for ST1 and ST3, R == 1 for ST2 and ST4. | ||||
106 | // opcode == 000 ST1 8-bit. | ||||
107 | // opcode == 010 ST1 16-bit. | ||||
108 | // opcode == 100 ST1 32 or 64-bit (Size determines which). | ||||
109 | static bool isST1SingleOpcode(uint32_t instr) { | ||||
110 | return (instr & 0x0040e000) == 0x00000000 || | ||||
111 | (instr & 0x0040e000) == 0x00004000 || | ||||
112 | (instr & 0x0040e000) == 0x00008000; | ||||
113 | } | ||||
114 | |||||
115 | static bool isST1Single(uint32_t instr) { | ||||
116 | return (instr & 0xbfff0000) == 0x0d000000 && isST1SingleOpcode(instr); | ||||
117 | } | ||||
118 | |||||
119 | // Writes to Rn (writeback). | ||||
120 | static bool isST1SinglePost(uint32_t instr) { | ||||
121 | return (instr & 0xbfe00000) == 0x0d800000 && isST1SingleOpcode(instr); | ||||
122 | } | ||||
123 | |||||
124 | static bool isST1(uint32_t instr) { | ||||
125 | return isST1Multiple(instr) || isST1MultiplePost(instr) || | ||||
126 | isST1Single(instr) || isST1SinglePost(instr); | ||||
127 | } | ||||
128 | |||||
129 | // Load/store exclusive | ||||
130 | // | size (2) 00 | 1000 | o2 L o1 | Rs (5) | o0 | Rt2 (5) | Rn (5) | Rt (5) | | ||||
131 | // L == 0 for Stores. | ||||
132 | static bool isLoadStoreExclusive(uint32_t instr) { | ||||
133 | return (instr & 0x3f000000) == 0x08000000; | ||||
134 | } | ||||
135 | |||||
136 | static bool isLoadExclusive(uint32_t instr) { | ||||
137 | return (instr & 0x3f400000) == 0x08400000; | ||||
138 | } | ||||
139 | |||||
140 | // Load register literal | ||||
141 | // | opc (2) 01 | 1 V 00 | imm19 | Rt (5) | | ||||
142 | static bool isLoadLiteral(uint32_t instr) { | ||||
143 | return (instr & 0x3b000000) == 0x18000000; | ||||
144 | } | ||||
145 | |||||
146 | // Load/store no-allocate pair | ||||
147 | // (offset) | ||||
148 | // | opc (2) 10 | 1 V 00 | 0 L | imm7 | Rt2 (5) | Rn (5) | Rt (5) | | ||||
149 | // L == 0 for stores. | ||||
150 | // Never writes to register | ||||
151 | static bool isSTNP(uint32_t instr) { | ||||
152 | return (instr & 0x3bc00000) == 0x28000000; | ||||
153 | } | ||||
154 | |||||
155 | // Load/store register pair | ||||
156 | // (post-indexed) | ||||
157 | // | opc (2) 10 | 1 V 00 | 1 L | imm7 | Rt2 (5) | Rn (5) | Rt (5) | | ||||
158 | // L == 0 for stores, V == 0 for Scalar, V == 1 for Simd/FP | ||||
159 | // Writes to Rn. | ||||
160 | static bool isSTPPost(uint32_t instr) { | ||||
161 | return (instr & 0x3bc00000) == 0x28800000; | ||||
162 | } | ||||
163 | |||||
164 | // (offset) | ||||
165 | // | opc (2) 10 | 1 V 01 | 0 L | imm7 | Rt2 (5) | Rn (5) | Rt (5) | | ||||
166 | static bool isSTPOffset(uint32_t instr) { | ||||
167 | return (instr & 0x3bc00000) == 0x29000000; | ||||
168 | } | ||||
169 | |||||
170 | // (pre-index) | ||||
171 | // | opc (2) 10 | 1 V 01 | 1 L | imm7 | Rt2 (5) | Rn (5) | Rt (5) | | ||||
172 | // Writes to Rn. | ||||
173 | static bool isSTPPre(uint32_t instr) { | ||||
174 | return (instr & 0x3bc00000) == 0x29800000; | ||||
175 | } | ||||
176 | |||||
177 | static bool isSTP(uint32_t instr) { | ||||
178 | return isSTPPost(instr) || isSTPOffset(instr) || isSTPPre(instr); | ||||
179 | } | ||||
180 | |||||
181 | // Load/store register (unscaled immediate) | ||||
182 | // | size (2) 11 | 1 V 00 | opc (2) 0 | imm9 | 00 | Rn (5) | Rt (5) | | ||||
183 | // V == 0 for Scalar, V == 1 for Simd/FP. | ||||
184 | static bool isLoadStoreUnscaled(uint32_t instr) { | ||||
185 | return (instr & 0x3b000c00) == 0x38000000; | ||||
186 | } | ||||
187 | |||||
188 | // Load/store register (immediate post-indexed) | ||||
189 | // | size (2) 11 | 1 V 00 | opc (2) 0 | imm9 | 01 | Rn (5) | Rt (5) | | ||||
190 | static bool isLoadStoreImmediatePost(uint32_t instr) { | ||||
191 | return (instr & 0x3b200c00) == 0x38000400; | ||||
192 | } | ||||
193 | |||||
194 | // Load/store register (unprivileged) | ||||
195 | // | size (2) 11 | 1 V 00 | opc (2) 0 | imm9 | 10 | Rn (5) | Rt (5) | | ||||
196 | static bool isLoadStoreUnpriv(uint32_t instr) { | ||||
197 | return (instr & 0x3b200c00) == 0x38000800; | ||||
198 | } | ||||
199 | |||||
200 | // Load/store register (immediate pre-indexed) | ||||
201 | // | size (2) 11 | 1 V 00 | opc (2) 0 | imm9 | 11 | Rn (5) | Rt (5) | | ||||
202 | static bool isLoadStoreImmediatePre(uint32_t instr) { | ||||
203 | return (instr & 0x3b200c00) == 0x38000c00; | ||||
204 | } | ||||
205 | |||||
206 | // Load/store register (register offset) | ||||
207 | // | size (2) 11 | 1 V 00 | opc (2) 1 | Rm (5) | option (3) S | 10 | Rn | Rt | | ||||
208 | static bool isLoadStoreRegisterOff(uint32_t instr) { | ||||
209 | return (instr & 0x3b200c00) == 0x38200800; | ||||
210 | } | ||||
211 | |||||
212 | // Load/store register (unsigned immediate) | ||||
213 | // | size (2) 11 | 1 V 01 | opc (2) | imm12 | Rn (5) | Rt (5) | | ||||
214 | static bool isLoadStoreRegisterUnsigned(uint32_t instr) { | ||||
215 | return (instr & 0x3b000000) == 0x39000000; | ||||
216 | } | ||||
217 | |||||
218 | // Rt is always in bit position 0 - 4. | ||||
219 | static uint32_t getRt(uint32_t instr) { return (instr & 0x1f); } | ||||
220 | |||||
221 | // Rn is always in bit position 5 - 9. | ||||
222 | static uint32_t getRn(uint32_t instr) { return (instr >> 5) & 0x1f; } | ||||
223 | |||||
224 | // C4.1.2 Branches, Exception Generating and System instructions | ||||
225 | // | op0 (3) 1 | 01 op1 (4) | x (22) | | ||||
226 | // op0 == 010 101 op1 == 0xxx Conditional Branch. | ||||
227 | // op0 == 110 101 op1 == 1xxx Unconditional Branch Register. | ||||
228 | // op0 == x00 101 op1 == xxxx Unconditional Branch immediate. | ||||
229 | // op0 == x01 101 op1 == 0xxx Compare and branch immediate. | ||||
230 | // op0 == x01 101 op1 == 1xxx Test and branch immediate. | ||||
231 | static bool isBranch(uint32_t instr) { | ||||
232 | return ((instr & 0xfe000000) == 0xd6000000) || // Cond branch. | ||||
233 | ((instr & 0xfe000000) == 0x54000000) || // Uncond branch reg. | ||||
234 | ((instr & 0x7c000000) == 0x14000000) || // Uncond branch imm. | ||||
235 | ((instr & 0x7c000000) == 0x34000000); // Compare and test branch. | ||||
236 | } | ||||
237 | |||||
238 | static bool isV8SingleRegisterNonStructureLoadStore(uint32_t instr) { | ||||
239 | return isLoadStoreUnscaled(instr) || isLoadStoreImmediatePost(instr) || | ||||
240 | isLoadStoreUnpriv(instr) || isLoadStoreImmediatePre(instr) || | ||||
241 | isLoadStoreRegisterOff(instr) || isLoadStoreRegisterUnsigned(instr); | ||||
242 | } | ||||
243 | |||||
244 | // Note that this function refers to v8.0 only and does not include the | ||||
245 | // additional load and store instructions added for in later revisions of | ||||
246 | // the architecture such as the Atomic memory operations introduced | ||||
247 | // in v8.1. | ||||
248 | static bool isV8NonStructureLoad(uint32_t instr) { | ||||
249 | if (isLoadExclusive(instr)) | ||||
250 | return true; | ||||
251 | if (isLoadLiteral(instr)) | ||||
252 | return true; | ||||
253 | else if (isV8SingleRegisterNonStructureLoadStore(instr)) { | ||||
254 | // For Load and Store single register, Loads are derived from a | ||||
255 | // combination of the Size, V and Opc fields. | ||||
256 | uint32_t size = (instr >> 30) & 0xff; | ||||
257 | uint32_t v = (instr >> 26) & 0x1; | ||||
258 | uint32_t opc = (instr >> 22) & 0x3; | ||||
259 | // For the load and store instructions that we are decoding. | ||||
260 | // Opc == 0 are all stores. | ||||
261 | // Opc == 1 with a couple of exceptions are loads. The exceptions are: | ||||
262 | // Size == 00 (0), V == 1, Opc == 10 (2) which is a store and | ||||
263 | // Size == 11 (3), V == 0, Opc == 10 (2) which is a prefetch. | ||||
264 | return opc != 0 && !(size == 0 && v == 1 && opc == 2) && | ||||
265 | !(size == 3 && v == 0 && opc == 2); | ||||
266 | } | ||||
267 | return false; | ||||
268 | } | ||||
269 | |||||
270 | // The following decode instructions are only complete up to the instructions | ||||
271 | // needed for errata 843419. | ||||
272 | |||||
273 | // Instruction with writeback updates the index register after the load/store. | ||||
274 | static bool hasWriteback(uint32_t instr) { | ||||
275 | return isLoadStoreImmediatePre(instr) || isLoadStoreImmediatePost(instr) || | ||||
276 | isSTPPre(instr) || isSTPPost(instr) || isST1SinglePost(instr) || | ||||
277 | isST1MultiplePost(instr); | ||||
278 | } | ||||
279 | |||||
280 | // For the load and store class of instructions, a load can write to the | ||||
281 | // destination register, a load and a store can write to the base register when | ||||
282 | // the instruction has writeback. | ||||
283 | static bool doesLoadStoreWriteToReg(uint32_t instr, uint32_t reg) { | ||||
284 | return (isV8NonStructureLoad(instr) && getRt(instr) == reg) || | ||||
285 | (hasWriteback(instr) && getRn(instr) == reg); | ||||
286 | } | ||||
287 | |||||
288 | // Scanner for Cortex-A53 errata 843419 | ||||
289 | // Full details are available in the Cortex A53 MPCore revision 0 Software | ||||
290 | // Developers Errata Notice (ARM-EPM-048406). | ||||
291 | // | ||||
292 | // The instruction sequence that triggers the erratum is common in compiled | ||||
293 | // AArch64 code, however it is sensitive to the offset of the sequence within | ||||
294 | // a 4k page. This means that by scanning and fixing the patch after we have | ||||
295 | // assigned addresses we only need to disassemble and fix instances of the | ||||
296 | // sequence in the range of affected offsets. | ||||
297 | // | ||||
298 | // In summary the erratum conditions are a series of 4 instructions: | ||||
299 | // 1.) An ADRP instruction that writes to register Rn with low 12 bits of | ||||
300 | // address of instruction either 0xff8 or 0xffc. | ||||
301 | // 2.) A load or store instruction that can be: | ||||
302 | // - A single register load or store, of either integer or vector registers. | ||||
303 | // - An STP or STNP, of either integer or vector registers. | ||||
304 | // - An Advanced SIMD ST1 store instruction. | ||||
305 | // - Must not write to Rn, but may optionally read from it. | ||||
306 | // 3.) An optional instruction that is not a branch and does not write to Rn. | ||||
307 | // 4.) A load or store from the Load/store register (unsigned immediate) class | ||||
308 | // that uses Rn as the base address register. | ||||
309 | // | ||||
310 | // Note that we do not attempt to scan for Sequence 2 as described in the | ||||
311 | // Software Developers Errata Notice as this has been assessed to be extremely | ||||
312 | // unlikely to occur in compiled code. This matches gold and ld.bfd behavior. | ||||
313 | |||||
314 | // Return true if the Instruction sequence Adrp, Instr2, and Instr4 match | ||||
315 | // the erratum sequence. The Adrp, Instr2 and Instr4 correspond to 1.), 2.), | ||||
316 | // and 4.) in the Scanner for Cortex-A53 errata comment above. | ||||
317 | static bool is843419ErratumSequence(uint32_t instr1, uint32_t instr2, | ||||
318 | uint32_t instr4) { | ||||
319 | if (!isADRP(instr1)) | ||||
320 | return false; | ||||
321 | |||||
322 | uint32_t rn = getRt(instr1); | ||||
323 | return isLoadStoreClass(instr2) && | ||||
324 | (isLoadStoreExclusive(instr2) || isLoadLiteral(instr2) || | ||||
325 | isV8SingleRegisterNonStructureLoadStore(instr2) || isSTP(instr2) || | ||||
326 | isSTNP(instr2) || isST1(instr2)) && | ||||
327 | !doesLoadStoreWriteToReg(instr2, rn) && | ||||
328 | isLoadStoreRegisterUnsigned(instr4) && getRn(instr4) == rn; | ||||
329 | } | ||||
330 | |||||
331 | // Scan the instruction sequence starting at Offset Off from the base of | ||||
332 | // InputSection isec. We update Off in this function rather than in the caller | ||||
333 | // as we can skip ahead much further into the section when we know how many | ||||
334 | // instructions we've scanned. | ||||
335 | // Return the offset of the load or store instruction in isec that we want to | ||||
336 | // patch or 0 if no patch required. | ||||
337 | static uint64_t scanCortexA53Errata843419(InputSection *isec, uint64_t &off, | ||||
338 | uint64_t limit) { | ||||
339 | uint64_t isecAddr = isec->getVA(0); | ||||
340 | |||||
341 | // Advance Off so that (isecAddr + Off) modulo 0x1000 is at least 0xff8. | ||||
342 | uint64_t initialPageOff = (isecAddr + off) & 0xfff; | ||||
343 | if (initialPageOff < 0xff8) | ||||
344 | off += 0xff8 - initialPageOff; | ||||
345 | |||||
346 | bool optionalAllowed = limit - off > 12; | ||||
347 | if (off >= limit || limit - off < 12) { | ||||
348 | // Need at least 3 4-byte sized instructions to trigger erratum. | ||||
349 | off = limit; | ||||
350 | return 0; | ||||
351 | } | ||||
352 | |||||
353 | uint64_t patchOff = 0; | ||||
354 | const uint8_t *buf = isec->data().begin(); | ||||
355 | const ulittle32_t *instBuf = reinterpret_cast<const ulittle32_t *>(buf + off); | ||||
356 | uint32_t instr1 = *instBuf++; | ||||
357 | uint32_t instr2 = *instBuf++; | ||||
358 | uint32_t instr3 = *instBuf++; | ||||
359 | if (is843419ErratumSequence(instr1, instr2, instr3)) { | ||||
360 | patchOff = off + 8; | ||||
361 | } else if (optionalAllowed && !isBranch(instr3)) { | ||||
362 | uint32_t instr4 = *instBuf++; | ||||
363 | if (is843419ErratumSequence(instr1, instr2, instr4)) | ||||
364 | patchOff = off + 12; | ||||
365 | } | ||||
366 | if (((isecAddr + off) & 0xfff) == 0xff8) | ||||
367 | off += 4; | ||||
368 | else | ||||
369 | off += 0xffc; | ||||
370 | return patchOff; | ||||
371 | } | ||||
372 | |||||
373 | class elf::Patch843419Section : public SyntheticSection { | ||||
374 | public: | ||||
375 | Patch843419Section(InputSection *p, uint64_t off); | ||||
376 | |||||
377 | void writeTo(uint8_t *buf) override; | ||||
378 | |||||
379 | size_t getSize() const override { return 8; } | ||||
380 | |||||
381 | uint64_t getLDSTAddr() const; | ||||
382 | |||||
383 | static bool classof(const SectionBase *d) { | ||||
384 | return d->kind() == InputSectionBase::Synthetic && d->name == ".text.patch"; | ||||
385 | } | ||||
386 | |||||
387 | // The Section we are patching. | ||||
388 | const InputSection *patchee; | ||||
389 | // The offset of the instruction in the patchee section we are patching. | ||||
390 | uint64_t patcheeOffset; | ||||
391 | // A label for the start of the Patch that we can use as a relocation target. | ||||
392 | Symbol *patchSym; | ||||
393 | }; | ||||
394 | |||||
395 | Patch843419Section::Patch843419Section(InputSection *p, uint64_t off) | ||||
396 | : SyntheticSection(SHF_ALLOC | SHF_EXECINSTR, SHT_PROGBITS, 4, | ||||
397 | ".text.patch"), | ||||
398 | patchee(p), patcheeOffset(off) { | ||||
399 | this->parent = p->getParent(); | ||||
400 | patchSym = addSyntheticLocal( | ||||
401 | saver.save("__CortexA53843419_" + utohexstr(getLDSTAddr())), STT_FUNC, 0, | ||||
402 | getSize(), *this); | ||||
403 | addSyntheticLocal(saver.save("$x"), STT_NOTYPE, 0, 0, *this); | ||||
404 | } | ||||
405 | |||||
406 | uint64_t Patch843419Section::getLDSTAddr() const { | ||||
407 | return patchee->getVA(patcheeOffset); | ||||
408 | } | ||||
409 | |||||
410 | void Patch843419Section::writeTo(uint8_t *buf) { | ||||
411 | // Copy the instruction that we will be replacing with a branch in the | ||||
412 | // patchee Section. | ||||
413 | write32le(buf, read32le(patchee->data().begin() + patcheeOffset)); | ||||
414 | |||||
415 | // Apply any relocation transferred from the original patchee section. | ||||
416 | relocateAlloc(buf, buf + getSize()); | ||||
417 | |||||
418 | // Return address is the next instruction after the one we have just copied. | ||||
419 | uint64_t s = getLDSTAddr() + 4; | ||||
420 | uint64_t p = patchSym->getVA() + 4; | ||||
421 | target->relocateNoSym(buf + 4, R_AARCH64_JUMP26, s - p); | ||||
422 | } | ||||
423 | |||||
424 | void AArch64Err843419Patcher::init() { | ||||
425 | // The AArch64 ABI permits data in executable sections. We must avoid scanning | ||||
426 | // this data as if it were instructions to avoid false matches. We use the | ||||
427 | // mapping symbols in the InputObjects to identify this data, caching the | ||||
428 | // results in sectionMap so we don't have to recalculate it each pass. | ||||
429 | |||||
430 | // The ABI Section 4.5.4 Mapping symbols; defines local symbols that describe | ||||
431 | // half open intervals [Symbol Value, Next Symbol Value) of code and data | ||||
432 | // within sections. If there is no next symbol then the half open interval is | ||||
433 | // [Symbol Value, End of section). The type, code or data, is determined by | ||||
434 | // the mapping symbol name, $x for code, $d for data. | ||||
435 | auto isCodeMapSymbol = [](const Symbol *b) { | ||||
436 | return b->getName() == "$x" || b->getName().startswith("$x."); | ||||
437 | }; | ||||
438 | auto isDataMapSymbol = [](const Symbol *b) { | ||||
439 | return b->getName() == "$d" || b->getName().startswith("$d."); | ||||
440 | }; | ||||
441 | |||||
442 | // Collect mapping symbols for every executable InputSection. | ||||
443 | for (InputFile *file : objectFiles) { | ||||
444 | auto *f = cast<ObjFile<ELF64LE>>(file); | ||||
445 | for (Symbol *b : f->getLocalSymbols()) { | ||||
446 | auto *def = dyn_cast<Defined>(b); | ||||
447 | if (!def) | ||||
448 | continue; | ||||
449 | if (!isCodeMapSymbol(def) && !isDataMapSymbol(def)) | ||||
450 | continue; | ||||
451 | if (auto *sec = dyn_cast_or_null<InputSection>(def->section)) | ||||
452 | if (sec->flags & SHF_EXECINSTR) | ||||
453 | sectionMap[sec].push_back(def); | ||||
454 | } | ||||
455 | } | ||||
456 | // For each InputSection make sure the mapping symbols are in sorted in | ||||
457 | // ascending order and free from consecutive runs of mapping symbols with | ||||
458 | // the same type. For example we must remove the redundant $d.1 from $x.0 | ||||
459 | // $d.0 $d.1 $x.1. | ||||
460 | for (auto &kv : sectionMap) { | ||||
461 | std::vector<const Defined *> &mapSyms = kv.second; | ||||
462 | llvm::stable_sort(mapSyms, [](const Defined *a, const Defined *b) { | ||||
463 | return a->value < b->value; | ||||
464 | }); | ||||
465 | mapSyms.erase( | ||||
466 | std::unique(mapSyms.begin(), mapSyms.end(), | ||||
467 | [=](const Defined *a, const Defined *b) { | ||||
468 | return isCodeMapSymbol(a) == isCodeMapSymbol(b); | ||||
469 | }), | ||||
470 | mapSyms.end()); | ||||
471 | // Always start with a Code Mapping Symbol. | ||||
472 | if (!mapSyms.empty() && !isCodeMapSymbol(mapSyms.front())) | ||||
473 | mapSyms.erase(mapSyms.begin()); | ||||
474 | } | ||||
475 | initialized = true; | ||||
476 | } | ||||
477 | |||||
478 | // Insert the PatchSections we have created back into the | ||||
479 | // InputSectionDescription. As inserting patches alters the addresses of | ||||
480 | // InputSections that follow them, we try and place the patches after all the | ||||
481 | // executable sections, although we may need to insert them earlier if the | ||||
482 | // InputSectionDescription is larger than the maximum branch range. | ||||
483 | void AArch64Err843419Patcher::insertPatches( | ||||
484 | InputSectionDescription &isd, std::vector<Patch843419Section *> &patches) { | ||||
485 | uint64_t isecLimit; | ||||
486 | uint64_t prevIsecLimit = isd.sections.front()->outSecOff; | ||||
487 | uint64_t patchUpperBound = prevIsecLimit + target->getThunkSectionSpacing(); | ||||
488 | uint64_t outSecAddr = isd.sections.front()->getParent()->addr; | ||||
489 | |||||
490 | // Set the outSecOff of patches to the place where we want to insert them. | ||||
491 | // We use a similar strategy to Thunk placement. Place patches roughly | ||||
492 | // every multiple of maximum branch range. | ||||
493 | auto patchIt = patches.begin(); | ||||
494 | auto patchEnd = patches.end(); | ||||
495 | for (const InputSection *isec : isd.sections) { | ||||
496 | isecLimit = isec->outSecOff + isec->getSize(); | ||||
497 | if (isecLimit > patchUpperBound) { | ||||
498 | while (patchIt != patchEnd) { | ||||
499 | if ((*patchIt)->getLDSTAddr() - outSecAddr >= prevIsecLimit) | ||||
500 | break; | ||||
501 | (*patchIt)->outSecOff = prevIsecLimit; | ||||
502 | ++patchIt; | ||||
503 | } | ||||
504 | patchUpperBound = prevIsecLimit + target->getThunkSectionSpacing(); | ||||
505 | } | ||||
506 | prevIsecLimit = isecLimit; | ||||
507 | } | ||||
508 | for (; patchIt != patchEnd; ++patchIt) { | ||||
509 | (*patchIt)->outSecOff = isecLimit; | ||||
| |||||
510 | } | ||||
511 | |||||
512 | // Merge all patch sections. We use the outSecOff assigned above to | ||||
513 | // determine the insertion point. This is ok as we only merge into an | ||||
514 | // InputSectionDescription once per pass, and at the end of the pass | ||||
515 | // assignAddresses() will recalculate all the outSecOff values. | ||||
516 | std::vector<InputSection *> tmp; | ||||
517 | tmp.reserve(isd.sections.size() + patches.size()); | ||||
518 | auto mergeCmp = [](const InputSection *a, const InputSection *b) { | ||||
519 | if (a->outSecOff != b->outSecOff) | ||||
520 | return a->outSecOff < b->outSecOff; | ||||
521 | return isa<Patch843419Section>(a) && !isa<Patch843419Section>(b); | ||||
522 | }; | ||||
523 | std::merge(isd.sections.begin(), isd.sections.end(), patches.begin(), | ||||
524 | patches.end(), std::back_inserter(tmp), mergeCmp); | ||||
525 | isd.sections = std::move(tmp); | ||||
526 | } | ||||
527 | |||||
528 | // Given an erratum sequence that starts at address adrpAddr, with an | ||||
529 | // instruction that we need to patch at patcheeOffset from the start of | ||||
530 | // InputSection isec, create a Patch843419 Section and add it to the | ||||
531 | // Patches that we need to insert. | ||||
532 | static void implementPatch(uint64_t adrpAddr, uint64_t patcheeOffset, | ||||
533 | InputSection *isec, | ||||
534 | std::vector<Patch843419Section *> &patches) { | ||||
535 | // There may be a relocation at the same offset that we are patching. There | ||||
536 | // are four cases that we need to consider. | ||||
537 | // Case 1: R_AARCH64_JUMP26 branch relocation. We have already patched this | ||||
538 | // instance of the erratum on a previous patch and altered the relocation. We | ||||
539 | // have nothing more to do. | ||||
540 | // Case 2: A TLS Relaxation R_RELAX_TLS_IE_TO_LE. In this case the ADRP that | ||||
541 | // we read will be transformed into a MOVZ later so we actually don't match | ||||
542 | // the sequence and have nothing more to do. | ||||
543 | // Case 3: A load/store register (unsigned immediate) class relocation. There | ||||
544 | // are two of these R_AARCH_LD64_ABS_LO12_NC and R_AARCH_LD64_GOT_LO12_NC and | ||||
545 | // they are both absolute. We need to add the same relocation to the patch, | ||||
546 | // and replace the relocation with a R_AARCH_JUMP26 branch relocation. | ||||
547 | // Case 4: No relocation. We must create a new R_AARCH64_JUMP26 branch | ||||
548 | // relocation at the offset. | ||||
549 | auto relIt = llvm::find_if(isec->relocations, [=](const Relocation &r) { | ||||
550 | return r.offset == patcheeOffset; | ||||
551 | }); | ||||
552 | if (relIt != isec->relocations.end() && | ||||
553 | (relIt->type == R_AARCH64_JUMP26 || relIt->expr == R_RELAX_TLS_IE_TO_LE)) | ||||
554 | return; | ||||
555 | |||||
556 | log("detected cortex-a53-843419 erratum sequence starting at " + | ||||
557 | utohexstr(adrpAddr) + " in unpatched output."); | ||||
558 | |||||
559 | auto *ps = make<Patch843419Section>(isec, patcheeOffset); | ||||
560 | patches.push_back(ps); | ||||
561 | |||||
562 | auto makeRelToPatch = [](uint64_t offset, Symbol *patchSym) { | ||||
563 | return Relocation{R_PC, R_AARCH64_JUMP26, offset, 0, patchSym}; | ||||
564 | }; | ||||
565 | |||||
566 | if (relIt != isec->relocations.end()) { | ||||
567 | ps->relocations.push_back( | ||||
568 | {relIt->expr, relIt->type, 0, relIt->addend, relIt->sym}); | ||||
569 | *relIt = makeRelToPatch(patcheeOffset, ps->patchSym); | ||||
570 | } else | ||||
571 | isec->relocations.push_back(makeRelToPatch(patcheeOffset, ps->patchSym)); | ||||
572 | } | ||||
573 | |||||
574 | // Scan all the instructions in InputSectionDescription, for each instance of | ||||
575 | // the erratum sequence create a Patch843419Section. We return the list of | ||||
576 | // Patch843419Sections that need to be applied to the InputSectionDescription. | ||||
577 | std::vector<Patch843419Section *> | ||||
578 | AArch64Err843419Patcher::patchInputSectionDescription( | ||||
579 | InputSectionDescription &isd) { | ||||
580 | std::vector<Patch843419Section *> patches; | ||||
581 | for (InputSection *isec : isd.sections) { | ||||
582 | // LLD doesn't use the erratum sequence in SyntheticSections. | ||||
583 | if (isa<SyntheticSection>(isec)) | ||||
584 | continue; | ||||
585 | // Use sectionMap to make sure we only scan code and not inline data. | ||||
586 | // We have already sorted MapSyms in ascending order and removed consecutive | ||||
587 | // mapping symbols of the same type. Our range of executable instructions to | ||||
588 | // scan is therefore [codeSym->value, dataSym->value) or [codeSym->value, | ||||
589 | // section size). | ||||
590 | std::vector<const Defined *> &mapSyms = sectionMap[isec]; | ||||
591 | |||||
592 | auto codeSym = mapSyms.begin(); | ||||
593 | while (codeSym != mapSyms.end()) { | ||||
594 | auto dataSym = std::next(codeSym); | ||||
595 | uint64_t off = (*codeSym)->value; | ||||
596 | uint64_t limit = | ||||
597 | (dataSym == mapSyms.end()) ? isec->data().size() : (*dataSym)->value; | ||||
598 | |||||
599 | while (off < limit) { | ||||
600 | uint64_t startAddr = isec->getVA(off); | ||||
601 | if (uint64_t patcheeOffset = | ||||
602 | scanCortexA53Errata843419(isec, off, limit)) | ||||
603 | implementPatch(startAddr, patcheeOffset, isec, patches); | ||||
604 | } | ||||
605 | if (dataSym == mapSyms.end()) | ||||
606 | break; | ||||
607 | codeSym = std::next(dataSym); | ||||
608 | } | ||||
609 | } | ||||
610 | return patches; | ||||
611 | } | ||||
612 | |||||
613 | // For each InputSectionDescription make one pass over the executable sections | ||||
614 | // looking for the erratum sequence; creating a synthetic Patch843419Section | ||||
615 | // for each instance found. We insert these synthetic patch sections after the | ||||
616 | // executable code in each InputSectionDescription. | ||||
617 | // | ||||
618 | // PreConditions: | ||||
619 | // The Output and Input Sections have had their final addresses assigned. | ||||
620 | // | ||||
621 | // PostConditions: | ||||
622 | // Returns true if at least one patch was added. The addresses of the | ||||
623 | // Output and Input Sections may have been changed. | ||||
624 | // Returns false if no patches were required and no changes were made. | ||||
625 | bool AArch64Err843419Patcher::createFixes() { | ||||
626 | if (!initialized) | ||||
| |||||
627 | init(); | ||||
628 | |||||
629 | bool addressesChanged = false; | ||||
630 | for (OutputSection *os : outputSections) { | ||||
631 | if (!(os->flags & SHF_ALLOC) || !(os->flags & SHF_EXECINSTR)) | ||||
632 | continue; | ||||
633 | for (BaseCommand *bc : os->sectionCommands) | ||||
634 | if (auto *isd
| ||||
635 | std::vector<Patch843419Section *> patches = | ||||
636 | patchInputSectionDescription(*isd); | ||||
637 | if (!patches.empty()) { | ||||
638 | insertPatches(*isd, patches); | ||||
639 | addressesChanged = true; | ||||
640 | } | ||||
641 | } | ||||
642 | } | ||||
643 | return addressesChanged; | ||||
644 | } |
1 | // -*- C++ -*- |
2 | //===----------------------------------------------------------------------===// |
3 | // |
4 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
5 | // See https://llvm.org/LICENSE.txt for license information. |
6 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
7 | // |
8 | //===----------------------------------------------------------------------===// |
9 | |
10 | #ifndef _LIBCPP___ITERATOR_WRAP_ITER_H |
11 | #define _LIBCPP___ITERATOR_WRAP_ITER_H |
12 | |
13 | #include <__config> |
14 | #include <__debug> |
15 | #include <__iterator/iterator_traits.h> |
16 | #include <__memory/pointer_traits.h> // __to_address |
17 | #include <type_traits> |
18 | |
19 | #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) |
20 | #pragma GCC system_header |
21 | #endif |
22 | |
23 | _LIBCPP_PUSH_MACROSpush_macro("min") push_macro("max") |
24 | #include <__undef_macros> |
25 | |
26 | _LIBCPP_BEGIN_NAMESPACE_STDnamespace std { inline namespace __1 { |
27 | |
28 | template <class _Iter> |
29 | class __wrap_iter |
30 | { |
31 | public: |
32 | typedef _Iter iterator_type; |
33 | typedef typename iterator_traits<iterator_type>::value_type value_type; |
34 | typedef typename iterator_traits<iterator_type>::difference_type difference_type; |
35 | typedef typename iterator_traits<iterator_type>::pointer pointer; |
36 | typedef typename iterator_traits<iterator_type>::reference reference; |
37 | typedef typename iterator_traits<iterator_type>::iterator_category iterator_category; |
38 | #if _LIBCPP_STD_VER14 > 17 |
39 | typedef contiguous_iterator_tag iterator_concept; |
40 | #endif |
41 | |
42 | private: |
43 | iterator_type __i; |
44 | public: |
45 | _LIBCPP_INLINE_VISIBILITY__attribute__ ((__visibility__("hidden"))) __attribute__ ((__exclude_from_explicit_instantiation__ )) _LIBCPP_CONSTEXPR_IF_NODEBUGconstexpr __wrap_iter() _NOEXCEPTnoexcept |
46 | #if _LIBCPP_STD_VER14 > 11 |
47 | : __i{} |
48 | #endif |
49 | { |
50 | #if _LIBCPP_DEBUG_LEVEL0 == 2 |
51 | __get_db()->__insert_i(this); |
52 | #endif |
53 | } |
54 | template <class _Up> _LIBCPP_INLINE_VISIBILITY__attribute__ ((__visibility__("hidden"))) __attribute__ ((__exclude_from_explicit_instantiation__ )) _LIBCPP_CONSTEXPR_IF_NODEBUGconstexpr |
55 | __wrap_iter(const __wrap_iter<_Up>& __u, |
56 | typename enable_if<is_convertible<_Up, iterator_type>::value>::type* = nullptr) _NOEXCEPTnoexcept |
57 | : __i(__u.base()) |
58 | { |
59 | #if _LIBCPP_DEBUG_LEVEL0 == 2 |
60 | __get_db()->__iterator_copy(this, &__u); |
61 | #endif |
62 | } |
63 | #if _LIBCPP_DEBUG_LEVEL0 == 2 |
64 | _LIBCPP_INLINE_VISIBILITY__attribute__ ((__visibility__("hidden"))) __attribute__ ((__exclude_from_explicit_instantiation__ )) _LIBCPP_CONSTEXPR_IF_NODEBUGconstexpr |
65 | __wrap_iter(const __wrap_iter& __x) |
66 | : __i(__x.base()) |
67 | { |
68 | __get_db()->__iterator_copy(this, &__x); |
69 | } |
70 | _LIBCPP_INLINE_VISIBILITY__attribute__ ((__visibility__("hidden"))) __attribute__ ((__exclude_from_explicit_instantiation__ )) _LIBCPP_CONSTEXPR_IF_NODEBUGconstexpr |
71 | __wrap_iter& operator=(const __wrap_iter& __x) |
72 | { |
73 | if (this != &__x) |
74 | { |
75 | __get_db()->__iterator_copy(this, &__x); |
76 | __i = __x.__i; |
77 | } |
78 | return *this; |
79 | } |
80 | _LIBCPP_INLINE_VISIBILITY__attribute__ ((__visibility__("hidden"))) __attribute__ ((__exclude_from_explicit_instantiation__ )) _LIBCPP_CONSTEXPR_IF_NODEBUGconstexpr |
81 | ~__wrap_iter() |
82 | { |
83 | __get_db()->__erase_i(this); |
84 | } |
85 | #endif |
86 | _LIBCPP_INLINE_VISIBILITY__attribute__ ((__visibility__("hidden"))) __attribute__ ((__exclude_from_explicit_instantiation__ )) _LIBCPP_CONSTEXPR_IF_NODEBUGconstexpr reference operator*() const _NOEXCEPTnoexcept |
87 | { |
88 | #if _LIBCPP_DEBUG_LEVEL0 == 2 |
89 | _LIBCPP_ASSERT(__get_const_db()->__dereferenceable(this),((void)0) |
90 | "Attempted to dereference a non-dereferenceable iterator")((void)0); |
91 | #endif |
92 | return *__i; |
93 | } |
94 | _LIBCPP_INLINE_VISIBILITY__attribute__ ((__visibility__("hidden"))) __attribute__ ((__exclude_from_explicit_instantiation__ )) _LIBCPP_CONSTEXPR_IF_NODEBUGconstexpr pointer operator->() const _NOEXCEPTnoexcept |
95 | { |
96 | #if _LIBCPP_DEBUG_LEVEL0 == 2 |
97 | _LIBCPP_ASSERT(__get_const_db()->__dereferenceable(this),((void)0) |
98 | "Attempted to dereference a non-dereferenceable iterator")((void)0); |
99 | #endif |
100 | return _VSTDstd::__1::__to_address(__i); |
101 | } |
102 | _LIBCPP_INLINE_VISIBILITY__attribute__ ((__visibility__("hidden"))) __attribute__ ((__exclude_from_explicit_instantiation__ )) _LIBCPP_CONSTEXPR_IF_NODEBUGconstexpr __wrap_iter& operator++() _NOEXCEPTnoexcept |
103 | { |
104 | #if _LIBCPP_DEBUG_LEVEL0 == 2 |
105 | _LIBCPP_ASSERT(__get_const_db()->__dereferenceable(this),((void)0) |
106 | "Attempted to increment a non-incrementable iterator")((void)0); |
107 | #endif |
108 | ++__i; |
109 | return *this; |
110 | } |
111 | _LIBCPP_INLINE_VISIBILITY__attribute__ ((__visibility__("hidden"))) __attribute__ ((__exclude_from_explicit_instantiation__ )) _LIBCPP_CONSTEXPR_IF_NODEBUGconstexpr __wrap_iter operator++(int) _NOEXCEPTnoexcept |
112 | {__wrap_iter __tmp(*this); ++(*this); return __tmp;} |
113 | |
114 | _LIBCPP_INLINE_VISIBILITY__attribute__ ((__visibility__("hidden"))) __attribute__ ((__exclude_from_explicit_instantiation__ )) _LIBCPP_CONSTEXPR_IF_NODEBUGconstexpr __wrap_iter& operator--() _NOEXCEPTnoexcept |
115 | { |
116 | #if _LIBCPP_DEBUG_LEVEL0 == 2 |
117 | _LIBCPP_ASSERT(__get_const_db()->__decrementable(this),((void)0) |
118 | "Attempted to decrement a non-decrementable iterator")((void)0); |
119 | #endif |
120 | --__i; |
121 | return *this; |
122 | } |
123 | _LIBCPP_INLINE_VISIBILITY__attribute__ ((__visibility__("hidden"))) __attribute__ ((__exclude_from_explicit_instantiation__ )) _LIBCPP_CONSTEXPR_IF_NODEBUGconstexpr __wrap_iter operator--(int) _NOEXCEPTnoexcept |
124 | {__wrap_iter __tmp(*this); --(*this); return __tmp;} |
125 | _LIBCPP_INLINE_VISIBILITY__attribute__ ((__visibility__("hidden"))) __attribute__ ((__exclude_from_explicit_instantiation__ )) _LIBCPP_CONSTEXPR_IF_NODEBUGconstexpr __wrap_iter operator+ (difference_type __n) const _NOEXCEPTnoexcept |
126 | {__wrap_iter __w(*this); __w += __n; return __w;} |
127 | _LIBCPP_INLINE_VISIBILITY__attribute__ ((__visibility__("hidden"))) __attribute__ ((__exclude_from_explicit_instantiation__ )) _LIBCPP_CONSTEXPR_IF_NODEBUGconstexpr __wrap_iter& operator+=(difference_type __n) _NOEXCEPTnoexcept |
128 | { |
129 | #if _LIBCPP_DEBUG_LEVEL0 == 2 |
130 | _LIBCPP_ASSERT(__get_const_db()->__addable(this, __n),((void)0) |
131 | "Attempted to add/subtract an iterator outside its valid range")((void)0); |
132 | #endif |
133 | __i += __n; |
134 | return *this; |
135 | } |
136 | _LIBCPP_INLINE_VISIBILITY__attribute__ ((__visibility__("hidden"))) __attribute__ ((__exclude_from_explicit_instantiation__ )) _LIBCPP_CONSTEXPR_IF_NODEBUGconstexpr __wrap_iter operator- (difference_type __n) const _NOEXCEPTnoexcept |
137 | {return *this + (-__n);} |
138 | _LIBCPP_INLINE_VISIBILITY__attribute__ ((__visibility__("hidden"))) __attribute__ ((__exclude_from_explicit_instantiation__ )) _LIBCPP_CONSTEXPR_IF_NODEBUGconstexpr __wrap_iter& operator-=(difference_type __n) _NOEXCEPTnoexcept |
139 | {*this += -__n; return *this;} |
140 | _LIBCPP_INLINE_VISIBILITY__attribute__ ((__visibility__("hidden"))) __attribute__ ((__exclude_from_explicit_instantiation__ )) _LIBCPP_CONSTEXPR_IF_NODEBUGconstexpr reference operator[](difference_type __n) const _NOEXCEPTnoexcept |
141 | { |
142 | #if _LIBCPP_DEBUG_LEVEL0 == 2 |
143 | _LIBCPP_ASSERT(__get_const_db()->__subscriptable(this, __n),((void)0) |
144 | "Attempted to subscript an iterator outside its valid range")((void)0); |
145 | #endif |
146 | return __i[__n]; |
147 | } |
148 | |
149 | _LIBCPP_INLINE_VISIBILITY__attribute__ ((__visibility__("hidden"))) __attribute__ ((__exclude_from_explicit_instantiation__ )) _LIBCPP_CONSTEXPR_IF_NODEBUGconstexpr iterator_type base() const _NOEXCEPTnoexcept {return __i;} |
150 | |
151 | private: |
152 | #if _LIBCPP_DEBUG_LEVEL0 == 2 |
153 | _LIBCPP_INLINE_VISIBILITY__attribute__ ((__visibility__("hidden"))) __attribute__ ((__exclude_from_explicit_instantiation__ )) _LIBCPP_CONSTEXPR_IF_NODEBUGconstexpr __wrap_iter(const void* __p, iterator_type __x) : __i(__x) |
154 | { |
155 | __get_db()->__insert_ic(this, __p); |
156 | } |
157 | #else |
158 | _LIBCPP_INLINE_VISIBILITY__attribute__ ((__visibility__("hidden"))) __attribute__ ((__exclude_from_explicit_instantiation__ )) _LIBCPP_CONSTEXPR_IF_NODEBUGconstexpr __wrap_iter(iterator_type __x) _NOEXCEPTnoexcept : __i(__x) {} |
159 | #endif |
160 | |
161 | template <class _Up> friend class __wrap_iter; |
162 | template <class _CharT, class _Traits, class _Alloc> friend class basic_string; |
163 | template <class _Tp, class _Alloc> friend class _LIBCPP_TEMPLATE_VIS__attribute__ ((__type_visibility__("default"))) vector; |
164 | template <class _Tp, size_t> friend class _LIBCPP_TEMPLATE_VIS__attribute__ ((__type_visibility__("default"))) span; |
165 | }; |
166 | |
167 | template <class _Iter1> |
168 | _LIBCPP_INLINE_VISIBILITY__attribute__ ((__visibility__("hidden"))) __attribute__ ((__exclude_from_explicit_instantiation__ )) _LIBCPP_CONSTEXPR_IF_NODEBUGconstexpr |
169 | bool operator==(const __wrap_iter<_Iter1>& __x, const __wrap_iter<_Iter1>& __y) _NOEXCEPTnoexcept |
170 | { |
171 | return __x.base() == __y.base(); |
172 | } |
173 | |
174 | template <class _Iter1, class _Iter2> |
175 | _LIBCPP_INLINE_VISIBILITY__attribute__ ((__visibility__("hidden"))) __attribute__ ((__exclude_from_explicit_instantiation__ )) _LIBCPP_CONSTEXPR_IF_NODEBUGconstexpr |
176 | bool operator==(const __wrap_iter<_Iter1>& __x, const __wrap_iter<_Iter2>& __y) _NOEXCEPTnoexcept |
177 | { |
178 | return __x.base() == __y.base(); |
179 | } |
180 | |
181 | template <class _Iter1> |
182 | _LIBCPP_INLINE_VISIBILITY__attribute__ ((__visibility__("hidden"))) __attribute__ ((__exclude_from_explicit_instantiation__ )) _LIBCPP_CONSTEXPR_IF_NODEBUGconstexpr |
183 | bool operator<(const __wrap_iter<_Iter1>& __x, const __wrap_iter<_Iter1>& __y) _NOEXCEPTnoexcept |
184 | { |
185 | #if _LIBCPP_DEBUG_LEVEL0 == 2 |
186 | _LIBCPP_ASSERT(__get_const_db()->__less_than_comparable(&__x, &__y),((void)0) |
187 | "Attempted to compare incomparable iterators")((void)0); |
188 | #endif |
189 | return __x.base() < __y.base(); |
190 | } |
191 | |
192 | template <class _Iter1, class _Iter2> |
193 | _LIBCPP_INLINE_VISIBILITY__attribute__ ((__visibility__("hidden"))) __attribute__ ((__exclude_from_explicit_instantiation__ )) _LIBCPP_CONSTEXPR_IF_NODEBUGconstexpr |
194 | bool operator<(const __wrap_iter<_Iter1>& __x, const __wrap_iter<_Iter2>& __y) _NOEXCEPTnoexcept |
195 | { |
196 | #if _LIBCPP_DEBUG_LEVEL0 == 2 |
197 | _LIBCPP_ASSERT(__get_const_db()->__less_than_comparable(&__x, &__y),((void)0) |
198 | "Attempted to compare incomparable iterators")((void)0); |
199 | #endif |
200 | return __x.base() < __y.base(); |
201 | } |
202 | |
203 | template <class _Iter1> |
204 | _LIBCPP_INLINE_VISIBILITY__attribute__ ((__visibility__("hidden"))) __attribute__ ((__exclude_from_explicit_instantiation__ )) _LIBCPP_CONSTEXPR_IF_NODEBUGconstexpr |
205 | bool operator!=(const __wrap_iter<_Iter1>& __x, const __wrap_iter<_Iter1>& __y) _NOEXCEPTnoexcept |
206 | { |
207 | return !(__x == __y); |
208 | } |
209 | |
210 | template <class _Iter1, class _Iter2> |
211 | _LIBCPP_INLINE_VISIBILITY__attribute__ ((__visibility__("hidden"))) __attribute__ ((__exclude_from_explicit_instantiation__ )) _LIBCPP_CONSTEXPR_IF_NODEBUGconstexpr |
212 | bool operator!=(const __wrap_iter<_Iter1>& __x, const __wrap_iter<_Iter2>& __y) _NOEXCEPTnoexcept |
213 | { |
214 | return !(__x == __y); |
215 | } |
216 | |
217 | template <class _Iter1> |
218 | _LIBCPP_INLINE_VISIBILITY__attribute__ ((__visibility__("hidden"))) __attribute__ ((__exclude_from_explicit_instantiation__ )) _LIBCPP_CONSTEXPR_IF_NODEBUGconstexpr |
219 | bool operator>(const __wrap_iter<_Iter1>& __x, const __wrap_iter<_Iter1>& __y) _NOEXCEPTnoexcept |
220 | { |
221 | return __y < __x; |
222 | } |
223 | |
224 | template <class _Iter1, class _Iter2> |
225 | _LIBCPP_INLINE_VISIBILITY__attribute__ ((__visibility__("hidden"))) __attribute__ ((__exclude_from_explicit_instantiation__ )) _LIBCPP_CONSTEXPR_IF_NODEBUGconstexpr |
226 | bool operator>(const __wrap_iter<_Iter1>& __x, const __wrap_iter<_Iter2>& __y) _NOEXCEPTnoexcept |
227 | { |
228 | return __y < __x; |
229 | } |
230 | |
231 | template <class _Iter1> |
232 | _LIBCPP_INLINE_VISIBILITY__attribute__ ((__visibility__("hidden"))) __attribute__ ((__exclude_from_explicit_instantiation__ )) _LIBCPP_CONSTEXPR_IF_NODEBUGconstexpr |
233 | bool operator>=(const __wrap_iter<_Iter1>& __x, const __wrap_iter<_Iter1>& __y) _NOEXCEPTnoexcept |
234 | { |
235 | return !(__x < __y); |
236 | } |
237 | |
238 | template <class _Iter1, class _Iter2> |
239 | _LIBCPP_INLINE_VISIBILITY__attribute__ ((__visibility__("hidden"))) __attribute__ ((__exclude_from_explicit_instantiation__ )) _LIBCPP_CONSTEXPR_IF_NODEBUGconstexpr |
240 | bool operator>=(const __wrap_iter<_Iter1>& __x, const __wrap_iter<_Iter2>& __y) _NOEXCEPTnoexcept |
241 | { |
242 | return !(__x < __y); |
243 | } |
244 | |
245 | template <class _Iter1> |
246 | _LIBCPP_INLINE_VISIBILITY__attribute__ ((__visibility__("hidden"))) __attribute__ ((__exclude_from_explicit_instantiation__ )) _LIBCPP_CONSTEXPR_IF_NODEBUGconstexpr |
247 | bool operator<=(const __wrap_iter<_Iter1>& __x, const __wrap_iter<_Iter1>& __y) _NOEXCEPTnoexcept |
248 | { |
249 | return !(__y < __x); |
250 | } |
251 | |
252 | template <class _Iter1, class _Iter2> |
253 | _LIBCPP_INLINE_VISIBILITY__attribute__ ((__visibility__("hidden"))) __attribute__ ((__exclude_from_explicit_instantiation__ )) _LIBCPP_CONSTEXPR_IF_NODEBUGconstexpr |
254 | bool operator<=(const __wrap_iter<_Iter1>& __x, const __wrap_iter<_Iter2>& __y) _NOEXCEPTnoexcept |
255 | { |
256 | return !(__y < __x); |
257 | } |
258 | |
259 | template <class _Iter1, class _Iter2> |
260 | _LIBCPP_INLINE_VISIBILITY__attribute__ ((__visibility__("hidden"))) __attribute__ ((__exclude_from_explicit_instantiation__ )) _LIBCPP_CONSTEXPR_IF_NODEBUGconstexpr |
261 | #ifndef _LIBCPP_CXX03_LANG |
262 | auto operator-(const __wrap_iter<_Iter1>& __x, const __wrap_iter<_Iter2>& __y) _NOEXCEPTnoexcept |
263 | -> decltype(__x.base() - __y.base()) |
264 | #else |
265 | typename __wrap_iter<_Iter1>::difference_type |
266 | operator-(const __wrap_iter<_Iter1>& __x, const __wrap_iter<_Iter2>& __y) _NOEXCEPTnoexcept |
267 | #endif // C++03 |
268 | { |
269 | #if _LIBCPP_DEBUG_LEVEL0 == 2 |
270 | _LIBCPP_ASSERT(__get_const_db()->__less_than_comparable(&__x, &__y),((void)0) |
271 | "Attempted to subtract incompatible iterators")((void)0); |
272 | #endif |
273 | return __x.base() - __y.base(); |
274 | } |
275 | |
276 | template <class _Iter1> |
277 | _LIBCPP_INLINE_VISIBILITY__attribute__ ((__visibility__("hidden"))) __attribute__ ((__exclude_from_explicit_instantiation__ )) _LIBCPP_CONSTEXPR_IF_NODEBUGconstexpr |
278 | __wrap_iter<_Iter1> operator+(typename __wrap_iter<_Iter1>::difference_type __n, __wrap_iter<_Iter1> __x) _NOEXCEPTnoexcept |
279 | { |
280 | __x += __n; |
281 | return __x; |
282 | } |
283 | |
284 | #if _LIBCPP_STD_VER14 <= 17 |
285 | template <class _It> |
286 | struct __is_cpp17_contiguous_iterator<__wrap_iter<_It> > : true_type {}; |
287 | #endif |
288 | |
289 | template <class _Iter> |
290 | _LIBCPP_CONSTEXPRconstexpr |
291 | decltype(_VSTDstd::__1::__to_address(declval<_Iter>())) |
292 | __to_address(__wrap_iter<_Iter> __w) _NOEXCEPTnoexcept { |
293 | return _VSTDstd::__1::__to_address(__w.base()); |
294 | } |
295 | |
296 | _LIBCPP_END_NAMESPACE_STD} } |
297 | |
298 | _LIBCPP_POP_MACROSpop_macro("min") pop_macro("max") |
299 | |
300 | #endif // _LIBCPP___ITERATOR_WRAP_ITER_H |