rustc_codegen_llvm/
builder.rs

1use std::borrow::{Borrow, Cow};
2use std::iter;
3use std::ops::Deref;
4
5use rustc_ast::expand::typetree::FncTree;
6pub(crate) mod autodiff;
7pub(crate) mod gpu_offload;
8
9use libc::{c_char, c_uint};
10use rustc_abi as abi;
11use rustc_abi::{Align, Size, WrappingRange};
12use rustc_codegen_ssa::MemFlags;
13use rustc_codegen_ssa::common::{IntPredicate, RealPredicate, SynchronizationScope, TypeKind};
14use rustc_codegen_ssa::mir::operand::{OperandRef, OperandValue};
15use rustc_codegen_ssa::mir::place::PlaceRef;
16use rustc_codegen_ssa::traits::*;
17use rustc_data_structures::small_c_str::SmallCStr;
18use rustc_hir::def_id::DefId;
19use rustc_middle::middle::codegen_fn_attrs::{CodegenFnAttrs, TargetFeature, TargetFeatureKind};
20use rustc_middle::ty::layout::{
21    FnAbiError, FnAbiOfHelpers, FnAbiRequest, HasTypingEnv, LayoutError, LayoutOfHelpers,
22    TyAndLayout,
23};
24use rustc_middle::ty::{self, Instance, Ty, TyCtxt};
25use rustc_sanitizers::{cfi, kcfi};
26use rustc_session::config::OptLevel;
27use rustc_span::Span;
28use rustc_target::callconv::{FnAbi, PassMode};
29use rustc_target::spec::{Arch, HasTargetSpec, SanitizerSet, Target};
30use smallvec::SmallVec;
31use tracing::{debug, instrument};
32
33use crate::abi::FnAbiLlvmExt;
34use crate::attributes;
35use crate::common::Funclet;
36use crate::context::{CodegenCx, FullCx, GenericCx, SCx};
37use crate::llvm::{
38    self, AtomicOrdering, AtomicRmwBinOp, BasicBlock, FromGeneric, GEPNoWrapFlags, Metadata, TRUE,
39    ToLlvmBool, Type, Value,
40};
41use crate::type_of::LayoutLlvmExt;
42
43#[must_use]
44pub(crate) struct GenericBuilder<'a, 'll, CX: Borrow<SCx<'ll>>> {
45    pub llbuilder: &'ll mut llvm::Builder<'ll>,
46    pub cx: &'a GenericCx<'ll, CX>,
47}
48
49pub(crate) type SBuilder<'a, 'll> = GenericBuilder<'a, 'll, SCx<'ll>>;
50pub(crate) type Builder<'a, 'll, 'tcx> = GenericBuilder<'a, 'll, FullCx<'ll, 'tcx>>;
51
52impl<'a, 'll, CX: Borrow<SCx<'ll>>> Drop for GenericBuilder<'a, 'll, CX> {
53    fn drop(&mut self) {
54        unsafe {
55            llvm::LLVMDisposeBuilder(&mut *(self.llbuilder as *mut _));
56        }
57    }
58}
59
60impl<'a, 'll> SBuilder<'a, 'll> {
61    pub(crate) fn call(
62        &mut self,
63        llty: &'ll Type,
64        llfn: &'ll Value,
65        args: &[&'ll Value],
66        funclet: Option<&Funclet<'ll>>,
67    ) -> &'ll Value {
68        debug!("call {:?} with args ({:?})", llfn, args);
69
70        let args = self.check_call("call", llty, llfn, args);
71        let funclet_bundle = funclet.map(|funclet| funclet.bundle());
72        let mut bundles: SmallVec<[_; 2]> = SmallVec::new();
73        if let Some(funclet_bundle) = funclet_bundle {
74            bundles.push(funclet_bundle);
75        }
76
77        let call = unsafe {
78            llvm::LLVMBuildCallWithOperandBundles(
79                self.llbuilder,
80                llty,
81                llfn,
82                args.as_ptr() as *const &llvm::Value,
83                args.len() as c_uint,
84                bundles.as_ptr(),
85                bundles.len() as c_uint,
86                c"".as_ptr(),
87            )
88        };
89        call
90    }
91}
92
93impl<'a, 'll, CX: Borrow<SCx<'ll>>> GenericBuilder<'a, 'll, CX> {
94    fn with_cx(scx: &'a GenericCx<'ll, CX>) -> Self {
95        // Create a fresh builder from the simple context.
96        let llbuilder = unsafe { llvm::LLVMCreateBuilderInContext(scx.deref().borrow().llcx) };
97        GenericBuilder { llbuilder, cx: scx }
98    }
99
100    pub(crate) fn bitcast(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
101        unsafe { llvm::LLVMBuildBitCast(self.llbuilder, val, dest_ty, UNNAMED) }
102    }
103
104    pub(crate) fn ret_void(&mut self) {
105        llvm::LLVMBuildRetVoid(self.llbuilder);
106    }
107
108    pub(crate) fn ret(&mut self, v: &'ll Value) {
109        unsafe {
110            llvm::LLVMBuildRet(self.llbuilder, v);
111        }
112    }
113
114    pub(crate) fn build(cx: &'a GenericCx<'ll, CX>, llbb: &'ll BasicBlock) -> Self {
115        let bx = Self::with_cx(cx);
116        unsafe {
117            llvm::LLVMPositionBuilderAtEnd(bx.llbuilder, llbb);
118        }
119        bx
120    }
121
122    // The generic builder has less functionality and thus (unlike the other alloca) we can not
123    // easily jump to the beginning of the function to place our allocas there. We trust the user
124    // to manually do that. FIXME(offload): improve the genericCx and add more llvm wrappers to
125    // handle this.
126    pub(crate) fn direct_alloca(&mut self, ty: &'ll Type, align: Align, name: &str) -> &'ll Value {
127        let val = unsafe {
128            let alloca = llvm::LLVMBuildAlloca(self.llbuilder, ty, UNNAMED);
129            llvm::LLVMSetAlignment(alloca, align.bytes() as c_uint);
130            // Cast to default addrspace if necessary
131            llvm::LLVMBuildPointerCast(self.llbuilder, alloca, self.cx.type_ptr(), UNNAMED)
132        };
133        if name != "" {
134            let name = std::ffi::CString::new(name).unwrap();
135            llvm::set_value_name(val, &name.as_bytes());
136        }
137        val
138    }
139
140    pub(crate) fn inbounds_gep(
141        &mut self,
142        ty: &'ll Type,
143        ptr: &'ll Value,
144        indices: &[&'ll Value],
145    ) -> &'ll Value {
146        unsafe {
147            llvm::LLVMBuildGEPWithNoWrapFlags(
148                self.llbuilder,
149                ty,
150                ptr,
151                indices.as_ptr(),
152                indices.len() as c_uint,
153                UNNAMED,
154                GEPNoWrapFlags::InBounds,
155            )
156        }
157    }
158
159    pub(crate) fn store(&mut self, val: &'ll Value, ptr: &'ll Value, align: Align) -> &'ll Value {
160        debug!("Store {:?} -> {:?}", val, ptr);
161        assert_eq!(self.cx.type_kind(self.cx.val_ty(ptr)), TypeKind::Pointer);
162        unsafe {
163            let store = llvm::LLVMBuildStore(self.llbuilder, val, ptr);
164            llvm::LLVMSetAlignment(store, align.bytes() as c_uint);
165            store
166        }
167    }
168
169    pub(crate) fn load(&mut self, ty: &'ll Type, ptr: &'ll Value, align: Align) -> &'ll Value {
170        unsafe {
171            let load = llvm::LLVMBuildLoad2(self.llbuilder, ty, ptr, UNNAMED);
172            llvm::LLVMSetAlignment(load, align.bytes() as c_uint);
173            load
174        }
175    }
176
177    fn memset(&mut self, ptr: &'ll Value, fill_byte: &'ll Value, size: &'ll Value, align: Align) {
178        unsafe {
179            llvm::LLVMRustBuildMemSet(
180                self.llbuilder,
181                ptr,
182                align.bytes() as c_uint,
183                fill_byte,
184                size,
185                false,
186            );
187        }
188    }
189}
190
191/// Empty string, to be used where LLVM expects an instruction name, indicating
192/// that the instruction is to be left unnamed (i.e. numbered, in textual IR).
193// FIXME(eddyb) pass `&CStr` directly to FFI once it's a thin pointer.
194pub(crate) const UNNAMED: *const c_char = c"".as_ptr();
195
196impl<'ll, CX: Borrow<SCx<'ll>>> BackendTypes for GenericBuilder<'_, 'll, CX> {
197    type Value = <GenericCx<'ll, CX> as BackendTypes>::Value;
198    type Metadata = <GenericCx<'ll, CX> as BackendTypes>::Metadata;
199    type Function = <GenericCx<'ll, CX> as BackendTypes>::Function;
200    type BasicBlock = <GenericCx<'ll, CX> as BackendTypes>::BasicBlock;
201    type Type = <GenericCx<'ll, CX> as BackendTypes>::Type;
202    type Funclet = <GenericCx<'ll, CX> as BackendTypes>::Funclet;
203
204    type DIScope = <GenericCx<'ll, CX> as BackendTypes>::DIScope;
205    type DILocation = <GenericCx<'ll, CX> as BackendTypes>::DILocation;
206    type DIVariable = <GenericCx<'ll, CX> as BackendTypes>::DIVariable;
207}
208
209impl abi::HasDataLayout for Builder<'_, '_, '_> {
210    fn data_layout(&self) -> &abi::TargetDataLayout {
211        self.cx.data_layout()
212    }
213}
214
215impl<'tcx> ty::layout::HasTyCtxt<'tcx> for Builder<'_, '_, 'tcx> {
216    #[inline]
217    fn tcx(&self) -> TyCtxt<'tcx> {
218        self.cx.tcx
219    }
220}
221
222impl<'tcx> ty::layout::HasTypingEnv<'tcx> for Builder<'_, '_, 'tcx> {
223    fn typing_env(&self) -> ty::TypingEnv<'tcx> {
224        self.cx.typing_env()
225    }
226}
227
228impl HasTargetSpec for Builder<'_, '_, '_> {
229    #[inline]
230    fn target_spec(&self) -> &Target {
231        self.cx.target_spec()
232    }
233}
234
235impl<'tcx> LayoutOfHelpers<'tcx> for Builder<'_, '_, 'tcx> {
236    #[inline]
237    fn handle_layout_err(&self, err: LayoutError<'tcx>, span: Span, ty: Ty<'tcx>) -> ! {
238        self.cx.handle_layout_err(err, span, ty)
239    }
240}
241
242impl<'tcx> FnAbiOfHelpers<'tcx> for Builder<'_, '_, 'tcx> {
243    #[inline]
244    fn handle_fn_abi_err(
245        &self,
246        err: FnAbiError<'tcx>,
247        span: Span,
248        fn_abi_request: FnAbiRequest<'tcx>,
249    ) -> ! {
250        self.cx.handle_fn_abi_err(err, span, fn_abi_request)
251    }
252}
253
254impl<'ll, 'tcx> Deref for Builder<'_, 'll, 'tcx> {
255    type Target = CodegenCx<'ll, 'tcx>;
256
257    #[inline]
258    fn deref(&self) -> &Self::Target {
259        self.cx
260    }
261}
262
263macro_rules! math_builder_methods {
264    ($($name:ident($($arg:ident),*) => $llvm_capi:ident),+ $(,)?) => {
265        $(fn $name(&mut self, $($arg: &'ll Value),*) -> &'ll Value {
266            unsafe {
267                llvm::$llvm_capi(self.llbuilder, $($arg,)* UNNAMED)
268            }
269        })+
270    }
271}
272
273macro_rules! set_math_builder_methods {
274    ($($name:ident($($arg:ident),*) => ($llvm_capi:ident, $llvm_set_math:ident)),+ $(,)?) => {
275        $(fn $name(&mut self, $($arg: &'ll Value),*) -> &'ll Value {
276            unsafe {
277                let instr = llvm::$llvm_capi(self.llbuilder, $($arg,)* UNNAMED);
278                llvm::$llvm_set_math(instr);
279                instr
280            }
281        })+
282    }
283}
284
285impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
286    type CodegenCx = CodegenCx<'ll, 'tcx>;
287
288    fn build(cx: &'a CodegenCx<'ll, 'tcx>, llbb: &'ll BasicBlock) -> Self {
289        let bx = Builder::with_cx(cx);
290        unsafe {
291            llvm::LLVMPositionBuilderAtEnd(bx.llbuilder, llbb);
292        }
293        bx
294    }
295
296    fn cx(&self) -> &CodegenCx<'ll, 'tcx> {
297        self.cx
298    }
299
300    fn llbb(&self) -> &'ll BasicBlock {
301        unsafe { llvm::LLVMGetInsertBlock(self.llbuilder) }
302    }
303
304    fn set_span(&mut self, _span: Span) {}
305
306    fn append_block(cx: &'a CodegenCx<'ll, 'tcx>, llfn: &'ll Value, name: &str) -> &'ll BasicBlock {
307        unsafe {
308            let name = SmallCStr::new(name);
309            llvm::LLVMAppendBasicBlockInContext(cx.llcx, llfn, name.as_ptr())
310        }
311    }
312
313    fn append_sibling_block(&mut self, name: &str) -> &'ll BasicBlock {
314        Self::append_block(self.cx, self.llfn(), name)
315    }
316
317    fn switch_to_block(&mut self, llbb: Self::BasicBlock) {
318        *self = Self::build(self.cx, llbb)
319    }
320
321    fn ret_void(&mut self) {
322        llvm::LLVMBuildRetVoid(self.llbuilder);
323    }
324
325    fn ret(&mut self, v: &'ll Value) {
326        unsafe {
327            llvm::LLVMBuildRet(self.llbuilder, v);
328        }
329    }
330
331    fn br(&mut self, dest: &'ll BasicBlock) {
332        unsafe {
333            llvm::LLVMBuildBr(self.llbuilder, dest);
334        }
335    }
336
337    fn cond_br(
338        &mut self,
339        cond: &'ll Value,
340        then_llbb: &'ll BasicBlock,
341        else_llbb: &'ll BasicBlock,
342    ) {
343        unsafe {
344            llvm::LLVMBuildCondBr(self.llbuilder, cond, then_llbb, else_llbb);
345        }
346    }
347
348    fn switch(
349        &mut self,
350        v: &'ll Value,
351        else_llbb: &'ll BasicBlock,
352        cases: impl ExactSizeIterator<Item = (u128, &'ll BasicBlock)>,
353    ) {
354        let switch =
355            unsafe { llvm::LLVMBuildSwitch(self.llbuilder, v, else_llbb, cases.len() as c_uint) };
356        for (on_val, dest) in cases {
357            let on_val = self.const_uint_big(self.val_ty(v), on_val);
358            unsafe { llvm::LLVMAddCase(switch, on_val, dest) }
359        }
360    }
361
362    fn switch_with_weights(
363        &mut self,
364        v: Self::Value,
365        else_llbb: Self::BasicBlock,
366        else_is_cold: bool,
367        cases: impl ExactSizeIterator<Item = (u128, Self::BasicBlock, bool)>,
368    ) {
369        if self.cx.sess().opts.optimize == rustc_session::config::OptLevel::No {
370            self.switch(v, else_llbb, cases.map(|(val, dest, _)| (val, dest)));
371            return;
372        }
373
374        let id = self.cx.create_metadata(b"branch_weights");
375
376        // For switch instructions with 2 targets, the `llvm.expect` intrinsic is used.
377        // This function handles switch instructions with more than 2 targets and it needs to
378        // emit branch weights metadata instead of using the intrinsic.
379        // The values 1 and 2000 are the same as the values used by the `llvm.expect` intrinsic.
380        let cold_weight = llvm::LLVMValueAsMetadata(self.cx.const_u32(1));
381        let hot_weight = llvm::LLVMValueAsMetadata(self.cx.const_u32(2000));
382        let weight =
383            |is_cold: bool| -> &Metadata { if is_cold { cold_weight } else { hot_weight } };
384
385        let mut md: SmallVec<[&Metadata; 16]> = SmallVec::with_capacity(cases.len() + 2);
386        md.push(id);
387        md.push(weight(else_is_cold));
388
389        let switch =
390            unsafe { llvm::LLVMBuildSwitch(self.llbuilder, v, else_llbb, cases.len() as c_uint) };
391        for (on_val, dest, is_cold) in cases {
392            let on_val = self.const_uint_big(self.val_ty(v), on_val);
393            unsafe { llvm::LLVMAddCase(switch, on_val, dest) }
394            md.push(weight(is_cold));
395        }
396
397        self.cx.set_metadata_node(switch, llvm::MD_prof, &md);
398    }
399
400    fn invoke(
401        &mut self,
402        llty: &'ll Type,
403        fn_attrs: Option<&CodegenFnAttrs>,
404        fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>,
405        llfn: &'ll Value,
406        args: &[&'ll Value],
407        then: &'ll BasicBlock,
408        catch: &'ll BasicBlock,
409        funclet: Option<&Funclet<'ll>>,
410        instance: Option<Instance<'tcx>>,
411    ) -> &'ll Value {
412        debug!("invoke {:?} with args ({:?})", llfn, args);
413
414        let args = self.check_call("invoke", llty, llfn, args);
415        let funclet_bundle = funclet.map(|funclet| funclet.bundle());
416        let mut bundles: SmallVec<[_; 2]> = SmallVec::new();
417        if let Some(funclet_bundle) = funclet_bundle {
418            bundles.push(funclet_bundle);
419        }
420
421        // Emit CFI pointer type membership test
422        self.cfi_type_test(fn_attrs, fn_abi, instance, llfn);
423
424        // Emit KCFI operand bundle
425        let kcfi_bundle = self.kcfi_operand_bundle(fn_attrs, fn_abi, instance, llfn);
426        if let Some(kcfi_bundle) = kcfi_bundle.as_ref().map(|b| b.as_ref()) {
427            bundles.push(kcfi_bundle);
428        }
429
430        let invoke = unsafe {
431            llvm::LLVMBuildInvokeWithOperandBundles(
432                self.llbuilder,
433                llty,
434                llfn,
435                args.as_ptr(),
436                args.len() as c_uint,
437                then,
438                catch,
439                bundles.as_ptr(),
440                bundles.len() as c_uint,
441                UNNAMED,
442            )
443        };
444        if let Some(fn_abi) = fn_abi {
445            fn_abi.apply_attrs_callsite(self, invoke);
446        }
447        invoke
448    }
449
450    fn unreachable(&mut self) {
451        unsafe {
452            llvm::LLVMBuildUnreachable(self.llbuilder);
453        }
454    }
455
456    math_builder_methods! {
457        add(a, b) => LLVMBuildAdd,
458        fadd(a, b) => LLVMBuildFAdd,
459        sub(a, b) => LLVMBuildSub,
460        fsub(a, b) => LLVMBuildFSub,
461        mul(a, b) => LLVMBuildMul,
462        fmul(a, b) => LLVMBuildFMul,
463        udiv(a, b) => LLVMBuildUDiv,
464        exactudiv(a, b) => LLVMBuildExactUDiv,
465        sdiv(a, b) => LLVMBuildSDiv,
466        exactsdiv(a, b) => LLVMBuildExactSDiv,
467        fdiv(a, b) => LLVMBuildFDiv,
468        urem(a, b) => LLVMBuildURem,
469        srem(a, b) => LLVMBuildSRem,
470        frem(a, b) => LLVMBuildFRem,
471        shl(a, b) => LLVMBuildShl,
472        lshr(a, b) => LLVMBuildLShr,
473        ashr(a, b) => LLVMBuildAShr,
474        and(a, b) => LLVMBuildAnd,
475        or(a, b) => LLVMBuildOr,
476        xor(a, b) => LLVMBuildXor,
477        neg(x) => LLVMBuildNeg,
478        fneg(x) => LLVMBuildFNeg,
479        not(x) => LLVMBuildNot,
480        unchecked_sadd(x, y) => LLVMBuildNSWAdd,
481        unchecked_uadd(x, y) => LLVMBuildNUWAdd,
482        unchecked_ssub(x, y) => LLVMBuildNSWSub,
483        unchecked_usub(x, y) => LLVMBuildNUWSub,
484        unchecked_smul(x, y) => LLVMBuildNSWMul,
485        unchecked_umul(x, y) => LLVMBuildNUWMul,
486    }
487
488    fn unchecked_suadd(&mut self, a: &'ll Value, b: &'ll Value) -> &'ll Value {
489        unsafe {
490            let add = llvm::LLVMBuildAdd(self.llbuilder, a, b, UNNAMED);
491            if llvm::LLVMIsAInstruction(add).is_some() {
492                llvm::LLVMSetNUW(add, TRUE);
493                llvm::LLVMSetNSW(add, TRUE);
494            }
495            add
496        }
497    }
498    fn unchecked_susub(&mut self, a: &'ll Value, b: &'ll Value) -> &'ll Value {
499        unsafe {
500            let sub = llvm::LLVMBuildSub(self.llbuilder, a, b, UNNAMED);
501            if llvm::LLVMIsAInstruction(sub).is_some() {
502                llvm::LLVMSetNUW(sub, TRUE);
503                llvm::LLVMSetNSW(sub, TRUE);
504            }
505            sub
506        }
507    }
508    fn unchecked_sumul(&mut self, a: &'ll Value, b: &'ll Value) -> &'ll Value {
509        unsafe {
510            let mul = llvm::LLVMBuildMul(self.llbuilder, a, b, UNNAMED);
511            if llvm::LLVMIsAInstruction(mul).is_some() {
512                llvm::LLVMSetNUW(mul, TRUE);
513                llvm::LLVMSetNSW(mul, TRUE);
514            }
515            mul
516        }
517    }
518
519    fn or_disjoint(&mut self, a: &'ll Value, b: &'ll Value) -> &'ll Value {
520        unsafe {
521            let or = llvm::LLVMBuildOr(self.llbuilder, a, b, UNNAMED);
522
523            // If a and b are both values, then `or` is a value, rather than
524            // an instruction, so we need to check before setting the flag.
525            // (See also `LLVMBuildNUWNeg` which also needs a check.)
526            if llvm::LLVMIsAInstruction(or).is_some() {
527                llvm::LLVMSetIsDisjoint(or, TRUE);
528            }
529            or
530        }
531    }
532
533    set_math_builder_methods! {
534        fadd_fast(x, y) => (LLVMBuildFAdd, LLVMRustSetFastMath),
535        fsub_fast(x, y) => (LLVMBuildFSub, LLVMRustSetFastMath),
536        fmul_fast(x, y) => (LLVMBuildFMul, LLVMRustSetFastMath),
537        fdiv_fast(x, y) => (LLVMBuildFDiv, LLVMRustSetFastMath),
538        frem_fast(x, y) => (LLVMBuildFRem, LLVMRustSetFastMath),
539        fadd_algebraic(x, y) => (LLVMBuildFAdd, LLVMRustSetAlgebraicMath),
540        fsub_algebraic(x, y) => (LLVMBuildFSub, LLVMRustSetAlgebraicMath),
541        fmul_algebraic(x, y) => (LLVMBuildFMul, LLVMRustSetAlgebraicMath),
542        fdiv_algebraic(x, y) => (LLVMBuildFDiv, LLVMRustSetAlgebraicMath),
543        frem_algebraic(x, y) => (LLVMBuildFRem, LLVMRustSetAlgebraicMath),
544    }
545
546    fn checked_binop(
547        &mut self,
548        oop: OverflowOp,
549        ty: Ty<'tcx>,
550        lhs: Self::Value,
551        rhs: Self::Value,
552    ) -> (Self::Value, Self::Value) {
553        let (size, signed) = ty.int_size_and_signed(self.tcx);
554        let width = size.bits();
555
556        if !signed {
557            match oop {
558                OverflowOp::Sub => {
559                    // Emit sub and icmp instead of llvm.usub.with.overflow. LLVM considers these
560                    // to be the canonical form. It will attempt to reform llvm.usub.with.overflow
561                    // in the backend if profitable.
562                    let sub = self.sub(lhs, rhs);
563                    let cmp = self.icmp(IntPredicate::IntULT, lhs, rhs);
564                    return (sub, cmp);
565                }
566                OverflowOp::Add => {
567                    // Like with sub above, using icmp is the preferred form. See
568                    // <https://rust-lang.zulipchat.com/#narrow/channel/187780-t-compiler.2Fllvm/topic/.60uadd.2Ewith.2Eoverflow.60.20.28again.29/near/533041085>
569                    let add = self.add(lhs, rhs);
570                    let cmp = self.icmp(IntPredicate::IntULT, add, lhs);
571                    return (add, cmp);
572                }
573                OverflowOp::Mul => {}
574            }
575        }
576
577        let oop_str = match oop {
578            OverflowOp::Add => "add",
579            OverflowOp::Sub => "sub",
580            OverflowOp::Mul => "mul",
581        };
582
583        let name = format!("llvm.{}{oop_str}.with.overflow", if signed { 's' } else { 'u' });
584
585        let res = self.call_intrinsic(name, &[self.type_ix(width)], &[lhs, rhs]);
586        (self.extract_value(res, 0), self.extract_value(res, 1))
587    }
588
589    fn from_immediate(&mut self, val: Self::Value) -> Self::Value {
590        if self.cx().val_ty(val) == self.cx().type_i1() {
591            self.zext(val, self.cx().type_i8())
592        } else {
593            val
594        }
595    }
596
597    fn to_immediate_scalar(&mut self, val: Self::Value, scalar: abi::Scalar) -> Self::Value {
598        if scalar.is_bool() {
599            return self.unchecked_utrunc(val, self.cx().type_i1());
600        }
601        val
602    }
603
604    fn alloca(&mut self, size: Size, align: Align) -> &'ll Value {
605        let mut bx = Builder::with_cx(self.cx);
606        bx.position_at_start(unsafe { llvm::LLVMGetFirstBasicBlock(self.llfn()) });
607        let ty = self.cx().type_array(self.cx().type_i8(), size.bytes());
608        unsafe {
609            let alloca = llvm::LLVMBuildAlloca(bx.llbuilder, ty, UNNAMED);
610            llvm::LLVMSetAlignment(alloca, align.bytes() as c_uint);
611            // Cast to default addrspace if necessary
612            llvm::LLVMBuildPointerCast(bx.llbuilder, alloca, self.cx().type_ptr(), UNNAMED)
613        }
614    }
615
616    fn scalable_alloca(&mut self, elt: u64, align: Align, element_ty: Ty<'_>) -> Self::Value {
617        let mut bx = Builder::with_cx(self.cx);
618        bx.position_at_start(unsafe { llvm::LLVMGetFirstBasicBlock(self.llfn()) });
619        let llvm_ty = match element_ty.kind() {
620            ty::Bool => bx.type_i1(),
621            ty::Int(int_ty) => self.cx.type_int_from_ty(*int_ty),
622            ty::Uint(uint_ty) => self.cx.type_uint_from_ty(*uint_ty),
623            ty::Float(float_ty) => self.cx.type_float_from_ty(*float_ty),
624            _ => unreachable!("scalable vectors can only contain a bool, int, uint or float"),
625        };
626
627        unsafe {
628            let ty = llvm::LLVMScalableVectorType(llvm_ty, elt.try_into().unwrap());
629            let alloca = llvm::LLVMBuildAlloca(&bx.llbuilder, ty, UNNAMED);
630            llvm::LLVMSetAlignment(alloca, align.bytes() as c_uint);
631            alloca
632        }
633    }
634
635    fn load(&mut self, ty: &'ll Type, ptr: &'ll Value, align: Align) -> &'ll Value {
636        unsafe {
637            let load = llvm::LLVMBuildLoad2(self.llbuilder, ty, ptr, UNNAMED);
638            let align = align.min(self.cx().tcx.sess.target.max_reliable_alignment());
639            llvm::LLVMSetAlignment(load, align.bytes() as c_uint);
640            load
641        }
642    }
643
644    fn volatile_load(&mut self, ty: &'ll Type, ptr: &'ll Value) -> &'ll Value {
645        unsafe {
646            let load = llvm::LLVMBuildLoad2(self.llbuilder, ty, ptr, UNNAMED);
647            llvm::LLVMSetVolatile(load, llvm::TRUE);
648            load
649        }
650    }
651
652    fn atomic_load(
653        &mut self,
654        ty: &'ll Type,
655        ptr: &'ll Value,
656        order: rustc_middle::ty::AtomicOrdering,
657        size: Size,
658    ) -> &'ll Value {
659        unsafe {
660            let load = llvm::LLVMBuildLoad2(self.llbuilder, ty, ptr, UNNAMED);
661            // Set atomic ordering
662            llvm::LLVMSetOrdering(load, AtomicOrdering::from_generic(order));
663            // LLVM requires the alignment of atomic loads to be at least the size of the type.
664            llvm::LLVMSetAlignment(load, size.bytes() as c_uint);
665            load
666        }
667    }
668
669    #[instrument(level = "trace", skip(self))]
670    fn load_operand(&mut self, place: PlaceRef<'tcx, &'ll Value>) -> OperandRef<'tcx, &'ll Value> {
671        if place.layout.is_unsized() {
672            let tail = self.tcx.struct_tail_for_codegen(place.layout.ty, self.typing_env());
673            if matches!(tail.kind(), ty::Foreign(..)) {
674                // Unsized locals and, at least conceptually, even unsized arguments must be copied
675                // around, which requires dynamically determining their size. Therefore, we cannot
676                // allow `extern` types here. Consult t-opsem before removing this check.
677                panic!("unsized locals must not be `extern` types");
678            }
679        }
680        assert_eq!(place.val.llextra.is_some(), place.layout.is_unsized());
681
682        if place.layout.is_zst() {
683            return OperandRef::zero_sized(place.layout);
684        }
685
686        #[instrument(level = "trace", skip(bx))]
687        fn scalar_load_metadata<'a, 'll, 'tcx>(
688            bx: &mut Builder<'a, 'll, 'tcx>,
689            load: &'ll Value,
690            scalar: abi::Scalar,
691            layout: TyAndLayout<'tcx>,
692            offset: Size,
693        ) {
694            if bx.cx.sess().opts.optimize == OptLevel::No {
695                // Don't emit metadata we're not going to use
696                return;
697            }
698
699            if !scalar.is_uninit_valid() {
700                bx.noundef_metadata(load);
701            }
702
703            match scalar.primitive() {
704                abi::Primitive::Int(..) => {
705                    if !scalar.is_always_valid(bx) {
706                        bx.range_metadata(load, scalar.valid_range(bx));
707                    }
708                }
709                abi::Primitive::Pointer(_) => {
710                    if !scalar.valid_range(bx).contains(0) {
711                        bx.nonnull_metadata(load);
712                    }
713
714                    if let Some(pointee) = layout.pointee_info_at(bx, offset)
715                        && let Some(_) = pointee.safe
716                    {
717                        bx.align_metadata(load, pointee.align);
718                    }
719                }
720                abi::Primitive::Float(_) => {}
721            }
722        }
723
724        let val = if let Some(_) = place.val.llextra {
725            // FIXME: Merge with the `else` below?
726            OperandValue::Ref(place.val)
727        } else if place.layout.is_llvm_immediate() {
728            let mut const_llval = None;
729            let llty = place.layout.llvm_type(self);
730            if let Some(global) = llvm::LLVMIsAGlobalVariable(place.val.llval) {
731                if llvm::LLVMIsGlobalConstant(global).is_true() {
732                    if let Some(init) = llvm::LLVMGetInitializer(global) {
733                        if self.val_ty(init) == llty {
734                            const_llval = Some(init);
735                        }
736                    }
737                }
738            }
739
740            let llval = const_llval.unwrap_or_else(|| {
741                let load = self.load(llty, place.val.llval, place.val.align);
742                if let abi::BackendRepr::Scalar(scalar) = place.layout.backend_repr {
743                    scalar_load_metadata(self, load, scalar, place.layout, Size::ZERO);
744                    self.to_immediate_scalar(load, scalar)
745                } else {
746                    load
747                }
748            });
749            OperandValue::Immediate(llval)
750        } else if let abi::BackendRepr::ScalarPair(a, b) = place.layout.backend_repr {
751            let b_offset = a.size(self).align_to(b.align(self).abi);
752
753            let mut load = |i, scalar: abi::Scalar, layout, align, offset| {
754                let llptr = if i == 0 {
755                    place.val.llval
756                } else {
757                    self.inbounds_ptradd(place.val.llval, self.const_usize(b_offset.bytes()))
758                };
759                let llty = place.layout.scalar_pair_element_llvm_type(self, i, false);
760                let load = self.load(llty, llptr, align);
761                scalar_load_metadata(self, load, scalar, layout, offset);
762                self.to_immediate_scalar(load, scalar)
763            };
764
765            OperandValue::Pair(
766                load(0, a, place.layout, place.val.align, Size::ZERO),
767                load(1, b, place.layout, place.val.align.restrict_for_offset(b_offset), b_offset),
768            )
769        } else {
770            OperandValue::Ref(place.val)
771        };
772
773        OperandRef { val, layout: place.layout, move_annotation: None }
774    }
775
776    fn write_operand_repeatedly(
777        &mut self,
778        cg_elem: OperandRef<'tcx, &'ll Value>,
779        count: u64,
780        dest: PlaceRef<'tcx, &'ll Value>,
781    ) {
782        if self.cx.sess().opts.optimize == OptLevel::No {
783            // To let debuggers single-step over lines like
784            //
785            //     let foo = ["bar"; 42];
786            //
787            // we need the debugger-friendly LLVM IR that `_unoptimized()`
788            // provides. The `_optimized()` version generates trickier LLVM IR.
789            // See PR #148058 for a failed attempt at handling that.
790            self.write_operand_repeatedly_unoptimized(cg_elem, count, dest);
791        } else {
792            self.write_operand_repeatedly_optimized(cg_elem, count, dest);
793        }
794    }
795
796    fn range_metadata(&mut self, load: &'ll Value, range: WrappingRange) {
797        if self.cx.sess().opts.optimize == OptLevel::No {
798            // Don't emit metadata we're not going to use
799            return;
800        }
801
802        let llty = self.cx.val_ty(load);
803        let md = [
804            llvm::LLVMValueAsMetadata(self.cx.const_uint_big(llty, range.start)),
805            llvm::LLVMValueAsMetadata(self.cx.const_uint_big(llty, range.end.wrapping_add(1))),
806        ];
807        self.set_metadata_node(load, llvm::MD_range, &md);
808    }
809
810    fn nonnull_metadata(&mut self, load: &'ll Value) {
811        self.set_metadata_node(load, llvm::MD_nonnull, &[]);
812    }
813
814    fn store(&mut self, val: &'ll Value, ptr: &'ll Value, align: Align) -> &'ll Value {
815        self.store_with_flags(val, ptr, align, MemFlags::empty())
816    }
817
818    fn store_with_flags(
819        &mut self,
820        val: &'ll Value,
821        ptr: &'ll Value,
822        align: Align,
823        flags: MemFlags,
824    ) -> &'ll Value {
825        debug!("Store {:?} -> {:?} ({:?})", val, ptr, flags);
826        assert_eq!(self.cx.type_kind(self.cx.val_ty(ptr)), TypeKind::Pointer);
827        unsafe {
828            let store = llvm::LLVMBuildStore(self.llbuilder, val, ptr);
829            let align = align.min(self.cx().tcx.sess.target.max_reliable_alignment());
830            let align =
831                if flags.contains(MemFlags::UNALIGNED) { 1 } else { align.bytes() as c_uint };
832            llvm::LLVMSetAlignment(store, align);
833            if flags.contains(MemFlags::VOLATILE) {
834                llvm::LLVMSetVolatile(store, llvm::TRUE);
835            }
836            if flags.contains(MemFlags::NONTEMPORAL) {
837                // Make sure that the current target architectures supports "sane" non-temporal
838                // stores, i.e., non-temporal stores that are equivalent to regular stores except
839                // for performance. LLVM doesn't seem to care about this, and will happily treat
840                // `!nontemporal` stores as-if they were normal stores (for reordering optimizations
841                // etc) even on x86, despite later lowering them to MOVNT which do *not* behave like
842                // regular stores but require special fences. So we keep a list of architectures
843                // where `!nontemporal` is known to be truly just a hint, and use regular stores
844                // everywhere else. (In the future, we could alternatively ensure that an sfence
845                // gets emitted after a sequence of movnt before any kind of synchronizing
846                // operation. But it's not clear how to do that with LLVM.)
847                // For more context, see <https://github.com/rust-lang/rust/issues/114582> and
848                // <https://github.com/llvm/llvm-project/issues/64521>.
849                let use_nontemporal = matches!(
850                    self.cx.tcx.sess.target.arch,
851                    Arch::AArch64 | Arch::Arm | Arch::RiscV32 | Arch::RiscV64
852                );
853                if use_nontemporal {
854                    // According to LLVM [1] building a nontemporal store must
855                    // *always* point to a metadata value of the integer 1.
856                    //
857                    // [1]: https://llvm.org/docs/LangRef.html#store-instruction
858                    let one = llvm::LLVMValueAsMetadata(self.cx.const_i32(1));
859                    self.set_metadata_node(store, llvm::MD_nontemporal, &[one]);
860                }
861            }
862            store
863        }
864    }
865
866    fn atomic_store(
867        &mut self,
868        val: &'ll Value,
869        ptr: &'ll Value,
870        order: rustc_middle::ty::AtomicOrdering,
871        size: Size,
872    ) {
873        debug!("Store {:?} -> {:?}", val, ptr);
874        assert_eq!(self.cx.type_kind(self.cx.val_ty(ptr)), TypeKind::Pointer);
875        unsafe {
876            let store = llvm::LLVMBuildStore(self.llbuilder, val, ptr);
877            // Set atomic ordering
878            llvm::LLVMSetOrdering(store, AtomicOrdering::from_generic(order));
879            // LLVM requires the alignment of atomic stores to be at least the size of the type.
880            llvm::LLVMSetAlignment(store, size.bytes() as c_uint);
881        }
882    }
883
884    fn gep(&mut self, ty: &'ll Type, ptr: &'ll Value, indices: &[&'ll Value]) -> &'ll Value {
885        unsafe {
886            llvm::LLVMBuildGEPWithNoWrapFlags(
887                self.llbuilder,
888                ty,
889                ptr,
890                indices.as_ptr(),
891                indices.len() as c_uint,
892                UNNAMED,
893                GEPNoWrapFlags::default(),
894            )
895        }
896    }
897
898    fn inbounds_gep(
899        &mut self,
900        ty: &'ll Type,
901        ptr: &'ll Value,
902        indices: &[&'ll Value],
903    ) -> &'ll Value {
904        unsafe {
905            llvm::LLVMBuildGEPWithNoWrapFlags(
906                self.llbuilder,
907                ty,
908                ptr,
909                indices.as_ptr(),
910                indices.len() as c_uint,
911                UNNAMED,
912                GEPNoWrapFlags::InBounds,
913            )
914        }
915    }
916
917    fn inbounds_nuw_gep(
918        &mut self,
919        ty: &'ll Type,
920        ptr: &'ll Value,
921        indices: &[&'ll Value],
922    ) -> &'ll Value {
923        unsafe {
924            llvm::LLVMBuildGEPWithNoWrapFlags(
925                self.llbuilder,
926                ty,
927                ptr,
928                indices.as_ptr(),
929                indices.len() as c_uint,
930                UNNAMED,
931                GEPNoWrapFlags::InBounds | GEPNoWrapFlags::NUW,
932            )
933        }
934    }
935
936    /* Casts */
937    fn trunc(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
938        unsafe { llvm::LLVMBuildTrunc(self.llbuilder, val, dest_ty, UNNAMED) }
939    }
940
941    fn unchecked_utrunc(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
942        debug_assert_ne!(self.val_ty(val), dest_ty);
943
944        let trunc = self.trunc(val, dest_ty);
945        unsafe {
946            if llvm::LLVMIsAInstruction(trunc).is_some() {
947                llvm::LLVMSetNUW(trunc, TRUE);
948            }
949        }
950        trunc
951    }
952
953    fn unchecked_strunc(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
954        debug_assert_ne!(self.val_ty(val), dest_ty);
955
956        let trunc = self.trunc(val, dest_ty);
957        unsafe {
958            if llvm::LLVMIsAInstruction(trunc).is_some() {
959                llvm::LLVMSetNSW(trunc, TRUE);
960            }
961        }
962        trunc
963    }
964
965    fn sext(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
966        unsafe { llvm::LLVMBuildSExt(self.llbuilder, val, dest_ty, UNNAMED) }
967    }
968
969    fn fptoui_sat(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
970        self.call_intrinsic("llvm.fptoui.sat", &[dest_ty, self.val_ty(val)], &[val])
971    }
972
973    fn fptosi_sat(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
974        self.call_intrinsic("llvm.fptosi.sat", &[dest_ty, self.val_ty(val)], &[val])
975    }
976
977    fn fptoui(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
978        // On WebAssembly the `fptoui` and `fptosi` instructions currently have
979        // poor codegen. The reason for this is that the corresponding wasm
980        // instructions, `i32.trunc_f32_s` for example, will trap when the float
981        // is out-of-bounds, infinity, or nan. This means that LLVM
982        // automatically inserts control flow around `fptoui` and `fptosi`
983        // because the LLVM instruction `fptoui` is defined as producing a
984        // poison value, not having UB on out-of-bounds values.
985        //
986        // This method, however, is only used with non-saturating casts that
987        // have UB on out-of-bounds values. This means that it's ok if we use
988        // the raw wasm instruction since out-of-bounds values can do whatever
989        // we like. To ensure that LLVM picks the right instruction we choose
990        // the raw wasm intrinsic functions which avoid LLVM inserting all the
991        // other control flow automatically.
992        if self.sess().target.is_like_wasm {
993            let src_ty = self.cx.val_ty(val);
994            if self.cx.type_kind(src_ty) != TypeKind::Vector {
995                let float_width = self.cx.float_width(src_ty);
996                let int_width = self.cx.int_width(dest_ty);
997                if matches!((int_width, float_width), (32 | 64, 32 | 64)) {
998                    return self.call_intrinsic(
999                        "llvm.wasm.trunc.unsigned",
1000                        &[dest_ty, src_ty],
1001                        &[val],
1002                    );
1003                }
1004            }
1005        }
1006        unsafe { llvm::LLVMBuildFPToUI(self.llbuilder, val, dest_ty, UNNAMED) }
1007    }
1008
1009    fn fptosi(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1010        // see `fptoui` above for why wasm is different here
1011        if self.sess().target.is_like_wasm {
1012            let src_ty = self.cx.val_ty(val);
1013            if self.cx.type_kind(src_ty) != TypeKind::Vector {
1014                let float_width = self.cx.float_width(src_ty);
1015                let int_width = self.cx.int_width(dest_ty);
1016                if matches!((int_width, float_width), (32 | 64, 32 | 64)) {
1017                    return self.call_intrinsic(
1018                        "llvm.wasm.trunc.signed",
1019                        &[dest_ty, src_ty],
1020                        &[val],
1021                    );
1022                }
1023            }
1024        }
1025        unsafe { llvm::LLVMBuildFPToSI(self.llbuilder, val, dest_ty, UNNAMED) }
1026    }
1027
1028    fn uitofp(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1029        unsafe { llvm::LLVMBuildUIToFP(self.llbuilder, val, dest_ty, UNNAMED) }
1030    }
1031
1032    fn sitofp(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1033        unsafe { llvm::LLVMBuildSIToFP(self.llbuilder, val, dest_ty, UNNAMED) }
1034    }
1035
1036    fn fptrunc(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1037        unsafe { llvm::LLVMBuildFPTrunc(self.llbuilder, val, dest_ty, UNNAMED) }
1038    }
1039
1040    fn fpext(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1041        unsafe { llvm::LLVMBuildFPExt(self.llbuilder, val, dest_ty, UNNAMED) }
1042    }
1043
1044    fn ptrtoint(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1045        unsafe { llvm::LLVMBuildPtrToInt(self.llbuilder, val, dest_ty, UNNAMED) }
1046    }
1047
1048    fn inttoptr(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1049        unsafe { llvm::LLVMBuildIntToPtr(self.llbuilder, val, dest_ty, UNNAMED) }
1050    }
1051
1052    fn bitcast(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1053        unsafe { llvm::LLVMBuildBitCast(self.llbuilder, val, dest_ty, UNNAMED) }
1054    }
1055
1056    fn intcast(&mut self, val: &'ll Value, dest_ty: &'ll Type, is_signed: bool) -> &'ll Value {
1057        unsafe {
1058            llvm::LLVMBuildIntCast2(self.llbuilder, val, dest_ty, is_signed.to_llvm_bool(), UNNAMED)
1059        }
1060    }
1061
1062    fn pointercast(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1063        unsafe { llvm::LLVMBuildPointerCast(self.llbuilder, val, dest_ty, UNNAMED) }
1064    }
1065
1066    /* Comparisons */
1067    fn icmp(&mut self, op: IntPredicate, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
1068        let op = llvm::IntPredicate::from_generic(op);
1069        unsafe { llvm::LLVMBuildICmp(self.llbuilder, op as c_uint, lhs, rhs, UNNAMED) }
1070    }
1071
1072    fn fcmp(&mut self, op: RealPredicate, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
1073        let op = llvm::RealPredicate::from_generic(op);
1074        unsafe { llvm::LLVMBuildFCmp(self.llbuilder, op as c_uint, lhs, rhs, UNNAMED) }
1075    }
1076
1077    fn three_way_compare(
1078        &mut self,
1079        ty: Ty<'tcx>,
1080        lhs: Self::Value,
1081        rhs: Self::Value,
1082    ) -> Self::Value {
1083        let size = ty.primitive_size(self.tcx);
1084        let name = if ty.is_signed() { "llvm.scmp" } else { "llvm.ucmp" };
1085
1086        self.call_intrinsic(name, &[self.type_i8(), self.type_ix(size.bits())], &[lhs, rhs])
1087    }
1088
1089    /* Miscellaneous instructions */
1090    fn memcpy(
1091        &mut self,
1092        dst: &'ll Value,
1093        dst_align: Align,
1094        src: &'ll Value,
1095        src_align: Align,
1096        size: &'ll Value,
1097        flags: MemFlags,
1098        tt: Option<FncTree>,
1099    ) {
1100        assert!(!flags.contains(MemFlags::NONTEMPORAL), "non-temporal memcpy not supported");
1101        let size = self.intcast(size, self.type_isize(), false);
1102        let is_volatile = flags.contains(MemFlags::VOLATILE);
1103        let memcpy = unsafe {
1104            llvm::LLVMRustBuildMemCpy(
1105                self.llbuilder,
1106                dst,
1107                dst_align.bytes() as c_uint,
1108                src,
1109                src_align.bytes() as c_uint,
1110                size,
1111                is_volatile,
1112            )
1113        };
1114
1115        // TypeTree metadata for memcpy is especially important: when Enzyme encounters
1116        // a memcpy during autodiff, it needs to know the structure of the data being
1117        // copied to properly track derivatives. For example, copying an array of floats
1118        // vs. copying a struct with mixed types requires different derivative handling.
1119        // The TypeTree tells Enzyme exactly what memory layout to expect.
1120        if let Some(tt) = tt {
1121            crate::typetree::add_tt(self.cx().llmod, self.cx().llcx, memcpy, tt);
1122        }
1123    }
1124
1125    fn memmove(
1126        &mut self,
1127        dst: &'ll Value,
1128        dst_align: Align,
1129        src: &'ll Value,
1130        src_align: Align,
1131        size: &'ll Value,
1132        flags: MemFlags,
1133    ) {
1134        assert!(!flags.contains(MemFlags::NONTEMPORAL), "non-temporal memmove not supported");
1135        let size = self.intcast(size, self.type_isize(), false);
1136        let is_volatile = flags.contains(MemFlags::VOLATILE);
1137        unsafe {
1138            llvm::LLVMRustBuildMemMove(
1139                self.llbuilder,
1140                dst,
1141                dst_align.bytes() as c_uint,
1142                src,
1143                src_align.bytes() as c_uint,
1144                size,
1145                is_volatile,
1146            );
1147        }
1148    }
1149
1150    fn memset(
1151        &mut self,
1152        ptr: &'ll Value,
1153        fill_byte: &'ll Value,
1154        size: &'ll Value,
1155        align: Align,
1156        flags: MemFlags,
1157    ) {
1158        assert!(!flags.contains(MemFlags::NONTEMPORAL), "non-temporal memset not supported");
1159        let is_volatile = flags.contains(MemFlags::VOLATILE);
1160        unsafe {
1161            llvm::LLVMRustBuildMemSet(
1162                self.llbuilder,
1163                ptr,
1164                align.bytes() as c_uint,
1165                fill_byte,
1166                size,
1167                is_volatile,
1168            );
1169        }
1170    }
1171
1172    fn select(
1173        &mut self,
1174        cond: &'ll Value,
1175        then_val: &'ll Value,
1176        else_val: &'ll Value,
1177    ) -> &'ll Value {
1178        unsafe { llvm::LLVMBuildSelect(self.llbuilder, cond, then_val, else_val, UNNAMED) }
1179    }
1180
1181    fn va_arg(&mut self, list: &'ll Value, ty: &'ll Type) -> &'ll Value {
1182        unsafe { llvm::LLVMBuildVAArg(self.llbuilder, list, ty, UNNAMED) }
1183    }
1184
1185    fn extract_element(&mut self, vec: &'ll Value, idx: &'ll Value) -> &'ll Value {
1186        unsafe { llvm::LLVMBuildExtractElement(self.llbuilder, vec, idx, UNNAMED) }
1187    }
1188
1189    fn vector_splat(&mut self, num_elts: usize, elt: &'ll Value) -> &'ll Value {
1190        unsafe {
1191            let elt_ty = self.cx.val_ty(elt);
1192            let undef = llvm::LLVMGetUndef(self.type_vector(elt_ty, num_elts as u64));
1193            let vec = self.insert_element(undef, elt, self.cx.const_i32(0));
1194            let vec_i32_ty = self.type_vector(self.type_i32(), num_elts as u64);
1195            self.shuffle_vector(vec, undef, self.const_null(vec_i32_ty))
1196        }
1197    }
1198
1199    fn extract_value(&mut self, agg_val: &'ll Value, idx: u64) -> &'ll Value {
1200        assert_eq!(idx as c_uint as u64, idx);
1201        unsafe { llvm::LLVMBuildExtractValue(self.llbuilder, agg_val, idx as c_uint, UNNAMED) }
1202    }
1203
1204    fn insert_value(&mut self, agg_val: &'ll Value, elt: &'ll Value, idx: u64) -> &'ll Value {
1205        assert_eq!(idx as c_uint as u64, idx);
1206        unsafe { llvm::LLVMBuildInsertValue(self.llbuilder, agg_val, elt, idx as c_uint, UNNAMED) }
1207    }
1208
1209    fn set_personality_fn(&mut self, personality: &'ll Value) {
1210        unsafe {
1211            llvm::LLVMSetPersonalityFn(self.llfn(), personality);
1212        }
1213    }
1214
1215    fn cleanup_landing_pad(&mut self, pers_fn: &'ll Value) -> (&'ll Value, &'ll Value) {
1216        let ty = self.type_struct(&[self.type_ptr(), self.type_i32()], false);
1217        let landing_pad = self.landing_pad(ty, pers_fn, 0);
1218        unsafe {
1219            llvm::LLVMSetCleanup(landing_pad, llvm::TRUE);
1220        }
1221        (self.extract_value(landing_pad, 0), self.extract_value(landing_pad, 1))
1222    }
1223
1224    fn filter_landing_pad(&mut self, pers_fn: &'ll Value) {
1225        let ty = self.type_struct(&[self.type_ptr(), self.type_i32()], false);
1226        let landing_pad = self.landing_pad(ty, pers_fn, 1);
1227        self.add_clause(landing_pad, self.const_array(self.type_ptr(), &[]));
1228    }
1229
1230    fn resume(&mut self, exn0: &'ll Value, exn1: &'ll Value) {
1231        let ty = self.type_struct(&[self.type_ptr(), self.type_i32()], false);
1232        let mut exn = self.const_poison(ty);
1233        exn = self.insert_value(exn, exn0, 0);
1234        exn = self.insert_value(exn, exn1, 1);
1235        unsafe {
1236            llvm::LLVMBuildResume(self.llbuilder, exn);
1237        }
1238    }
1239
1240    fn cleanup_pad(&mut self, parent: Option<&'ll Value>, args: &[&'ll Value]) -> Funclet<'ll> {
1241        let ret = unsafe {
1242            llvm::LLVMBuildCleanupPad(
1243                self.llbuilder,
1244                parent,
1245                args.as_ptr(),
1246                args.len() as c_uint,
1247                c"cleanuppad".as_ptr(),
1248            )
1249        };
1250        Funclet::new(ret.expect("LLVM does not have support for cleanuppad"))
1251    }
1252
1253    fn cleanup_ret(&mut self, funclet: &Funclet<'ll>, unwind: Option<&'ll BasicBlock>) {
1254        unsafe {
1255            llvm::LLVMBuildCleanupRet(self.llbuilder, funclet.cleanuppad(), unwind)
1256                .expect("LLVM does not have support for cleanupret");
1257        }
1258    }
1259
1260    fn catch_pad(&mut self, parent: &'ll Value, args: &[&'ll Value]) -> Funclet<'ll> {
1261        let ret = unsafe {
1262            llvm::LLVMBuildCatchPad(
1263                self.llbuilder,
1264                parent,
1265                args.as_ptr(),
1266                args.len() as c_uint,
1267                c"catchpad".as_ptr(),
1268            )
1269        };
1270        Funclet::new(ret.expect("LLVM does not have support for catchpad"))
1271    }
1272
1273    fn catch_switch(
1274        &mut self,
1275        parent: Option<&'ll Value>,
1276        unwind: Option<&'ll BasicBlock>,
1277        handlers: &[&'ll BasicBlock],
1278    ) -> &'ll Value {
1279        let ret = unsafe {
1280            llvm::LLVMBuildCatchSwitch(
1281                self.llbuilder,
1282                parent,
1283                unwind,
1284                handlers.len() as c_uint,
1285                c"catchswitch".as_ptr(),
1286            )
1287        };
1288        let ret = ret.expect("LLVM does not have support for catchswitch");
1289        for handler in handlers {
1290            unsafe {
1291                llvm::LLVMAddHandler(ret, handler);
1292            }
1293        }
1294        ret
1295    }
1296
1297    // Atomic Operations
1298    fn atomic_cmpxchg(
1299        &mut self,
1300        dst: &'ll Value,
1301        cmp: &'ll Value,
1302        src: &'ll Value,
1303        order: rustc_middle::ty::AtomicOrdering,
1304        failure_order: rustc_middle::ty::AtomicOrdering,
1305        weak: bool,
1306    ) -> (&'ll Value, &'ll Value) {
1307        unsafe {
1308            let value = llvm::LLVMBuildAtomicCmpXchg(
1309                self.llbuilder,
1310                dst,
1311                cmp,
1312                src,
1313                AtomicOrdering::from_generic(order),
1314                AtomicOrdering::from_generic(failure_order),
1315                llvm::FALSE, // SingleThreaded
1316            );
1317            llvm::LLVMSetWeak(value, weak.to_llvm_bool());
1318            let val = self.extract_value(value, 0);
1319            let success = self.extract_value(value, 1);
1320            (val, success)
1321        }
1322    }
1323
1324    fn atomic_rmw(
1325        &mut self,
1326        op: rustc_codegen_ssa::common::AtomicRmwBinOp,
1327        dst: &'ll Value,
1328        src: &'ll Value,
1329        order: rustc_middle::ty::AtomicOrdering,
1330        ret_ptr: bool,
1331    ) -> &'ll Value {
1332        // FIXME: If `ret_ptr` is true and `src` is not a pointer, we *should* tell LLVM that the
1333        // LHS is a pointer and the operation should be provenance-preserving, but LLVM does not
1334        // currently support that (https://github.com/llvm/llvm-project/issues/120837).
1335        let mut res = unsafe {
1336            llvm::LLVMBuildAtomicRMW(
1337                self.llbuilder,
1338                AtomicRmwBinOp::from_generic(op),
1339                dst,
1340                src,
1341                AtomicOrdering::from_generic(order),
1342                llvm::FALSE, // SingleThreaded
1343            )
1344        };
1345        if ret_ptr && self.val_ty(res) != self.type_ptr() {
1346            res = self.inttoptr(res, self.type_ptr());
1347        }
1348        res
1349    }
1350
1351    fn atomic_fence(
1352        &mut self,
1353        order: rustc_middle::ty::AtomicOrdering,
1354        scope: SynchronizationScope,
1355    ) {
1356        let single_threaded = match scope {
1357            SynchronizationScope::SingleThread => true,
1358            SynchronizationScope::CrossThread => false,
1359        };
1360        unsafe {
1361            llvm::LLVMBuildFence(
1362                self.llbuilder,
1363                AtomicOrdering::from_generic(order),
1364                single_threaded.to_llvm_bool(),
1365                UNNAMED,
1366            );
1367        }
1368    }
1369
1370    fn set_invariant_load(&mut self, load: &'ll Value) {
1371        self.set_metadata_node(load, llvm::MD_invariant_load, &[]);
1372    }
1373
1374    fn lifetime_start(&mut self, ptr: &'ll Value, size: Size) {
1375        self.call_lifetime_intrinsic("llvm.lifetime.start", ptr, size);
1376    }
1377
1378    fn lifetime_end(&mut self, ptr: &'ll Value, size: Size) {
1379        self.call_lifetime_intrinsic("llvm.lifetime.end", ptr, size);
1380    }
1381
1382    fn call(
1383        &mut self,
1384        llty: &'ll Type,
1385        fn_call_attrs: Option<&CodegenFnAttrs>,
1386        fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>,
1387        llfn: &'ll Value,
1388        args: &[&'ll Value],
1389        funclet: Option<&Funclet<'ll>>,
1390        instance: Option<Instance<'tcx>>,
1391    ) -> &'ll Value {
1392        debug!("call {:?} with args ({:?})", llfn, args);
1393
1394        let args = self.check_call("call", llty, llfn, args);
1395        let funclet_bundle = funclet.map(|funclet| funclet.bundle());
1396        let mut bundles: SmallVec<[_; 2]> = SmallVec::new();
1397        if let Some(funclet_bundle) = funclet_bundle {
1398            bundles.push(funclet_bundle);
1399        }
1400
1401        // Emit CFI pointer type membership test
1402        self.cfi_type_test(fn_call_attrs, fn_abi, instance, llfn);
1403
1404        // Emit KCFI operand bundle
1405        let kcfi_bundle = self.kcfi_operand_bundle(fn_call_attrs, fn_abi, instance, llfn);
1406        if let Some(kcfi_bundle) = kcfi_bundle.as_ref().map(|b| b.as_ref()) {
1407            bundles.push(kcfi_bundle);
1408        }
1409
1410        let call = unsafe {
1411            llvm::LLVMBuildCallWithOperandBundles(
1412                self.llbuilder,
1413                llty,
1414                llfn,
1415                args.as_ptr() as *const &llvm::Value,
1416                args.len() as c_uint,
1417                bundles.as_ptr(),
1418                bundles.len() as c_uint,
1419                c"".as_ptr(),
1420            )
1421        };
1422
1423        if let Some(instance) = instance {
1424            // Attributes on the function definition being called
1425            let fn_defn_attrs = self.cx.tcx.codegen_fn_attrs(instance.def_id());
1426            if let Some(fn_call_attrs) = fn_call_attrs
1427                // If there is an inline attribute and a target feature that matches
1428                // we will add the attribute to the callsite otherwise we'll omit
1429                // this and not add the attribute to prevent soundness issues.
1430                && let Some(inlining_rule) = attributes::inline_attr(&self.cx, self.cx.tcx, instance)
1431                && self.cx.tcx.is_target_feature_call_safe(
1432                    &fn_defn_attrs.target_features,
1433                    &fn_call_attrs.target_features.iter().cloned().chain(
1434                        self.cx.tcx.sess.target_features.iter().map(|feat| TargetFeature {
1435                            name: *feat,
1436                            kind: TargetFeatureKind::Implied,
1437                        })
1438                    ).collect::<Vec<_>>(),
1439                )
1440            {
1441                attributes::apply_to_callsite(
1442                    call,
1443                    llvm::AttributePlace::Function,
1444                    &[inlining_rule],
1445                );
1446            }
1447        }
1448
1449        if let Some(fn_abi) = fn_abi {
1450            fn_abi.apply_attrs_callsite(self, call);
1451        }
1452        call
1453    }
1454
1455    fn tail_call(
1456        &mut self,
1457        llty: Self::Type,
1458        fn_attrs: Option<&CodegenFnAttrs>,
1459        fn_abi: &FnAbi<'tcx, Ty<'tcx>>,
1460        llfn: Self::Value,
1461        args: &[Self::Value],
1462        funclet: Option<&Self::Funclet>,
1463        instance: Option<Instance<'tcx>>,
1464    ) {
1465        let call = self.call(llty, fn_attrs, Some(fn_abi), llfn, args, funclet, instance);
1466        llvm::LLVMSetTailCallKind(call, llvm::TailCallKind::MustTail);
1467
1468        match &fn_abi.ret.mode {
1469            PassMode::Ignore | PassMode::Indirect { .. } => self.ret_void(),
1470            PassMode::Direct(_) | PassMode::Pair { .. } | PassMode::Cast { .. } => self.ret(call),
1471        }
1472    }
1473
1474    fn zext(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1475        unsafe { llvm::LLVMBuildZExt(self.llbuilder, val, dest_ty, UNNAMED) }
1476    }
1477
1478    fn apply_attrs_to_cleanup_callsite(&mut self, llret: &'ll Value) {
1479        // Cleanup is always the cold path.
1480        let cold_inline = llvm::AttributeKind::Cold.create_attr(self.llcx);
1481        attributes::apply_to_callsite(llret, llvm::AttributePlace::Function, &[cold_inline]);
1482    }
1483}
1484
1485impl<'ll> StaticBuilderMethods for Builder<'_, 'll, '_> {
1486    fn get_static(&mut self, def_id: DefId) -> &'ll Value {
1487        // Forward to the `get_static` method of `CodegenCx`
1488        let global = self.cx().get_static(def_id);
1489        if self.cx().tcx.is_thread_local_static(def_id) {
1490            let pointer =
1491                self.call_intrinsic("llvm.threadlocal.address", &[self.val_ty(global)], &[global]);
1492            // Cast to default address space if globals are in a different addrspace
1493            self.pointercast(pointer, self.type_ptr())
1494        } else {
1495            // Cast to default address space if globals are in a different addrspace
1496            self.cx().const_pointercast(global, self.type_ptr())
1497        }
1498    }
1499}
1500
1501impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
1502    pub(crate) fn llfn(&self) -> &'ll Value {
1503        unsafe { llvm::LLVMGetBasicBlockParent(self.llbb()) }
1504    }
1505}
1506
1507impl<'a, 'll, CX: Borrow<SCx<'ll>>> GenericBuilder<'a, 'll, CX> {
1508    fn position_at_start(&mut self, llbb: &'ll BasicBlock) {
1509        unsafe {
1510            llvm::LLVMRustPositionBuilderAtStart(self.llbuilder, llbb);
1511        }
1512    }
1513}
1514impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
1515    fn align_metadata(&mut self, load: &'ll Value, align: Align) {
1516        let md = [llvm::LLVMValueAsMetadata(self.cx.const_u64(align.bytes()))];
1517        self.set_metadata_node(load, llvm::MD_align, &md);
1518    }
1519
1520    fn noundef_metadata(&mut self, load: &'ll Value) {
1521        self.set_metadata_node(load, llvm::MD_noundef, &[]);
1522    }
1523
1524    pub(crate) fn set_unpredictable(&mut self, inst: &'ll Value) {
1525        self.set_metadata_node(inst, llvm::MD_unpredictable, &[]);
1526    }
1527
1528    fn write_operand_repeatedly_optimized(
1529        &mut self,
1530        cg_elem: OperandRef<'tcx, &'ll Value>,
1531        count: u64,
1532        dest: PlaceRef<'tcx, &'ll Value>,
1533    ) {
1534        let zero = self.const_usize(0);
1535        let count = self.const_usize(count);
1536
1537        let header_bb = self.append_sibling_block("repeat_loop_header");
1538        let body_bb = self.append_sibling_block("repeat_loop_body");
1539        let next_bb = self.append_sibling_block("repeat_loop_next");
1540
1541        self.br(header_bb);
1542
1543        let mut header_bx = Self::build(self.cx, header_bb);
1544        let i = header_bx.phi(self.val_ty(zero), &[zero], &[self.llbb()]);
1545
1546        let keep_going = header_bx.icmp(IntPredicate::IntULT, i, count);
1547        header_bx.cond_br(keep_going, body_bb, next_bb);
1548
1549        let mut body_bx = Self::build(self.cx, body_bb);
1550        let dest_elem = dest.project_index(&mut body_bx, i);
1551        cg_elem.val.store(&mut body_bx, dest_elem);
1552
1553        let next = body_bx.unchecked_uadd(i, self.const_usize(1));
1554        body_bx.br(header_bb);
1555        header_bx.add_incoming_to_phi(i, next, body_bb);
1556
1557        *self = Self::build(self.cx, next_bb);
1558    }
1559
1560    fn write_operand_repeatedly_unoptimized(
1561        &mut self,
1562        cg_elem: OperandRef<'tcx, &'ll Value>,
1563        count: u64,
1564        dest: PlaceRef<'tcx, &'ll Value>,
1565    ) {
1566        let zero = self.const_usize(0);
1567        let count = self.const_usize(count);
1568        let start = dest.project_index(self, zero).val.llval;
1569        let end = dest.project_index(self, count).val.llval;
1570
1571        let header_bb = self.append_sibling_block("repeat_loop_header");
1572        let body_bb = self.append_sibling_block("repeat_loop_body");
1573        let next_bb = self.append_sibling_block("repeat_loop_next");
1574
1575        self.br(header_bb);
1576
1577        let mut header_bx = Self::build(self.cx, header_bb);
1578        let current = header_bx.phi(self.val_ty(start), &[start], &[self.llbb()]);
1579
1580        let keep_going = header_bx.icmp(IntPredicate::IntNE, current, end);
1581        header_bx.cond_br(keep_going, body_bb, next_bb);
1582
1583        let mut body_bx = Self::build(self.cx, body_bb);
1584        let align = dest.val.align.restrict_for_offset(dest.layout.field(self.cx(), 0).size);
1585        cg_elem
1586            .val
1587            .store(&mut body_bx, PlaceRef::new_sized_aligned(current, cg_elem.layout, align));
1588
1589        let next = body_bx.inbounds_gep(
1590            self.backend_type(cg_elem.layout),
1591            current,
1592            &[self.const_usize(1)],
1593        );
1594        body_bx.br(header_bb);
1595        header_bx.add_incoming_to_phi(current, next, body_bb);
1596
1597        *self = Self::build(self.cx, next_bb);
1598    }
1599
1600    pub(crate) fn minnum(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
1601        self.call_intrinsic("llvm.minnum", &[self.val_ty(lhs)], &[lhs, rhs])
1602    }
1603
1604    pub(crate) fn maxnum(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
1605        self.call_intrinsic("llvm.maxnum", &[self.val_ty(lhs)], &[lhs, rhs])
1606    }
1607
1608    pub(crate) fn insert_element(
1609        &mut self,
1610        vec: &'ll Value,
1611        elt: &'ll Value,
1612        idx: &'ll Value,
1613    ) -> &'ll Value {
1614        unsafe { llvm::LLVMBuildInsertElement(self.llbuilder, vec, elt, idx, UNNAMED) }
1615    }
1616
1617    pub(crate) fn shuffle_vector(
1618        &mut self,
1619        v1: &'ll Value,
1620        v2: &'ll Value,
1621        mask: &'ll Value,
1622    ) -> &'ll Value {
1623        unsafe { llvm::LLVMBuildShuffleVector(self.llbuilder, v1, v2, mask, UNNAMED) }
1624    }
1625
1626    pub(crate) fn vector_reduce_fadd(&mut self, acc: &'ll Value, src: &'ll Value) -> &'ll Value {
1627        self.call_intrinsic("llvm.vector.reduce.fadd", &[self.val_ty(src)], &[acc, src])
1628    }
1629    pub(crate) fn vector_reduce_fmul(&mut self, acc: &'ll Value, src: &'ll Value) -> &'ll Value {
1630        self.call_intrinsic("llvm.vector.reduce.fmul", &[self.val_ty(src)], &[acc, src])
1631    }
1632    pub(crate) fn vector_reduce_fadd_reassoc(
1633        &mut self,
1634        acc: &'ll Value,
1635        src: &'ll Value,
1636    ) -> &'ll Value {
1637        unsafe {
1638            let instr =
1639                self.call_intrinsic("llvm.vector.reduce.fadd", &[self.val_ty(src)], &[acc, src]);
1640            llvm::LLVMRustSetAllowReassoc(instr);
1641            instr
1642        }
1643    }
1644    pub(crate) fn vector_reduce_fmul_reassoc(
1645        &mut self,
1646        acc: &'ll Value,
1647        src: &'ll Value,
1648    ) -> &'ll Value {
1649        unsafe {
1650            let instr =
1651                self.call_intrinsic("llvm.vector.reduce.fmul", &[self.val_ty(src)], &[acc, src]);
1652            llvm::LLVMRustSetAllowReassoc(instr);
1653            instr
1654        }
1655    }
1656    pub(crate) fn vector_reduce_add(&mut self, src: &'ll Value) -> &'ll Value {
1657        self.call_intrinsic("llvm.vector.reduce.add", &[self.val_ty(src)], &[src])
1658    }
1659    pub(crate) fn vector_reduce_mul(&mut self, src: &'ll Value) -> &'ll Value {
1660        self.call_intrinsic("llvm.vector.reduce.mul", &[self.val_ty(src)], &[src])
1661    }
1662    pub(crate) fn vector_reduce_and(&mut self, src: &'ll Value) -> &'ll Value {
1663        self.call_intrinsic("llvm.vector.reduce.and", &[self.val_ty(src)], &[src])
1664    }
1665    pub(crate) fn vector_reduce_or(&mut self, src: &'ll Value) -> &'ll Value {
1666        self.call_intrinsic("llvm.vector.reduce.or", &[self.val_ty(src)], &[src])
1667    }
1668    pub(crate) fn vector_reduce_xor(&mut self, src: &'ll Value) -> &'ll Value {
1669        self.call_intrinsic("llvm.vector.reduce.xor", &[self.val_ty(src)], &[src])
1670    }
1671    pub(crate) fn vector_reduce_fmin(&mut self, src: &'ll Value) -> &'ll Value {
1672        self.call_intrinsic("llvm.vector.reduce.fmin", &[self.val_ty(src)], &[src])
1673    }
1674    pub(crate) fn vector_reduce_fmax(&mut self, src: &'ll Value) -> &'ll Value {
1675        self.call_intrinsic("llvm.vector.reduce.fmax", &[self.val_ty(src)], &[src])
1676    }
1677    pub(crate) fn vector_reduce_min(&mut self, src: &'ll Value, is_signed: bool) -> &'ll Value {
1678        self.call_intrinsic(
1679            if is_signed { "llvm.vector.reduce.smin" } else { "llvm.vector.reduce.umin" },
1680            &[self.val_ty(src)],
1681            &[src],
1682        )
1683    }
1684    pub(crate) fn vector_reduce_max(&mut self, src: &'ll Value, is_signed: bool) -> &'ll Value {
1685        self.call_intrinsic(
1686            if is_signed { "llvm.vector.reduce.smax" } else { "llvm.vector.reduce.umax" },
1687            &[self.val_ty(src)],
1688            &[src],
1689        )
1690    }
1691}
1692impl<'a, 'll, CX: Borrow<SCx<'ll>>> GenericBuilder<'a, 'll, CX> {
1693    pub(crate) fn add_clause(&mut self, landing_pad: &'ll Value, clause: &'ll Value) {
1694        unsafe {
1695            llvm::LLVMAddClause(landing_pad, clause);
1696        }
1697    }
1698
1699    pub(crate) fn catch_ret(
1700        &mut self,
1701        funclet: &Funclet<'ll>,
1702        unwind: &'ll BasicBlock,
1703    ) -> &'ll Value {
1704        let ret = unsafe { llvm::LLVMBuildCatchRet(self.llbuilder, funclet.cleanuppad(), unwind) };
1705        ret.expect("LLVM does not have support for catchret")
1706    }
1707
1708    fn check_call<'b>(
1709        &mut self,
1710        typ: &str,
1711        fn_ty: &'ll Type,
1712        llfn: &'ll Value,
1713        args: &'b [&'ll Value],
1714    ) -> Cow<'b, [&'ll Value]> {
1715        assert!(
1716            self.cx.type_kind(fn_ty) == TypeKind::Function,
1717            "builder::{typ} not passed a function, but {fn_ty:?}"
1718        );
1719
1720        let param_tys = self.cx.func_params_types(fn_ty);
1721
1722        let all_args_match = iter::zip(&param_tys, args.iter().map(|&v| self.cx.val_ty(v)))
1723            .all(|(expected_ty, actual_ty)| *expected_ty == actual_ty);
1724
1725        if all_args_match {
1726            return Cow::Borrowed(args);
1727        }
1728
1729        let casted_args: Vec<_> = iter::zip(param_tys, args)
1730            .enumerate()
1731            .map(|(i, (expected_ty, &actual_val))| {
1732                let actual_ty = self.cx.val_ty(actual_val);
1733                if expected_ty != actual_ty {
1734                    debug!(
1735                        "type mismatch in function call of {:?}. \
1736                            Expected {:?} for param {}, got {:?}; injecting bitcast",
1737                        llfn, expected_ty, i, actual_ty
1738                    );
1739                    self.bitcast(actual_val, expected_ty)
1740                } else {
1741                    actual_val
1742                }
1743            })
1744            .collect();
1745
1746        Cow::Owned(casted_args)
1747    }
1748
1749    pub(crate) fn va_arg(&mut self, list: &'ll Value, ty: &'ll Type) -> &'ll Value {
1750        unsafe { llvm::LLVMBuildVAArg(self.llbuilder, list, ty, UNNAMED) }
1751    }
1752}
1753
1754impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
1755    pub(crate) fn call_intrinsic(
1756        &mut self,
1757        base_name: impl Into<Cow<'static, str>>,
1758        type_params: &[&'ll Type],
1759        args: &[&'ll Value],
1760    ) -> &'ll Value {
1761        let (ty, f) = self.cx.get_intrinsic(base_name.into(), type_params);
1762        self.call(ty, None, None, f, args, None, None)
1763    }
1764
1765    fn call_lifetime_intrinsic(&mut self, intrinsic: &'static str, ptr: &'ll Value, size: Size) {
1766        let size = size.bytes();
1767        if size == 0 {
1768            return;
1769        }
1770
1771        if !self.cx().sess().emit_lifetime_markers() {
1772            return;
1773        }
1774
1775        if crate::llvm_util::get_version() >= (22, 0, 0) {
1776            self.call_intrinsic(intrinsic, &[self.val_ty(ptr)], &[ptr]);
1777        } else {
1778            self.call_intrinsic(intrinsic, &[self.val_ty(ptr)], &[self.cx.const_u64(size), ptr]);
1779        }
1780    }
1781}
1782impl<'a, 'll, CX: Borrow<SCx<'ll>>> GenericBuilder<'a, 'll, CX> {
1783    pub(crate) fn phi(
1784        &mut self,
1785        ty: &'ll Type,
1786        vals: &[&'ll Value],
1787        bbs: &[&'ll BasicBlock],
1788    ) -> &'ll Value {
1789        assert_eq!(vals.len(), bbs.len());
1790        let phi = unsafe { llvm::LLVMBuildPhi(self.llbuilder, ty, UNNAMED) };
1791        unsafe {
1792            llvm::LLVMAddIncoming(phi, vals.as_ptr(), bbs.as_ptr(), vals.len() as c_uint);
1793            phi
1794        }
1795    }
1796
1797    fn add_incoming_to_phi(&mut self, phi: &'ll Value, val: &'ll Value, bb: &'ll BasicBlock) {
1798        unsafe {
1799            llvm::LLVMAddIncoming(phi, &val, &bb, 1 as c_uint);
1800        }
1801    }
1802}
1803impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
1804    pub(crate) fn landing_pad(
1805        &mut self,
1806        ty: &'ll Type,
1807        pers_fn: &'ll Value,
1808        num_clauses: usize,
1809    ) -> &'ll Value {
1810        // Use LLVMSetPersonalityFn to set the personality. It supports arbitrary Consts while,
1811        // LLVMBuildLandingPad requires the argument to be a Function (as of LLVM 12). The
1812        // personality lives on the parent function anyway.
1813        self.set_personality_fn(pers_fn);
1814        unsafe {
1815            llvm::LLVMBuildLandingPad(self.llbuilder, ty, None, num_clauses as c_uint, UNNAMED)
1816        }
1817    }
1818
1819    pub(crate) fn callbr(
1820        &mut self,
1821        llty: &'ll Type,
1822        fn_attrs: Option<&CodegenFnAttrs>,
1823        fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>,
1824        llfn: &'ll Value,
1825        args: &[&'ll Value],
1826        default_dest: &'ll BasicBlock,
1827        indirect_dest: &[&'ll BasicBlock],
1828        funclet: Option<&Funclet<'ll>>,
1829        instance: Option<Instance<'tcx>>,
1830    ) -> &'ll Value {
1831        debug!("invoke {:?} with args ({:?})", llfn, args);
1832
1833        let args = self.check_call("callbr", llty, llfn, args);
1834        let funclet_bundle = funclet.map(|funclet| funclet.bundle());
1835        let mut bundles: SmallVec<[_; 2]> = SmallVec::new();
1836        if let Some(funclet_bundle) = funclet_bundle {
1837            bundles.push(funclet_bundle);
1838        }
1839
1840        // Emit CFI pointer type membership test
1841        self.cfi_type_test(fn_attrs, fn_abi, instance, llfn);
1842
1843        // Emit KCFI operand bundle
1844        let kcfi_bundle = self.kcfi_operand_bundle(fn_attrs, fn_abi, instance, llfn);
1845        if let Some(kcfi_bundle) = kcfi_bundle.as_ref().map(|b| b.as_ref()) {
1846            bundles.push(kcfi_bundle);
1847        }
1848
1849        let callbr = unsafe {
1850            llvm::LLVMBuildCallBr(
1851                self.llbuilder,
1852                llty,
1853                llfn,
1854                default_dest,
1855                indirect_dest.as_ptr(),
1856                indirect_dest.len() as c_uint,
1857                args.as_ptr(),
1858                args.len() as c_uint,
1859                bundles.as_ptr(),
1860                bundles.len() as c_uint,
1861                UNNAMED,
1862            )
1863        };
1864        if let Some(fn_abi) = fn_abi {
1865            fn_abi.apply_attrs_callsite(self, callbr);
1866        }
1867        callbr
1868    }
1869
1870    // Emits CFI pointer type membership tests.
1871    fn cfi_type_test(
1872        &mut self,
1873        fn_attrs: Option<&CodegenFnAttrs>,
1874        fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>,
1875        instance: Option<Instance<'tcx>>,
1876        llfn: &'ll Value,
1877    ) {
1878        let is_indirect_call = unsafe { llvm::LLVMRustIsNonGVFunctionPointerTy(llfn) };
1879        if self.tcx.sess.is_sanitizer_cfi_enabled()
1880            && let Some(fn_abi) = fn_abi
1881            && is_indirect_call
1882        {
1883            if let Some(fn_attrs) = fn_attrs
1884                && fn_attrs.sanitizers.disabled.contains(SanitizerSet::CFI)
1885            {
1886                return;
1887            }
1888
1889            let mut options = cfi::TypeIdOptions::empty();
1890            if self.tcx.sess.is_sanitizer_cfi_generalize_pointers_enabled() {
1891                options.insert(cfi::TypeIdOptions::GENERALIZE_POINTERS);
1892            }
1893            if self.tcx.sess.is_sanitizer_cfi_normalize_integers_enabled() {
1894                options.insert(cfi::TypeIdOptions::NORMALIZE_INTEGERS);
1895            }
1896
1897            let typeid = if let Some(instance) = instance {
1898                cfi::typeid_for_instance(self.tcx, instance, options)
1899            } else {
1900                cfi::typeid_for_fnabi(self.tcx, fn_abi, options)
1901            };
1902            let typeid_metadata = self.cx.create_metadata(typeid.as_bytes());
1903            let dbg_loc = self.get_dbg_loc();
1904
1905            // Test whether the function pointer is associated with the type identifier using the
1906            // llvm.type.test intrinsic. The LowerTypeTests link-time optimization pass replaces
1907            // calls to this intrinsic with code to test type membership.
1908            let typeid = self.get_metadata_value(typeid_metadata);
1909            let cond = self.call_intrinsic("llvm.type.test", &[], &[llfn, typeid]);
1910            let bb_pass = self.append_sibling_block("type_test.pass");
1911            let bb_fail = self.append_sibling_block("type_test.fail");
1912            self.cond_br(cond, bb_pass, bb_fail);
1913
1914            self.switch_to_block(bb_fail);
1915            if let Some(dbg_loc) = dbg_loc {
1916                self.set_dbg_loc(dbg_loc);
1917            }
1918            self.abort();
1919            self.unreachable();
1920
1921            self.switch_to_block(bb_pass);
1922            if let Some(dbg_loc) = dbg_loc {
1923                self.set_dbg_loc(dbg_loc);
1924            }
1925        }
1926    }
1927
1928    // Emits KCFI operand bundles.
1929    fn kcfi_operand_bundle(
1930        &mut self,
1931        fn_attrs: Option<&CodegenFnAttrs>,
1932        fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>,
1933        instance: Option<Instance<'tcx>>,
1934        llfn: &'ll Value,
1935    ) -> Option<llvm::OperandBundleBox<'ll>> {
1936        let is_indirect_call = unsafe { llvm::LLVMRustIsNonGVFunctionPointerTy(llfn) };
1937        let kcfi_bundle = if self.tcx.sess.is_sanitizer_kcfi_enabled()
1938            && let Some(fn_abi) = fn_abi
1939            && is_indirect_call
1940        {
1941            if let Some(fn_attrs) = fn_attrs
1942                && fn_attrs.sanitizers.disabled.contains(SanitizerSet::KCFI)
1943            {
1944                return None;
1945            }
1946
1947            let mut options = kcfi::TypeIdOptions::empty();
1948            if self.tcx.sess.is_sanitizer_cfi_generalize_pointers_enabled() {
1949                options.insert(kcfi::TypeIdOptions::GENERALIZE_POINTERS);
1950            }
1951            if self.tcx.sess.is_sanitizer_cfi_normalize_integers_enabled() {
1952                options.insert(kcfi::TypeIdOptions::NORMALIZE_INTEGERS);
1953            }
1954
1955            let kcfi_typeid = if let Some(instance) = instance {
1956                kcfi::typeid_for_instance(self.tcx, instance, options)
1957            } else {
1958                kcfi::typeid_for_fnabi(self.tcx, fn_abi, options)
1959            };
1960
1961            Some(llvm::OperandBundleBox::new("kcfi", &[self.const_u32(kcfi_typeid)]))
1962        } else {
1963            None
1964        };
1965        kcfi_bundle
1966    }
1967
1968    /// Emits a call to `llvm.instrprof.increment`. Used by coverage instrumentation.
1969    #[instrument(level = "debug", skip(self))]
1970    pub(crate) fn instrprof_increment(
1971        &mut self,
1972        fn_name: &'ll Value,
1973        hash: &'ll Value,
1974        num_counters: &'ll Value,
1975        index: &'ll Value,
1976    ) {
1977        self.call_intrinsic("llvm.instrprof.increment", &[], &[fn_name, hash, num_counters, index]);
1978    }
1979}
rustc_codegen_llvm/builder.rs

rustc_codegen_llvm/
builder.rs
