diff --git a/compiler/rustc_codegen_llvm/src/attributes.rs b/compiler/rustc_codegen_llvm/src/attributes.rs index 4073d521293a9..b1e270fd8fe70 100644 --- a/compiler/rustc_codegen_llvm/src/attributes.rs +++ b/compiler/rustc_codegen_llvm/src/attributes.rs @@ -467,6 +467,8 @@ pub(crate) fn default_optimisation_attrs<'ll>( sess: &Session, ) -> SmallVec<[&'ll Attribute; 2]> { let mut attrs = SmallVec::new(); + // Either attribute satisfies LLVM's combined size predicate, but the software pipeliner reads + // `optsize` alone, so a `z` build needs both. match sess.opts.optimize { OptLevel::Size => { attrs.push(llvm::AttributeKind::OptimizeForSize.create_attr(cx.llcx)); diff --git a/compiler/rustc_codegen_llvm/src/back/write.rs b/compiler/rustc_codegen_llvm/src/back/write.rs index 41a04c637b231..98eaf5e850f50 100644 --- a/compiler/rustc_codegen_llvm/src/back/write.rs +++ b/compiler/rustc_codegen_llvm/src/back/write.rs @@ -121,15 +121,15 @@ pub(crate) fn create_target_machine(tcx: TyCtxt<'_>, mod_name: &str) -> OwnedTar target_machine_factory(tcx.sess, tcx.backend_optimization_level(()))(tcx.dcx(), config) } -fn to_llvm_opt_settings(cfg: config::OptLevel) -> (llvm::CodeGenOptLevel, llvm::CodeGenOptSize) { +// `s` and `z` are the same codegen level; what tells LLVM that they are size builds is the +// per-function attribute, not this setting. +fn to_llvm_opt_level(cfg: config::OptLevel) -> llvm::CodeGenOptLevel { use self::config::OptLevel::*; match cfg { - No => (llvm::CodeGenOptLevel::None, llvm::CodeGenOptSizeNone), - Less => (llvm::CodeGenOptLevel::Less, llvm::CodeGenOptSizeNone), - More => (llvm::CodeGenOptLevel::Default, llvm::CodeGenOptSizeNone), - Aggressive => (llvm::CodeGenOptLevel::Aggressive, llvm::CodeGenOptSizeNone), - Size => (llvm::CodeGenOptLevel::Default, llvm::CodeGenOptSizeDefault), - SizeMin => (llvm::CodeGenOptLevel::Default, llvm::CodeGenOptSizeAggressive), + No => llvm::CodeGenOptLevel::None, + Less => llvm::CodeGenOptLevel::Less, + More | Size | SizeMin => llvm::CodeGenOptLevel::Default, + Aggressive => llvm::CodeGenOptLevel::Aggressive, } } @@ -186,7 +186,7 @@ pub(crate) fn target_machine_factory( let reloc_model = to_llvm_relocation_model(sess.relocation_model()); - let (opt_level, _) = to_llvm_opt_settings(optlvl); + let opt_level = to_llvm_opt_level(optlvl); let float_abi = to_llvm_float_abi(sess.target.llvm_floatabi); let ffunction_sections = @@ -592,8 +592,7 @@ pub(crate) unsafe fn llvm_optimize<'ll>( vectorize_slp = false; vectorize_loop = false; } else { - unroll_loops = - opt_level != config::OptLevel::Size && opt_level != config::OptLevel::SizeMin; + unroll_loops = opt_level.objective() != config::OptimizationObjective::Size; merge_functions = config.merge_functions; vectorize_slp = config.vectorize_slp; vectorize_loop = config.vectorize_loop; diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs index 1a323e550623b..74367dc5faae7 100644 --- a/compiler/rustc_codegen_llvm/src/context.rs +++ b/compiler/rustc_codegen_llvm/src/context.rs @@ -1067,6 +1067,11 @@ impl<'ll, 'tcx> MiscCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { attributes::apply_to_llfn(llfn, llvm::AttributePlace::Function, &attrs); } + fn apply_default_optimisation_attrs(&self, llfn: &'ll Value) { + let attrs = attributes::default_optimisation_attrs(self, self.sess()); + attributes::apply_to_llfn(llfn, llvm::AttributePlace::Function, &attrs); + } + fn declare_c_main(&self, fn_type: Self::Type) -> Option { let entry_name = self.sess().target.entry_name.as_ref(); if self.get_declared_value(entry_name).is_none() { diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index 04368201b4576..c73b6a5a28d5c 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -1748,6 +1748,7 @@ fn gen_fn<'a, 'll, 'tcx>( let llfn = cx.declare_fn(name, fn_abi, None); cx.set_frame_pointer_type(llfn); cx.apply_target_cpu_attr(llfn); + cx.apply_default_optimisation_attrs(llfn); // FIXME(eddyb) find a nicer way to do this. llvm::set_linkage(llfn, llvm::Linkage::InternalLinkage); let llbb = Builder::append_block(cx, llfn, "entry-block"); diff --git a/compiler/rustc_codegen_llvm/src/llvm/mod.rs b/compiler/rustc_codegen_llvm/src/llvm/mod.rs index e189dc758d8e7..377f788d3462e 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/mod.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/mod.rs @@ -10,7 +10,6 @@ use rustc_abi::{AddressSpace, Align, Size, WrappingRange}; use rustc_llvm::RustString; pub(crate) use self::CallConv::*; -pub(crate) use self::CodeGenOptSize::*; pub(crate) use self::conversions::*; pub(crate) use self::ffi::*; pub(crate) use self::metadata_kind::*; @@ -185,14 +184,6 @@ impl AttributePlace { } } -#[derive(Copy, Clone, PartialEq)] -#[repr(C)] -pub(crate) enum CodeGenOptSize { - CodeGenOptSizeNone = 0, - CodeGenOptSizeDefault = 1, - CodeGenOptSizeAggressive = 2, -} - pub(crate) fn SetInstructionCallConv(instr: &Value, cc: CallConv) { unsafe { LLVMSetInstructionCallConv(instr, cc as c_uint); diff --git a/compiler/rustc_codegen_ssa/src/back/linker.rs b/compiler/rustc_codegen_ssa/src/back/linker.rs index ffb9130ed1a08..b92e79c8f43ec 100644 --- a/compiler/rustc_codegen_ssa/src/back/linker.rs +++ b/compiler/rustc_codegen_ssa/src/back/linker.rs @@ -407,6 +407,7 @@ impl<'a> GccLinker<'a> { self.link_arg(&arg); } + // The plugin has no distinct size setting: `s` and `z` both map to `O2`, as `More` does. let opt_level = match self.sess.opts.optimize { config::OptLevel::No => "O0", config::OptLevel::Less => "O1", @@ -698,9 +699,7 @@ impl<'a> Linker for GccLinker<'a> { // GNU-style linkers support optimization with -O. GNU ld doesn't // need a numeric argument, but other linkers do. - if self.sess.opts.optimize == config::OptLevel::More - || self.sess.opts.optimize == config::OptLevel::Aggressive - { + if self.sess.optimization_objective() == config::OptimizationObjective::Speed { self.link_arg("-O1"); } } @@ -1021,7 +1020,7 @@ impl<'a> Linker for MsvcLinker<'a> { // MSVC's ICF (Identical COMDAT Folding) link optimization is // slow for Rust and thus we disable it by default when not in // optimization build. - if self.sess.opts.optimize != config::OptLevel::No { + if self.sess.optimization_objective() != config::OptimizationObjective::None { self.link_arg("/OPT:REF,ICF"); } else { // It is necessary to specify NOICF here, because /OPT:REF diff --git a/compiler/rustc_codegen_ssa/src/back/write.rs b/compiler/rustc_codegen_ssa/src/back/write.rs index 18968f0f89960..d070d39fda55e 100644 --- a/compiler/rustc_codegen_ssa/src/back/write.rs +++ b/compiler/rustc_codegen_ssa/src/back/write.rs @@ -23,7 +23,8 @@ use rustc_metadata::fs::copy_to_stdout; use rustc_middle::dep_graph::{WorkProduct, WorkProductMap}; use rustc_middle::ty::TyCtxt; use rustc_session::config::{ - self, Lto, OptLevel, OutFileName, OutputFilenames, OutputType, Passes, SwitchWithOptPath, + self, Lto, OptLevel, OptimizationObjective, OutFileName, OutputFilenames, OutputType, Passes, + SwitchWithOptPath, }; use rustc_session::{IncrCompSession, Session}; use rustc_span::source_map::SourceMap; @@ -230,8 +231,7 @@ impl ModuleConfig { // Copy what clang does by turning on loop vectorization at O2 and // slp vectorization at O3. vectorize_loop: !sess.opts.cg.no_vectorize_loops - && (sess.opts.optimize == config::OptLevel::More - || sess.opts.optimize == config::OptLevel::Aggressive), + && sess.optimization_objective() == OptimizationObjective::Speed, vectorize_slp: !sess.opts.cg.no_vectorize_slp && sess.opts.optimize == config::OptLevel::Aggressive, @@ -246,13 +246,10 @@ impl ModuleConfig { // O2 and O3) since it can be useful for reducing code size. merge_functions: match sess.merge_functions() { MergeFunctions::Disabled => false, - MergeFunctions::Trampolines | MergeFunctions::Aliases => { - use config::OptLevel::*; - match sess.opts.optimize { - Aggressive | More | SizeMin | Size => true, - Less | No => false, - } - } + MergeFunctions::Trampolines | MergeFunctions::Aliases => matches!( + sess.optimization_objective(), + OptimizationObjective::Speed | OptimizationObjective::Size + ), }, emit_lifetime_markers: sess.emit_lifetime_markers(), diff --git a/compiler/rustc_codegen_ssa/src/base.rs b/compiler/rustc_codegen_ssa/src/base.rs index 14e07a9c89395..14357f9255311 100644 --- a/compiler/rustc_codegen_ssa/src/base.rs +++ b/compiler/rustc_codegen_ssa/src/base.rs @@ -557,6 +557,8 @@ pub fn maybe_create_entry_wrapper<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( // `main` should respect same config for frame pointer elimination as rest of code cx.set_frame_pointer_type(llfn); cx.apply_target_cpu_attr(llfn); + // The shim has no item behind it, so it does not go through the per-item attribute path. + cx.apply_default_optimisation_attrs(llfn); let llbb = Bx::append_block(cx, llfn, "top"); let mut bx = Bx::build(cx, llbb); @@ -1219,23 +1221,16 @@ impl CrateInfo { pub(crate) fn provide(providers: &mut Providers) { providers.backend_optimization_level = |tcx, cratenum| { - let for_speed = match tcx.sess.opts.optimize { - // If globally no optimisation is done, #[optimize] has no effect. - // - // This is done because if we ended up "upgrading" to `-O2` here, we’d populate the - // pass manager and it is likely that some module-wide passes (such as inliner or - // cross-function constant propagation) would ignore the `optnone` annotation we put - // on the functions, thus necessarily involving these functions into optimisations. - config::OptLevel::No => return config::OptLevel::No, - // If globally optimise-speed is already specified, just use that level. - config::OptLevel::Less => return config::OptLevel::Less, - config::OptLevel::More => return config::OptLevel::More, - config::OptLevel::Aggressive => return config::OptLevel::Aggressive, - // If globally optimize-for-size has been requested, use -O2 instead (if optimize(size) - // are present). - config::OptLevel::Size => config::OptLevel::More, - config::OptLevel::SizeMin => config::OptLevel::More, - }; + let global = tcx.sess.opts.optimize; + + // Nothing to promote outside a size build. At `-Copt-level=0` the upgrade would populate + // the pass manager, and its module-wide passes ignore the `optnone` annotation. + if global.objective() != config::OptimizationObjective::Size { + return global; + } + + // A speed-marked body in a size build needs a speed level, and `-O2` is the lowest one. + let for_speed = config::OptLevel::More; let defids = tcx.collect_and_partition_mono_items(cratenum).all_mono_items; @@ -1244,11 +1239,7 @@ pub(crate) fn provide(providers: &mut Providers) { matches!(optimize, OptimizeAttr::Speed) }); - if any_for_speed { - return for_speed; - } - - tcx.sess.opts.optimize + if any_for_speed { for_speed } else { global } }; } diff --git a/compiler/rustc_codegen_ssa/src/traits/misc.rs b/compiler/rustc_codegen_ssa/src/traits/misc.rs index add7128a2974b..7e48e86290ee8 100644 --- a/compiler/rustc_codegen_ssa/src/traits/misc.rs +++ b/compiler/rustc_codegen_ssa/src/traits/misc.rs @@ -28,6 +28,9 @@ pub trait MiscCodegenMethods<'tcx>: BackendTypes { fn sess(&self) -> &Session; fn set_frame_pointer_type(&self, llfn: Self::Function); fn apply_target_cpu_attr(&self, llfn: Self::Function); + /// Applies the compilation's optimization attributes to a function that no item produced, + /// such as the C entry shim. + fn apply_default_optimisation_attrs(&self, _llfn: Self::Function) {} /// Declares the extern "C" main function for the entry point. Returns None if the symbol /// already exists. fn declare_c_main(&self, fn_type: Self::FunctionSignature) -> Option; diff --git a/compiler/rustc_session/src/config.rs b/compiler/rustc_session/src/config.rs index 7e274c66f6f75..028173daaf743 100644 --- a/compiler/rustc_session/src/config.rs +++ b/compiler/rustc_session/src/config.rs @@ -126,6 +126,33 @@ impl OptLevel { _ => 2, } } + + /// Maps a general opt-level onto the objective it is trying to achieve. + pub fn objective(&self) -> OptimizationObjective { + match self { + OptLevel::No => OptimizationObjective::None, + OptLevel::Less => OptimizationObjective::Balanced, + OptLevel::More | OptLevel::Aggressive => OptimizationObjective::Speed, + OptLevel::Size | OptLevel::SizeMin => OptimizationObjective::Size, + } + } +} + +/// What a compilation, or a single item, is being asked to optimize *for*. +/// +/// Unlike [`OptLevel`], speed and size are alternatives here rather than degrees of the same +/// thing. Sites that map a level onto an LLVM or linker flag, where `s` and `z` produce different +/// flags, still match on [`OptLevel`]. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum OptimizationObjective { + /// Nothing is optimized, so there is no objective (`-Copt-level=0`). + None, + /// Optimize, without a strong preference for speed over size (`-Copt-level=1`). + Balanced, + /// Favor runtime speed over binary size (`-Copt-level=2`/`3`, `#[optimize(speed)]`). + Speed, + /// Favor binary size over runtime speed (`-Copt-level=s`/`z`, `#[optimize(size)]`). + Size, } /// This is what the `LtoCli` values get mapped to after resolving defaults and @@ -1568,10 +1595,9 @@ impl Options { pub fn share_generics(&self) -> bool { match self.unstable_opts.share_generics { Some(setting) => setting, - None => match self.optimize { - OptLevel::No | OptLevel::Less | OptLevel::Size | OptLevel::SizeMin => true, - OptLevel::More | OptLevel::Aggressive => false, - }, + // Sharing instantiations means fewer copies, at the cost of leaving them opaque to + // speed optimizations. That is a good trade unless speed is what is asked for. + None => self.optimize.objective() != OptimizationObjective::Speed, } } diff --git a/compiler/rustc_session/src/session.rs b/compiler/rustc_session/src/session.rs index f068aaa584dce..d33ebfc2e8181 100644 --- a/compiler/rustc_session/src/session.rs +++ b/compiler/rustc_session/src/session.rs @@ -983,6 +983,11 @@ impl Session { self.opts.unstable_opts.mir_opt_level.unwrap_or_else(|| self.opts.optimize.mir_opt_level()) } + /// What this compilation optimizes for; see [`config::OptimizationObjective`]. + pub fn optimization_objective(&self) -> config::OptimizationObjective { + self.opts.optimize.objective() + } + /// Calculates the flavor of LTO to use for this compilation. pub fn lto(&self) -> config::Lto { // If our target has codegen requirements ignore the command line diff --git a/tests/codegen-llvm/loop-vectorize-in-opt-sz.rs b/tests/codegen-llvm/loop-vectorize-in-opt-sz.rs new file mode 100644 index 0000000000000..f26a558f17ea4 --- /dev/null +++ b/tests/codegen-llvm/loop-vectorize-in-opt-sz.rs @@ -0,0 +1,23 @@ +//@ revisions: O2 S Z +//@[O2] compile-flags: -Copt-level=2 +//@[S] compile-flags: -Copt-level=s +//@[Z] compile-flags: -Copt-level=z +//@ only-x86_64 + +// Loop vectorization is enabled only for the speed objective, so both `-Copt-level=s` and `z` leave +// it off. The loop below is long enough that LLVM at `-O2` does vectorize it. + +#![crate_type = "lib"] + +// CHECK-LABEL: @scale_add( +// O2: fmul <4 x float> +// O2: fadd <4 x float> +// O2: store <4 x float> +// S-NOT: <4 x float> +// Z-NOT: <4 x float> +#[no_mangle] +pub fn scale_add(a: &[f32], b: &[f32], c: &mut [f32]) { + for i in 0..64 { + c[i] = a[i] * b[i] + 3.0; + } +} diff --git a/tests/codegen-llvm/merge-functions.rs b/tests/codegen-llvm/merge-functions.rs index b9d3727ce112e..8c15791c5ae74 100644 --- a/tests/codegen-llvm/merge-functions.rs +++ b/tests/codegen-llvm/merge-functions.rs @@ -1,5 +1,6 @@ -//@ revisions: O Os +//@ revisions: O Os Oz //@[Os] compile-flags: -Copt-level=s +//@[Oz] compile-flags: -Copt-level=z //@[O] compile-flags: -Copt-level=3 #![crate_type = "lib"] diff --git a/tests/codegen-llvm/opt-size-attr-shim.rs b/tests/codegen-llvm/opt-size-attr-shim.rs new file mode 100644 index 0000000000000..a82a3219ca5b1 --- /dev/null +++ b/tests/codegen-llvm/opt-size-attr-shim.rs @@ -0,0 +1,30 @@ +//@ revisions: O2 S Z +//@[O2] compile-flags: -Copt-level=2 +//@[S] compile-flags: -Copt-level=s +//@[Z] compile-flags: -Copt-level=z +//@ ignore-wasi wasi codegens the main symbol differently + +// The C entry shim has no item behind it, so it does not go through the per-item attribute path. +// It still has to carry the size attributes, or `main` is the one definition in a size build that +// LLVM optimizes for speed. + +#![crate_type = "bin"] + +// CHECK-LABEL: define {{.*}} @main( +// CHECK-SAME: [[MAIN_ATTRS:#[0-9]+]] + +// O2: attributes [[MAIN_ATTRS]] = +// O2-NOT: optsize +// O2-NOT: minsize + +// S: attributes [[MAIN_ATTRS]] = +// S-NOT: minsize +// S-SAME: optsize + +// Z: attributes [[MAIN_ATTRS]] = +// Z-SAME: minsize +// Z-SAME: optsize + +fn main() { + std::hint::black_box(0); +} diff --git a/tests/codegen-llvm/opt-size-attr.rs b/tests/codegen-llvm/opt-size-attr.rs new file mode 100644 index 0000000000000..7784e8d1c8e47 --- /dev/null +++ b/tests/codegen-llvm/opt-size-attr.rs @@ -0,0 +1,29 @@ +//@ revisions: O2 S Z +//@[O2] compile-flags: -Copt-level=2 +//@[S] compile-flags: -Copt-level=s +//@[Z] compile-flags: -Copt-level=z + +// `s` gives `optsize` while `z` gives both `minsize` and `optsize`. This is the only coverage of +// the `-Copt-level` path; `optimize-attr-1.rs` reaches these via `#[optimize(size)]`. + +#![crate_type = "lib"] + +// CHECK-LABEL: @f( +// CHECK-SAME: [[F_ATTRS:#[0-9]+]] + +// O2: attributes [[F_ATTRS]] = +// O2-NOT: optsize +// O2-NOT: minsize + +// S: attributes [[F_ATTRS]] = +// S-NOT: minsize +// S-SAME: optsize + +// Z: attributes [[F_ATTRS]] = +// Z-SAME: minsize +// Z-SAME: optsize + +#[no_mangle] +pub fn f(x: u32) -> u32 { + x + 1 +} diff --git a/tests/codegen-llvm/unroll-in-opt-sz.rs b/tests/codegen-llvm/unroll-in-opt-sz.rs new file mode 100644 index 0000000000000..a76c8b3fba51c --- /dev/null +++ b/tests/codegen-llvm/unroll-in-opt-sz.rs @@ -0,0 +1,31 @@ +//@ revisions: O2 S Z +//@[O2] compile-flags: -Copt-level=2 +//@[S] compile-flags: -Copt-level=s +//@[Z] compile-flags: -Copt-level=z + +// `-Copt-level=s`/`z` pass `PTO.LoopUnrolling = false`, so LLVM's unroller runs `OnlyWhenForced`. +// The O2 side is asserted too, so disabling unrolling everywhere cannot pass. Calls are counted +// because this knob emits no `llvm.loop.unroll.*` metadata. + +#![crate_type = "lib"] + +unsafe extern "C" { + fn side_effect(i: u32); +} + +// O2-LABEL: @four_iters( +// O2-COUNT-4: tail call void @side_effect +// S-LABEL: @four_iters( +// S: phi i32 +// S: tail call void @side_effect +// S-NOT: tail call void @side_effect +// Z-LABEL: @four_iters( +// Z: phi i32 +// Z: tail call void @side_effect +// Z-NOT: tail call void @side_effect +#[no_mangle] +pub fn four_iters() { + for i in 0..4u32 { + unsafe { side_effect(i) } + } +}