mir_transform: implement forced inlining

Adds `#[inline(must)]` and `#[inline(required)]` which is similar
to always inlining but reports an error and warning respectively if
the inlining was not possible, and which always attempts to inline
required-annotated items, regardless of optimisation levels.

Author: davidtwco

compiler/rustc_attr/src/builtin.rs

@@ -47,6 +47,27 @@ pub enum InlineAttr {
     Hint,
     Always,
     Never,
+    /// `#[inline(must)]` always attempts to inline in the MIR inliner, regardless of optimisation
+    /// level, and emits a warn-by-default lint if this is not possible.
+    Must {
+        attr_span: Span,
+        reason: Option<Symbol>,
+    },
+    /// `#[inline(required)]` always attempts to inline in the MIR inliner, regardless of
+    /// optimisation level, and emits a error-by-default lint if this is not possible.
+    Required {
+        attr_span: Span,
+        reason: Option<Symbol>,
+    },
+}
+
+impl InlineAttr {
+    pub fn always(&self) -> bool {
+        match self {
+            InlineAttr::Always | InlineAttr::Must { .. } | InlineAttr::Required { .. } => true,
+            InlineAttr::None | InlineAttr::Hint | InlineAttr::Never => false,
+        }
+    }
 }
 
 #[derive(Clone, Encodable, Decodable, Debug, PartialEq, Eq, HashStable_Generic)]

compiler/rustc_codegen_gcc/src/attributes.rs

@@ -20,7 +20,9 @@ fn inline_attr<'gcc, 'tcx>(
 ) -> Option<FnAttribute<'gcc>> {
     match inline {
         InlineAttr::Hint => Some(FnAttribute::Inline),
-        InlineAttr::Always => Some(FnAttribute::AlwaysInline),
+        InlineAttr::Always | InlineAttr::Required { .. } | InlineAttr::Must { .. } => {
+            Some(FnAttribute::AlwaysInline)
+        }
         InlineAttr::Never => {
             if cx.sess().target.arch != "amdgpu" {
                 Some(FnAttribute::NoInline)

compiler/rustc_codegen_llvm/src/attributes.rs

@@ -37,7 +37,9 @@ fn inline_attr<'ll>(cx: &CodegenCx<'ll, '_>, inline: InlineAttr) -> Option<&'ll
     }
     match inline {
         InlineAttr::Hint => Some(AttributeKind::InlineHint.create_attr(cx.llcx)),
-        InlineAttr::Always => Some(AttributeKind::AlwaysInline.create_attr(cx.llcx)),
+        InlineAttr::Always | InlineAttr::Required { .. } | InlineAttr::Must { .. } => {
+            Some(AttributeKind::AlwaysInline.create_attr(cx.llcx))
+        }
         InlineAttr::Never => {
             if cx.sess().target.arch != "amdgpu" {
                 Some(AttributeKind::NoInline.create_attr(cx.llcx))

compiler/rustc_codegen_ssa/src/codegen_attrs.rs

@@ -1,4 +1,4 @@
-use rustc_ast::{MetaItemInner, MetaItemKind, ast, attr};
+use rustc_ast::{MetaItem, MetaItemInner, MetaItemKind, ast, attr};
 use rustc_attr::{InlineAttr, InstructionSetAttr, OptimizeAttr, list_contains_name};
 use rustc_data_structures::fx::FxHashMap;
 use rustc_errors::codes::*;
@@ -17,7 +17,7 @@ use rustc_middle::ty::{self as ty, TyCtxt};
 use rustc_session::parse::feature_err;
 use rustc_session::{Session, lint};
 use rustc_span::symbol::Ident;
-use rustc_span::{Span, sym};
+use rustc_span::{Span, Symbol, sym};
 use rustc_target::spec::{SanitizerSet, abi};
 
 use crate::errors::{self, MissingFeatures, TargetFeatureDisableOrEnable};
@@ -525,10 +525,36 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs {
                     struct_span_code_err!(tcx.dcx(), attr.span, E0534, "expected one argument")
                         .emit();
                     InlineAttr::None
+                } else if list_contains_name(items, sym::must) && tcx.features().required_inlining {
+                    parse_inline_must_required(
+                        tcx,
+                        items,
+                        attr.span,
+                        sym::must,
+                        |attr_span, reason| InlineAttr::Must { attr_span, reason },
+                    )
+                } else if list_contains_name(items, sym::required)
+                    && tcx.features().required_inlining
+                {
+                    parse_inline_must_required(
+                        tcx,
+                        items,
+                        attr.span,
+                        sym::required,
+                        |attr_span, reason| InlineAttr::Required { attr_span, reason },
+                    )
                 } else if list_contains_name(items, sym::always) {
                     InlineAttr::Always
                 } else if list_contains_name(items, sym::never) {
                     InlineAttr::Never
+                } else if tcx.features().required_inlining {
+                    struct_span_code_err!(tcx.dcx(), items[0].span(), E0535, "invalid argument")
+                        .with_help(
+                            "valid inline arguments are `required`, `must`, `always` and `never`",
+                        )
+                        .emit();
+
+                    InlineAttr::None
                 } else {
                     struct_span_code_err!(tcx.dcx(), items[0].span(), E0535, "invalid argument")
                         .with_help("valid inline arguments are `always` and `never`")
@@ -586,7 +612,7 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs {
     // is probably a poor usage of `#[inline(always)]` and easily avoided by not using the attribute.
     if tcx.features().target_feature_11
         && tcx.is_closure_like(did.to_def_id())
-        && codegen_fn_attrs.inline != InlineAttr::Always
+        && !codegen_fn_attrs.inline.always()
     {
         let owner_id = tcx.parent(did.to_def_id());
         if tcx.def_kind(owner_id).has_codegen_attrs() {
@@ -600,8 +626,7 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs {
     // purpose functions as they wouldn't have the right target features
     // enabled. For that reason we also forbid #[inline(always)] as it can't be
     // respected.
-    if !codegen_fn_attrs.target_features.is_empty() && codegen_fn_attrs.inline == InlineAttr::Always
-    {
+    if !codegen_fn_attrs.target_features.is_empty() && codegen_fn_attrs.inline.always() {
         if let Some(span) = inline_span {
             tcx.dcx().span_err(
                 span,
@@ -611,7 +636,7 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs {
         }
     }
 
-    if !codegen_fn_attrs.no_sanitize.is_empty() && codegen_fn_attrs.inline == InlineAttr::Always {
+    if !codegen_fn_attrs.no_sanitize.is_empty() && codegen_fn_attrs.inline.always() {
         if let (Some(no_sanitize_span), Some(inline_span)) = (no_sanitize_span, inline_span) {
             let hir_id = tcx.local_def_id_to_hir_id(did);
             tcx.node_span_lint(
@@ -706,6 +731,37 @@ pub fn check_tied_features(
     None
 }
 
+fn parse_inline_must_required<'tcx>(
+    tcx: TyCtxt<'tcx>,
+    items: &[MetaItemInner],
+    attr_span: Span,
+    expected_symbol: Symbol,
+    create: impl Fn(Span, Option<Symbol>) -> InlineAttr,
+) -> InlineAttr {
+    match items.iter().find(|i| i.has_name(expected_symbol)).expect("called on items w/out sym") {
+        MetaItemInner::MetaItem(mi @ MetaItem { kind: MetaItemKind::Word, .. }) => {
+            debug_assert!(mi.has_name(expected_symbol));
+            create(attr_span, None)
+        }
+        nested => {
+            if let Some((found_symbol, reason)) = nested.singleton_lit_list()
+                && reason.kind.is_str()
+            {
+                debug_assert_eq!(found_symbol, expected_symbol);
+                create(attr_span, reason.kind.str())
+            } else {
+                struct_span_code_err!(tcx.dcx(), attr_span, E0535, "invalid argument")
+                    .with_help(format!(
+                        "expected one string argument to `#[inline({})]`",
+                        expected_symbol.as_str()
+                    ))
+                    .emit();
+                create(attr_span, None)
+            }
+        }
+    }
+}
+
 /// Checks if the provided DefId is a method in a trait impl for a trait which has track_caller
 /// applied to the method prototype.
 fn should_inherit_track_caller(tcx: TyCtxt<'_>, def_id: DefId) -> bool {

compiler/rustc_feature/src/unstable.rs

@@ -221,6 +221,8 @@ declare_features! (
     (internal, prelude_import, "1.2.0", None),
     /// Used to identify crates that contain the profiler runtime.
     (internal, profiler_runtime, "1.18.0", None),
+    /// Allows using `#[inline(required)]`/`#[inline(must)]`
+    (internal, required_inlining, "CURRENT_RUSTC_VERSION", None),
     /// Allows using `rustc_*` attributes (RFC 572).
     (internal, rustc_attrs, "1.0.0", None),
     /// Allows using the `#[stable]` and `#[unstable]` attributes.

compiler/rustc_lint_defs/src/builtin.rs

@@ -67,6 +67,7 @@ declare_lint_pass! {
         MISSING_ABI,
         MISSING_FRAGMENT_SPECIFIER,
         MISSING_UNSAFE_ON_EXTERN,
+        MUST_INLINE,
         MUST_NOT_SUSPEND,
         NAMED_ARGUMENTS_USED_POSITIONALLY,
         NEVER_TYPE_FALLBACK_FLOWING_INTO_UNSAFE,
@@ -88,6 +89,7 @@ declare_lint_pass! {
         REFINING_IMPL_TRAIT_REACHABLE,
         RENAMED_AND_REMOVED_LINTS,
         REPR_TRANSPARENT_EXTERNAL_PRIVATE_FIELDS,
+        REQUIRED_INLINE,
         RUST_2021_INCOMPATIBLE_CLOSURE_CAPTURES,
         RUST_2021_INCOMPATIBLE_OR_PATTERNS,
         RUST_2021_PREFIXES_INCOMPATIBLE_SYNTAX,
@@ -5082,3 +5084,39 @@ declare_lint! {
     };
     crate_level_only
 }
+
+declare_lint! {
+    /// The `must_inline` lint is emitted when a function annotated with
+    /// `#[inline(must)]` was not able to be inlined.
+    ///
+    /// ### Explanation
+    ///
+    /// Functions can be marked as `#[inline(must)]` in the standard
+    /// library if they must be inlined in order to guarantee performance
+    /// characteristics or some other similar guarantee.
+    ///
+    /// In some circumstances, these functions cannot be inlined and a
+    /// reason will be provided, this can either be rectified or the
+    /// lint can be silenced if the risk is acceptable.
+    pub MUST_INLINE,
+    Warn,
+    "`#[inline(must)]`-annotated function could not be inlined"
+}
+
+declare_lint! {
+    /// The `required_inline` lint is emitted when a function annotated with
+    /// `#[inline(required)]` was not able to be inlined.
+    ///
+    /// ### Explanation
+    ///
+    /// Functions can be marked as `#[inline(required)]` in the standard
+    /// library if they are required to be inlined in order to uphold
+    /// security properties or some other similar guarantee.
+    ///
+    /// In some circumstances, these functions cannot be inlined and a
+    /// reason will be provided, this can either be rectified or the
+    /// lint can be silenced if the risk is acceptable.
+    pub REQUIRED_INLINE,
+    Deny,
+    "`#[inline(required)]`-annotated function could not be inlined"
+}

compiler/rustc_middle/src/mir/mono.rs

@@ -1,7 +1,6 @@
 use std::fmt;
 use std::hash::Hash;
 
-use rustc_attr::InlineAttr;
 use rustc_data_structures::base_n::{BaseNString, CASE_INSENSITIVE, ToBaseN};
 use rustc_data_structures::fingerprint::Fingerprint;
 use rustc_data_structures::fx::FxIndexMap;
@@ -138,9 +137,10 @@ impl<'tcx> MonoItem<'tcx> {
                 // creating one copy of this `#[inline]` function which may
                 // conflict with upstream crates as it could be an exported
                 // symbol.
-                match tcx.codegen_fn_attrs(instance.def_id()).inline {
-                    InlineAttr::Always => InstantiationMode::LocalCopy,
-                    _ => InstantiationMode::GloballyShared { may_conflict: true },
+                if tcx.codegen_fn_attrs(instance.def_id()).inline.always() {
+                    InstantiationMode::LocalCopy
+                } else {
+                    InstantiationMode::GloballyShared { may_conflict: true }
                 }
             }
             MonoItem::Static(..) | MonoItem::GlobalAsm(..) => {

compiler/rustc_mir_transform/messages.ftl

@@ -25,6 +25,17 @@ mir_transform_must_not_suspend = {$pre}`{$def_path}`{$post} held across a suspen
     .help = consider using a block (`{"{ ... }"}`) to shrink the value's scope, ending before the suspend point
 mir_transform_operation_will_panic = this operation will panic at runtime
 
+mir_transform_required_inline =
+    `{$callee}` could not be inlined into `{$caller}` but {$requires_or_must} inlined
+    .call = ...`{$callee}` called here
+    .attr = inlining due to this annotation
+    .caller = within `{$caller}`...
+    .callee = `{$callee}` defined here
+    .note = could not be inlined due to: {$reason}
+
+mir_transform_required_inline_justification =
+    `{$callee}` {$requires_or_must} inlined to: {$sym}
+
 mir_transform_unaligned_packed_ref = reference to packed field is unaligned
     .note = packed structs are only aligned by one byte, and many modern architectures penalize unaligned field accesses
     .note_ub = creating a misaligned reference is undefined behavior (even if that reference is never dereferenced)

compiler/rustc_mir_transform/src/cross_crate_inline.rs

@@ -8,8 +8,6 @@ use rustc_middle::ty::TyCtxt;
 use rustc_session::config::{InliningThreshold, OptLevel};
 use rustc_span::sym;
 
-use crate::{inline, pass_manager as pm};
-
 pub(super) fn provide(providers: &mut Providers) {
     providers.cross_crate_inlinable = cross_crate_inlinable;
 }
@@ -46,7 +44,10 @@ fn cross_crate_inlinable(tcx: TyCtxt<'_>, def_id: LocalDefId) -> bool {
     // #[inline(never)] to force code generation.
     match codegen_fn_attrs.inline {
         InlineAttr::Never => return false,
-        InlineAttr::Hint | InlineAttr::Always => return true,
+        InlineAttr::Hint
+        | InlineAttr::Always
+        | InlineAttr::Required { .. }
+        | InlineAttr::Must { .. } => return true,
         _ => {}
     }
 
@@ -59,7 +60,8 @@ fn cross_crate_inlinable(tcx: TyCtxt<'_>, def_id: LocalDefId) -> bool {
     // Don't do any inference if codegen optimizations are disabled and also MIR inlining is not
     // enabled. This ensures that we do inference even if someone only passes -Zinline-mir,
     // which is less confusing than having to also enable -Copt-level=1.
-    if matches!(tcx.sess.opts.optimize, OptLevel::No) && !pm::should_run_pass(tcx, &inline::Inline)
+    if matches!(tcx.sess.opts.optimize, OptLevel::No)
+        && !crate::inline::should_run_pass_for_item(tcx, def_id.to_def_id())
     {
         return false;
     }

compiler/rustc_mir_transform/src/errors.rs

@@ -4,8 +4,8 @@ use rustc_macros::{Diagnostic, LintDiagnostic, Subdiagnostic};
 use rustc_middle::mir::AssertKind;
 use rustc_middle::ty::TyCtxt;
 use rustc_session::lint::{self, Lint};
-use rustc_span::Span;
 use rustc_span::def_id::DefId;
+use rustc_span::{Span, Symbol};
 
 use crate::fluent_generated as fluent;
 
@@ -136,3 +136,29 @@ pub(crate) struct MustNotSuspendReason {
 #[note(mir_transform_note2)]
 #[help]
 pub(crate) struct UndefinedTransmute;
+
+#[derive(LintDiagnostic)]
+#[diag(mir_transform_required_inline)]
+#[note]
+pub(crate) struct RequiredInline {
+    pub requires_or_must: &'static str,
+    #[label(mir_transform_caller)]
+    pub caller_span: Span,
+    #[label(mir_transform_callee)]
+    pub callee_span: Span,
+    #[label(mir_transform_attr)]
+    pub attr_span: Span,
+    #[label(mir_transform_call)]
+    pub call_span: Span,
+    pub callee: String,
+    pub caller: String,
+    pub reason: &'static str,
+    #[subdiagnostic]
+    pub justification: Option<RequiredInlineJustification>,
+}
+
+#[derive(Subdiagnostic)]
+#[note(mir_transform_required_inline_justification)]
+pub(crate) struct RequiredInlineJustification {
+    pub sym: Symbol,
+}