MIR move elimination [4/6]: MoveElimination

@@ -374,6 +374,12 @@ impl<R: Idx, C: Step + Idx> SparseIntervalMatrix<R, C> {         self.rows.iter_enumerated()     } +    pub fn clear_row(&mut self, row: R) {+        if let Some(row) = self.rows.get_mut(row) {+            row.clear();+        }+    }+     fn ensure_row(&mut self, row: R) -> &mut IntervalSet<C> {         self.rows.ensure_contains_elem(row, || IntervalSet::new(self.column_size))     }@@ -397,6 +403,16 @@ impl<R: Idx, C: Step + Idx> SparseIntervalMatrix<R, C> {         write_row.union(read_row)     } +    pub fn disjoint_rows(&self, a: R, b: R) -> bool+    where+        C: Step,+    {+        let (Some(a), Some(b)) = (self.rows.get(a), self.rows.get(b)) else {+            return true;+        };+        a.disjoint(b)+    }+     pub fn insert_all_into_row(&mut self, row: R) {         self.ensure_row(row).insert_all();     }

@@ -127,7 +127,6 @@ pub enum RuntimePhase {     /// disallowed:     /// * [`TerminatorKind::Yield`]     /// * [`TerminatorKind::CoroutineDrop`]-    /// * [`Rvalue::Aggregate`] for any `AggregateKind` except `Array`     /// * [`Rvalue::CopyForDeref`]     /// * [`PlaceElem::OpaqueCast`]     /// * [`LocalInfo::DerefTemp`](super::LocalInfo::DerefTemp)@@ -1449,9 +1448,6 @@ pub enum Rvalue<'tcx> {     /// This is needed because dataflow analysis needs to distinguish     /// `dest = Foo { x: ..., y: ... }` from `dest.x = ...; dest.y = ...;` in the case that `Foo`     /// has a destructor.-    ///-    /// Disallowed after deaggregation for all aggregate kinds except `Array` and `Coroutine`. After-    /// coroutine lowering, `Coroutine` aggregate kinds are disallowed too.     Aggregate(Box<AggregateKind<'tcx>>, IndexVec<FieldIdx, Operand<'tcx>>),      /// A CopyForDeref is equivalent to a read from a place at the

@@ -212,6 +212,19 @@ impl<'tcx> AdtDef<'tcx> {         self.0.0.repr     } +    /// Whether MIR may project into the fields of this ADT.+    ///+    /// MCP#838 bans field projections into SIMD types, including scalable+    /// vectors.+    ///+    /// Codegen requires `DynMetadata` to be manipulated as a whole, with its+    /// insides being treated as opaque..+    pub fn can_project_fields(self, tcx: TyCtxt<'tcx>) -> bool {+        !self.repr().simd()+            && !self.repr().scalable()+            && !tcx.is_lang_item(self.did(), LangItem::DynMetadata)+    }+     pub fn field_representing_type_info(         self,         tcx: TyCtxt<'tcx>,

@@ -155,7 +155,9 @@ pub(super) struct DestinationPropagation;  impl<'tcx> crate::MirPass<'tcx> for DestinationPropagation {     fn policy(&self, ctx: &crate::PassCtx<'_>) -> PassPolicy {-        PassPolicy::optional(ctx.mir_opt_level() >= 2)+        PassPolicy::optional(+            ctx.mir_opt_level() >= 2 && !ctx.opts.unstable_opts.mir_move_elimination,+        )     }      #[tracing::instrument(level = "trace", skip(self, tcx, body))]

@@ -165,6 +165,7 @@ declare_passes! {     mod lower_slice_len : LowerSliceLenCalls;     mod match_branches : MatchBranchSimplification;     mod mentioned_items : MentionedItems;+    mod move_elimination : MoveElimination;     mod multiple_return_terminators : MultipleReturnTerminators;     mod post_drop_elaboration : CheckLiveDrops;     mod prettify : ReorderBasicBlocks, ReorderLocals;@@ -770,6 +771,7 @@ pub(crate) fn run_optimization_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'             &dead_store_elimination::DeadStoreElimination::Final,             &dest_prop::DestinationPropagation,             &tail_copy_to_move::TailCopyToMove,+            &move_elimination::MoveElimination,             &simplify::SimplifyLocals::Final,             &multiple_return_terminators::MultipleReturnTerminators,             // Some cleanup necessary at least for LLVM and potentially other codegen backends.

@@ -0,0 +1,911 @@+//! Eliminates copies and moves by unifying MIR places whose allocation ranges+//! are disjoint.+//!+//! See RFC 3943 for the local-lifetime semantics that make this optimization+//! possible.+//!+//! # Motivation+//!+//! MIR building can insert a lot of redundant copies, and Rust code in general+//! often tends to move values around a lot. The result is a lot of assignments+//! of the form `dest = {move} src;` in MIR. MIR building for constants in+//! particular tends to create additional locals that are only used inside a+//! single block to shuffle a value around unnecessarily.+//!+//! Additionally, Rust constructs nested aggregates by repeatedly moving values+//! into fields. For example, a function may build an inner value in a local,+//! move it into an outer aggregate, then move that aggregate into the caller's+//! destination. If these intermediate source and destination places have+//! different addresses, each layer needs an actual copy or move of the bytes.+//!+//! LLVM cannot remove these copies when both the source and destination+//! addresses are observed because merging the allocations would be an+//! observable change: the program could see that two addresses which were+//! previously distinct have become the same. This pass removes the copies+//! earlier, while MIR still has the information needed to prove that the two+//! allocation ranges do not overlap.+//!+//! # Optimization+//!+//! The basis of this optimization is place unification. If the source and+//! destination of an assignment have the same address, then the assignment is a+//! no-op. The same idea applies to aggregate construction: if a field operand+//! is already located at the corresponding field of the destination, then the+//! aggregate assignment does not need to copy those fields.+//!+//! The pass represents each unification as a mapping from a local to the place+//! that should replace it. Mappings are transitive, so `_3` can be resolved+//! through `_2.1` to `_1.0.1` if earlier mappings established those+//! relationships.+//!+//! The mapping is built by scanning the MIR for assignment statements. For+//! simple `Use` assignments, it tries to unify the source and destination+//! places. For `Aggregate` assignments, it tries to map each field operand to+//! the corresponding field in the assignment destination. Once all mappings+//! have been chosen, they are applied with one rewrite pass over the body.+//!+//! # Constraints+//!+//! Adding a mapping must preserve these conditions:+//!+//! * At least one side of the candidate pair must be a bare local. The pass can+//!   map a local to a place with projections, but it cannot map between two+//!   places that both already have projections.+//!+//! * Any projections in the mapped place must be stable everywhere the local is+//!   used. `Deref` and `Index` projections are rejected because they may refer+//!   to different memory at different points in the function.+//!+//! * The allocation ranges of the source and destination places must not+//!   overlap. This is checked using `PreciseLiveness`, which computes the+//!   points where each local must have a distinct allocation. The non-overlap+//!   proof is required so that the operational semantics can allow both places+//!   to have the same address.+//!+//! * Special-use locals such as arguments and the return place must keep their+//!   roles. Temps may be mapped into an argument or return place, but two+//!   special-use locals are not mapped into each other.+//!+//! * Some locals are used in contexts where projections cannot be added, such+//!   as `Index` projections. These locals may only be replaced by another bare+//!   local.+//!+//! # Storage lifetimes+//!+//! The original storage markers no longer describe the merged locals, so they+//! are removed. Locals without storage markers have storage for the full function.+//!+//! # Aliasing fixup+//!+//! MIR assignments currently require source and destination places not to+//! overlap for types that are not treated as scalars in codegen. After local+//! unification, some assignments may violate that invariant, so a final phase+//! rewrites them into a form codegen can handle. For each assignment:+//!+//! * Self-assignments, such as `_1 = _1`, are deleted.+//!+//! * Simple `Use` assignments whose source and destination overlap but are not+//!   identical are routed through a temporary: the source is read into the+//!   temporary first, and the temporary is then moved into the destination.+//!+//! * Aggregate assignments with any operand that aliases the destination are+//!   decomposed into per-field assignments. Field self-assignments are dropped.+//!   Other aliasing fields are read into temporaries first, then all+//!   destination fields are written. For enum aggregates the discriminant is+//!   set after fields are written.+//!+//! * Other rvalues, such as `Repeat` and `Cast`, are hoisted into a temporary+//!   if any place they access aliases the destination.+//!+//! * Rvalues that operate only on scalar types, such as binary and unary ops,+//!   `Discriminant`, `Ref`, and `RawPtr`, are left untouched because their+//!   codegen does not rely on the no-aliasing assumption.++use rustc_abi::{ExternAbi, FieldIdx, VariantIdx};+use rustc_const_eval::util::most_packed_projection;+use rustc_index::IndexVec;+use rustc_index::bit_set::DenseBitSet;+use rustc_index::interval::SparseIntervalMatrix;+use rustc_middle::mir::visit::{MutVisitor, NonUseContext, PlaceContext, VisitPlacesWith, Visitor};+use rustc_middle::mir::*;+use rustc_middle::ty::{Ty, TyCtxt};+use rustc_mir_dataflow::impls::{SplitPointIndex, dump_liveness_matrix, liveness_matrix};+use rustc_mir_dataflow::points::DenseLocationMap;+use tracing::{debug, trace};++use crate::PassPolicy;+use crate::patch::MirPatch;++pub(super) struct MoveElimination;++impl<'tcx> crate::MirPass<'tcx> for MoveElimination {+    fn policy(&self, ctx: &crate::PassCtx<'_>) -> PassPolicy {+        PassPolicy::optional(+            ctx.mir_opt_level() >= 2 && ctx.opts.unstable_opts.mir_move_elimination,+        )+    }++    #[tracing::instrument(level = "trace", skip(self, tcx, body))]+    fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {+        let def_id = body.source.def_id();+        trace!(?def_id);++        let points = DenseLocationMap::new(body);+        let mut liveness_matrix =+            liveness_matrix(tcx, body, &points, Some("MoveElimination.liveness"));++        dump_liveness_matrix(tcx, body, "MoveElimination.pre-liveness", &points, &liveness_matrix);++        let unprojectable_locals = UnprojectableLocals::find(body);+        trace!(?unprojectable_locals);++        let rust_call_tuples = find_rust_call_tuples(tcx, body);+        trace!(?rust_call_tuples);++        let remapped_locals = PlaceUnification::run(+            tcx,+            body,+            &mut liveness_matrix,+            unprojectable_locals,+            rust_call_tuples,+        );++        apply_mappings(tcx, body, &remapped_locals);++        dump_liveness_matrix(tcx, body, "MoveElimination.post-liveness", &points, &liveness_matrix);++        apply_alias_fixup(tcx, body);+    }+}++////////////////////////////////////////////////////////////////////////////////+// Unprojectable locals++/// Set of locals which can only be replaced with another local, instead of+/// an arbitrary place. This is usually because it is used directly as a+/// `Local` outside of a place (e.g. `Index` projections).+#[derive(Debug)]+struct UnprojectableLocals {+    locals: DenseBitSet<Local>,+}++impl UnprojectableLocals {+    fn find(body: &Body<'_>) -> DenseBitSet<Local> {+        let mut out = Self { locals: DenseBitSet::new_empty(body.local_decls.len()) };++        // Arguments and return places have fixed roles and cannot be replaced+        // with projected locals.+        out.locals.insert(RETURN_PLACE);+        for arg in body.args_iter() {+            out.locals.insert(arg);+        }++        out.visit_body(body);+        out.locals+    }+}++impl<'tcx> Visitor<'tcx> for UnprojectableLocals {+    fn visit_place(&mut self, place: &Place<'tcx>, context: PlaceContext, location: Location) {+        // We can't add more projections before a first position Deref projection.+        if place.is_indirect() {+            trace!(+                "unprojectable local {:?} due to use as deref base at {location:?}",+                place.local+            );+            self.locals.insert(place.local);+        }++        // Only call visit_local for projections, not the base local.+        self.visit_projection(place.as_ref(), context, location);+    }++    fn visit_local(&mut self, local: Local, context: PlaceContext, location: Location) {+        // Storage markers are removed.+        if let PlaceContext::NonUse(NonUseContext::StorageLive | NonUseContext::StorageDead) =+            context+        {+            return;+        }++        // If this is reached, it means that this is a bare local used outside+        // of a place, which means it cannot be replaced with a projection of+        // another local.+        trace!("unprojectable local {local:?} at {location:?} ({context:?})");+        self.locals.insert(local);+    }+}++////////////////////////////////////////////////////////////////////////////////+// "rust-call" tuple handling++/// Search for tuple locals passed to calls using the "rust-call" ABI.+///+/// For rust-call ABI calls, caller-side MIR passes the logical arguments as a+/// tuple operand. We want to avoid remapping other locals into fields of that+/// tuple, especially if one of those locals is borrowed.+///+/// Since the tuple itself is never borrowed, it is trivial for LLVM alias+/// analysis to see that accesses to one argument do not affect the others, but+/// merging the arguments into tuple fields from the start can hide that+/// independence.+fn find_rust_call_tuples<'tcx>(tcx: TyCtxt<'tcx>, body: &Body<'tcx>) -> DenseBitSet<Local> {+    let mut rust_call_tuples = DenseBitSet::new_empty(body.local_decls.len());++    for block in body.basic_blocks.iter() {+        let terminator = block.terminator();+        let (func, args) = match &terminator.kind {+            TerminatorKind::Call { func, args, .. }+            | TerminatorKind::TailCall { func, args, .. } => (func, args),+            _ => continue,+        };++        let sig = func.ty(&body.local_decls, tcx).fn_sig(tcx);+        if sig.abi() != ExternAbi::RustCall {+            continue;+        }++        let arg_tuple = args.last().expect("rust-call ABI requires a tuple argument");+        let (Operand::Copy(place) | Operand::Move(place)) = arg_tuple.node else {+            continue;+        };+        if let Some(local) = place.as_local() {+            rust_call_tuples.insert(local);+        }+    }++    rust_call_tuples+}++////////////////////////////////////////////////////////////////////////////////+// Local unification++struct PlaceUnification<'a, 'tcx> {+    tcx: TyCtxt<'tcx>,+    body: &'a Body<'tcx>,+    liveness_matrix: &'a mut SparseIntervalMatrix<Local, SplitPointIndex>,+    unprojectable_locals: DenseBitSet<Local>,+    rust_call_tuples: DenseBitSet<Local>,+    remapped_locals: IndexVec<Local, Option<Place<'tcx>>>,+}++impl<'tcx> PlaceUnification<'_, 'tcx> {+    fn run(+        tcx: TyCtxt<'tcx>,+        body: &Body<'tcx>,+        liveness_matrix: &mut SparseIntervalMatrix<Local, SplitPointIndex>,+        unprojectable_locals: DenseBitSet<Local>,+        rust_call_tuples: DenseBitSet<Local>,+    ) -> IndexVec<Local, Option<Place<'tcx>>> {+        let mut visitor = PlaceUnification {+            tcx,+            body,+            liveness_matrix,+            unprojectable_locals,+            rust_call_tuples,+            remapped_locals: IndexVec::from_elem_n(None, body.local_decls.len()),+        };+        visitor.visit_body(body);++        // Finalize the mappings by transitively resolving all locals to their+        // new final place.+        for local in visitor.remapped_locals.indices() {+            if let Some(place) = visitor.remapped_locals[local] {+                let place = visitor.resolve_place(place);+                visitor.remapped_locals[local] = Some(place);+                trace!("Remapped {local:?} to {place:?}");+            }+        }++        visitor.remapped_locals+    }++    #[tracing::instrument(ret, level = "trace", skip(self))]+    fn resolve_place(&self, mut place: Place<'tcx>) -> Place<'tcx> {+        while let Some(new_place) = self.remapped_locals[place.local] {+            place = new_place.project_deeper(place.projection, self.tcx);+        }+        place+    }++    #[tracing::instrument(ret, level = "trace", skip(self))]+    fn can_unify_places(&self, a: Place<'tcx>, b: Place<'tcx>) -> Option<(Local, Place<'tcx>)> {+        let a = self.resolve_place(a);+        let b = self.resolve_place(b);++        if a.local == b.local {+            if a.projection != b.projection {+                trace!("cannot unify same local with different projections");+            }+            return None;+        }++        if self.rust_call_tuples.contains(a.local) || self.rust_call_tuples.contains(b.local) {+            trace!("cannot unify {a:?} and {b:?} involving a rust-call tuple argument");+            return None;+        }++        let (local, place) = match (a.as_local(), b.as_local()) {+            (None, None) => {+                trace!("cannot unify 2 places that both have projections");+                return None;+            }+            (None, Some(b)) => {+                if self.unprojectable_locals.contains(b) {+                    trace!("cannot unify {b:?} which cannot be projected");+                    return None;+                }+                (b, a)+            }+            (Some(a), None) => {+                if self.unprojectable_locals.contains(a) {+                    trace!("cannot unify {a:?} which cannot be projected");+                    return None;+                }+                (a, b)+            }+            (Some(a), Some(b)) => match (self.body.local_kind(a), self.body.local_kind(b)) {+                (+                    LocalKind::Arg | LocalKind::ReturnPointer,+                    LocalKind::Arg | LocalKind::ReturnPointer,+                ) => {+                    trace!("cannot unify {a:?} and {b:?} which are both arguments or return place");+                    return None;+                }+                (LocalKind::Arg | LocalKind::ReturnPointer, LocalKind::Temp) => (b, a.into()),+                (LocalKind::Temp, _) => (a, b.into()),+            },+        };++        if most_packed_projection(self.tcx, &self.body.local_decls, place).is_some() {+            trace!("cannot unify {place:?} which has packed field projections");+            return None;+        }++        if !self.liveness_matrix.disjoint_rows(local, place.local) {+            trace!("cannot unify {a:?} and {b:?} which have overlapping live ranges");+            return None;+        }++        // FIXME(#112651): This can be removed afterwards.+        let local_ty = self.body.local_decls[local].ty;+        let place_ty = place.ty(&self.body.local_decls, self.tcx).ty;+        if local_ty != place_ty {+            trace!(+                "cannot unify {a:?} and {b:?} which have different types due to subtyping ({local_ty:?} vs {place_ty:?})"+            );+            return None;+        }++        Some((local, place))+    }++    #[tracing::instrument(level = "trace", skip(self))]+    fn remap_local(&mut self, local: Local, place: Place<'tcx>) {+        debug_assert!(self.remapped_locals[local].is_none());+        self.remapped_locals[local] = Some(place);++        self.liveness_matrix.union_rows(local, place.local);+        self.liveness_matrix.clear_row(local);++        // If the original local was unprojectable then this now also applies to+        // the mapped local.+        if self.unprojectable_locals.contains(local) {+            debug_assert!(place.projection.is_empty());+            self.unprojectable_locals.insert(place.local);+        }+    }++    fn visit_aggregate_assign(+        &mut self,+        dest: Place<'tcx>,+        project_field: impl Fn(TyCtxt<'tcx>, Place<'tcx>, FieldIdx, Ty<'tcx>) -> Place<'tcx>,+        operands: &IndexVec<FieldIdx, Operand<'tcx>>,+        location: Location,+    ) {+        // Attempt to unify each field operand with the corresponding field in+        // the destination place.+        let mut candidates = vec![];+        for (idx, operand) in operands.iter_enumerated() {+            let (Operand::Copy(src) | Operand::Move(src)) = *operand else {+                continue;+            };+            let Some(src) = src.as_local() else {+                continue;+            };+            let dest = project_field(self.tcx, dest, idx, self.body.local_decls[src].ty);+            trace!("Attempting to unify {dest:?} and {src:?} at {location:?}");+            if let Some((local, place)) = self.can_unify_places(dest, src.into()) {+                candidates.push((local, place));+            }+        }++        // Do the actual remapping *after* checking for live range overlaps.+        //+        // If we tried to do this immediately after `can_unify_places` for each+        // operand then this would only allow unifying with one operand: once+        // the first operand's live range is unified with the destination, all+        // remaining inputs conflict with the combined live range.+        //+        // By deferring remapping until all inputs are checked, we can unify+        // with all eligible inputs instead of just one. This is sound because+        // they will each be mapped to a disjoint field of the destination+        // place.+        for (local, place) in candidates {+            // Repeated operands can produce multiple candidates for the same+            // local. Only keep the first mapping.+            if self.remapped_locals[local].is_some() {+                continue;+            }+            self.remap_local(local, place);+        }+    }+}++/// Since we are replacing all uses of a local with another place, we need to+/// ensure that the projections on that place are stable no matter where it is+/// used in the body. Additional this local may be used in debuginfo, so ensure+/// that the projections are compatible with usage in debuginfo.+fn check_projections(place: Place<'_>) -> bool {+    place.projection.iter().all(|elem| elem.is_stable_offset() && elem.can_use_in_debuginfo())+}++impl<'tcx> Visitor<'tcx> for PlaceUnification<'_, 'tcx> {+    fn visit_assign(&mut self, dest: &Place<'tcx>, rvalue: &Rvalue<'tcx>, location: Location) {+        if !check_projections(*dest) {+            return;+        }+        match rvalue {+            Rvalue::Use(Operand::Copy(src) | Operand::Move(src), _) => {+                if !check_projections(*src) {+                    return;+                }++                trace!("Attempting to unify {dest:?} and {src:?} at {location:?}");+                if let Some((local, place)) = self.can_unify_places(*src, *dest) {+                    self.remap_local(local, place);+                }+            }+            Rvalue::Aggregate(aggregate_kind, operands) => match *aggregate_kind {+                AggregateKind::Array(_) => self.visit_aggregate_assign(+                    *dest,+                    |tcx, place, field_idx, _field_ty| {+                        place.project_deeper(+                            &[PlaceElem::ConstantIndex {+                                offset: field_idx.as_u32().into(),+                                min_length: field_idx.as_u32() as u64 + 1,+                                from_end: false,+                            }],+                            tcx,+                        )+                    },+                    operands,+                    location,+                ),+                AggregateKind::Tuple => self.visit_aggregate_assign(+                    *dest,+                    |tcx, place, field_idx, field_ty| {+                        place.project_deeper(&[PlaceElem::Field(field_idx, field_ty)], tcx)+                    },+                    operands,+                    location,+                ),+                AggregateKind::Adt(_, _, _, _, Some(union_field_idx)) => {+                    debug_assert_eq!(operands.len(), 1);+                    self.visit_aggregate_assign(+                        *dest,+                        |tcx, place, _, field_ty| {+                            place+                                .project_deeper(&[PlaceElem::Field(union_field_idx, field_ty)], tcx)+                        },+                        operands,+                        location,+                    )+                }+                AggregateKind::Adt(adt_did, var_idx, _, _, None) => {+                    let def = self.tcx.adt_def(adt_did);+                    if !def.can_project_fields(self.tcx) {+                        return;+                    }+                    self.visit_aggregate_assign(+                        *dest,+                        |tcx, place, field_idx, field_ty| {+                            if def.is_enum() {+                                place.project_deeper(+                                    &[+                                        PlaceElem::Downcast(None, var_idx),+                                        PlaceElem::Field(field_idx, field_ty),+                                    ],+                                    tcx,+                                )+                            } else {+                                place.project_deeper(&[PlaceElem::Field(field_idx, field_ty)], tcx)+                            }+                        },+                        operands,+                        location,+                    )+                }+                _ => {}+            },+            _ => {}+        };+    }+}++////////////////////////////////////////////////////////////////////////////////+// Apply place mappings to the MIR body.++fn apply_mappings<'tcx>(+    tcx: TyCtxt<'tcx>,+    body: &mut Body<'tcx>,+    remapped_locals: &IndexVec<Local, Option<Place<'tcx>>>,+) {+    let mut rewriter = PlaceUpdater { tcx, remapped_locals };+    rewriter.visit_body_preserves_cfg(body);+}++struct PlaceUpdater<'a, 'tcx> {+    tcx: TyCtxt<'tcx>,+    remapped_locals: &'a IndexVec<Local, Option<Place<'tcx>>>,+}++impl<'tcx> MutVisitor<'tcx> for PlaceUpdater<'_, 'tcx> {+    fn tcx(&self) -> TyCtxt<'tcx> {+        self.tcx+    }++    fn visit_local(&mut self, local: &mut Local, context: PlaceContext, location: Location) {+        if let Some(new_place) = self.remapped_locals[*local] {+            trace!("replacing {local:?} with {new_place:?} at {location:?} ({context:?})");+            *local = new_place.as_local().expect("mapped place shouldn't have projections");+        }+    }++    fn visit_place(&mut self, place: &mut Place<'tcx>, context: PlaceContext, location: Location) {+        if let Some(new_place) = self.remapped_locals[place.local] {+            trace!("replacing {place:?} with {new_place:?} at {location:?} ({context:?})");+            *place = new_place.project_deeper(place.projection, self.tcx)+        }++        // Only call visit_local for projections, not the base local.+        if let Some(new_projection) = self.process_projection(&place.projection, location) {+            place.projection = self.tcx().mk_place_elems(&new_projection);+        }+    }++    fn visit_statement(&mut self, statement: &mut Statement<'tcx>, location: Location) {+        match statement.kind {+            // Remove storage lifetime markers because they no longer describe+            // the merged locals. The later storage reconstruction pass rebuilds+            // these markers for all locals, which can also shorten the storage+            // lifetimes of locals that were not merged.+            StatementKind::StorageDead(_) | StatementKind::StorageLive(_) => {+                statement.make_nop(true);+                return;+            }+            _ => {}+        }++        self.super_statement(statement, location);+    }+}++////////////////////////////////////////////////////////////////////////////////+// Aliasing assignment fixup+//+// MIR assignments currently do not allow source and destination to alias, so+// fix this in post-processing.++fn apply_alias_fixup<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {+    let mut patcher = MirPatch::new(body);+    let mut fixup = AliasFixup { tcx, local_decls: &body.local_decls, patcher: &mut patcher };+    for (block, data) in body.basic_blocks.as_mut_preserves_cfg().iter_enumerated_mut() {+        fixup.visit_basic_block_data(block, data);+    }+    patcher.apply(body);+}++/// Returns whether 2 places alias, ignoring indirect places.+fn places_directly_alias<'tcx>(+    tcx: TyCtxt<'tcx>,+    local_decls: &IndexVec<Local, LocalDecl<'tcx>>,+    a: Place<'tcx>,+    b: Place<'tcx>,+) -> bool {+    // This function doesn't handle indirect aliasing.+    if a.local != b.local || a.is_indirect_first_projection() || b.is_indirect_first_projection() {+        return false;+    }++    for ((prefix, elem_a), (_, elem_b)) in a.iter_projections().zip(b.iter_projections()) {+        // Continue until we find the first mismatching projection.+        if elem_a == elem_b {+            continue;+        }++        match (elem_a, elem_b) {+            // Disjoint fields don't alias except if they are union fields.+            (PlaceElem::Field(_, _), PlaceElem::Field(_, _)) => {+                let ty = prefix.ty(local_decls, tcx).ty;+                return ty.is_union();+            }++            // Disjoint slice elements don't alias.+            (+                PlaceElem::ConstantIndex { offset: offset_a, min_length: _, from_end: from_end_a },+                PlaceElem::ConstantIndex { offset: offset_b, min_length: _, from_end: from_end_b },+            ) if from_end_a == from_end_b && offset_a != offset_b => {+                return false;+            }++            // Conservatively assume the places may alias.+            _ => return true,+        }+    }++    // If the projections are identical *or* one is a prefix of the other then+    // the places alias.+    true+}++struct AliasFixup<'a, 'tcx> {+    tcx: TyCtxt<'tcx>,+    local_decls: &'a IndexVec<Local, LocalDecl<'tcx>>,+    patcher: &'a mut MirPatch<'tcx>,+}++impl<'tcx> AliasFixup<'_, 'tcx> {+    fn isolate_rvalue_to_local(+        &mut self,+        rvalue: Rvalue<'tcx>,+        source_info: SourceInfo,+        location: Location,+    ) -> Place<'tcx> {+        let ty = rvalue.ty(self.local_decls, self.tcx);+        let temp = Place::from(self.patcher.new_temp(ty, source_info.span));+        trace!("isolating {rvalue:?} to {temp:?} due to conflict");+        self.patcher.add_statement(location, StatementKind::StorageLive(temp.local));+        self.patcher.add_assign(location, Place::from(temp), rvalue);+        self.patcher.add_statement(+            location.successor_within_block(),+            StatementKind::StorageDead(temp.local),+        );+        temp+    }++    fn visit_aggregate_assign(+        &mut self,+        dest: Place<'tcx>,+        enum_variant: Option<VariantIdx>,+        project_field: impl Fn(TyCtxt<'tcx>, Place<'tcx>, FieldIdx, Ty<'tcx>) -> Place<'tcx>,+        operands: &IndexVec<FieldIdx, Operand<'tcx>>,+        source_info: SourceInfo,+        location: Location,+    ) {+        // Fast path: if no direct operand aliases the destination, we're done.+        //+        // We only look for direct aliases here, which is sufficient because we+        // know the input MIR did not have any aliasing and we didn't introduce+        // any indirect aliasing in this pass.+        //+        // If the destination place is indirect then it cannot be the start of+        // a lifetime as per the RFC 3943 MIR semantics. This means that the+        // lifetime of the underlying allocation must have started earlier,+        // which overlaps the early point of the assignment statement. Therefore+        // we couldn't have unified any source operand with this destination+        // place.+        //+        // If the source place is indirect then a similar reasoning applies. The+        // only exception is if there are multiple source places (e.g.+        // aggregates). In that situation it's possible for an indirect source+        // to overlap the destination if and only if there is also a direct+        // source that overlaps it:+        //+        // _2 = &_1+        // _3 = (copy *_2, move _1) // _1 becomes _3.1 after unification+        //+        // We handle this here in 2 ways: if there is no direct alias, then+        // we're fine. Otherwise, treat all indirect sources as potentially+        // aliasing with the destination operand.+        let has_direct_alias = operands.iter().any(|op| match op {+            Operand::Copy(src) | Operand::Move(src) => {+                places_directly_alias(self.tcx, self.local_decls, dest, *src)+            }+            Operand::Constant(_) | Operand::RuntimeChecks(_) => false,+        });+        if !has_direct_alias {+            return;+        }++        debug!("splitting aggregate assignment at {location:?}");++        // Split into per-field assignments.+        let mut assignments = vec![];+        for (idx, op) in operands.iter_enumerated() {+            let field_ty = op.ty(self.local_decls, self.tcx);+            let dest_field = project_field(self.tcx, dest, idx, field_ty);++            let emit_op = match op {+                Operand::Copy(src) | Operand::Move(src) => {+                    if *src == dest_field {+                        // Skip identity assignments.+                        continue;+                    } else if src.is_indirect_first_projection()+                        || places_directly_alias(self.tcx, self.local_decls, dest, *src)+                    {+                        // Partial alias: hoist the source to a temp first so the+                        // per-field write no longer overlaps the dest. Indirect+                        // sources also need hoisting here because they may point+                        // at one of the direct aliasing operands.+                        Operand::Move(self.isolate_rvalue_to_local(+                            Rvalue::Use(op.clone(), WithRetag::No),+                            source_info,+                            location,+                        ))+                    } else {+                        op.clone()+                    }+                }+                Operand::Constant(_) | Operand::RuntimeChecks(_) => op.clone(),+            };+            assignments.push((dest_field, emit_op));+        }++        // Perform assignments *after* all aliasing fields have been read into+        // temporary locals.+        for (dest_field, emit_op) in assignments {+            self.patcher.add_assign(location, dest_field, Rvalue::Use(emit_op, WithRetag::No));+        }++        // Delete the original aggregate assignment.+        self.patcher.nop_statement(location);++        // For enum variants, set the discriminant after all field writes.+        if let Some(variant_index) = enum_variant {+            self.patcher.add_statement(+                location,+                StatementKind::SetDiscriminant { place: Box::new(dest), variant_index },+            );+        }+    }+}++impl<'tcx> MutVisitor<'tcx> for AliasFixup<'_, 'tcx> {+    fn tcx(&self) -> TyCtxt<'tcx> {+        self.tcx+    }++    fn visit_statement(&mut self, statement: &mut Statement<'tcx>, location: Location) {+        // Fixup the MIR to remove aliasing assignments.+        if let StatementKind::Assign((dest, rvalue)) = &mut statement.kind {+            match *rvalue {+                Rvalue::Use(Operand::Copy(src) | Operand::Move(src), with_retag) => {+                    if places_directly_alias(self.tcx, self.local_decls, *dest, src) {+                        if src == *dest {+                            debug!("{:?} turned into self-assignment, deleting", location);+                            statement.make_nop(true);+                        } else {+                            let temp = self.isolate_rvalue_to_local(+                                rvalue.clone(),+                                statement.source_info,+                                location,+                            );+                            *rvalue = Rvalue::Use(Operand::Move(temp), with_retag);+                        }+                    }+                }+                Rvalue::Aggregate(AggregateKind::Array(_), ref mut operands) => self+                    .visit_aggregate_assign(+                        *dest,+                        None,+                        |tcx, place, field_idx, _field_ty| {+                            place.project_deeper(+                                &[PlaceElem::ConstantIndex {+                                    offset: field_idx.as_u32().into(),+                                    min_length: field_idx.as_u32() as u64 + 1,+                                    from_end: false,+                                }],+                                tcx,+                            )+                        },+                        operands,+                        statement.source_info,+                        location,+                    ),+                Rvalue::Aggregate(AggregateKind::Tuple, ref mut operands) => self+                    .visit_aggregate_assign(+                        *dest,+                        None,+                        |tcx, place, field_idx, field_ty| {+                            place.project_deeper(&[PlaceElem::Field(field_idx, field_ty)], tcx)+                        },+                        operands,+                        statement.source_info,+                        location,+                    ),+                Rvalue::Aggregate(+                    AggregateKind::Adt(_, _, _, _, Some(union_field_idx)),+                    ref mut operands,+                ) => {+                    debug_assert_eq!(operands.len(), 1);+                    self.visit_aggregate_assign(+                        *dest,+                        None,+                        |tcx, place, _, field_ty| {+                            place+                                .project_deeper(&[PlaceElem::Field(union_field_idx, field_ty)], tcx)+                        },+                        operands,+                        statement.source_info,+                        location,+                    )+                }+                // Types that forbid field projections must use the general+                // aliasing fallback below.+                Rvalue::Aggregate(+                    AggregateKind::Adt(adt_did, var_idx, _, _, None),+                    ref mut operands,+                ) if self.tcx.adt_def(adt_did).can_project_fields(self.tcx) => {+                    let def = self.tcx.adt_def(adt_did);+                    self.visit_aggregate_assign(+                        *dest,+                        def.is_enum().then_some(var_idx),+                        |tcx, place, field_idx, field_ty| {+                            if def.is_enum() {+                                place.project_deeper(+                                    &[+                                        PlaceElem::Downcast(None, var_idx),+                                        PlaceElem::Field(field_idx, field_ty),+                                    ],+                                    tcx,+                                )+                            } else {+                                place.project_deeper(&[PlaceElem::Field(field_idx, field_ty)], tcx)+                            }+                        },+                        operands,+                        statement.source_info,+                        location,+                    )+                }++                // For other rvalues, don't try to split them into components+                // and instead just introduce a temporary if there is any+                // aliasing+                Rvalue::Aggregate(..)+                | Rvalue::Repeat(..)+                | Rvalue::Cast(..)+                | Rvalue::CopyForDeref(..)+                | Rvalue::WrapUnsafeBinder(..) => {+                    let mut overlaps_dest = false;+                    VisitPlacesWith(|place, _ctxt| {+                        if places_directly_alias(self.tcx, self.local_decls, *dest, place) {+                            overlaps_dest = true;+                        }+                    })+                    .visit_rvalue(rvalue, location);+                    if overlaps_dest {+                        let temp = self.isolate_rvalue_to_local(+                            rvalue.clone(),+                            statement.source_info,+                            location,+                        );+                        *rvalue = Rvalue::Use(Operand::Move(temp), WithRetag::No);+                    }+                }++                // These either cannot have aliasing, or allow it because they+                // only operate on scalar backend types.+                Rvalue::Use(Operand::Constant(..) | Operand::RuntimeChecks(..), _)+                | Rvalue::Ref(..)+                | Rvalue::ThreadLocalRef(..)+                | Rvalue::BinaryOp(..)+                | Rvalue::UnaryOp(..)+                | Rvalue::Discriminant(..)+                | Rvalue::RawPtr(..)+                | Rvalue::Reborrow(..) => {}+            }+        }+    }+}

@@ -1,5 +1,4 @@ use rustc_abi::FieldIdx;-use rustc_attr_ir::lang_items::LangItem; use rustc_data_structures::flat_map_in_place::FlatMapInPlace; use rustc_index::IndexVec; use rustc_index::bit_set::{DenseBitSet, GrowableBitSet};@@ -64,16 +63,8 @@ fn escaping_locals<'tcx>(             return true;         }         if let ty::Adt(def, _args) = ty.kind()-            && (def.repr().simd()-                || def.repr().scalable()-                || tcx.is_lang_item(def.did(), LangItem::DynMetadata))+            && !def.can_project_fields(tcx)         {-            // Exclude #[repr(simd)] types so that they are not de-optimized into an array-            // (MCP#838 banned projections into SIMD types, but if the value is unused-            // this pass sees "all the uses are of the fields" and expands it.)--            // codegen wants to see the `DynMetadata<T>`,-            // not the inner reference-to-opaque-type.             return true;         }         // Default for non-ADTs

@@ -716,24 +716,19 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> {                     // Debug info is allowed to project into pattern types                     ty::Pat(base, _) => check_equal(self, location, *base),                     ty::Adt(adt_def, args) => {-                        // see <https://github.com/rust-lang/rust/blob/7601adcc764d42c9f2984082b49948af652df986/compiler/rustc_middle/src/ty/layout.rs#L861-L864>-                        if self.tcx.is_lang_item(adt_def.did(), LangItem::DynMetadata) {-                            self.fail(-                                location,-                                format!(-                                    "You can't project to field {f:?} of `DynMetadata` because \+                        if !adt_def.can_project_fields(self.tcx) {+                            let message =+                                if self.tcx.is_lang_item(adt_def.did(), LangItem::DynMetadata) {+                                    format!(+                                        "You can't project to field {f:?} of `DynMetadata` because \                                      layout is weird and thinks it doesn't have fields."-                                ),-                            );-                        }--                        if adt_def.repr().simd() || adt_def.repr().scalable() {-                            self.fail(-                                location,-                                format!(-                                    "Projecting into SIMD type {adt_def:?} is banned by MCP#838"-                                ),-                            );+                                    )+                                } else {+                                    format!(+                                        "Projecting into SIMD type {adt_def:?} is banned by MCP#838"+                                    )+                                };+                            self.fail(location, message);                         }                          let var = parent_ty.variant_index.unwrap_or(FIRST_VARIANT);

@@ -551,9 +551,6 @@ pub enum Rvalue {     /// This is needed because dataflow analysis needs to distinguish     /// `dest = Foo { x: ..., y: ... }` from `dest.x = ...; dest.y = ...;` in the case that `Foo`     /// has a destructor.-    ///-    /// Disallowed after deaggregation for all aggregate kinds except `Array` and `Coroutine`. After-    /// coroutine lowering, `Coroutine` aggregate kinds are disallowed too.     Aggregate(AggregateKind, Vec<Operand>),      /// * `Offset` has the same semantics as `<*const T>::offset`, except that the second

@@ -2690,7 +2690,7 @@ options! {         "include extra comments in mir pretty printing, like line numbers and statement indices, \          details about types, etc. (boolean for all passes, 'nll' to enable in NLL MIR only, default: 'nll')"),     mir_move_elimination: bool = (false, parse_bool, [TRACKED],-        "enable the experimental MIR move elimination pass (default: no)"),+        "enable the experimental MIR semantics for the move elimination optimization (default: no)"),     mir_opt_bisect_limit: Option<usize> = (None, parse_opt_number, [TRACKED],         "limit the number of MIR optimization pass executions (global across all bodies). \         Pass executions after this limit are skipped and reported. (default: no limit)"),

@@ -0,0 +1,27 @@+- // MIR for `aggregate_indirect_source_alias` before MoveElimination++ // MIR for `aggregate_indirect_source_alias` after MoveElimination+  +  fn aggregate_indirect_source_alias() -> (u8, u8) {+      let mut _0: (u8, u8);+      let mut _1: u8;+      let mut _2: *const u8;+      let mut _3: (u8, u8);++     let mut _4: u8;+  +      bb0: {+-         _1 = const 1_u8;+-         _2 = &raw const _1;+-         _3 = (copy (*_2), move _1);+-         _0 = copy _3;++         (_0.1: u8) = const 1_u8;++         _2 = &raw const (_0.1: u8);++         StorageLive(_4);++         _4 = no_retag copy (*_2);++         (_0.0: u8) = no_retag move _4;++         nop;++         StorageDead(_4);++         nop;+          return;+      }+  }+  

@@ -0,0 +1,84 @@+- // MIR for `aggregate_swap` before MoveElimination++ // MIR for `aggregate_swap` after MoveElimination+  +  fn aggregate_swap(_1: u8, _2: u8) -> (u8, u8) {+      debug x => _1;+      debug y => _2;+      let mut _0: (u8, u8);+      let mut _3: (u8, u8);+      let mut _4: u8;+      let mut _5: u8;+      let mut _8: u8;+      let mut _9: u8;++     let mut _10: u8;+      scope 1 {+-         debug pair => _3;++         debug pair => _0;+          let _6: u8;+          scope 2 {+-             debug a => _6;++             debug a => _9;+              let _7: u8;+              scope 3 {+-                 debug b => _7;++                 debug b => (_0.1: u8);+              }+          }+      }+  +      bb0: {+-         StorageLive(_3);+-         StorageLive(_4);+-         _4 = copy _1;+-         StorageLive(_5);+-         _5 = copy _2;+-         _3 = (move _4, move _5);+-         StorageDead(_5);+-         StorageDead(_4);+-         StorageLive(_6);+-         _6 = copy (_3.0: u8);+-         StorageLive(_7);+-         _7 = copy (_3.1: u8);+-         StorageLive(_8);+-         _8 = copy _7;+-         StorageLive(_9);+-         _9 = copy _6;+-         _3 = (move _8, move _9);+-         StorageDead(_9);+-         StorageDead(_8);+-         _0 = copy _3;+-         StorageDead(_7);+-         StorageDead(_6);+-         StorageDead(_3);++         nop;++         nop;++         nop;++         nop;++         nop;++         _0 = (move _1, move _2);++         nop;++         nop;++         nop;++         _9 = copy (_0.0: u8);++         nop;++         nop;++         nop;++         nop;++         nop;++         nop;++         StorageLive(_10);++         _10 = no_retag move (_0.1: u8);++         (_0.0: u8) = no_retag move _10;++         (_0.1: u8) = no_retag move _9;++         nop;++         StorageDead(_10);++         nop;++         nop;++         nop;++         nop;++         nop;++         nop;+          return;+      }+  }+  

@@ -0,0 +1,55 @@+- // MIR for `array_swap` before MoveElimination++ // MIR for `array_swap` after MoveElimination+  +  fn array_swap(_1: [u8; 2]) -> [u8; 2] {+      debug array => _1;+      let mut _0: [u8; 2];+      let _2: u8;+      let _3: u8;+      let mut _4: u8;+      let mut _5: u8;++     let mut _6: u8;+      scope 1 {+-         debug a => _2;+-         debug b => _3;++         debug a => _5;++         debug b => _1[1 of 2];+      }+  +      bb0: {+-         StorageLive(_2);+-         _2 = copy _1[0 of 2];+-         StorageLive(_3);+-         _3 = copy _1[1 of 2];+-         StorageLive(_4);+-         _4 = copy _3;+-         StorageLive(_5);+-         _5 = copy _2;+-         _1 = [move _4, move _5];+-         StorageDead(_5);+-         StorageDead(_4);++         nop;++         _5 = copy _1[0 of 2];++         nop;++         nop;++         nop;++         nop;++         nop;++         nop;++         StorageLive(_6);++         _6 = no_retag move _1[1 of 2];++         _1[0 of 1] = no_retag move _6;++         _1[1 of 2] = no_retag move _5;++         nop;++         StorageDead(_6);++         nop;++         nop;+          _0 = copy _1;+-         StorageDead(_3);+-         StorageDead(_2);++         nop;++         nop;+          return;+      }+  }+  

@@ -0,0 +1,120 @@+- // MIR for `enum_alias` before MoveElimination++ // MIR for `enum_alias` after MoveElimination+  +  fn enum_alias(_1: bool, _2: u8, _3: u8) -> PairEnum {+      debug flag => _1;+      debug x => _2;+      debug y => _3;+      let mut _0: PairEnum;+      let _4: PairEnum;+      let mut _5: u8;+      let mut _6: u8;+      let mut _7: bool;+      let mut _8: isize;+      let mut _11: u8;+      let mut _12: u8;++     let mut _13: u8;+      scope 1 {+-         debug input => _4;++         debug input => _0;+          let _9: u8;+          let _10: u8;+          scope 2 {+-             debug a => _9;+-             debug b => _10;++             debug a => _12;++             debug b => ((_0 as Pair).1: u8);+          }+      }+  +      bb0: {+-         StorageLive(_4);+-         StorageLive(_5);+-         _5 = copy _2;+-         StorageLive(_6);+-         _6 = copy _3;+-         _4 = PairEnum::Pair(move _5, move _6);+-         StorageDead(_6);+-         StorageDead(_5);+-         StorageLive(_7);+-         _7 = copy _1;+-         switchInt(move _7) -> [0: bb2, otherwise: bb1];++         nop;++         nop;++         nop;++         nop;++         nop;++         _0 = PairEnum::Pair(move _2, move _3);++         nop;++         nop;++         nop;++         nop;++         switchInt(move _1) -> [0: bb2, otherwise: bb1];+      }+  +      bb1: {+-         _0 = move _4;++         nop;+          goto -> bb6;+      }+  +      bb2: {+-         _8 = discriminant(_4);++         _8 = discriminant(_0);+          switchInt(move _8) -> [0: bb4, 1: bb5, otherwise: bb3];+      }+  +      bb3: {+          unreachable;+      }+  +      bb4: {+          _0 = PairEnum::Empty;+          goto -> bb6;+      }+  +      bb5: {+-         StorageLive(_9);+-         _9 = copy ((_4 as Pair).0: u8);+-         StorageLive(_10);+-         _10 = copy ((_4 as Pair).1: u8);+-         StorageLive(_11);+-         _11 = copy _10;+-         StorageLive(_12);+-         _12 = copy _9;+-         _0 = PairEnum::Pair(move _11, move _12);+-         StorageDead(_12);+-         StorageDead(_11);+-         StorageDead(_10);+-         StorageDead(_9);++         nop;++         _12 = copy ((_0 as Pair).0: u8);++         nop;++         nop;++         nop;++         nop;++         nop;++         nop;++         StorageLive(_13);++         _13 = no_retag move ((_0 as Pair).1: u8);++         ((_0 as variant#1).0: u8) = no_retag move _13;++         ((_0 as variant#1).1: u8) = no_retag move _12;++         discriminant(_0) = 1;++         nop;++         StorageDead(_13);++         nop;++         nop;++         nop;++         nop;+          goto -> bb6;+      }+  +      bb6: {+-         StorageDead(_7);+-         StorageDead(_4);++         nop;++         nop;+          return;+      }+  }+  

@@ -0,0 +1,90 @@+- // MIR for `mixed_aggregate_aliasing` before MoveElimination++ // MIR for `mixed_aggregate_aliasing` after MoveElimination+  +  fn mixed_aggregate_aliasing(_1: bool, _2: u8) -> Triple {+      debug flag => _1;+      debug z => _2;+      let mut _0: Triple;+      let _3: Triple;+      let mut _5: bool;+      let mut _6: u8;+      let mut _7: u8;+      let mut _8: u8;++     let mut _9: u8;+      scope 1 {+-         debug input => _3;++         debug input => _0;+          let _4: Triple;+          scope 2 {+-             debug out => _4;++             debug out => _0;+          }+      }+  +      bb0: {+-         StorageLive(_3);+-         _3 = opaque_triple() -> [return: bb1, unwind unreachable];++         nop;++         _0 = opaque_triple() -> [return: bb1, unwind unreachable];+      }+  +      bb1: {+-         StorageLive(_4);+-         StorageLive(_5);+-         _5 = copy _1;+-         switchInt(move _5) -> [0: bb3, otherwise: bb2];++         nop;++         nop;++         nop;++         switchInt(move _1) -> [0: bb3, otherwise: bb2];+      }+  +      bb2: {+-         _4 = copy _3;++         nop;+          goto -> bb4;+      }+  +      bb3: {+-         StorageLive(_6);+-         _6 = copy (_3.1: u8);+-         StorageLive(_7);+-         _7 = copy (_3.0: u8);+-         StorageLive(_8);+-         _8 = copy _2;+-         _4 = Triple(move _6, move _7, move _8);+-         StorageDead(_8);+-         StorageDead(_7);+-         StorageDead(_6);++         nop;++         _6 = copy (_0.1: u8);++         nop;++         nop;++         nop;++         nop;++         StorageLive(_9);++         _9 = no_retag move (_0.0: u8);++         (_0.0: u8) = no_retag move _6;++         (_0.1: u8) = no_retag move _9;++         (_0.2: u8) = no_retag move _2;++         nop;++         StorageDead(_9);++         nop;++         nop;++         nop;+          goto -> bb4;+      }+  +      bb4: {+-         StorageDead(_5);+-         _0 = copy _4;+-         StorageDead(_4);+-         StorageDead(_3);++         nop;++         nop;++         nop;++         nop;+          return;+      }+  }+  

@@ -0,0 +1,156 @@+//@ test-mir-pass: MoveElimination+//@ compile-flags: -Cpanic=abort++#![feature(core_intrinsics, custom_mir, repr_simd)]+#![allow(dead_code)]+#![allow(internal_features)]++use std::intrinsics::mir::*;++#[derive(Copy, Clone)]+pub struct Triple(u8, u8, u8);++pub union U {+    a: [u8; 4],+    b: [u8; 4],+}++unsafe extern "C" {+    safe fn opaque_triple() -> Triple;+}++// EMIT_MIR alias_fixup.mixed_aggregate_aliasing.MoveElimination.diff+pub fn mixed_aggregate_aliasing(flag: bool, z: u8) -> Triple {+    // This checks an aggregate assignment on one branch after the other branch+    // remaps the input into the return place: overlapping field reads are+    // hoisted through temporaries before writing back into the return place.+    // CHECK-LABEL: fn mixed_aggregate_aliasing(+    // CHECK: debug z => _2;+    // CHECK: debug input => _0;+    // CHECK: debug out => _0;+    // CHECK: [[field1:_.*]] = copy (_0.1: u8);+    // CHECK: [[field0:_.*]] = no_retag move (_0.0: u8);+    // CHECK: (_0.0: u8) = no_retag move [[field1]];+    // CHECK: (_0.1: u8) = no_retag move [[field0]];+    // CHECK: (_0.2: u8) = no_retag move _2;+    let input = opaque_triple();+    let out = if flag { input } else { Triple(input.1, input.0, z) };+    out+}++// EMIT_MIR alias_fixup.aggregate_swap.MoveElimination.diff+pub fn aggregate_swap(x: u8, y: u8) -> (u8, u8) {+    // This checks that an aggregate swap-like assignment is safe after any+    // remapping that makes source fields share storage with destination fields.+    // CHECK-LABEL: fn aggregate_swap(+    // CHECK: debug pair => _0;+    // CHECK: [[saved:_.*]] = copy (_0.0: u8);+    // CHECK: [[tmp:_.*]] = no_retag move (_0.1: u8);+    // CHECK: (_0.0: u8) = no_retag move [[tmp]];+    // CHECK: (_0.1: u8) = no_retag move [[saved]];+    let mut pair = (x, y);+    let a = pair.0;+    let b = pair.1;+    pair = (b, a);+    pair+}++// EMIT_MIR alias_fixup.simple_partial_alias.MoveElimination.diff+pub fn simple_partial_alias(x: [u8; 4]) -> U {+    // This checks a non-aggregate assignment involving two same-typed union+    // fields, which are conservatively treated as aliasing.+    // CHECK-LABEL: fn simple_partial_alias(+    // CHECK: debug u => _0;+    // CHECK: _0 = U { a: move _1 };+    // CHECK: [[tmp:_.*]] = copy (_0.0: [u8; 4]);+    // CHECK: (_0.1: [u8; 4]) = move [[tmp]];+    let mut u = U { a: x };+    let tmp = unsafe { u.a };+    u.b = tmp;+    u+}++// EMIT_MIR alias_fixup.aggregate_indirect_source_alias.MoveElimination.diff+#[custom_mir(dialect = "runtime", phase = "post-cleanup")]+pub fn aggregate_indirect_source_alias() -> (u8, u8) {+    // This checks that, when an aggregate has a direct operand aliasing the+    // destination, indirect operands are also hoisted before field writes.+    // CHECK-LABEL: fn aggregate_indirect_source_alias(+    // CHECK: [[p:_.*]] = &raw const (_0.1: u8);+    // CHECK: [[tmp:_.*]] = no_retag copy (*[[p]]);+    // CHECK: (_0.0: u8) = no_retag move [[tmp]];+    mir! {+        let a: u8;+        let p: *const u8;+        let out: (u8, u8);++        {+            a = 1_u8;+            p = &raw const a;+            out = (*p, Move(a));+            RET = out;+            Return()+        }+    }+}++// Aliasing array elements must all be read before any element is overwritten.+// EMIT_MIR alias_fixup.array_swap.MoveElimination.diff+pub fn array_swap(mut array: [u8; 2]) -> [u8; 2] {+    // CHECK-LABEL: fn array_swap(+    // CHECK: debug array => _1;+    // CHECK: [[FIRST:_[0-9]+]] = copy _1[0 of 2];+    // CHECK: [[SECOND:_[0-9]+]] = no_retag move _1[1 of 2];+    // CHECK: _1[0 of 1] = no_retag move [[SECOND]];+    // CHECK: _1[1 of 2] = no_retag move [[FIRST]];+    let [a, b] = array;+    array = [b, a];+    array+}++pub enum PairEnum {+    Empty,+    Pair(u8, u8),+}++// Enum alias fixup must read both fields before writing the swapped payload,+// then set the discriminant after the field writes.+// EMIT_MIR alias_fixup.enum_alias.MoveElimination.diff+pub fn enum_alias(flag: bool, x: u8, y: u8) -> PairEnum {+    // CHECK-LABEL: fn enum_alias(+    // CHECK: debug input => _0;+    // CHECK: [[FIRST:_[0-9]+]] = copy ((_0 as Pair).0: u8);+    // CHECK: [[SECOND:_[0-9]+]] = no_retag move ((_0 as Pair).1: u8);+    // CHECK: ((_0 as variant#1).0: u8) = no_retag move [[SECOND]];+    // CHECK: ((_0 as variant#1).1: u8) = no_retag move [[FIRST]];+    // CHECK-NEXT: discriminant(_0) = 1;+    let input = PairEnum::Pair(x, y);+    if flag {+        input+    } else {+        match input {+            PairEnum::Pair(a, b) => PairEnum::Pair(b, a), // swap+            PairEnum::Empty => PairEnum::Empty,+        }+    }+}++#[repr(simd)]+#[derive(Copy, Clone)]+pub struct Vector([u32; 4]);++pub union VectorUnion {+    vector: Vector,+    array: [u32; 4],+}++// SIMD aggregates cannot be replaced with direct access to the inner field.+// EMIT_MIR alias_fixup.simd_alias.MoveElimination.diff+pub unsafe fn simd_alias(flag: bool) -> VectorUnion {+    // CHECK-LABEL: fn simd_alias(+    // CHECK: debug input => _0;+    // CHECK: [[TEMP:_[0-9]+]] = Vector(move (_0.1: [u32; 4]));+    // CHECK-NEXT: (_0.0: Vector) = no_retag move [[TEMP]];+    let input = VectorUnion { vector: Vector([0; 4]) };+    if flag { VectorUnion { vector: Vector(input.array) } } else { input }+}

@@ -0,0 +1,78 @@+- // MIR for `simd_alias` before MoveElimination++ // MIR for `simd_alias` after MoveElimination+  +  fn simd_alias(_1: bool) -> VectorUnion {+      debug flag => _1;+      let mut _0: VectorUnion;+      let _2: VectorUnion;+      let mut _3: Vector;+      let mut _4: [u32; 4];+      let mut _5: bool;+      let mut _6: Vector;+      let mut _7: [u32; 4];++     let mut _8: Vector;+      scope 1 {+-         debug input => _2;++         debug input => _0;+      }+  +      bb0: {+-         StorageLive(_2);+-         StorageLive(_3);+-         StorageLive(_4);++         nop;++         nop;++         nop;+          _4 = [const 0_u32; 4];+-         _3 = Vector(move _4);+-         StorageDead(_4);+-         _2 = VectorUnion { vector: move _3 };+-         StorageDead(_3);+-         StorageLive(_5);+-         _5 = copy _1;+-         switchInt(move _5) -> [0: bb2, otherwise: bb1];++         (_0.0: Vector) = Vector(move _4);++         nop;++         nop;++         nop;++         nop;++         nop;++         switchInt(move _1) -> [0: bb2, otherwise: bb1];+      }+  +      bb1: {+-         StorageLive(_6);+-         StorageLive(_7);+-         _7 = copy (_2.1: [u32; 4]);+-         _6 = Vector(move _7);+-         StorageDead(_7);+-         _0 = VectorUnion { vector: move _6 };+-         StorageDead(_6);++         nop;++         nop;++         nop;++         StorageLive(_8);++         _8 = Vector(move (_0.1: [u32; 4]));++         (_0.0: Vector) = no_retag move _8;++         StorageDead(_8);++         nop;++         nop;++         nop;+          goto -> bb3;+      }+  +      bb2: {+-         _0 = move _2;++         nop;+          goto -> bb3;+      }+  +      bb3: {+-         StorageDead(_5);+-         StorageDead(_2);++         nop;++         nop;+          return;+      }+  }+  

@@ -0,0 +1,52 @@+- // MIR for `simple_partial_alias` before MoveElimination++ // MIR for `simple_partial_alias` after MoveElimination+  +  fn simple_partial_alias(_1: [u8; 4]) -> U {+      debug x => _1;+      let mut _0: U;+      let mut _2: U;+      let mut _3: [u8; 4];+      let mut _5: [u8; 4];+      scope 1 {+-         debug u => _2;++         debug u => _0;+          let _4: [u8; 4];+          scope 2 {+-             debug tmp => _4;++             debug tmp => _5;+          }+      }+  +      bb0: {+-         StorageLive(_2);+-         StorageLive(_3);+-         _3 = copy _1;+-         _2 = U { a: move _3 };+-         StorageDead(_3);+-         StorageLive(_4);+-         _4 = copy (_2.0: [u8; 4]);+-         StorageLive(_5);+-         _5 = copy _4;+-         (_2.1: [u8; 4]) = move _5;+-         StorageDead(_5);+-         _0 = move _2;+-         StorageDead(_4);+-         StorageDead(_2);++         nop;++         nop;++         nop;++         _0 = U { a: move _1 };++         nop;++         nop;++         _5 = copy (_0.0: [u8; 4]);++         nop;++         nop;++         (_0.1: [u8; 4]) = move _5;++         nop;++         nop;++         nop;++         nop;+          return;+      }+  }+  

@@ -0,0 +1,67 @@+- // MIR for `array_aggregate` before MoveElimination++ // MIR for `array_aggregate` after MoveElimination+  +  fn array_aggregate() -> [[u8; 8]; 3] {+      let mut _0: [[u8; 8]; 3];+      let _1: [u8; 8];+      let mut _4: [u8; 8];+      let mut _5: [u8; 8];+      let mut _6: [u8; 8];+      scope 1 {+-         debug a => _1;++         debug a => _0[0 of 1];+          let _2: [u8; 8];+          scope 2 {+-             debug b => _2;++             debug b => _0[1 of 2];+              let _3: [u8; 8];+              scope 3 {+-                 debug c => _3;++                 debug c => _0[2 of 3];+              }+          }+      }+  +      bb0: {+-         StorageLive(_1);+-         _1 = [const 1_u8; 8];+-         StorageLive(_2);+-         _2 = [const 2_u8; 8];+-         StorageLive(_3);+-         _3 = [const 3_u8; 8];+-         StorageLive(_4);+-         _4 = move _1;+-         StorageLive(_5);+-         _5 = move _2;+-         StorageLive(_6);+-         _6 = move _3;+-         _0 = [move _4, move _5, move _6];+-         StorageDead(_6);+-         StorageDead(_5);+-         StorageDead(_4);+-         StorageDead(_3);+-         StorageDead(_2);+-         StorageDead(_1);++         nop;++         _0[0 of 1] = [const 1_u8; 8];++         nop;++         _0[1 of 2] = [const 2_u8; 8];++         nop;++         _0[2 of 3] = [const 3_u8; 8];++         nop;++         nop;++         nop;++         nop;++         nop;++         nop;++         nop;++         nop;++         nop;++         nop;++         nop;++         nop;++         nop;+          return;+      }+  }+  

@@ -0,0 +1,57 @@+- // MIR for `enum_aggregate` before MoveElimination++ // MIR for `enum_aggregate` after MoveElimination+  +  fn enum_aggregate() -> Result<([u8; 8], [u8; 8]), ()> {+      let mut _0: std::result::Result<([u8; 8], [u8; 8]), ()>;+      let _1: [u8; 8];+      let mut _3: ([u8; 8], [u8; 8]);+      let mut _4: [u8; 8];+      let mut _5: [u8; 8];+      scope 1 {+-         debug a => _1;++         debug a => (((_0 as variant#0).0: ([u8; 8], [u8; 8])).0: [u8; 8]);+          let _2: [u8; 8];+          scope 2 {+-             debug b => _2;++             debug b => (((_0 as variant#0).0: ([u8; 8], [u8; 8])).1: [u8; 8]);+          }+      }+  +      bb0: {+-         StorageLive(_1);+-         _1 = [const 1_u8; 8];+-         StorageLive(_2);+-         _2 = [const 2_u8; 8];+-         StorageLive(_3);+-         StorageLive(_4);+-         _4 = move _1;+-         StorageLive(_5);+-         _5 = move _2;+-         _3 = (move _4, move _5);+-         StorageDead(_5);+-         StorageDead(_4);+-         _0 = Result::<([u8; 8], [u8; 8]), ()>::Ok(move _3);+-         StorageDead(_3);+-         StorageDead(_2);+-         StorageDead(_1);++         nop;++         (((_0 as variant#0).0: ([u8; 8], [u8; 8])).0: [u8; 8]) = [const 1_u8; 8];++         nop;++         (((_0 as variant#0).0: ([u8; 8], [u8; 8])).1: [u8; 8]) = [const 2_u8; 8];++         nop;++         nop;++         nop;++         nop;++         nop;++         nop;++         nop;++         nop;++         discriminant(_0) = 0;++         nop;++         nop;++         nop;++         nop;+          return;+      }+  }+  

@@ -0,0 +1,46 @@+- // MIR for `nrvo_borrowed` before MoveElimination++ // MIR for `nrvo_borrowed` after MoveElimination+  +  fn nrvo_borrowed() -> [u8; 8] {+      let mut _0: [u8; 8];+      let mut _1: [u8; 8];+      let _2: ();+      let mut _3: &mut [u8; 8];+      let mut _4: &mut [u8; 8];+      scope 1 {+-         debug buf => _1;++         debug buf => _0;+      }+  +      bb0: {+-         StorageLive(_1);+-         _1 = [const 1_u8; 8];+-         StorageLive(_2);+-         StorageLive(_3);+-         StorageLive(_4);+-         _4 = &mut _1;++         nop;++         _0 = [const 1_u8; 8];++         nop;++         nop;++         nop;++         _4 = &mut _0;+          _3 = &mut (*_4);+          _2 = init(move _3) -> [return: bb1, unwind unreachable];+      }+  +      bb1: {+-         StorageDead(_3);+-         StorageDead(_4);+-         StorageDead(_2);+-         _0 = move _1;+-         StorageDead(_1);++         nop;++         nop;++         nop;++         nop;++         nop;+          return;+      }+  }+  

@@ -0,0 +1,24 @@+- // MIR for `nrvo_unborrowed` before MoveElimination++ // MIR for `nrvo_unborrowed` after MoveElimination+  +  fn nrvo_unborrowed() -> [u8; 8] {+      let mut _0: [u8; 8];+      let _1: [u8; 8];+      scope 1 {+-         debug buf => _1;++         debug buf => _0;+      }+  +      bb0: {+-         StorageLive(_1);+-         _1 = [const 1_u8; 8];+-         _0 = move _1;+-         StorageDead(_1);++         nop;++         _0 = [const 1_u8; 8];++         nop;++         nop;+          return;+      }+  }+  

@@ -0,0 +1,80 @@+//@ test-mir-pass: MoveElimination+//@ compile-flags: -Cpanic=abort -Zmir-enable-passes=+TailCopyToMove++struct Pair {+    a: [u8; 8],+    b: [u8; 8],+}++fn init(_: &mut [u8; 8]) {}++// EMIT_MIR basic.nrvo_unborrowed.MoveElimination.diff+pub fn nrvo_unborrowed() -> [u8; 8] {+    // This checks the simplest NRVO-style case: the local should be merged with+    // the return place even though it has `Copy` type.+    // CHECK-LABEL: fn nrvo_unborrowed(+    // CHECK: debug buf => _0;+    // CHECK: _0 = [const 1_u8; 8]+    let buf = [1; 8];+    buf+}++// EMIT_MIR basic.nrvo_borrowed.MoveElimination.diff+pub fn nrvo_borrowed() -> [u8; 8] {+    // This checks that taking a temporary mutable borrow does not prevent+    // merging a `Copy` local once the borrow has ended.+    // CHECK-LABEL: fn nrvo_borrowed(+    // CHECK: debug buf => _0;+    // CHECK: _0 = [const 1_u8; 8]+    // CHECK: init(move {{_.*}})+    // CHECK-NOT: _0 = move+    let mut buf = [1; 8];+    init(&mut buf);+    buf+}++// EMIT_MIR basic.struct_aggregate.MoveElimination.diff+pub fn struct_aggregate() -> Pair {+    // This checks aggregate field remapping: the field locals can live directly+    // in the return place's fields.+    // CHECK-LABEL: fn struct_aggregate(+    // CHECK: debug a => (_0.0: [u8; 8]);+    // CHECK: debug b => (_0.1: [u8; 8]);+    // CHECK: (_0.0: [u8; 8]) = [const 1_u8; 8];+    // CHECK: (_0.1: [u8; 8]) = [const 2_u8; 8];+    let a = [1; 8];+    let b = [2; 8];+    Pair { a, b }+}++// EMIT_MIR basic.enum_aggregate.MoveElimination.diff+pub fn enum_aggregate() -> Result<([u8; 8], [u8; 8]), ()> {+    // This checks aggregate field remapping for enums: the payload fields can+    // be written directly and then the discriminant is set for the variant.+    // CHECK-LABEL: fn enum_aggregate(+    // CHECK: debug a => (((_0 as variant#0).0: ([u8; 8], [u8; 8])).0: [u8; 8]);+    // CHECK: debug b => (((_0 as variant#0).0: ([u8; 8], [u8; 8])).1: [u8; 8]);+    // CHECK: (((_0 as variant#0).0: ([u8; 8], [u8; 8])).0: [u8; 8]) = [const 1_u8; 8];+    // CHECK: (((_0 as variant#0).0: ([u8; 8], [u8; 8])).1: [u8; 8]) = [const 2_u8; 8];+    // CHECK: discriminant(_0) = 0;+    let a = [1; 8];+    let b = [2; 8];+    Result::Ok((a, b))+}++// EMIT_MIR basic.array_aggregate.MoveElimination.diff+pub fn array_aggregate() -> [[u8; 8]; 3] {+    // This checks aggregate remapping for arrays, which uses ConstantIndex+    // projections rather than field projections.+    // CHECK-LABEL: fn array_aggregate(+    // CHECK: debug a => _0[0 of 1];+    // CHECK: debug b => _0[1 of 2];+    // CHECK: debug c => _0[2 of 3];+    // CHECK: _0[0 of 1] = [const 1_u8; 8];+    // CHECK: _0[1 of 2] = [const 2_u8; 8];+    // CHECK: _0[2 of 3] = [const 3_u8; 8];+    let a = [1; 8];+    let b = [2; 8];+    let c = [3; 8];+    [a, b, c]+}

@@ -0,0 +1,49 @@+- // MIR for `struct_aggregate` before MoveElimination++ // MIR for `struct_aggregate` after MoveElimination+  +  fn struct_aggregate() -> Pair {+      let mut _0: Pair;+      let _1: [u8; 8];+      let mut _3: [u8; 8];+      let mut _4: [u8; 8];+      scope 1 {+-         debug a => _1;++         debug a => (_0.0: [u8; 8]);+          let _2: [u8; 8];+          scope 2 {+-             debug b => _2;++             debug b => (_0.1: [u8; 8]);+          }+      }+  +      bb0: {+-         StorageLive(_1);+-         _1 = [const 1_u8; 8];+-         StorageLive(_2);+-         _2 = [const 2_u8; 8];+-         StorageLive(_3);+-         _3 = move _1;+-         StorageLive(_4);+-         _4 = move _2;+-         _0 = Pair { a: move _3, b: move _4 };+-         StorageDead(_4);+-         StorageDead(_3);+-         StorageDead(_2);+-         StorageDead(_1);++         nop;++         (_0.0: [u8; 8]) = [const 1_u8; 8];++         nop;++         (_0.1: [u8; 8]) = [const 2_u8; 8];++         nop;++         nop;++         nop;++         nop;++         nop;++         nop;++         nop;++         nop;++         nop;+          return;+      }+  }+  

@@ -0,0 +1,92 @@+- // MIR for `dse_guard` before MoveElimination++ // MIR for `dse_guard` after MoveElimination+  +  fn dse_guard() -> () {+      let mut _0: ();+      let mut _1: Fields;+      let mut _3: Fields;+      let mut _4: Fields;+      let _5: ();+      let mut _6: *const Fields;+      let mut _7: Fields;+      let _8: ();+      let mut _9: *const Fields;+      scope 1 {+-         debug a => _1;++         debug a => _7;+          let mut _2: Fields;+          scope 2 {+              debug b => _2;+          }+      }+  +      bb0: {+-         StorageLive(_1);+-         StorageLive(_2);+-         StorageLive(_3);+-         _3 = make_fields(const 0_u8) -> [return: bb1, unwind unreachable];++         nop;++         nop;++         nop;++         _2 = make_fields(const 0_u8) -> [return: bb1, unwind unreachable];+      }+  +      bb1: {+-         _2 = move _3;+-         StorageDead(_3);+-         StorageLive(_4);+-         _4 = make_fields(const 1_u8) -> [return: bb2, unwind unreachable];++         nop;++         nop;++         nop;++         _7 = make_fields(const 1_u8) -> [return: bb2, unwind unreachable];+      }+  +      bb2: {+-         _1 = move _4;+-         StorageDead(_4);+-         StorageLive(_5);+-         StorageLive(_6);+-         _6 = &raw const _1;++         nop;++         nop;++         nop;++         nop;++         _6 = &raw const _7;+          _5 = observe(move _6) -> [return: bb3, unwind unreachable];+      }+  +      bb3: {+-         StorageDead(_6);+-         StorageDead(_5);+-         StorageLive(_7);+-         _7 = move _1;++         nop;++         nop;++         nop;++         nop;+          _2 = move _7;+-         StorageDead(_7);+-         StorageLive(_8);+-         StorageLive(_9);++         nop;++         nop;++         nop;+          _9 = &raw const _2;+          _8 = observe(move _9) -> [return: bb4, unwind unreachable];+      }+  +      bb4: {+-         StorageDead(_9);+-         StorageDead(_8);++         nop;++         nop;+          _0 = const ();+-         StorageDead(_2);+-         StorageDead(_1);++         nop;++         nop;+          return;+      }+  }+  

@@ -0,0 +1,36 @@+//@ test-mir-pass: MoveElimination+//@ compile-flags: -Cpanic=abort -Zmir-enable-passes=+DeadStoreElimination-initial++pub struct Fields {+    data: [u8; 8],+    tag: u8,+}++unsafe extern "C" {+    safe fn observe(_: *const Fields);+    safe fn make_fields(_: u8) -> Fields;+}++// EMIT_MIR dse.dse_guard.MoveElimination.diff+pub fn dse_guard() {+    // This guards the RFC soundness hazard: DSE must not remove the first write+    // to `b`, because that write keeps `b`'s address-observed lifetime+    // overlapping with `a` and prevents the later move from being eliminated.+    // CHECK-LABEL: fn dse_guard(+    // CHECK: debug a => [[a:_.*]];+    // CHECK: debug b => [[b:_.*]];+    // CHECK: [[b]] = make_fields(const 0_u8)+    // CHECK: [[a]] = make_fields(const 1_u8)+    // CHECK: observe(move+    // CHECK: [[b]] = move [[a]]+    // CHECK: observe(move+    let mut a;+    let mut b;++    b = make_fields(0);++    a = make_fields(1);+    observe(&raw const a);+    b = a;+    observe(&raw const b);+}

@@ -0,0 +1,41 @@+- // MIR for `constant_index_projected` before MoveElimination++ // MIR for `constant_index_projected` after MoveElimination+  +  fn constant_index_projected(_1: [[u8; 4]; 2]) -> () {+      debug array => _1;+      let mut _0: ();+      let _2: [u8; 4];+      let _3: [u8; 4];+      let mut _4: [u8; 4];+      scope 1 {+-         debug first => _2;++         debug first => _1[0 of 2];+      }+  +      bb0: {+-         StorageLive(_2);+-         _2 = copy _1[0 of 2];+-         StorageLive(_3);+-         StorageLive(_4);+-         _4 = copy _2;+-         _3 = std::hint::black_box::<[u8; 4]>(move _4) -> [return: bb1, unwind unreachable];++         nop;++         nop;++         nop;++         nop;++         nop;++         _3 = std::hint::black_box::<[u8; 4]>(move _1[0 of 2]) -> [return: bb1, unwind unreachable];+      }+  +      bb1: {+-         StorageDead(_4);+-         StorageDead(_3);++         nop;++         nop;+          _0 = const ();+-         StorageDead(_2);++         nop;+          return;+      }+  }+  

@@ -0,0 +1,16 @@+- // MIR for `index_local_not_projected` before MoveElimination++ // MIR for `index_local_not_projected` after MoveElimination+  +  fn index_local_not_projected(_1: [usize; 4]) -> [usize; 1] {+      let mut _0: [usize; 1];+      let mut _2: usize;+      let mut _3: usize;+  +      bb0: {+          _2 = const 2_usize;+          _3 = copy _1[_2];+          _0 = [copy _2];+          return;+      }+  }+  

@@ -0,0 +1,126 @@+- // MIR for `overlapping_lifetimes` before MoveElimination++ // MIR for `overlapping_lifetimes` after MoveElimination+  +  fn overlapping_lifetimes(_1: bool) -> Fields {+      debug flag => _1;+      let mut _0: Fields;+      let _2: Fields;+      let _4: ();+      let mut _5: *const Fields;+      let _6: ();+      let mut _7: bool;+      let mut _8: Fields;+      let mut _9: Fields;+      let mut _10: Fields;+      let _11: ();+      let mut _12: *const Fields;+      scope 1 {+-         debug src => _2;++         debug src => _10;+          let mut _3: Fields;+          scope 2 {+-             debug dst => _3;++             debug dst => _0;+          }+      }+  +      bb0: {+-         StorageLive(_2);+-         _2 = make_fields(const 0_u8) -> [return: bb1, unwind unreachable];++         nop;++         _10 = make_fields(const 0_u8) -> [return: bb1, unwind unreachable];+      }+  +      bb1: {+-         StorageLive(_3);+-         StorageLive(_4);+-         StorageLive(_5);+-         _5 = &raw const _2;++         nop;++         nop;++         nop;++         _5 = &raw const _10;+          _4 = observe(move _5) -> [return: bb2, unwind unreachable];+      }+  +      bb2: {+-         StorageDead(_5);+-         StorageDead(_4);+-         StorageLive(_6);+-         StorageLive(_7);+-         _7 = copy _1;+-         switchInt(move _7) -> [0: bb5, otherwise: bb3];++         nop;++         nop;++         nop;++         nop;++         nop;++         switchInt(move _1) -> [0: bb5, otherwise: bb3];+      }+  +      bb3: {+-         StorageLive(_8);+-         _8 = make_fields(const 1_u8) -> [return: bb4, unwind unreachable];++         nop;++         _0 = make_fields(const 1_u8) -> [return: bb4, unwind unreachable];+      }+  +      bb4: {+-         _3 = move _8;+-         StorageDead(_8);+-         StorageLive(_9);+-         _9 = move _2;+-         _3 = move _9;+-         StorageDead(_9);++         nop;++         nop;++         nop;++         nop;++         _0 = move _10;++         nop;+          _6 = const ();+          goto -> bb6;+      }+  +      bb5: {+-         StorageLive(_10);+-         _10 = move _2;+-         _3 = move _10;+-         StorageDead(_10);++         nop;++         nop;++         _0 = move _10;++         nop;+          _6 = const ();+          goto -> bb6;+      }+  +      bb6: {+-         StorageDead(_7);+-         StorageDead(_6);+-         StorageLive(_11);+-         StorageLive(_12);+-         _12 = &raw const _3;++         nop;++         nop;++         nop;++         nop;++         _12 = &raw const _0;+          _11 = observe(move _12) -> [return: bb7, unwind unreachable];+      }+  +      bb7: {+-         StorageDead(_12);+-         StorageDead(_11);+-         _0 = move _3;+-         StorageDead(_3);+-         StorageDead(_2);++         nop;++         nop;++         nop;++         nop;++         nop;+          return;+      }+  }+  

@@ -0,0 +1,48 @@+- // MIR for `packed_fields_not_projected` before MoveElimination++ // MIR for `packed_fields_not_projected` after MoveElimination+  +  fn packed_fields_not_projected() -> Packed {+      let mut _0: Packed;+      let _1: [u8; 8];+      let mut _3: [u8; 8];+      let mut _4: [u8; 8];+      scope 1 {+-         debug a => _1;++         debug a => _3;+          let _2: [u8; 8];+          scope 2 {+-             debug b => _2;++             debug b => _4;+          }+      }+  +      bb0: {+-         StorageLive(_1);+-         _1 = [const 1_u8; 8];+-         StorageLive(_2);+-         _2 = [const 2_u8; 8];+-         StorageLive(_3);+-         _3 = copy _1;+-         StorageLive(_4);+-         _4 = copy _2;++         nop;++         _3 = [const 1_u8; 8];++         nop;++         _4 = [const 2_u8; 8];++         nop;++         nop;++         nop;++         nop;+          _0 = Packed { a: move _3, b: move _4 };+-         StorageDead(_4);+-         StorageDead(_3);+-         StorageDead(_2);+-         StorageDead(_1);++         nop;++         nop;++         nop;++         nop;+          return;+      }+  }+  

@@ -0,0 +1,139 @@+//@ test-mir-pass: MoveElimination+//@ compile-flags: -Cpanic=abort++#![feature(core_intrinsics, custom_mir, repr_simd)]+#![allow(internal_features)]++use std::intrinsics::mir::*;++pub struct Fields {+    data: [u8; 8],+    tag: u8,+}++#[repr(packed)]+struct Packed {+    a: [u8; 8],+    b: [u8; 8],+}++#[repr(simd)]+struct U32x4([u32; 4]);++unsafe extern "C" {+    safe fn observe(_: *const Fields);+    safe fn make_fields(_: u8) -> Fields;+}++// EMIT_MIR exclusions.index_local_not_projected.MoveElimination.diff+#[custom_mir(dialect = "runtime", phase = "post-cleanup")]+pub fn index_local_not_projected(a: [usize; 4]) -> [usize; 1] {+    // This checks that a local used as an array index is kept as a bare local,+    // because it cannot later be rewritten to a projection like `_0[0]`.+    // CHECK-LABEL: fn index_local_not_projected(+    // CHECK: [[idx:_.*]] = const 2_usize;+    // CHECK: {{.*}} = copy _1[{{.*}}[[idx]]{{.*}}];+    // CHECK: _0 = [copy [[idx]]];+    mir! {+        let idx: usize;+        let b: usize;++        {+            idx = 2usize;+            b = a[idx];+            RET = [idx];+            Return()+        }+    }+}++// EMIT_MIR exclusions.packed_fields_not_projected.MoveElimination.diff+pub fn packed_fields_not_projected() -> Packed {+    // This checks that aggregate fields are not remapped into packed struct+    // fields, which could create unaligned projected places.+    // CHECK-LABEL: fn packed_fields_not_projected(+    // CHECK: debug a => [[a:_.*]];+    // CHECK: debug b => [[b:_.*]];+    // CHECK: _0 = Packed { a: move [[a]], b: move [[b]] };+    let a = [1; 8];+    let b = [2; 8];+    Packed { a, b }+}++// EMIT_MIR exclusions.simd_field_not_projected.MoveElimination.diff+pub fn simd_field_not_projected() -> U32x4 {+    // This checks that aggregate fields are not remapped into repr(simd) ADTs,+    // since optimized MIR must not project into SIMD vectors.+    // CHECK-LABEL: fn simd_field_not_projected(+    // CHECK: debug lanes => [[lanes:_.*]];+    // CHECK: _0 = U32x4(move [[lanes]]);+    let lanes = [1, 2, 3, 4];+    U32x4(lanes)+}++// EMIT_MIR exclusions.overlapping_lifetimes.MoveElimination.diff+pub fn overlapping_lifetimes(flag: bool) -> Fields {+    // This checks the liveness-matrix overlap test for an address-observed+    // move-only local: `src` and `dst` only overlap on one branch, but that is+    // enough to reject merging them for the whole function.+    // CHECK-LABEL: fn overlapping_lifetimes(+    // CHECK: debug flag => _1;+    // CHECK: debug src => [[src:_[1-9][0-9]*]];+    // CHECK: debug dst => _0;+    // CHECK: &raw const [[src]];+    // CHECK: observe+    // CHECK: switchInt(move _1)+    // CHECK: _0 = make_fields(const 1_u8)+    // CHECK: _0 = move [[src]];+    // CHECK: &raw const _0;+    // CHECK: observe+    let src = make_fields(0);+    let mut dst;+    observe(&raw const src);+    if flag {+        dst = make_fields(1);+        dst = src;+    } else {+        dst = src;+    }+    observe(&raw const dst);+    dst+}++// EMIT_MIR exclusions.rust_call_tuple_not_projected.MoveElimination.diff+pub fn rust_call_tuple_not_projected<F: FnOnce([u8; 8], [u8; 8])>(f: F) {+    // This checks that locals are not remapped into the tuple argument passed+    // to a rust-call ABI function. If the tuple itself is never borrowed, alias+    // analysis can trivially see that accesses to one argument don't affect the+    // others. Merging the arguments into tuple fields from the start can hide+    // that independence.+    // CHECK-LABEL: fn rust_call_tuple_not_projected(+    // CHECK: debug a => [[a:_.*]];+    // CHECK: debug b => [[b:_.*]];+    // CHECK: [[tuple:_.*]] = (move [[a]], move [[b]]);+    // CHECK: <F as FnOnce<([u8; 8], [u8; 8])>>::call_once(move _1, move [[tuple]])+    let a = [1; 8];+    let b = [2; 8];+    f(a, b);+}++// Subslice projections cannot appear in debuginfo, so refuse to merge `array`+// and `local`.+// EMIT_MIR exclusions.subslice_not_projected.MoveElimination.diff+pub fn subslice_not_projected(array: [u8; 4]) {+    // CHECK-LABEL: fn subslice_not_projected(+    // CHECK: debug middle => [[MIDDLE:_[0-9]+]];+    // CHECK: [[MIDDLE]] = copy _1[1..3];+    let [_, middle @ .., _] = array;+    std::hint::black_box(middle);+}++// Forward constant indices are supported in debuginfo and allow remapping.+// EMIT_MIR exclusions.constant_index_projected.MoveElimination.diff+pub fn constant_index_projected(array: [[u8; 4]; 2]) {+    // CHECK-LABEL: fn constant_index_projected(+    // CHECK: debug first => _1[0 of 2];+    // CHECK: std::hint::black_box::<[u8; 4]>(move _1[0 of 2])+    let [first, ..] = array;+    std::hint::black_box(first);+}

@@ -0,0 +1,72 @@+- // MIR for `rust_call_tuple_not_projected` before MoveElimination++ // MIR for `rust_call_tuple_not_projected` after MoveElimination+  +  fn rust_call_tuple_not_projected(_1: F) -> () {+      debug f => _1;+      let mut _0: ();+      let _2: [u8; 8];+      let _4: ();+      let mut _5: F;+      let mut _6: ([u8; 8], [u8; 8]);+      let mut _7: [u8; 8];+      let mut _8: [u8; 8];+      scope 1 {+-         debug a => _2;++         debug a => _7;+          let _3: [u8; 8];+          scope 2 {+-             debug b => _3;++             debug b => _8;+          }+      }+  +      bb0: {+-         StorageLive(_2);+-         _2 = [const 1_u8; 8];+-         StorageLive(_3);+-         _3 = [const 2_u8; 8];+-         StorageLive(_4);+-         StorageLive(_5);+-         _5 = move _1;+-         StorageLive(_6);+-         StorageLive(_7);+-         _7 = copy _2;+-         StorageLive(_8);+-         _8 = copy _3;++         nop;++         _7 = [const 1_u8; 8];++         nop;++         _8 = [const 2_u8; 8];++         nop;++         nop;++         nop;++         nop;++         nop;++         nop;++         nop;++         nop;+          _6 = (move _7, move _8);+-         _4 = <F as FnOnce<([u8; 8], [u8; 8])>>::call_once(move _5, move _6) -> [return: bb1, unwind unreachable];++         _4 = <F as FnOnce<([u8; 8], [u8; 8])>>::call_once(move _1, move _6) -> [return: bb1, unwind unreachable];+      }+  +      bb1: {+-         StorageDead(_8);+-         StorageDead(_7);+-         StorageDead(_6);+-         StorageDead(_5);+-         StorageDead(_4);++         nop;++         nop;++         nop;++         nop;++         nop;+          _0 = const ();+-         StorageDead(_3);+-         StorageDead(_2);++         nop;++         nop;+          return;+      }+  }+  

@@ -0,0 +1,30 @@+- // MIR for `simd_field_not_projected` before MoveElimination++ // MIR for `simd_field_not_projected` after MoveElimination+  +  fn simd_field_not_projected() -> U32x4 {+      let mut _0: U32x4;+      let _1: [u32; 4];+      let mut _2: [u32; 4];+      scope 1 {+-         debug lanes => _1;++         debug lanes => _2;+      }+  +      bb0: {+-         StorageLive(_1);+-         _1 = [const 1_u32, const 2_u32, const 3_u32, const 4_u32];+-         StorageLive(_2);+-         _2 = copy _1;++         nop;++         _2 = [const 1_u32, const 2_u32, const 3_u32, const 4_u32];++         nop;++         nop;+          _0 = U32x4(move _2);+-         StorageDead(_2);+-         StorageDead(_1);++         nop;++         nop;+          return;+      }+  }+  

@@ -0,0 +1,40 @@+- // MIR for `subslice_not_projected` before MoveElimination++ // MIR for `subslice_not_projected` after MoveElimination+  +  fn subslice_not_projected(_1: [u8; 4]) -> () {+      debug array => _1;+      let mut _0: ();+      let _2: [u8; 2];+      let _3: [u8; 2];+      let mut _4: [u8; 2];+      scope 1 {+-         debug middle => _2;++         debug middle => _4;+      }+  +      bb0: {+-         StorageLive(_2);+-         _2 = copy _1[1..3];+-         StorageLive(_3);+-         StorageLive(_4);+-         _4 = copy _2;++         nop;++         _4 = copy _1[1..3];++         nop;++         nop;++         nop;+          _3 = std::hint::black_box::<[u8; 2]>(move _4) -> [return: bb1, unwind unreachable];+      }+  +      bb1: {+-         StorageDead(_4);+-         StorageDead(_3);++         nop;++         nop;+          _0 = const ();+-         StorageDead(_2);++         nop;+          return;+      }+  }+  

@@ -0,0 +1,27 @@+- // MIR for `call_destination` before MoveElimination++ // MIR for `call_destination` after MoveElimination+  +  fn call_destination() -> () {+      let mut _0: ();+      let mut _1: u32;+      let mut _2: u32;+      let mut _3: *const u32;+  +      bb0: {+          _1 = const 0_u32;+          _3 = &raw const _1;+          _1 = opaque() -> [return: bb1, unwind: bb2];+      }+  +      bb1: {+          _0 = ();+          return;+      }+  +      bb2 (cleanup): {+          _2 = const 2_u32;+          _1 = move _2;+          resume;+      }+  }+  

@@ -0,0 +1,41 @@+//@ test-mir-pass: MoveElimination+//@ compile-flags: -Cpanic=unwind++#![crate_type = "lib"]+#![feature(custom_mir, core_intrinsics)]+use std::intrinsics::mir::*;++#[inline(never)]+fn opaque() -> u32 {+    1+}++// EMIT_MIR unwind.call_destination.MoveElimination.diff+#[custom_mir(dialect = "runtime", phase = "post-cleanup")]+pub fn call_destination() {+    // CHECK-LABEL: fn call_destination(+    // CHECK: [[DEST:_[0-9]+]] = opaque()+    // CHECK: [[SRC:_[0-9]+]] = const 2_u32;+    // CHECK-NEXT: [[DEST]] = move [[SRC]];+    // Unwinding doesn't deallocate the call destination, it only leaves it+    // uninitialized. This checks that dest is not merged with src.+    mir! {+        let dest: u32;+        let src: u32;+        let p: *const u32;+        {+            dest = 0;+            p = &raw const dest;+            Call(dest = opaque(), ReturnTo(done), UnwindCleanup(cleanup))+        }+        done = {+            RET = ();+            Return()+        }+        cleanup (cleanup) = {+            src = 2;+            dest = Move(src);+            UnwindResume()+        }+    }+}