Implement block placement (#1549)

* Implement block placement

Implement a simple pass which re-orders cold blocks at the end of the
list of blocks in the CFG.

* Set PPTC version

* Use Array.Resize

Address gdkchan's feedback

ARMeilleure/CodeGen/Optimizations/BlockPlacement.cs

@@ -0,0 +1,66 @@
+using ARMeilleure.IntermediateRepresentation;
+using ARMeilleure.Translation;
+using System.Diagnostics;
+
+using static ARMeilleure.IntermediateRepresentation.OperandHelper;
+
+namespace ARMeilleure.CodeGen.Optimizations
+{
+    static class BlockPlacement
+    {
+        public static void RunPass(ControlFlowGraph cfg)
+        {
+            bool update = false;
+
+            BasicBlock block;
+            BasicBlock nextBlock;
+
+            BasicBlock lastBlock = cfg.Blocks.Last;
+
+            // Move cold blocks at the end of the list, so that they are emitted away from hot code.
+            for (block = cfg.Blocks.First; block != lastBlock; block = nextBlock)
+            {
+                nextBlock = block.ListNext;
+
+                if (block.Frequency == BasicBlockFrequency.Cold)
+                {
+                    cfg.Blocks.Remove(block);
+                    cfg.Blocks.AddLast(block);
+                }
+            }
+
+            for (block = cfg.Blocks.First; block != null; block = nextBlock)
+            {
+                nextBlock = block.ListNext;
+
+                if (block.SuccessorCount == 2 && block.Operations.Last is Operation branchOp)
+                {
+                    Debug.Assert(branchOp.Instruction == Instruction.BranchIf);
+
+                    BasicBlock falseSucc = block.GetSuccessor(0);
+                    BasicBlock trueSucc = block.GetSuccessor(1);
+
+                    // If true successor is next block in list, invert the condition. We avoid extra branching by
+                    // making the true side the fallthrough (i.e, convert it to the false side).
+                    if (trueSucc == block.ListNext)
+                    {
+                        Comparison comp = (Comparison)branchOp.GetSource(2).AsInt32();
+                        Comparison compInv = comp.Invert();
+
+                        branchOp.SetSource(2, Const((int)compInv));
+
+                        block.SetSuccessor(0, trueSucc);
+                        block.SetSuccessor(1, falseSucc);
+
+                        update = true;
+                    }
+                }
+            }
+
+            if (update)
+            {
+                cfg.Update(removeUnreachableBlocks: false);
+            }
+        }
+    }
+}

ARMeilleure/CodeGen/X86/CodeGenerator.cs

@@ -106,6 +106,8 @@ namespace ARMeilleure.CodeGen.X86
 
             X86Optimizer.RunPass(cfg);
 
+            BlockPlacement.RunPass(cfg);
+
             Logger.EndPass(PassName.Optimization, cfg);
 
             Logger.StartPass(PassName.PreAllocation);
@@ -186,9 +188,11 @@ namespace ARMeilleure.CodeGen.X86
                     }
                 }
 
+                byte[] code = context.GetCode();
+
                 Logger.EndPass(PassName.CodeGeneration);
 
-                return new CompiledFunction(context.GetCode(), unwindInfo);
+                return new CompiledFunction(code, unwindInfo);
             }
         }
 

ARMeilleure/Diagnostics/IRDumper.cs

@@ -57,6 +57,11 @@ namespace ARMeilleure.Diagnostics
         {
             DumpBlockName(block);
 
+            if (block.Frequency == BasicBlockFrequency.Cold)
+            {
+                _builder.Append(" cold");
+            }
+
             if (block.SuccessorCount > 0)
             {
                 _builder.Append(" (");

ARMeilleure/Instructions/InstEmitFlowHelper.cs

@@ -174,7 +174,7 @@ namespace ARMeilleure.Instructions
                 Operand lblContinue = context.GetLabel(nextAddr.Value);
 
                 // We need to clear out the call flag for the return address before comparing it.
-                context.BranchIf(lblContinue, context.BitwiseAnd(returnAddress, Const(~CallFlag)), nextAddr, Comparison.Equal);
+                context.BranchIf(lblContinue, context.BitwiseAnd(returnAddress, Const(~CallFlag)), nextAddr, Comparison.Equal, BasicBlockFrequency.Cold);
 
                 context.Return(returnAddress);
             }

ARMeilleure/Instructions/InstEmitMemoryHelper.cs

@@ -147,7 +147,7 @@ namespace ARMeilleure.Instructions
 
             context.Branch(lblEnd);
 
-            context.MarkLabel(lblSlowPath);
+            context.MarkLabel(lblSlowPath, BasicBlockFrequency.Cold);
 
             EmitReadIntFallback(context, address, rt, size);
 
@@ -165,7 +165,7 @@ namespace ARMeilleure.Instructions
 
             Operand lblFastPath = Label();
 
-            context.BranchIfFalse(lblFastPath, isUnalignedAddr);
+            context.BranchIfFalse(lblFastPath, isUnalignedAddr, BasicBlockFrequency.Cold);
 
             // The call is not expected to return (it should throw).
             context.Call(typeof(NativeInterface).GetMethod(nameof(NativeInterface.ThrowInvalidMemoryAccess)), address);
@@ -216,7 +216,7 @@ namespace ARMeilleure.Instructions
 
             context.Branch(lblEnd);
 
-            context.MarkLabel(lblSlowPath);
+            context.MarkLabel(lblSlowPath, BasicBlockFrequency.Cold);
 
             EmitReadVectorFallback(context, address, vector, rt, elem, size);
 
@@ -256,7 +256,7 @@ namespace ARMeilleure.Instructions
 
             context.Branch(lblEnd);
 
-            context.MarkLabel(lblSlowPath);
+            context.MarkLabel(lblSlowPath, BasicBlockFrequency.Cold);
 
             EmitWriteIntFallback(context, address, rt, size);
 
@@ -274,7 +274,7 @@ namespace ARMeilleure.Instructions
 
             Operand lblFastPath = Label();
 
-            context.BranchIfFalse(lblFastPath, isUnalignedAddr);
+            context.BranchIfFalse(lblFastPath, isUnalignedAddr, BasicBlockFrequency.Cold);
 
             // The call is not expected to return (it should throw).
             context.Call(typeof(NativeInterface).GetMethod(nameof(NativeInterface.ThrowInvalidMemoryAccess)), address);
@@ -331,7 +331,7 @@ namespace ARMeilleure.Instructions
 
             context.Branch(lblEnd);
 
-            context.MarkLabel(lblSlowPath);
+            context.MarkLabel(lblSlowPath, BasicBlockFrequency.Cold);
 
             EmitWriteVectorFallback(context, address, rt, elem, size);
 
@@ -402,7 +402,7 @@ namespace ARMeilleure.Instructions
                     Operand lblNotWatched = Label();
 
                     // Is the page currently being monitored for modifications? If so we need to call MarkRegionAsModified.
-                    context.BranchIf(lblNotWatched, pte, Const(0L), Comparison.GreaterOrEqual);
+                    context.BranchIf(lblNotWatched, pte, Const(0L), Comparison.GreaterOrEqual, BasicBlockFrequency.Cold);
 
                     // Mark the region as modified. Size here doesn't matter as address is assumed to be size aligned here.
                     context.Call(typeof(NativeInterface).GetMethod(nameof(NativeInterface.MarkRegionAsModified)), address, Const(1UL));
@@ -412,7 +412,7 @@ namespace ARMeilleure.Instructions
                 Operand lblNonNull = Label();
 
                 // Skip exception if the PTE address is non-null (not zero).
-                context.BranchIfTrue(lblNonNull, pte);
+                context.BranchIfTrue(lblNonNull, pte, BasicBlockFrequency.Cold);
 
                 // The call is not expected to return (it should throw).
                 context.Call(typeof(NativeInterface).GetMethod(nameof(NativeInterface.ThrowInvalidMemoryAccess)), address);

ARMeilleure/IntermediateRepresentation/BasicBlock.cs

@@ -5,10 +5,12 @@ namespace ARMeilleure.IntermediateRepresentation
 {
     class BasicBlock : IIntrusiveListNode<BasicBlock>
     {
-        private readonly List<BasicBlock> _successors = new List<BasicBlock>();
+        private readonly List<BasicBlock> _successors;
 
         public int Index { get; set; }
 
+        public BasicBlockFrequency Frequency { get; set; }
+
         public BasicBlock ListPrevious { get; set; }
         public BasicBlock ListNext { get; set; }
 
@@ -25,6 +27,8 @@ namespace ARMeilleure.IntermediateRepresentation
 
         public BasicBlock(int index)
         {
+            _successors = new List<BasicBlock>();
+
             Operations = new IntrusiveList<Node>();
             Predecessors = new List<BasicBlock>();
             DominanceFrontiers = new HashSet<BasicBlock>();

ARMeilleure/IntermediateRepresentation/BasicBlockFrequency.cs

@@ -0,0 +1,8 @@
+namespace ARMeilleure.IntermediateRepresentation
+{
+    enum BasicBlockFrequency
+    {
+        Default,
+        Cold
+    }
+}

ARMeilleure/Translation/ControlFlowGraph.cs

@@ -7,26 +7,37 @@ namespace ARMeilleure.Translation
 {
     class ControlFlowGraph
     {
+        private BasicBlock[] _postOrderBlocks;
+        private int[] _postOrderMap;
+
         public BasicBlock Entry { get; }
         public IntrusiveList<BasicBlock> Blocks { get; }
-        public BasicBlock[] PostOrderBlocks { get; }
-        public int[] PostOrderMap { get; }
+        public BasicBlock[] PostOrderBlocks => _postOrderBlocks;
+        public int[] PostOrderMap => _postOrderMap; 
 
         public ControlFlowGraph(BasicBlock entry, IntrusiveList<BasicBlock> blocks)
         {
             Entry = entry;
             Blocks = blocks;
 
-            RemoveUnreachableBlocks(blocks);
+            Update(removeUnreachableBlocks: true);
+        }
+
+        public void Update(bool removeUnreachableBlocks)
+        {
+            if (removeUnreachableBlocks)
+            {
+                RemoveUnreachableBlocks(Blocks);
+            }
 
             var visited = new HashSet<BasicBlock>();
             var blockStack = new Stack<BasicBlock>();
 
-            PostOrderBlocks = new BasicBlock[blocks.Count];
-            PostOrderMap = new int[blocks.Count];
+            Array.Resize(ref _postOrderBlocks, Blocks.Count);
+            Array.Resize(ref _postOrderMap, Blocks.Count);
 
-            visited.Add(entry);
-            blockStack.Push(entry);
+            visited.Add(Entry);
+            blockStack.Push(Entry);
 
             int index = 0;
 

ARMeilleure/Translation/EmitterContext.cs

@@ -20,6 +20,7 @@ namespace ARMeilleure.Translation
         private BasicBlock _ifBlock;
 
         private bool _needsNewBlock;
+        private BasicBlockFrequency _nextBlockFreq;
 
         public EmitterContext()
         {
@@ -27,6 +28,7 @@ namespace ARMeilleure.Translation
             _irBlocks = new IntrusiveList<BasicBlock>();
 
             _needsNewBlock = true;
+            _nextBlockFreq = BasicBlockFrequency.Default;
         }
 
         public Operand Add(Operand op1, Operand op2)
@@ -58,24 +60,24 @@ namespace ARMeilleure.Translation
         {
             NewNextBlockIfNeeded();
 
-            BranchToLabel(label, uncond: true);
+            BranchToLabel(label, uncond: true, BasicBlockFrequency.Default);
         }
 
-        public void BranchIf(Operand label, Operand op1, Operand op2, Comparison comp)
+        public void BranchIf(Operand label, Operand op1, Operand op2, Comparison comp, BasicBlockFrequency falseFreq = default)
         {
             Add(Instruction.BranchIf, null, op1, op2, Const((int)comp));
 
-            BranchToLabel(label, uncond: false);
+            BranchToLabel(label, uncond: false, falseFreq);
         }
 
-        public void BranchIfFalse(Operand label, Operand op1)
+        public void BranchIfFalse(Operand label, Operand op1, BasicBlockFrequency falseFreq = default)
         {
-            BranchIf(label, op1, Const(op1.Type, 0), Comparison.Equal);
+            BranchIf(label, op1, Const(op1.Type, 0), Comparison.Equal, falseFreq);
         }
 
-        public void BranchIfTrue(Operand label, Operand op1)
+        public void BranchIfTrue(Operand label, Operand op1, BasicBlockFrequency falseFreq = default)
         {
-            BranchIf(label, op1, Const(op1.Type, 0), Comparison.NotEqual);
+            BranchIf(label, op1, Const(op1.Type, 0), Comparison.NotEqual, falseFreq);
         }
 
         public Operand ByteSwap(Operand op1)
@@ -582,7 +584,7 @@ namespace ARMeilleure.Translation
             return dest;
         }
 
-        private void BranchToLabel(Operand label, bool uncond)
+        private void BranchToLabel(Operand label, bool uncond, BasicBlockFrequency nextFreq)
         {
             if (!_irLabels.TryGetValue(label, out BasicBlock branchBlock))
             {
@@ -602,10 +604,13 @@ namespace ARMeilleure.Translation
             }
 
             _needsNewBlock = true;
+            _nextBlockFreq = nextFreq;
         }
 
-        public void MarkLabel(Operand label)
+        public void MarkLabel(Operand label, BasicBlockFrequency nextFreq = default)
         {
+            _nextBlockFreq = nextFreq;
+
             if (_irLabels.TryGetValue(label, out BasicBlock nextBlock))
             {
                 nextBlock.Index = _irBlocks.Count;
@@ -633,7 +638,7 @@ namespace ARMeilleure.Translation
 
         private void NextBlock(BasicBlock nextBlock)
         {
-            if (_irBlock != null && _irBlock.SuccessorCount == 0 && !EndsWithUnconditional(_irBlock))
+            if (_irBlock?.SuccessorCount == 0 && !EndsWithUnconditional(_irBlock))
             {
                 _irBlock.AddSuccessor(nextBlock);
 
@@ -646,8 +651,10 @@ namespace ARMeilleure.Translation
             }
 
             _irBlock = nextBlock;
+            _irBlock.Frequency = _nextBlockFreq;
 
             _needsNewBlock = false;
+            _nextBlockFreq = BasicBlockFrequency.Default;
         }
 
         private static bool EndsWithUnconditional(BasicBlock block)

ARMeilleure/Translation/PTC/Ptc.cs

@@ -21,7 +21,7 @@ namespace ARMeilleure.Translation.PTC
     {
         private const string HeaderMagic = "PTChd";
 
-        private const int InternalVersion = 1535; //! To be incremented manually for each change to the ARMeilleure project.
+        private const int InternalVersion = 1549; //! To be incremented manually for each change to the ARMeilleure project.
 
         private const string ActualDir = "0";
         private const string BackupDir = "1";

ARMeilleure/Translation/Translator.cs

@@ -301,11 +301,11 @@ namespace ARMeilleure.Translation
             Operand lblNonZero = Label();
             Operand lblExit    = Label();
 
-            context.BranchIfTrue(lblNonZero, count);
+            context.BranchIfTrue(lblNonZero, count, BasicBlockFrequency.Cold);
 
             Operand running = context.Call(typeof(NativeInterface).GetMethod(nameof(NativeInterface.CheckSynchronization)));
 
-            context.BranchIfTrue(lblExit, running);
+            context.BranchIfTrue(lblExit, running, BasicBlockFrequency.Cold);
 
             context.Return(Const(0L));