ryujinx/src/ARMeilleure/CodeGen/Arm64/CodeGeneratorIntrinsic.cs

using ARMeilleure.IntermediateRepresentation;
using System;
using System.Diagnostics;

namespace ARMeilleure.CodeGen.Arm64
{
    static class CodeGeneratorIntrinsic
    {
        public static void GenerateOperation(CodeGenContext context, Operation operation)
        {
            Intrinsic intrin = operation.Intrinsic;

            IntrinsicInfo info = IntrinsicTable.GetInfo(intrin & ~(Intrinsic.Arm64VTypeMask | Intrinsic.Arm64VSizeMask));

            switch (info.Type)
            {
                case IntrinsicType.ScalarUnary:
                    GenerateVectorUnary(
                        context,
                        0,
                        (uint)(intrin & Intrinsic.Arm64VSizeMask) >> (int)Intrinsic.Arm64VSizeShift,
                        info.Inst,
                        operation.Destination,
                        operation.GetSource(0));
                    break;
                case IntrinsicType.ScalarUnaryByElem:
                    Debug.Assert(operation.GetSource(1).Kind == OperandKind.Constant);

                    GenerateVectorUnaryByElem(
                        context,
                        0,
                        (uint)(intrin & Intrinsic.Arm64VSizeMask) >> (int)Intrinsic.Arm64VSizeShift,
                        info.Inst,
                        (uint)operation.GetSource(1).AsInt32(),
                        operation.Destination,
                        operation.GetSource(0));
                    break;
                case IntrinsicType.ScalarBinary:
                    GenerateVectorBinary(
                        context,
                        0,
                        (uint)(intrin & Intrinsic.Arm64VSizeMask) >> (int)Intrinsic.Arm64VSizeShift,
                        info.Inst,
                        operation.Destination,
                        operation.GetSource(0),
                        operation.GetSource(1));
                    break;
                case IntrinsicType.ScalarBinaryFPByElem:
                    Debug.Assert(operation.GetSource(2).Kind == OperandKind.Constant);

                    GenerateVectorBinaryFPByElem(
                        context,
                        0,
                        (uint)(intrin & Intrinsic.Arm64VSizeMask) >> (int)Intrinsic.Arm64VSizeShift,
                        info.Inst,
                        (uint)operation.GetSource(2).AsInt32(),
                        operation.Destination,
                        operation.GetSource(0),
                        operation.GetSource(1));
                    break;
                case IntrinsicType.ScalarBinaryRd:
                    GenerateVectorUnary(
                        context,
                        0,
                        (uint)(intrin & Intrinsic.Arm64VSizeMask) >> (int)Intrinsic.Arm64VSizeShift,
                        info.Inst,
                        operation.Destination,
                        operation.GetSource(1));
                    break;
                case IntrinsicType.ScalarBinaryShl:
                    Debug.Assert(operation.GetSource(1).Kind == OperandKind.Constant);

                    GenerateVectorBinaryShlImm(
                        context,
                        0,
                        (uint)(intrin & Intrinsic.Arm64VSizeMask) >> (int)Intrinsic.Arm64VSizeShift,
                        info.Inst,
                        operation.Destination,
                        operation.GetSource(0),
                        (uint)operation.GetSource(1).AsInt32());
                    break;
                case IntrinsicType.ScalarBinaryShr:
                    Debug.Assert(operation.GetSource(1).Kind == OperandKind.Constant);

                    GenerateVectorBinaryShrImm(
                        context,
                        0,
                        (uint)(intrin & Intrinsic.Arm64VSizeMask) >> (int)Intrinsic.Arm64VSizeShift,
                        info.Inst,
                        operation.Destination,
                        operation.GetSource(0),
                        (uint)operation.GetSource(1).AsInt32());
                    break;
                case IntrinsicType.ScalarFPCompare:
                    GenerateScalarFPCompare(
                        context,
                        (uint)(intrin & Intrinsic.Arm64VSizeMask) >> (int)Intrinsic.Arm64VSizeShift,
                        info.Inst,
                        operation.Destination,
                        operation.GetSource(0),
                        operation.GetSource(1));
                    break;
                case IntrinsicType.ScalarFPConvFixed:
                    Debug.Assert(operation.GetSource(1).Kind == OperandKind.Constant);

                    GenerateVectorBinaryShrImm(
                        context,
                        0,
                        ((uint)(intrin & Intrinsic.Arm64VSizeMask) >> (int)Intrinsic.Arm64VSizeShift) + 2u,
                        info.Inst,
                        operation.Destination,
                        operation.GetSource(0),
                        (uint)operation.GetSource(1).AsInt32());
                    break;
                case IntrinsicType.ScalarFPConvFixedGpr:
                    Debug.Assert(operation.GetSource(1).Kind == OperandKind.Constant);

                    GenerateScalarFPConvGpr(
                        context,
                        (uint)(intrin & Intrinsic.Arm64VSizeMask) >> (int)Intrinsic.Arm64VSizeShift,
                        info.Inst,
                        operation.Destination,
                        operation.GetSource(0),
                        (uint)operation.GetSource(1).AsInt32());
                    break;
                case IntrinsicType.ScalarFPConvGpr:
                    GenerateScalarFPConvGpr(
                        context,
                        (uint)(intrin & Intrinsic.Arm64VSizeMask) >> (int)Intrinsic.Arm64VSizeShift,
                        info.Inst,
                        operation.Destination,
                        operation.GetSource(0));
                    break;
                case IntrinsicType.ScalarTernary:
                    GenerateScalarTernary(
                        context,
                        (uint)(intrin & Intrinsic.Arm64VSizeMask) >> (int)Intrinsic.Arm64VSizeShift,
                        info.Inst,
                        operation.Destination,
                        operation.GetSource(1),
                        operation.GetSource(2),
                        operation.GetSource(0));
                    break;
                case IntrinsicType.ScalarTernaryFPRdByElem:
                    Debug.Assert(operation.GetSource(3).Kind == OperandKind.Constant);

                    GenerateVectorBinaryFPByElem(
                        context,
                        0,
                        (uint)(intrin & Intrinsic.Arm64VSizeMask) >> (int)Intrinsic.Arm64VSizeShift,
                        info.Inst,
                        (uint)operation.GetSource(3).AsInt32(),
                        operation.Destination,
                        operation.GetSource(1),
                        operation.GetSource(2));
                    break;
                case IntrinsicType.ScalarTernaryShlRd:
                    Debug.Assert(operation.GetSource(2).Kind == OperandKind.Constant);

                    GenerateVectorBinaryShlImm(
                        context,
                        0,
                        (uint)(intrin & Intrinsic.Arm64VSizeMask) >> (int)Intrinsic.Arm64VSizeShift,
                        info.Inst,
                        operation.Destination,
                        operation.GetSource(1),
                        (uint)operation.GetSource(2).AsInt32());
                    break;
                case IntrinsicType.ScalarTernaryShrRd:
                    Debug.Assert(operation.GetSource(2).Kind == OperandKind.Constant);

                    GenerateVectorBinaryShrImm(
                        context,
                        0,
                        (uint)(intrin & Intrinsic.Arm64VSizeMask) >> (int)Intrinsic.Arm64VSizeShift,
                        info.Inst,
                        operation.Destination,
                        operation.GetSource(1),
                        (uint)operation.GetSource(2).AsInt32());
                    break;

                case IntrinsicType.VectorUnary:
                    GenerateVectorUnary(
                        context,
                        (uint)(intrin & Intrinsic.Arm64VTypeMask) >> (int)Intrinsic.Arm64VTypeShift,
                        (uint)(intrin & Intrinsic.Arm64VSizeMask) >> (int)Intrinsic.Arm64VSizeShift,
                        info.Inst,
                        operation.Destination,
                        operation.GetSource(0));
                    break;
                case IntrinsicType.VectorUnaryByElem:
                    Debug.Assert(operation.GetSource(1).Kind == OperandKind.Constant);

                    GenerateVectorUnaryByElem(
                        context,
                        (uint)(intrin & Intrinsic.Arm64VTypeMask) >> (int)Intrinsic.Arm64VTypeShift,
                        (uint)(intrin & Intrinsic.Arm64VSizeMask) >> (int)Intrinsic.Arm64VSizeShift,
                        info.Inst,
                        (uint)operation.GetSource(1).AsInt32(),
                        operation.Destination,
                        operation.GetSource(0));
                    break;
                case IntrinsicType.VectorBinary:
                    GenerateVectorBinary(
                        context,
                        (uint)(intrin & Intrinsic.Arm64VTypeMask) >> (int)Intrinsic.Arm64VTypeShift,
                        (uint)(intrin & Intrinsic.Arm64VSizeMask) >> (int)Intrinsic.Arm64VSizeShift,
                        info.Inst,
                        operation.Destination,
                        operation.GetSource(0),
                        operation.GetSource(1));
                    break;
                case IntrinsicType.VectorBinaryBitwise:
                    GenerateVectorBinary(
                        context,
                        (uint)(intrin & Intrinsic.Arm64VTypeMask) >> (int)Intrinsic.Arm64VTypeShift,
                        info.Inst,
                        operation.Destination,
                        operation.GetSource(0),
                        operation.GetSource(1));
                    break;
                case IntrinsicType.VectorBinaryByElem:
                    Debug.Assert(operation.GetSource(2).Kind == OperandKind.Constant);

                    GenerateVectorBinaryByElem(
                        context,
                        (uint)(intrin & Intrinsic.Arm64VTypeMask) >> (int)Intrinsic.Arm64VTypeShift,
                        (uint)(intrin & Intrinsic.Arm64VSizeMask) >> (int)Intrinsic.Arm64VSizeShift,
                        info.Inst,
                        (uint)operation.GetSource(2).AsInt32(),
                        operation.Destination,
                        operation.GetSource(0),
                        operation.GetSource(1));
                    break;
                case IntrinsicType.VectorBinaryFPByElem:
                    Debug.Assert(operation.GetSource(2).Kind == OperandKind.Constant);

                    GenerateVectorBinaryFPByElem(
                        context,
                        (uint)(intrin & Intrinsic.Arm64VTypeMask) >> (int)Intrinsic.Arm64VTypeShift,
                        (uint)(intrin & Intrinsic.Arm64VSizeMask) >> (int)Intrinsic.Arm64VSizeShift,
                        info.Inst,
                        (uint)operation.GetSource(2).AsInt32(),
                        operation.Destination,
                        operation.GetSource(0),
                        operation.GetSource(1));
                    break;
                case IntrinsicType.VectorBinaryRd:
                    GenerateVectorUnary(
                        context,
                        (uint)(intrin & Intrinsic.Arm64VTypeMask) >> (int)Intrinsic.Arm64VTypeShift,
                        (uint)(intrin & Intrinsic.Arm64VSizeMask) >> (int)Intrinsic.Arm64VSizeShift,
                        info.Inst,
                        operation.Destination,
                        operation.GetSource(1));
                    break;
                case IntrinsicType.VectorBinaryShl:
                    Debug.Assert(operation.GetSource(1).Kind == OperandKind.Constant);

                    GenerateVectorBinaryShlImm(
                        context,
                        (uint)(intrin & Intrinsic.Arm64VTypeMask) >> (int)Intrinsic.Arm64VTypeShift,
                        (uint)(intrin & Intrinsic.Arm64VSizeMask) >> (int)Intrinsic.Arm64VSizeShift,
                        info.Inst,
                        operation.Destination,
                        operation.GetSource(0),
                        (uint)operation.GetSource(1).AsInt32());
                    break;
                case IntrinsicType.VectorBinaryShr:
                    Debug.Assert(operation.GetSource(1).Kind == OperandKind.Constant);

                    GenerateVectorBinaryShrImm(
                        context,
                        (uint)(intrin & Intrinsic.Arm64VTypeMask) >> (int)Intrinsic.Arm64VTypeShift,
                        (uint)(intrin & Intrinsic.Arm64VSizeMask) >> (int)Intrinsic.Arm64VSizeShift,
                        info.Inst,
                        operation.Destination,
                        operation.GetSource(0),
                        (uint)operation.GetSource(1).AsInt32());
                    break;
                case IntrinsicType.VectorFPConvFixed:
                    Debug.Assert(operation.GetSource(1).Kind == OperandKind.Constant);

                    GenerateVectorBinaryShrImm(
                        context,
                        (uint)(intrin & Intrinsic.Arm64VTypeMask) >> (int)Intrinsic.Arm64VTypeShift,
                        ((uint)(intrin & Intrinsic.Arm64VSizeMask) >> (int)Intrinsic.Arm64VSizeShift) + 2u,
                        info.Inst,
                        operation.Destination,
                        operation.GetSource(0),
                        (uint)operation.GetSource(1).AsInt32());
                    break;
                case IntrinsicType.VectorInsertByElem:
                    Debug.Assert(operation.GetSource(1).Kind == OperandKind.Constant);
                    Debug.Assert(operation.GetSource(3).Kind == OperandKind.Constant);

                    GenerateVectorInsertByElem(
                        context,
                        (uint)(intrin & Intrinsic.Arm64VSizeMask) >> (int)Intrinsic.Arm64VSizeShift,
                        info.Inst,
                        (uint)operation.GetSource(3).AsInt32(),
                        (uint)operation.GetSource(1).AsInt32(),
                        operation.Destination,
                        operation.GetSource(2));
                    break;
                case IntrinsicType.VectorLookupTable:
                    Debug.Assert((uint)(operation.SourcesCount - 2) <= 3);

                    for (int i = 1; i < operation.SourcesCount - 1; i++)
                    {
                        Register currReg = operation.GetSource(i).GetRegister();
                        Register prevReg = operation.GetSource(i - 1).GetRegister();

                        Debug.Assert(prevReg.Index + 1 == currReg.Index && currReg.Type == RegisterType.Vector);
                    }

                    GenerateVectorBinary(
                        context,
                        (uint)(intrin & Intrinsic.Arm64VTypeMask) >> (int)Intrinsic.Arm64VTypeShift,
                        info.Inst | ((uint)(operation.SourcesCount - 2) << 13),
                        operation.Destination,
                        operation.GetSource(0),
                        operation.GetSource(operation.SourcesCount - 1));
                    break;
                case IntrinsicType.VectorTernaryFPRdByElem:
                    Debug.Assert(operation.GetSource(3).Kind == OperandKind.Constant);

                    GenerateVectorBinaryFPByElem(
                        context,
                        (uint)(intrin & Intrinsic.Arm64VTypeMask) >> (int)Intrinsic.Arm64VTypeShift,
                        (uint)(intrin & Intrinsic.Arm64VSizeMask) >> (int)Intrinsic.Arm64VSizeShift,
                        info.Inst,
                        (uint)operation.GetSource(3).AsInt32(),
                        operation.Destination,
                        operation.GetSource(1),
                        operation.GetSource(2));
                    break;
                case IntrinsicType.VectorTernaryRd:
                    GenerateVectorBinary(
                        context,
                        (uint)(intrin & Intrinsic.Arm64VTypeMask) >> (int)Intrinsic.Arm64VTypeShift,
                        (uint)(intrin & Intrinsic.Arm64VSizeMask) >> (int)Intrinsic.Arm64VSizeShift,
                        info.Inst,
                        operation.Destination,
                        operation.GetSource(1),
                        operation.GetSource(2));
                    break;
                case IntrinsicType.VectorTernaryRdBitwise:
                    GenerateVectorBinary(
                        context,
                        (uint)(intrin & Intrinsic.Arm64VTypeMask) >> (int)Intrinsic.Arm64VTypeShift,
                        info.Inst,
                        operation.Destination,
                        operation.GetSource(1),
                        operation.GetSource(2));
                    break;
                case IntrinsicType.VectorTernaryRdByElem:
                    Debug.Assert(operation.GetSource(3).Kind == OperandKind.Constant);

                    GenerateVectorBinaryByElem(
                        context,
                        (uint)(intrin & Intrinsic.Arm64VTypeMask) >> (int)Intrinsic.Arm64VTypeShift,
                        (uint)(intrin & Intrinsic.Arm64VSizeMask) >> (int)Intrinsic.Arm64VSizeShift,
                        info.Inst,
                        (uint)operation.GetSource(3).AsInt32(),
                        operation.Destination,
                        operation.GetSource(1),
                        operation.GetSource(2));
                    break;
                case IntrinsicType.VectorTernaryShlRd:
                    Debug.Assert(operation.GetSource(2).Kind == OperandKind.Constant);

                    GenerateVectorBinaryShlImm(
                        context,
                        (uint)(intrin & Intrinsic.Arm64VTypeMask) >> (int)Intrinsic.Arm64VTypeShift,
                        (uint)(intrin & Intrinsic.Arm64VSizeMask) >> (int)Intrinsic.Arm64VSizeShift,
                        info.Inst,
                        operation.Destination,
                        operation.GetSource(1),
                        (uint)operation.GetSource(2).AsInt32());
                    break;
                case IntrinsicType.VectorTernaryShrRd:
                    Debug.Assert(operation.GetSource(2).Kind == OperandKind.Constant);

                    GenerateVectorBinaryShrImm(
                        context,
                        (uint)(intrin & Intrinsic.Arm64VTypeMask) >> (int)Intrinsic.Arm64VTypeShift,
                        (uint)(intrin & Intrinsic.Arm64VSizeMask) >> (int)Intrinsic.Arm64VSizeShift,
                        info.Inst,
                        operation.Destination,
                        operation.GetSource(1),
                        (uint)operation.GetSource(2).AsInt32());
                    break;

                case IntrinsicType.GetRegister:
                    context.Assembler.WriteInstruction(info.Inst, operation.Destination);
                    break;
                case IntrinsicType.SetRegister:
                    context.Assembler.WriteInstruction(info.Inst, operation.GetSource(0));
                    break;

                default:
                    throw new NotImplementedException(info.Type.ToString());
            }
        }

        private static void GenerateScalarFPCompare(
            CodeGenContext context,
            uint sz,
            uint instruction,
            Operand dest,
            Operand rn,
            Operand rm)
        {
            instruction |= (sz << 22);

            if (rm.Kind == OperandKind.Constant && rm.Value == 0)
            {
                instruction |= 0b1000;
                rm = rn;
            }

            context.Assembler.WriteInstructionRm16NoRet(instruction, rn, rm);
            context.Assembler.Mrs(dest, 1, 3, 4, 2, 0);
        }

        private static void GenerateScalarFPConvGpr(
            CodeGenContext context,
            uint sz,
            uint instruction,
            Operand rd,
            Operand rn)
        {
            instruction |= (sz << 22);

            if (rd.Type.IsInteger())
            {
                context.Assembler.WriteInstructionAuto(instruction, rd, rn);
            }
            else
            {
                if (rn.Type == OperandType.I64)
                {
                    instruction |= Assembler.SfFlag;
                }

                context.Assembler.WriteInstruction(instruction, rd, rn);
            }
        }

        private static void GenerateScalarFPConvGpr(
            CodeGenContext context,
            uint sz,
            uint instruction,
            Operand rd,
            Operand rn,
            uint fBits)
        {
            Debug.Assert(fBits <= 64);

            instruction |= (sz << 22);
            instruction |= (64 - fBits) << 10;

            if (rd.Type.IsInteger())
            {
                Debug.Assert(rd.Type != OperandType.I32 || fBits <= 32);

                context.Assembler.WriteInstructionAuto(instruction, rd, rn);
            }
            else
            {
                if (rn.Type == OperandType.I64)
                {
                    instruction |= Assembler.SfFlag;
                }
                else
                {
                    Debug.Assert(fBits <= 32);
                }

                context.Assembler.WriteInstruction(instruction, rd, rn);
            }

        }

        private static void GenerateScalarTernary(
            CodeGenContext context,
            uint sz,
            uint instruction,
            Operand rd,
            Operand rn,
            Operand rm,
            Operand ra)
        {
            instruction |= (sz << 22);

            context.Assembler.WriteInstruction(instruction, rd, rn, rm, ra);
        }

        private static void GenerateVectorUnary(
            CodeGenContext context,
            uint q,
            uint sz,
            uint instruction,
            Operand rd,
            Operand rn)
        {
            instruction |= (q << 30) | (sz << 22);

            context.Assembler.WriteInstruction(instruction, rd, rn);
        }

        private static void GenerateVectorUnaryByElem(
            CodeGenContext context,
            uint q,
            uint sz,
            uint instruction,
            uint srcIndex,
            Operand rd,
            Operand rn)
        {
            uint imm5 = (srcIndex << ((int)sz + 1)) | (1u << (int)sz);

            instruction |= (q << 30) | (imm5 << 16);

            context.Assembler.WriteInstruction(instruction, rd, rn);
        }

        private static void GenerateVectorBinary(
            CodeGenContext context,
            uint q,
            uint instruction,
            Operand rd,
            Operand rn,
            Operand rm)
        {
            instruction |= (q << 30);

            context.Assembler.WriteInstructionRm16(instruction, rd, rn, rm);
        }

        private static void GenerateVectorBinary(
            CodeGenContext context,
            uint q,
            uint sz,
            uint instruction,
            Operand rd,
            Operand rn,
            Operand rm)
        {
            instruction |= (q << 30) | (sz << 22);

            context.Assembler.WriteInstructionRm16(instruction, rd, rn, rm);
        }

        private static void GenerateVectorBinaryByElem(
            CodeGenContext context,
            uint q,
            uint size,
            uint instruction,
            uint srcIndex,
            Operand rd,
            Operand rn,
            Operand rm)
        {
            instruction |= (q << 30) | (size << 22);

            if (size == 2)
            {
                instruction |= ((srcIndex & 1) << 21) | ((srcIndex & 2) << 10);
            }
            else
            {
                instruction |= ((srcIndex & 3) << 20) | ((srcIndex & 4) << 9);
            }

            context.Assembler.WriteInstructionRm16(instruction, rd, rn, rm);
        }

        private static void GenerateVectorBinaryFPByElem(
            CodeGenContext context,
            uint q,
            uint sz,
            uint instruction,
            uint srcIndex,
            Operand rd,
            Operand rn,
            Operand rm)
        {
            instruction |= (q << 30) | (sz << 22);

            if (sz != 0)
            {
                instruction |= (srcIndex & 1) << 11;
            }
            else
            {
                instruction |= ((srcIndex & 1) << 21) | ((srcIndex & 2) << 10);
            }

            context.Assembler.WriteInstructionRm16(instruction, rd, rn, rm);
        }

        private static void GenerateVectorBinaryShlImm(
            CodeGenContext context,
            uint q,
            uint sz,
            uint instruction,
            Operand rd,
            Operand rn,
            uint shift)
        {
            instruction |= (q << 30);

            Debug.Assert(shift >= 0 && shift < (8u << (int)sz));

            uint imm = (8u << (int)sz) | (shift & (0x3fu >> (int)(3 - sz)));

            instruction |= (imm << 16);

            context.Assembler.WriteInstruction(instruction, rd, rn);
        }

        private static void GenerateVectorBinaryShrImm(
            CodeGenContext context,
            uint q,
            uint sz,
            uint instruction,
            Operand rd,
            Operand rn,
            uint shift)
        {
            instruction |= (q << 30);

            Debug.Assert(shift > 0 && shift <= (8u << (int)sz));

            uint imm = (8u << (int)sz) | ((8u << (int)sz) - shift);

            instruction |= (imm << 16);

            context.Assembler.WriteInstruction(instruction, rd, rn);
        }

        private static void GenerateVectorInsertByElem(
            CodeGenContext context,
            uint sz,
            uint instruction,
            uint srcIndex,
            uint dstIndex,
            Operand rd,
            Operand rn)
        {
            uint imm4 = srcIndex << (int)sz;
            uint imm5 = (dstIndex << ((int)sz + 1)) | (1u << (int)sz);

            instruction |= imm4 << 11;
            instruction |= imm5 << 16;

            context.Assembler.WriteInstruction(instruction, rd, rn);
        }
    }
}