Use libhac for loading NSO and KIP (#1047)

* Use libhac for loading NSOs and KIPs

* Fix formatting

* Refactor KIP and NSO executables for libhac

* Fix up formatting

* Remove Ryujinx.HLE.Loaders.Compression

* Remove reference to Ryujinx.HLE.Loaders.Compression in NxStaticObject.cs

* Remove reference to Ryujinx.HLE.Loaders.Compression in KernelInitialProcess.cs

* Rename classes in Ryujinx.HLE.Loaders.Executables

* Fix space alignment

* Fix up formatting

Ryujinx.HLE/HOS/Horizon.cs

@@ -32,7 +32,7 @@ using System.Reflection;
 using System.Threading;
 
 using TimeServiceManager = Ryujinx.HLE.HOS.Services.Time.TimeManager;
-using NxStaticObject     = Ryujinx.HLE.Loaders.Executables.NxStaticObject;
+using NsoExecutable      = Ryujinx.HLE.Loaders.Executables.NsoExecutable;
 
 using static LibHac.Fs.ApplicationSaveDataManagement;
 
@@ -271,9 +271,9 @@ namespace Ryujinx.HLE.HOS
 
         public void LoadKip(string kipFile)
         {
-            using (FileStream fs = new FileStream(kipFile, FileMode.Open))
+            using (IStorage fs = new LocalStorage(kipFile, FileAccess.Read))
             {
-                ProgramLoader.LoadKernelInitalProcess(this, new KernelInitialProcess(fs));
+                ProgramLoader.LoadKernelInitalProcess(this, new KipExecutable(fs));
             }
         }
 
@@ -543,9 +543,9 @@ namespace Ryujinx.HLE.HOS
 
                     Logger.PrintInfo(LogClass.Loader, $"Loading {file.Name}...");
 
-                    codeFs.OpenFile(out IFile nsoFile, file.FullPath.ToU8Span(), OpenMode.Read).ThrowIfFailure();
-
-                    NxStaticObject staticObject = new NxStaticObject(nsoFile.AsStream());
+                    codeFs.OpenFile(out IFile nsoFile, file.FullPath.ToU8Span(), OpenMode.Read).ThrowIfFailure();          
+                         
+                    NsoExecutable staticObject = new NsoExecutable(nsoFile.AsStorage());
 
                     staticObjects.Add(staticObject);
                 }
@@ -569,13 +569,13 @@ namespace Ryujinx.HLE.HOS
 
             bool isNro = Path.GetExtension(filePath).ToLower() == ".nro";
 
-            FileStream input = new FileStream(filePath, FileMode.Open);
 
             IExecutable staticObject;
 
             if (isNro)
             {
-                NxRelocatableObject obj = new NxRelocatableObject(input);
+                FileStream input = new FileStream(filePath, FileMode.Open);
+                NroExecutable obj = new NroExecutable(input);
                 staticObject = obj;
 
                 // homebrew NRO can actually have some data after the actual NRO
@@ -648,7 +648,7 @@ namespace Ryujinx.HLE.HOS
             }
             else
             {
-                staticObject = new NxStaticObject(input);
+                staticObject = new NsoExecutable(new LocalStorage(filePath, FileAccess.Read));
             }
 
             ContentManager.LoadEntries(Device);

Ryujinx.HLE/HOS/ProgramLoader.cs

@@ -1,4 +1,5 @@
 using ARMeilleure.Memory;
+using LibHac;
 using Ryujinx.Common;
 using Ryujinx.Common.Logging;
 using Ryujinx.HLE.HOS.Kernel.Common;
@@ -17,7 +18,7 @@ namespace Ryujinx.HLE.HOS
         private const int ArgsDataSize   = 0x9000;
         private const int ArgsTotalSize  = ArgsHeaderSize + ArgsDataSize;
 
-        public static bool LoadKernelInitalProcess(Horizon system, KernelInitialProcess kip)
+        public static bool LoadKernelInitalProcess(Horizon system, KipExecutable kip)
         {
             int endOffset = kip.DataOffset + kip.Data.Length;
 
@@ -30,7 +31,7 @@ namespace Ryujinx.HLE.HOS
 
             int codePagesCount = codeSize / KMemoryManager.PageSize;
 
-            ulong codeBaseAddress = kip.Addr39Bits ? 0x8000000UL : 0x200000UL;
+            ulong codeBaseAddress = (kip.Header.Flags & 0x10) != 0 ? 0x8000000UL : 0x200000UL;
 
             ulong codeAddress = codeBaseAddress + (ulong)kip.TextOffset;
 
@@ -43,27 +44,27 @@ namespace Ryujinx.HLE.HOS
                 mmuFlags |= 0x20;
             }
 
-            if (kip.Addr39Bits)
+            if ((kip.Header.Flags & 0x10) != 0)
             {
                 mmuFlags |= (int)AddressSpaceType.Addr39Bits << 1;
             }
 
-            if (kip.Is64Bits)
+            if ((kip.Header.Flags & 0x08) != 0)
             {
                 mmuFlags |= 1;
             }
 
             ProcessCreationInfo creationInfo = new ProcessCreationInfo(
-                kip.Name,
-                kip.ProcessCategory,
-                kip.TitleId,
+                kip.Header.Name,
+                kip.Header.ProcessCategory,
+                kip.Header.TitleId,
                 codeAddress,
                 codePagesCount,
                 mmuFlags,
                 0,
                 0);
 
-            MemoryRegion memoryRegion = kip.IsService
+            MemoryRegion memoryRegion = (kip.Header.Flags & 0x20) != 0
                 ? MemoryRegion.Service
                 : MemoryRegion.Application;
 
@@ -103,9 +104,9 @@ namespace Ryujinx.HLE.HOS
                 return false;
             }
 
-            process.DefaultCpuCore = kip.DefaultProcessorId;
+            process.DefaultCpuCore = kip.Header.DefaultCore;
 
-            result = process.Start(kip.MainThreadPriority, (ulong)kip.MainThreadStackSize);
+            result = process.Start(kip.Header.MainThreadPriority, (ulong)kip.Header.Sections[1].Attribute);
 
             if (result != KernelResult.Success)
             {
@@ -309,4 +310,4 @@ namespace Ryujinx.HLE.HOS
             return SetProcessMemoryPermission(dataStart, end - dataStart, MemoryPermission.ReadAndWrite);
         }
     }
-}
+}

Ryujinx.HLE/HOS/Services/Ro/IRoInterface.cs

@@ -157,7 +157,7 @@ namespace Ryujinx.HLE.HOS.Services.Ro
 
             stream.Position = 0;
 
-            NxRelocatableObject executable = new NxRelocatableObject(stream, nroAddress, bssAddress);
+            NroExecutable executable = new NroExecutable(stream, nroAddress, bssAddress);
 
             // check if everything is page align.
             if ((executable.Text.Length & 0xFFF) != 0 || (executable.Ro.Length & 0xFFF) != 0 ||

Ryujinx.HLE/HOS/Services/Ro/Types/NroInfo.cs

@@ -4,7 +4,7 @@ namespace Ryujinx.HLE.HOS.Services.Ro
 {
     class NroInfo
     {
-        public NxRelocatableObject Executable { get; private set; }
+        public NroExecutable Executable { get; private set; }
 
         public byte[] Hash             { get; private set; }
         public ulong  NroAddress       { get; private set; }
@@ -15,7 +15,7 @@ namespace Ryujinx.HLE.HOS.Services.Ro
         public ulong  NroMappedAddress { get; set;         }
 
         public NroInfo(
-            NxRelocatableObject executable,
+            NroExecutable       executable,
             byte[]              hash,
             ulong               nroAddress,
             ulong               nroSize,

Ryujinx.HLE/Loaders/Compression/BackwardsLz.cs

@@ -1,107 +0,0 @@
-using System;
-
-namespace Ryujinx.HLE.Loaders.Compression
-{
-    static class BackwardsLz
-    {
-        private class BackwardsReader
-        {
-            private byte[] _data;
-
-            private int _position;
-
-            public int Position => _position;
-
-            public BackwardsReader(byte[] data, int end)
-            {
-                _data     = data;
-                _position = end;
-            }
-
-            public void SeekCurrent(int offset)
-            {
-                _position += offset;
-            }
-
-            public byte ReadByte()
-            {
-                return _data[--_position];
-            }
-
-            public short ReadInt16()
-            {
-                return (short)((ReadByte() << 8) | (ReadByte() << 0));
-            }
-
-            public int ReadInt32()
-            {
-                return ((ReadByte() << 24) |
-                        (ReadByte() << 16) |
-                        (ReadByte() << 8)  |
-                        (ReadByte() << 0));
-            }
-        }
-
-        public static void DecompressInPlace(byte[] buffer, int headerEnd)
-        {
-            BackwardsReader reader = new BackwardsReader(buffer, headerEnd);
-
-            int additionalDecLength = reader.ReadInt32();
-            int startOffset         = reader.ReadInt32();
-            int compressedLength    = reader.ReadInt32();
-
-            reader.SeekCurrent(12 - startOffset);
-
-            int decBase = headerEnd - compressedLength;
-
-            int decPos = compressedLength + additionalDecLength;
-
-            byte mask   = 0;
-            byte header = 0;
-
-            while (decPos > 0)
-            {
-                if ((mask >>= 1) == 0)
-                {
-                    header = reader.ReadByte();
-                    mask   = 0x80;
-                }
-
-                if ((header & mask) == 0)
-                {
-                    buffer[decBase + --decPos] = reader.ReadByte();
-                }
-                else
-                {
-                    ushort pair = (ushort)reader.ReadInt16();
-
-                    int length   = (pair >> 12)   + 3;
-                    int position = (pair & 0xfff) + 3;
-
-                    if (length > decPos)
-                    {
-                        length = decPos;
-                    }
-
-                    decPos -= length;
-
-                    int dstPos = decBase + decPos;
-
-                    if (length <= position)
-                    {
-                        int srcPos = dstPos + position;
-
-                        Buffer.BlockCopy(buffer, srcPos, buffer, dstPos, length);
-                    }
-                    else
-                    {
-                        for (int offset = 0; offset < length; offset++)
-                        {
-                            buffer[dstPos + offset] = buffer[dstPos + position + offset];
-                        }
-                    }
-                }
-            }
-        }
-    }
-}

Ryujinx.HLE/Loaders/Compression/Lz4.cs

@@ -1,78 +0,0 @@
-using System;
-
-namespace Ryujinx.HLE.Loaders.Compression
-{
-    static class Lz4
-    {
-        public static byte[] Decompress(byte[] cmp, int decLength)
-        {
-            byte[] dec = new byte[decLength];
-
-            int cmpPos = 0;
-            int decPos = 0;
-
-            int GetLength(int length)
-            {
-                byte sum;
-
-                if (length == 0xf)
-                {
-                    do
-                    {
-                        length += (sum = cmp[cmpPos++]);
-                    }
-                    while (sum == 0xff);
-                }
-
-                return length;
-            }
-
-            do
-            {
-                byte token = cmp[cmpPos++];
-
-                int encCount = (token >> 0) & 0xf;
-                int litCount = (token >> 4) & 0xf;
-
-                // Copy literal chunk
-                litCount = GetLength(litCount);
-
-                Buffer.BlockCopy(cmp, cmpPos, dec, decPos, litCount);
-
-                cmpPos += litCount;
-                decPos += litCount;
-
-                if (cmpPos >= cmp.Length)
-                {
-                    break;
-                }
-
-                // Copy compressed chunk
-                int back = cmp[cmpPos++] << 0 |
-                           cmp[cmpPos++] << 8;
-
-                encCount = GetLength(encCount) + 4;
-
-                int encPos = decPos - back;
-
-                if (encCount <= back)
-                {
-                    Buffer.BlockCopy(dec, encPos, dec, decPos, encCount);
-
-                    decPos += encCount;
-                }
-                else
-                {
-                    while (encCount-- > 0)
-                    {
-                        dec[decPos++] = dec[encPos++];
-                    }
-                }
-            }
-            while (cmpPos < cmp.Length &&
-                   decPos < dec.Length);
-
-            return dec;
-        }
-    }
-}

Ryujinx.HLE/Loaders/Executables/KernelInitialProcess.cs

@@ -1,147 +0,0 @@
-using Ryujinx.HLE.Loaders.Compression;
-using System.IO;
-
-namespace Ryujinx.HLE.Loaders.Executables
-{
-    class KernelInitialProcess : IExecutable
-    {
-        public string Name { get; private set; }
-
-        public ulong TitleId { get; private set; }
-
-        public int ProcessCategory { get; private set; }
-
-        public byte MainThreadPriority { get; private set; }
-        public byte DefaultProcessorId { get; private set; }
-
-        public bool Is64Bits   { get; private set; }
-        public bool Addr39Bits { get; private set; }
-        public bool IsService  { get; private set; }
-
-        public byte[] Text { get; private set; }
-        public byte[] Ro   { get; private set; }
-        public byte[] Data { get; private set; }
-
-        public int TextOffset { get; private set; }
-        public int RoOffset   { get; private set; }
-        public int DataOffset { get; private set; }
-        public int BssOffset  { get; private set; }
-        public int BssSize    { get; private set; }
-
-        public int MainThreadStackSize { get; private set; }
-
-        public int[] Capabilities { get; private set; }
-
-        private struct SegmentHeader
-        {
-            public int Offset           { get; private set; }
-            public int DecompressedSize { get; private set; }
-            public int CompressedSize   { get; private set; }
-            public int Attribute        { get; private set; }
-
-            public SegmentHeader(
-                int offset,
-                int decompressedSize,
-                int compressedSize,
-                int attribute)
-            {
-                Offset           = offset;
-                DecompressedSize = decompressedSize;
-                CompressedSize   = compressedSize;
-                Attribute        = attribute;
-            }
-        }
-
-        public KernelInitialProcess(Stream input)
-        {
-            BinaryReader reader = new BinaryReader(input);
-
-            string magic = ReadString(reader, 4);
-
-            if (magic != "KIP1")
-            {
-
-            }
-
-            Name = ReadString(reader, 12);
-
-            TitleId = reader.ReadUInt64();
-
-            ProcessCategory = reader.ReadInt32();
-
-            MainThreadPriority = reader.ReadByte();
-            DefaultProcessorId = reader.ReadByte();
-
-            byte reserved = reader.ReadByte();
-            byte flags    = reader.ReadByte();
-
-            Is64Bits   = (flags & 0x08) != 0;
-            Addr39Bits = (flags & 0x10) != 0;
-            IsService  = (flags & 0x20) != 0;
-
-            SegmentHeader[] segments = new SegmentHeader[6];
-
-            for (int index = 0; index < segments.Length; index++)
-            {
-                segments[index] = new SegmentHeader(
-                    reader.ReadInt32(),
-                    reader.ReadInt32(),
-                    reader.ReadInt32(),
-                    reader.ReadInt32());
-            }
-
-            TextOffset = segments[0].Offset;
-            RoOffset   = segments[1].Offset;
-            DataOffset = segments[2].Offset;
-            BssOffset  = segments[3].Offset;
-            BssSize    = segments[3].DecompressedSize;
-
-            MainThreadStackSize = segments[1].Attribute;
-
-            Capabilities = new int[32];
-
-            for (int index = 0; index < Capabilities.Length; index++)
-            {
-                Capabilities[index] = reader.ReadInt32();
-            }
-
-            input.Seek(0x100, SeekOrigin.Begin);
-
-            Text = ReadSegment(segments[0], input);
-            Ro   = ReadSegment(segments[1], input);
-            Data = ReadSegment(segments[2], input);
-        }
-
-        private byte[] ReadSegment(SegmentHeader header, Stream input)
-        {
-            byte[] data = new byte[header.DecompressedSize];
-
-            input.Read(data, 0, header.CompressedSize);
-
-            BackwardsLz.DecompressInPlace(data, header.CompressedSize);
-
-            return data;
-        }
-
-        private static string ReadString(BinaryReader reader, int maxSize)
-        {
-            string value = string.Empty;
-
-            for (int index = 0; index < maxSize; index++)
-            {
-                char chr = (char)reader.ReadByte();
-
-                if (chr == '\0')
-                {
-                    reader.BaseStream.Seek(maxSize - index - 1, SeekOrigin.Current);
-
-                    break;
-                }
-
-                value += chr;
-            }
-
-            return value;
-        }
-    }
-}

Ryujinx.HLE/Loaders/Executables/KipExecutable.cs

@@ -0,0 +1,35 @@
+using LibHac;
+using LibHac.Fs;
+using System.IO;
+
+namespace Ryujinx.HLE.Loaders.Executables
+{
+    class KipExecutable : Kip, IExecutable
+    {
+        public byte[] Text { get; }
+        public byte[] Ro { get; }
+        public byte[] Data { get; }
+
+        public int TextOffset => Header.Sections[0].OutOffset;
+        public int RoOffset => Header.Sections[1].OutOffset;
+        public int DataOffset => Header.Sections[2].OutOffset;
+        public int BssOffset => Header.Sections[3].OutOffset;
+        public int BssSize => Header.Sections[3].DecompressedSize;
+
+        public int[] Capabilities { get; }
+
+        public KipExecutable(IStorage inStorage) : base(inStorage)
+        {
+            Capabilities = new int[32];
+
+            for (int index = 0; index < Capabilities.Length; index++)
+            {
+                Capabilities[index] = System.BitConverter.ToInt32(Header.Capabilities, index * 4);
+            }
+
+            Text = DecompressSection(0);
+            Ro = DecompressSection(1);
+            Data = DecompressSection(2);
+        }
+    }
+}

Ryujinx.HLE/Loaders/Executables/NxRelocatableObject.cs → Ryujinx.HLE/Loaders/Executables/NroExecutable.cs

@@ -2,7 +2,7 @@ using System.IO;
 
 namespace Ryujinx.HLE.Loaders.Executables
 {
-    class NxRelocatableObject : IExecutable
+    class NroExecutable : IExecutable
     {
         public byte[] Text { get; private set; }
         public byte[] Ro   { get; private set; }
@@ -20,7 +20,7 @@ namespace Ryujinx.HLE.Loaders.Executables
         public ulong SourceAddress { get; private set; }
         public ulong BssAddress    { get; private set; }
 
-        public NxRelocatableObject(Stream input, ulong sourceAddress = 0, ulong bssAddress = 0)
+        public NroExecutable(Stream input, ulong sourceAddress = 0, ulong bssAddress = 0)
         {
             SourceAddress = sourceAddress;
             BssAddress    = bssAddress;

Ryujinx.HLE/Loaders/Executables/NsoExecutable.cs

@@ -0,0 +1,28 @@
+using LibHac;
+using LibHac.Fs;
+using System;
+using System.IO;
+
+namespace Ryujinx.HLE.Loaders.Executables
+{
+    class NsoExecutable : Nso, IExecutable
+    {
+        public byte[] Text { get; }
+        public byte[] Ro { get; }
+        public byte[] Data { get; }
+
+        public int TextOffset => (int)Sections[0].MemoryOffset;
+        public int RoOffset => (int)Sections[1].MemoryOffset;
+        public int DataOffset => (int)Sections[2].MemoryOffset;
+        public int BssOffset => DataOffset + Data.Length;
+
+        public new int BssSize => (int)base.BssSize;
+
+        public NsoExecutable(IStorage inStorage) : base(inStorage)
+        {
+            Text = Sections[0].DecompressSection();
+            Ro = Sections[1].DecompressSection();
+            Data =  Sections[2].DecompressSection();
+        }
+    }
+}

Ryujinx.HLE/Loaders/Executables/NxStaticObject.cs

@@ -1,109 +0,0 @@
-using Ryujinx.HLE.Loaders.Compression;
-using System;
-using System.IO;
-
-namespace Ryujinx.HLE.Loaders.Executables
-{
-    class NxStaticObject : IExecutable
-    {
-        public byte[] Text { get; private set; }
-        public byte[] Ro   { get; private set; }
-        public byte[] Data { get; private set; }
-
-        public int TextOffset { get; private set; }
-        public int RoOffset   { get; private set; }
-        public int DataOffset { get; private set; }
-        public int BssSize    { get; private set; }
-
-        public int BssOffset => DataOffset + Data.Length;
-
-        [Flags]
-        private enum NsoFlags
-        {
-            IsTextCompressed = 1 << 0,
-            IsRoCompressed   = 1 << 1,
-            IsDataCompressed = 1 << 2,
-            HasTextHash      = 1 << 3,
-            HasRoHash        = 1 << 4,
-            HasDataHash      = 1 << 5
-        }
-
-        public NxStaticObject(Stream input)
-        {
-            BinaryReader reader = new BinaryReader(input);
-
-            input.Seek(0, SeekOrigin.Begin);
-
-            int nsoMagic      = reader.ReadInt32();
-            int version       = reader.ReadInt32();
-            int reserved      = reader.ReadInt32();
-            int flagsMsk      = reader.ReadInt32();
-            int textOffset    = reader.ReadInt32();
-            int textMemOffset = reader.ReadInt32();
-            int textDecSize   = reader.ReadInt32();
-            int modNameOffset = reader.ReadInt32();
-            int roOffset      = reader.ReadInt32();
-            int roMemOffset   = reader.ReadInt32();
-            int roDecSize     = reader.ReadInt32();
-            int modNameSize   = reader.ReadInt32();
-            int dataOffset    = reader.ReadInt32();
-            int dataMemOffset = reader.ReadInt32();
-            int dataDecSize   = reader.ReadInt32();
-            int bssSize       = reader.ReadInt32();
-
-            byte[] buildId = reader.ReadBytes(0x20);
-
-            int textSize = reader.ReadInt32();
-            int roSize   = reader.ReadInt32();
-            int dataSize = reader.ReadInt32();
-
-            input.Seek(0x24, SeekOrigin.Current);
-
-            int dynStrOffset = reader.ReadInt32();
-            int dynStrSize   = reader.ReadInt32();
-            int dynSymOffset = reader.ReadInt32();
-            int dynSymSize   = reader.ReadInt32();
-
-            byte[] textHash = reader.ReadBytes(0x20);
-            byte[] roHash   = reader.ReadBytes(0x20);
-            byte[] dataHash = reader.ReadBytes(0x20);
-
-            NsoFlags flags = (NsoFlags)flagsMsk;
-
-            TextOffset = textMemOffset;
-            RoOffset   = roMemOffset;
-            DataOffset = dataMemOffset;
-            BssSize    = bssSize;
-
-            // Text segment
-            input.Seek(textOffset, SeekOrigin.Begin);
-
-            Text = reader.ReadBytes(textSize);
-
-            if (flags.HasFlag(NsoFlags.IsTextCompressed) && textSize != 0)
-            {
-                Text = Lz4.Decompress(Text, textDecSize);
-            }
-
-            // Read-only data segment
-            input.Seek(roOffset, SeekOrigin.Begin);
-
-            Ro = reader.ReadBytes(roSize);
-
-            if (flags.HasFlag(NsoFlags.IsRoCompressed) && roSize != 0)
-            {
-                Ro = Lz4.Decompress(Ro, roDecSize);
-            }
-
-            // Data segment
-            input.Seek(dataOffset, SeekOrigin.Begin);
-
-            Data = reader.ReadBytes(dataSize);
-
-            if (flags.HasFlag(NsoFlags.IsDataCompressed) && dataSize != 0)
-            {
-                Data = Lz4.Decompress(Data, dataDecSize);
-            }
-        }
-    }
-}