Implement aoc:u and support loading AddOnContent (#1221)

* Initial rebased AddOnContent support

* Fix bounds calculation
* Use existing GameCard in VFS per Xpl0itR's suggestion
+ Add dummy IPurchaseEventManager per AcK's suggestion

* Support multiple containers

* Add option to selectively disable addons

* Import tickets from AOC FS

* Load all nsps in base directory automatically

* Revert LoadNsp renaming

Removes conflicts with Mods PR. Not much is lost, old names were fine.

* Address AcK's comments

* Address Thog's comments

Dispose opened nsp files
Fix potential bug by clearing metadata on load

Ryujinx.HLE/FileSystem/Content/ContentManager.cs

@@ -4,6 +4,8 @@ using LibHac.Fs;
 using LibHac.FsSystem;
 using LibHac.FsSystem.NcaUtils;
 using LibHac.Ncm;
+using LibHac.Spl;
+using Ryujinx.Common.Logging;
 using Ryujinx.HLE.Exceptions;
 using Ryujinx.HLE.HOS.Services.Time;
 using Ryujinx.HLE.Utilities;
@@ -28,6 +30,22 @@ namespace Ryujinx.HLE.FileSystem.Content
 
         private SortedDictionary<(ulong titleId, NcaContentType type), string> _contentDictionary;
 
+        private struct AocItem
+        {
+            public readonly string ContainerPath;
+            public readonly string NcaPath;
+            public bool Enabled;
+
+            public AocItem(string containerPath, string ncaPath, bool enabled)
+            {
+                ContainerPath = containerPath;
+                NcaPath = ncaPath;
+                Enabled = enabled;
+            }
+        }
+
+        private SortedList<ulong, AocItem> _aocData { get; }
+
         private VirtualFileSystem _virtualFileSystem;
 
         private readonly object _lock = new object();
@@ -68,6 +86,8 @@ namespace Ryujinx.HLE.FileSystem.Content
             };
 
             _virtualFileSystem = virtualFileSystem;
+
+            _aocData = new SortedList<ulong, AocItem>();
         }
 
         public void LoadEntries(Switch device = null)
@@ -176,6 +196,89 @@ namespace Ryujinx.HLE.FileSystem.Content
             }
         }
 
+        // fs must contain AOC nca files in its root
+        public void AddAocData(IFileSystem fs, string containerPath, ulong aocBaseId)
+        {
+            _virtualFileSystem.ImportTickets(fs);
+
+            foreach (var ncaPath in fs.EnumerateEntries("*.cnmt.nca", SearchOptions.Default))
+            {
+                fs.OpenFile(out IFile ncaFile, ncaPath.FullPath.ToU8Span(), OpenMode.Read);
+                using (ncaFile)
+                {
+                    var nca = new Nca(_virtualFileSystem.KeySet, ncaFile.AsStorage());
+                    if (nca.Header.ContentType != NcaContentType.Meta)
+                    {
+                        Logger.PrintWarning(LogClass.Application, $"{ncaPath} is not a valid metadata file");
+
+                        continue;
+                    }
+
+                    using var pfs0 = nca.OpenFileSystem(0, Switch.GetIntegrityCheckLevel());
+
+                    pfs0.OpenFile(out IFile cnmtFile, pfs0.EnumerateEntries().Single().FullPath.ToU8Span(), OpenMode.Read);
+
+                    using (cnmtFile)
+                    {
+                        var cnmt = new Cnmt(cnmtFile.AsStream());
+
+                        if (cnmt.Type != ContentMetaType.AddOnContent || (cnmt.TitleId & 0xFFFFFFFFFFFFE000) != aocBaseId)
+                        {
+                            continue;
+                        }
+
+                        string ncaId = BitConverter.ToString(cnmt.ContentEntries[0].NcaId).Replace("-", "").ToLower();
+                        if (!_aocData.TryAdd(cnmt.TitleId, new AocItem(containerPath, $"{ncaId}.nca", true)))
+                        {
+                            Logger.PrintWarning(LogClass.Application, $"Duplicate AddOnContent detected. TitleId {cnmt.TitleId:X16}");
+                        }
+                        else
+                        {
+                            Logger.PrintInfo(LogClass.Application, $"Found AddOnContent with TitleId {cnmt.TitleId:X16}");
+                        }
+                    }
+                }
+            }
+        }
+
+        public void ClearAocData() => _aocData.Clear();
+
+        public int GetAocCount() => _aocData.Where(e => e.Value.Enabled).Count();
+
+        public IList<ulong> GetAocTitleIds() => _aocData.Where(e => e.Value.Enabled).Select(e => e.Key).ToList();
+
+        public bool GetAocDataStorage(ulong aocTitleId, out IStorage aocStorage)
+        {
+            aocStorage = null;
+
+            if (_aocData.TryGetValue(aocTitleId, out AocItem aoc) && aoc.Enabled)
+            {
+                var file = new FileStream(aoc.ContainerPath, FileMode.Open, FileAccess.Read);
+                PartitionFileSystem pfs;
+                IFile ncaFile;
+
+                switch (Path.GetExtension(aoc.ContainerPath))
+                {
+                    case ".xci":
+                        pfs = new Xci(_virtualFileSystem.KeySet, file.AsStorage()).OpenPartition(XciPartitionType.Secure);
+                        pfs.OpenFile(out ncaFile, aoc.NcaPath.ToU8Span(), OpenMode.Read);
+                        break;
+                    case ".nsp":
+                        pfs = new PartitionFileSystem(file.AsStorage());
+                        pfs.OpenFile(out ncaFile, aoc.NcaPath.ToU8Span(), OpenMode.Read);
+                        break;
+                    default:
+                        return false; // Print error?
+                }
+
+                aocStorage = new Nca(_virtualFileSystem.KeySet, ncaFile.AsStorage()).OpenStorage(NcaSectionType.Data, Switch.GetIntegrityCheckLevel());
+                
+                return true;
+            }
+
+            return false;
+        }
+
         public void ClearEntry(long titleId, NcaContentType contentType, StorageId storageId)
         {
             lock (_lock)

Ryujinx.HLE/HOS/ApplicationLoader.cs

@@ -146,6 +146,20 @@ namespace Ryujinx.HLE.HOS
 
             _contentManager.LoadEntries(_device);
 
+            _contentManager.ClearAocData();
+            _contentManager.AddAocData(securePartition, xciFile, mainNca.Header.TitleId);
+
+            // Check all nsp's in the base directory for AOC
+            foreach (var fn in new FileInfo(xciFile).Directory.EnumerateFiles("*.nsp"))
+            {
+                using (FileStream fs = fn.OpenRead())
+                using (IStorage storage = fs.AsStorage())
+                using (PartitionFileSystem pfs = new PartitionFileSystem(storage))
+                {
+                    _contentManager.AddAocData(pfs, fn.FullName, mainNca.Header.TitleId);
+                }
+            }
+
             LoadNca(mainNca, patchNca, controlNca);
         }
 
@@ -179,6 +193,21 @@ namespace Ryujinx.HLE.HOS
 
             if (mainNca != null)
             {
+                _contentManager.ClearAocData();
+                _contentManager.AddAocData(nsp, nspFile, mainNca.Header.TitleId);
+
+                // Check all nsp's in the base directory for AOC
+                foreach (var fn in new FileInfo(nspFile).Directory.EnumerateFiles("*.nsp"))
+                {
+                    if (fn.FullName == nspFile) continue;
+                    using (FileStream fs = fn.OpenRead())
+                    using (IStorage storage = fs.AsStorage())
+                    using (PartitionFileSystem pfs = new PartitionFileSystem(storage))
+                    {
+                        _contentManager.AddAocData(pfs, fn.FullName, mainNca.Header.TitleId);
+                    }
+                }
+
                 LoadNca(mainNca, patchNca, controlNca);
 
                 return;

Ryujinx.HLE/HOS/Services/Fs/IFileSystemProxy.cs

@@ -372,17 +372,26 @@ namespace Ryujinx.HLE.HOS.Services.Fs
             byte[] padding = context.RequestData.ReadBytes(7);
             long titleId = context.RequestData.ReadInt64();
 
+            // We do a mitm here to find if the request is for an AOC.
+            // This is because AOC can be distributed over multiple containers in the emulator.
+            if (context.Device.System.ContentManager.GetAocDataStorage((ulong)titleId, out LibHac.Fs.IStorage aocStorage))
+            {
+                Logger.PrintInfo(LogClass.Loader, $"Opened AddOnContent Data TitleID={titleId:X16}");
+
+                MakeObject(context, new FileSystemProxy.IStorage(aocStorage));
+                
+                return ResultCode.Success;
+            }
+
             NcaContentType contentType = NcaContentType.Data;
 
-            StorageId installedStorage =
-                context.Device.System.ContentManager.GetInstalledStorage(titleId, contentType, storageId);
+            StorageId installedStorage = context.Device.System.ContentManager.GetInstalledStorage(titleId, contentType, storageId);
 
             if (installedStorage == StorageId.None)
             {
                 contentType = NcaContentType.PublicData;
 
-                installedStorage =
-                    context.Device.System.ContentManager.GetInstalledStorage(titleId, contentType, storageId);
+                installedStorage = context.Device.System.ContentManager.GetInstalledStorage(titleId, contentType, storageId);
             }
 
             if (installedStorage != StorageId.None)

Ryujinx.HLE/HOS/Services/Ns/IAddOnContentManager.cs

@@ -1,32 +1,186 @@
 using Ryujinx.Common.Logging;
+using Ryujinx.HLE.HOS.Ipc;
+using Ryujinx.HLE.HOS.Kernel.Common;
+using Ryujinx.HLE.HOS.Kernel.Threading;
+using System;
 
 namespace Ryujinx.HLE.HOS.Services.Ns
 {
     [Service("aoc:u")]
     class IAddOnContentManager : IpcService
     {
-        public IAddOnContentManager(ServiceCtx context) { }
+        KEvent _addOnContentListChangedEvent;
+
+        public IAddOnContentManager(ServiceCtx context)
+        {
+            _addOnContentListChangedEvent = new KEvent(context.Device.System.KernelContext);
+        }
 
         [Command(2)]
-        // CountAddOnContent(u64, pid) -> u32
-        public static ResultCode CountAddOnContent(ServiceCtx context)
+        // CountAddOnContent(pid) -> u32
+        public ResultCode CountAddOnContent(ServiceCtx context)
         {
-            context.ResponseData.Write(0);
+            long pid = context.Process.Pid;
 
-            Logger.PrintStub(LogClass.ServiceNs);
+            // Official code checks ApplicationControlProperty.RuntimeAddOnContentInstall 
+            // if true calls ns:am ListAvailableAddOnContent again to get updated count
+
+            byte runtimeAddOnContentInstall = context.Device.Application.ControlData.Value.RuntimeAddOnContentInstall;
+            if (runtimeAddOnContentInstall != 0)
+            {
+                Logger.PrintWarning(LogClass.ServiceNs, $"RuntimeAddOnContentInstall is true. Some DLC may be missing");
+            }
+
+            uint aocCount = CountAddOnContentImpl(context);
+
+            context.ResponseData.Write(aocCount);
+
+            Logger.PrintStub(LogClass.ServiceNs, new { aocCount, runtimeAddOnContentInstall });
 
             return ResultCode.Success;
         }
 
+        private static uint CountAddOnContentImpl(ServiceCtx context)
+        {
+            return (uint)context.Device.System.ContentManager.GetAocCount();
+        }
+
         [Command(3)]
-        // ListAddOnContent(u32, u32, u64, pid) -> (u32, buffer<u32, 6>)
-        public static ResultCode ListAddOnContent(ServiceCtx context)
+        // ListAddOnContent(u32, u32, pid) -> (u32, buffer<u32>)
+        public ResultCode ListAddOnContent(ServiceCtx context)
+        {
+            uint startIndex = context.RequestData.ReadUInt32();
+            uint bufferSize = context.RequestData.ReadUInt32();
+            long pid = context.Process.Pid;
+
+            var aocTitleIds = context.Device.System.ContentManager.GetAocTitleIds();
+
+            uint aocCount = CountAddOnContentImpl(context);
+
+            if (aocCount <= startIndex)
+            {
+                context.ResponseData.Write((uint)0);
+
+                return ResultCode.Success;
+            }
+
+            aocCount = Math.Min(aocCount - startIndex, bufferSize);
+
+            context.ResponseData.Write(aocCount);
+
+            ulong bufAddr = (ulong)context.Request.ReceiveBuff[0].Position;
+
+            ulong aocBaseId = GetAddOnContentBaseIdImpl(context);
+
+            for (int i = 0; i < aocCount; ++i)
+            {
+                context.Memory.Write(bufAddr + (ulong)i * 4, (int)(aocTitleIds[i + (int)startIndex] - aocBaseId));
+            }
+
+            Logger.PrintStub(LogClass.ServiceNs, new { bufferSize, startIndex, aocCount });
+
+            return ResultCode.Success;
+        }
+
+        [Command(5)]
+        // GetAddOnContentBaseId(pid) -> u64
+        public ResultCode GetAddonContentBaseId(ServiceCtx context)
+        {
+            long pid = context.Process.Pid;
+
+            // Official code calls arp:r GetApplicationControlProperty to get AddOnContentBaseId
+            // If the call fails, calls arp:r GetApplicationLaunchProperty to get App TitleId
+            ulong aocBaseId = GetAddOnContentBaseIdImpl(context);
+
+            context.ResponseData.Write(aocBaseId);
+
+            Logger.PrintStub(LogClass.ServiceNs, $"aocBaseId={aocBaseId:X16}");
+
+            // ResultCode will be error code of GetApplicationLaunchProperty if it fails
+            return ResultCode.Success;
+        }
+
+        private static ulong GetAddOnContentBaseIdImpl(ServiceCtx context)
+        {
+            ulong aocBaseId = context.Device.Application.ControlData.Value.AddOnContentBaseId;
+
+            if (aocBaseId == 0)
+            {
+                aocBaseId = context.Device.Application.TitleId + 0x1000;
+            }
+
+            return aocBaseId;
+        }
+
+        [Command(7)]
+        // PrepareAddOnContent(u32, pid)
+        public ResultCode PrepareAddOnContent(ServiceCtx context)
+        {
+            uint aocIndex = context.RequestData.ReadUInt32();
+            long pid = context.Process.Pid;
+
+            // Official Code calls a bunch of functions from arp:r for aocBaseId
+            // and ns:am RegisterContentsExternalKey?, GetOwnedApplicationContentMetaStatus? etc...
+
+            // Ideally, this should probably initialize the AocData values for the specified index
+
+            Logger.PrintStub(LogClass.ServiceNs, new { aocIndex });
+
+            return ResultCode.Success;
+        }
+
+        [Command(8)]
+        // GetAddOnContentListChangedEvent() -> handle<copy>
+        public ResultCode GetAddOnContentListChangedEvent(ServiceCtx context)
+        {
+            // Official code seems to make an internal call to ns:am Cmd 84 GetDynamicCommitEvent()
+
+            if (context.Process.HandleTable.GenerateHandle(_addOnContentListChangedEvent.ReadableEvent, out int handle) != KernelResult.Success)
+            {
+                throw new InvalidOperationException("Out of handles!");
+            }
+
+            context.Response.HandleDesc = IpcHandleDesc.MakeCopy(handle);
+
+            Logger.PrintStub(LogClass.ServiceNs);
+
+            return ResultCode.Success;
+        }
+
+
+        [Command(9)] // [10.0.0+] 
+        // GetAddOnContentLostErrorCode() -> u64
+        public ResultCode GetAddOnContentLostErrorCode(ServiceCtx context)
+        {
+            // Seems to calculate ((value & 0x1ff)) + 2000 on 0x7d0a4
+            // which gives 0x874 (2000+164). 164 being Module ID of `EC (Shop)`
+            context.ResponseData.Write(2164L);
+
+            Logger.PrintStub(LogClass.ServiceNs);
+
+            return ResultCode.Success;
+        }
+
+        [Command(100)]
+        // CreateEcPurchasedEventManager() -> object<nn::ec::IPurchaseEventManager>
+        public ResultCode CreateEcPurchasedEventManager(ServiceCtx context)
         {
+            MakeObject(context, new IPurchaseEventManager());
+
             Logger.PrintStub(LogClass.ServiceNs);
 
-            // TODO: This is supposed to write a u32 array aswell.
-            // It's unknown what it contains.
-            context.ResponseData.Write(0);
+            return ResultCode.Success;
+        }
+
+        [Command(101)]
+        // CreatePermanentEcPurchasedEventManager() -> object<nn::ec::IPurchaseEventManager>
+        public ResultCode CreatePermanentEcPurchasedEventManager(ServiceCtx context)
+        {
+            // Very similar to CreateEcPurchasedEventManager but with some extra code
+
+            MakeObject(context, new IPurchaseEventManager());
+
+            Logger.PrintStub(LogClass.ServiceNs);
 
             return ResultCode.Success;
         }

Ryujinx.HLE/HOS/Services/Ns/IPurchaseEventManager.cs

@@ -0,0 +1,8 @@
+namespace Ryujinx.HLE.HOS.Services.Ns
+{
+    class IPurchaseEventManager : IpcService
+    {
+        // TODO: Implement this
+        // Size seems to be atleast 0x7a8
+    }
+}