KThread.cs 32 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111
  1. using ARMeilleure.Memory;
  2. using Ryujinx.Common.Logging;
  3. using Ryujinx.HLE.HOS.Kernel.Common;
  4. using Ryujinx.HLE.HOS.Kernel.Process;
  5. using System;
  6. using System.Collections.Generic;
  7. using System.Linq;
  8. using System.Text;
  9. using System.Threading;
  10. namespace Ryujinx.HLE.HOS.Kernel.Threading
  11. {
  12. class KThread : KSynchronizationObject, IKFutureSchedulerObject
  13. {
  14. private int _hostThreadRunning;
  15. public Thread HostThread { get; private set; }
  16. public ARMeilleure.State.ExecutionContext Context { get; private set; }
  17. public long AffinityMask { get; set; }
  18. public long ThreadUid { get; private set; }
  19. public long TotalTimeRunning { get; set; }
  20. public KSynchronizationObject SignaledObj { get; set; }
  21. public ulong CondVarAddress { get; set; }
  22. private ulong _entrypoint;
  23. public ulong MutexAddress { get; set; }
  24. public KProcess Owner { get; private set; }
  25. private ulong _tlsAddress;
  26. public ulong TlsAddress => _tlsAddress;
  27. public ulong TlsDramAddress { get; private set; }
  28. public long LastScheduledTime { get; set; }
  29. public LinkedListNode<KThread>[] SiblingsPerCore { get; private set; }
  30. public LinkedList<KThread> Withholder { get; set; }
  31. public LinkedListNode<KThread> WithholderNode { get; set; }
  32. public LinkedListNode<KThread> ProcessListNode { get; set; }
  33. private LinkedList<KThread> _mutexWaiters;
  34. private LinkedListNode<KThread> _mutexWaiterNode;
  35. public KThread MutexOwner { get; private set; }
  36. public int ThreadHandleForUserMutex { get; set; }
  37. private ThreadSchedState _forcePauseFlags;
  38. public KernelResult ObjSyncResult { get; set; }
  39. public int DynamicPriority { get; set; }
  40. public int CurrentCore { get; set; }
  41. public int BasePriority { get; set; }
  42. public int PreferredCore { get; set; }
  43. private long _affinityMaskOverride;
  44. private int _preferredCoreOverride;
  45. private int _affinityOverrideCount;
  46. public ThreadSchedState SchedFlags { get; private set; }
  47. public bool ShallBeTerminated { get; private set; }
  48. public bool SyncCancelled { get; set; }
  49. public bool WaitingSync { get; set; }
  50. private bool _hasExited;
  51. private bool _hasBeenInitialized;
  52. private bool _hasBeenReleased;
  53. public bool WaitingInArbitration { get; set; }
  54. private KScheduler _scheduler;
  55. private KSchedulingData _schedulingData;
  56. public long LastPc { get; set; }
  57. public KThread(Horizon system) : base(system)
  58. {
  59. _scheduler = system.Scheduler;
  60. _schedulingData = system.Scheduler.SchedulingData;
  61. SiblingsPerCore = new LinkedListNode<KThread>[KScheduler.CpuCoresCount];
  62. _mutexWaiters = new LinkedList<KThread>();
  63. }
  64. public KernelResult Initialize(
  65. ulong entrypoint,
  66. ulong argsPtr,
  67. ulong stackTop,
  68. int priority,
  69. int defaultCpuCore,
  70. KProcess owner,
  71. ThreadType type = ThreadType.User)
  72. {
  73. if ((uint)type > 3)
  74. {
  75. throw new ArgumentException($"Invalid thread type \"{type}\".");
  76. }
  77. PreferredCore = defaultCpuCore;
  78. AffinityMask |= 1L << defaultCpuCore;
  79. SchedFlags = type == ThreadType.Dummy
  80. ? ThreadSchedState.Running
  81. : ThreadSchedState.None;
  82. CurrentCore = PreferredCore;
  83. DynamicPriority = priority;
  84. BasePriority = priority;
  85. ObjSyncResult = KernelResult.ThreadNotStarted;
  86. _entrypoint = entrypoint;
  87. if (type == ThreadType.User)
  88. {
  89. if (owner.AllocateThreadLocalStorage(out _tlsAddress) != KernelResult.Success)
  90. {
  91. return KernelResult.OutOfMemory;
  92. }
  93. TlsDramAddress = owner.MemoryManager.GetDramAddressFromVa(_tlsAddress);
  94. MemoryHelper.FillWithZeros(owner.CpuMemory, (long)_tlsAddress, KTlsPageInfo.TlsEntrySize);
  95. }
  96. bool is64Bits;
  97. if (owner != null)
  98. {
  99. Owner = owner;
  100. owner.IncrementReferenceCount();
  101. owner.IncrementThreadCount();
  102. is64Bits = (owner.MmuFlags & 1) != 0;
  103. }
  104. else
  105. {
  106. is64Bits = true;
  107. }
  108. HostThread = new Thread(() => ThreadStart(entrypoint));
  109. Context = new ARMeilleure.State.ExecutionContext();
  110. bool isAarch32 = (Owner.MmuFlags & 1) == 0;
  111. Context.IsAarch32 = isAarch32;
  112. Context.SetX(0, argsPtr);
  113. if (isAarch32)
  114. {
  115. Context.SetX(13, (uint)stackTop);
  116. }
  117. else
  118. {
  119. Context.SetX(31, stackTop);
  120. }
  121. Context.CntfrqEl0 = 19200000;
  122. Context.Tpidr = (long)_tlsAddress;
  123. owner.SubscribeThreadEventHandlers(Context);
  124. ThreadUid = System.GetThreadUid();
  125. _hasBeenInitialized = true;
  126. if (owner != null)
  127. {
  128. owner.AddThread(this);
  129. if (owner.IsPaused)
  130. {
  131. System.CriticalSection.Enter();
  132. if (ShallBeTerminated || SchedFlags == ThreadSchedState.TerminationPending)
  133. {
  134. System.CriticalSection.Leave();
  135. return KernelResult.Success;
  136. }
  137. _forcePauseFlags |= ThreadSchedState.ProcessPauseFlag;
  138. CombineForcePauseFlags();
  139. System.CriticalSection.Leave();
  140. }
  141. }
  142. return KernelResult.Success;
  143. }
  144. public KernelResult Start()
  145. {
  146. if (!System.KernelInitialized)
  147. {
  148. System.CriticalSection.Enter();
  149. if (!ShallBeTerminated && SchedFlags != ThreadSchedState.TerminationPending)
  150. {
  151. _forcePauseFlags |= ThreadSchedState.KernelInitPauseFlag;
  152. CombineForcePauseFlags();
  153. }
  154. System.CriticalSection.Leave();
  155. }
  156. KernelResult result = KernelResult.ThreadTerminating;
  157. System.CriticalSection.Enter();
  158. if (!ShallBeTerminated)
  159. {
  160. KThread currentThread = System.Scheduler.GetCurrentThread();
  161. while (SchedFlags != ThreadSchedState.TerminationPending &&
  162. currentThread.SchedFlags != ThreadSchedState.TerminationPending &&
  163. !currentThread.ShallBeTerminated)
  164. {
  165. if ((SchedFlags & ThreadSchedState.LowMask) != ThreadSchedState.None)
  166. {
  167. result = KernelResult.InvalidState;
  168. break;
  169. }
  170. if (currentThread._forcePauseFlags == ThreadSchedState.None)
  171. {
  172. if (Owner != null && _forcePauseFlags != ThreadSchedState.None)
  173. {
  174. CombineForcePauseFlags();
  175. }
  176. SetNewSchedFlags(ThreadSchedState.Running);
  177. result = KernelResult.Success;
  178. break;
  179. }
  180. else
  181. {
  182. currentThread.CombineForcePauseFlags();
  183. System.CriticalSection.Leave();
  184. System.CriticalSection.Enter();
  185. if (currentThread.ShallBeTerminated)
  186. {
  187. break;
  188. }
  189. }
  190. }
  191. }
  192. System.CriticalSection.Leave();
  193. return result;
  194. }
  195. public void Exit()
  196. {
  197. // TODO: Debug event.
  198. if (Owner != null)
  199. {
  200. Owner.ResourceLimit?.Release(LimitableResource.Thread, 0, 1);
  201. _hasBeenReleased = true;
  202. }
  203. System.CriticalSection.Enter();
  204. _forcePauseFlags &= ~ThreadSchedState.ForcePauseMask;
  205. ExitImpl();
  206. System.CriticalSection.Leave();
  207. DecrementReferenceCount();
  208. }
  209. private void ExitImpl()
  210. {
  211. System.CriticalSection.Enter();
  212. SetNewSchedFlags(ThreadSchedState.TerminationPending);
  213. _hasExited = true;
  214. Signal();
  215. System.CriticalSection.Leave();
  216. }
  217. public KernelResult Sleep(long timeout)
  218. {
  219. System.CriticalSection.Enter();
  220. if (ShallBeTerminated || SchedFlags == ThreadSchedState.TerminationPending)
  221. {
  222. System.CriticalSection.Leave();
  223. return KernelResult.ThreadTerminating;
  224. }
  225. SetNewSchedFlags(ThreadSchedState.Paused);
  226. if (timeout > 0)
  227. {
  228. System.TimeManager.ScheduleFutureInvocation(this, timeout);
  229. }
  230. System.CriticalSection.Leave();
  231. if (timeout > 0)
  232. {
  233. System.TimeManager.UnscheduleFutureInvocation(this);
  234. }
  235. return 0;
  236. }
  237. public void Yield()
  238. {
  239. System.CriticalSection.Enter();
  240. if (SchedFlags != ThreadSchedState.Running)
  241. {
  242. System.CriticalSection.Leave();
  243. System.Scheduler.ContextSwitch();
  244. return;
  245. }
  246. if (DynamicPriority < KScheduler.PrioritiesCount)
  247. {
  248. // Move current thread to the end of the queue.
  249. _schedulingData.Reschedule(DynamicPriority, CurrentCore, this);
  250. }
  251. _scheduler.ThreadReselectionRequested = true;
  252. System.CriticalSection.Leave();
  253. System.Scheduler.ContextSwitch();
  254. }
  255. public void YieldWithLoadBalancing()
  256. {
  257. System.CriticalSection.Enter();
  258. if (SchedFlags != ThreadSchedState.Running)
  259. {
  260. System.CriticalSection.Leave();
  261. System.Scheduler.ContextSwitch();
  262. return;
  263. }
  264. int prio = DynamicPriority;
  265. int core = CurrentCore;
  266. KThread nextThreadOnCurrentQueue = null;
  267. if (DynamicPriority < KScheduler.PrioritiesCount)
  268. {
  269. // Move current thread to the end of the queue.
  270. _schedulingData.Reschedule(prio, core, this);
  271. Func<KThread, bool> predicate = x => x.DynamicPriority == prio;
  272. nextThreadOnCurrentQueue = _schedulingData.ScheduledThreads(core).FirstOrDefault(predicate);
  273. }
  274. IEnumerable<KThread> SuitableCandidates()
  275. {
  276. foreach (KThread thread in _schedulingData.SuggestedThreads(core))
  277. {
  278. int srcCore = thread.CurrentCore;
  279. if (srcCore >= 0)
  280. {
  281. KThread selectedSrcCore = _scheduler.CoreContexts[srcCore].SelectedThread;
  282. if (selectedSrcCore == thread || ((selectedSrcCore?.DynamicPriority ?? 2) < 2))
  283. {
  284. continue;
  285. }
  286. }
  287. // If the candidate was scheduled after the current thread, then it's not worth it,
  288. // unless the priority is higher than the current one.
  289. if (nextThreadOnCurrentQueue.LastScheduledTime >= thread.LastScheduledTime ||
  290. nextThreadOnCurrentQueue.DynamicPriority < thread.DynamicPriority)
  291. {
  292. yield return thread;
  293. }
  294. }
  295. }
  296. KThread dst = SuitableCandidates().FirstOrDefault(x => x.DynamicPriority <= prio);
  297. if (dst != null)
  298. {
  299. _schedulingData.TransferToCore(dst.DynamicPriority, core, dst);
  300. _scheduler.ThreadReselectionRequested = true;
  301. }
  302. if (this != nextThreadOnCurrentQueue)
  303. {
  304. _scheduler.ThreadReselectionRequested = true;
  305. }
  306. System.CriticalSection.Leave();
  307. System.Scheduler.ContextSwitch();
  308. }
  309. public void YieldAndWaitForLoadBalancing()
  310. {
  311. System.CriticalSection.Enter();
  312. if (SchedFlags != ThreadSchedState.Running)
  313. {
  314. System.CriticalSection.Leave();
  315. System.Scheduler.ContextSwitch();
  316. return;
  317. }
  318. int core = CurrentCore;
  319. _schedulingData.TransferToCore(DynamicPriority, -1, this);
  320. KThread selectedThread = null;
  321. if (!_schedulingData.ScheduledThreads(core).Any())
  322. {
  323. foreach (KThread thread in _schedulingData.SuggestedThreads(core))
  324. {
  325. if (thread.CurrentCore < 0)
  326. {
  327. continue;
  328. }
  329. KThread firstCandidate = _schedulingData.ScheduledThreads(thread.CurrentCore).FirstOrDefault();
  330. if (firstCandidate == thread)
  331. {
  332. continue;
  333. }
  334. if (firstCandidate == null || firstCandidate.DynamicPriority >= 2)
  335. {
  336. _schedulingData.TransferToCore(thread.DynamicPriority, core, thread);
  337. selectedThread = thread;
  338. }
  339. break;
  340. }
  341. }
  342. if (selectedThread != this)
  343. {
  344. _scheduler.ThreadReselectionRequested = true;
  345. }
  346. System.CriticalSection.Leave();
  347. System.Scheduler.ContextSwitch();
  348. }
  349. public void SetPriority(int priority)
  350. {
  351. System.CriticalSection.Enter();
  352. BasePriority = priority;
  353. UpdatePriorityInheritance();
  354. System.CriticalSection.Leave();
  355. }
  356. public KernelResult SetActivity(bool pause)
  357. {
  358. KernelResult result = KernelResult.Success;
  359. System.CriticalSection.Enter();
  360. ThreadSchedState lowNibble = SchedFlags & ThreadSchedState.LowMask;
  361. if (lowNibble != ThreadSchedState.Paused && lowNibble != ThreadSchedState.Running)
  362. {
  363. System.CriticalSection.Leave();
  364. return KernelResult.InvalidState;
  365. }
  366. System.CriticalSection.Enter();
  367. if (!ShallBeTerminated && SchedFlags != ThreadSchedState.TerminationPending)
  368. {
  369. if (pause)
  370. {
  371. // Pause, the force pause flag should be clear (thread is NOT paused).
  372. if ((_forcePauseFlags & ThreadSchedState.ThreadPauseFlag) == 0)
  373. {
  374. _forcePauseFlags |= ThreadSchedState.ThreadPauseFlag;
  375. CombineForcePauseFlags();
  376. }
  377. else
  378. {
  379. result = KernelResult.InvalidState;
  380. }
  381. }
  382. else
  383. {
  384. // Unpause, the force pause flag should be set (thread is paused).
  385. if ((_forcePauseFlags & ThreadSchedState.ThreadPauseFlag) != 0)
  386. {
  387. ThreadSchedState oldForcePauseFlags = _forcePauseFlags;
  388. _forcePauseFlags &= ~ThreadSchedState.ThreadPauseFlag;
  389. if ((oldForcePauseFlags & ~ThreadSchedState.ThreadPauseFlag) == ThreadSchedState.None)
  390. {
  391. ThreadSchedState oldSchedFlags = SchedFlags;
  392. SchedFlags &= ThreadSchedState.LowMask;
  393. AdjustScheduling(oldSchedFlags);
  394. }
  395. }
  396. else
  397. {
  398. result = KernelResult.InvalidState;
  399. }
  400. }
  401. }
  402. System.CriticalSection.Leave();
  403. System.CriticalSection.Leave();
  404. return result;
  405. }
  406. public void CancelSynchronization()
  407. {
  408. System.CriticalSection.Enter();
  409. if ((SchedFlags & ThreadSchedState.LowMask) != ThreadSchedState.Paused || !WaitingSync)
  410. {
  411. SyncCancelled = true;
  412. }
  413. else if (Withholder != null)
  414. {
  415. Withholder.Remove(WithholderNode);
  416. SetNewSchedFlags(ThreadSchedState.Running);
  417. Withholder = null;
  418. SyncCancelled = true;
  419. }
  420. else
  421. {
  422. SignaledObj = null;
  423. ObjSyncResult = KernelResult.Cancelled;
  424. SetNewSchedFlags(ThreadSchedState.Running);
  425. SyncCancelled = false;
  426. }
  427. System.CriticalSection.Leave();
  428. }
  429. public KernelResult SetCoreAndAffinityMask(int newCore, long newAffinityMask)
  430. {
  431. System.CriticalSection.Enter();
  432. bool useOverride = _affinityOverrideCount != 0;
  433. // The value -3 is "do not change the preferred core".
  434. if (newCore == -3)
  435. {
  436. newCore = useOverride ? _preferredCoreOverride : PreferredCore;
  437. if ((newAffinityMask & (1 << newCore)) == 0)
  438. {
  439. System.CriticalSection.Leave();
  440. return KernelResult.InvalidCombination;
  441. }
  442. }
  443. if (useOverride)
  444. {
  445. _preferredCoreOverride = newCore;
  446. _affinityMaskOverride = newAffinityMask;
  447. }
  448. else
  449. {
  450. long oldAffinityMask = AffinityMask;
  451. PreferredCore = newCore;
  452. AffinityMask = newAffinityMask;
  453. if (oldAffinityMask != newAffinityMask)
  454. {
  455. int oldCore = CurrentCore;
  456. if (CurrentCore >= 0 && ((AffinityMask >> CurrentCore) & 1) == 0)
  457. {
  458. if (PreferredCore < 0)
  459. {
  460. CurrentCore = HighestSetCore(AffinityMask);
  461. }
  462. else
  463. {
  464. CurrentCore = PreferredCore;
  465. }
  466. }
  467. AdjustSchedulingForNewAffinity(oldAffinityMask, oldCore);
  468. }
  469. }
  470. System.CriticalSection.Leave();
  471. return KernelResult.Success;
  472. }
  473. private static int HighestSetCore(long mask)
  474. {
  475. for (int core = KScheduler.CpuCoresCount - 1; core >= 0; core--)
  476. {
  477. if (((mask >> core) & 1) != 0)
  478. {
  479. return core;
  480. }
  481. }
  482. return -1;
  483. }
  484. private void CombineForcePauseFlags()
  485. {
  486. ThreadSchedState oldFlags = SchedFlags;
  487. ThreadSchedState lowNibble = SchedFlags & ThreadSchedState.LowMask;
  488. SchedFlags = lowNibble | _forcePauseFlags;
  489. AdjustScheduling(oldFlags);
  490. }
  491. private void SetNewSchedFlags(ThreadSchedState newFlags)
  492. {
  493. System.CriticalSection.Enter();
  494. ThreadSchedState oldFlags = SchedFlags;
  495. SchedFlags = (oldFlags & ThreadSchedState.HighMask) | newFlags;
  496. if ((oldFlags & ThreadSchedState.LowMask) != newFlags)
  497. {
  498. AdjustScheduling(oldFlags);
  499. }
  500. System.CriticalSection.Leave();
  501. }
  502. public void ReleaseAndResume()
  503. {
  504. System.CriticalSection.Enter();
  505. if ((SchedFlags & ThreadSchedState.LowMask) == ThreadSchedState.Paused)
  506. {
  507. if (Withholder != null)
  508. {
  509. Withholder.Remove(WithholderNode);
  510. SetNewSchedFlags(ThreadSchedState.Running);
  511. Withholder = null;
  512. }
  513. else
  514. {
  515. SetNewSchedFlags(ThreadSchedState.Running);
  516. }
  517. }
  518. System.CriticalSection.Leave();
  519. }
  520. public void Reschedule(ThreadSchedState newFlags)
  521. {
  522. System.CriticalSection.Enter();
  523. ThreadSchedState oldFlags = SchedFlags;
  524. SchedFlags = (oldFlags & ThreadSchedState.HighMask) |
  525. (newFlags & ThreadSchedState.LowMask);
  526. AdjustScheduling(oldFlags);
  527. System.CriticalSection.Leave();
  528. }
  529. public void AddMutexWaiter(KThread requester)
  530. {
  531. AddToMutexWaitersList(requester);
  532. requester.MutexOwner = this;
  533. UpdatePriorityInheritance();
  534. }
  535. public void RemoveMutexWaiter(KThread thread)
  536. {
  537. if (thread._mutexWaiterNode?.List != null)
  538. {
  539. _mutexWaiters.Remove(thread._mutexWaiterNode);
  540. }
  541. thread.MutexOwner = null;
  542. UpdatePriorityInheritance();
  543. }
  544. public KThread RelinquishMutex(ulong mutexAddress, out int count)
  545. {
  546. count = 0;
  547. if (_mutexWaiters.First == null)
  548. {
  549. return null;
  550. }
  551. KThread newMutexOwner = null;
  552. LinkedListNode<KThread> currentNode = _mutexWaiters.First;
  553. do
  554. {
  555. // Skip all threads that are not waiting for this mutex.
  556. while (currentNode != null && currentNode.Value.MutexAddress != mutexAddress)
  557. {
  558. currentNode = currentNode.Next;
  559. }
  560. if (currentNode == null)
  561. {
  562. break;
  563. }
  564. LinkedListNode<KThread> nextNode = currentNode.Next;
  565. _mutexWaiters.Remove(currentNode);
  566. currentNode.Value.MutexOwner = newMutexOwner;
  567. if (newMutexOwner != null)
  568. {
  569. // New owner was already selected, re-insert on new owner list.
  570. newMutexOwner.AddToMutexWaitersList(currentNode.Value);
  571. }
  572. else
  573. {
  574. // New owner not selected yet, use current thread.
  575. newMutexOwner = currentNode.Value;
  576. }
  577. count++;
  578. currentNode = nextNode;
  579. }
  580. while (currentNode != null);
  581. if (newMutexOwner != null)
  582. {
  583. UpdatePriorityInheritance();
  584. newMutexOwner.UpdatePriorityInheritance();
  585. }
  586. return newMutexOwner;
  587. }
  588. private void UpdatePriorityInheritance()
  589. {
  590. // If any of the threads waiting for the mutex has
  591. // higher priority than the current thread, then
  592. // the current thread inherits that priority.
  593. int highestPriority = BasePriority;
  594. if (_mutexWaiters.First != null)
  595. {
  596. int waitingDynamicPriority = _mutexWaiters.First.Value.DynamicPriority;
  597. if (waitingDynamicPriority < highestPriority)
  598. {
  599. highestPriority = waitingDynamicPriority;
  600. }
  601. }
  602. if (highestPriority != DynamicPriority)
  603. {
  604. int oldPriority = DynamicPriority;
  605. DynamicPriority = highestPriority;
  606. AdjustSchedulingForNewPriority(oldPriority);
  607. if (MutexOwner != null)
  608. {
  609. // Remove and re-insert to ensure proper sorting based on new priority.
  610. MutexOwner._mutexWaiters.Remove(_mutexWaiterNode);
  611. MutexOwner.AddToMutexWaitersList(this);
  612. MutexOwner.UpdatePriorityInheritance();
  613. }
  614. }
  615. }
  616. private void AddToMutexWaitersList(KThread thread)
  617. {
  618. LinkedListNode<KThread> nextPrio = _mutexWaiters.First;
  619. int currentPriority = thread.DynamicPriority;
  620. while (nextPrio != null && nextPrio.Value.DynamicPriority <= currentPriority)
  621. {
  622. nextPrio = nextPrio.Next;
  623. }
  624. if (nextPrio != null)
  625. {
  626. thread._mutexWaiterNode = _mutexWaiters.AddBefore(nextPrio, thread);
  627. }
  628. else
  629. {
  630. thread._mutexWaiterNode = _mutexWaiters.AddLast(thread);
  631. }
  632. }
  633. private void AdjustScheduling(ThreadSchedState oldFlags)
  634. {
  635. if (oldFlags == SchedFlags)
  636. {
  637. return;
  638. }
  639. if (oldFlags == ThreadSchedState.Running)
  640. {
  641. // Was running, now it's stopped.
  642. if (CurrentCore >= 0)
  643. {
  644. _schedulingData.Unschedule(DynamicPriority, CurrentCore, this);
  645. }
  646. for (int core = 0; core < KScheduler.CpuCoresCount; core++)
  647. {
  648. if (core != CurrentCore && ((AffinityMask >> core) & 1) != 0)
  649. {
  650. _schedulingData.Unsuggest(DynamicPriority, core, this);
  651. }
  652. }
  653. }
  654. else if (SchedFlags == ThreadSchedState.Running)
  655. {
  656. // Was stopped, now it's running.
  657. if (CurrentCore >= 0)
  658. {
  659. _schedulingData.Schedule(DynamicPriority, CurrentCore, this);
  660. }
  661. for (int core = 0; core < KScheduler.CpuCoresCount; core++)
  662. {
  663. if (core != CurrentCore && ((AffinityMask >> core) & 1) != 0)
  664. {
  665. _schedulingData.Suggest(DynamicPriority, core, this);
  666. }
  667. }
  668. }
  669. _scheduler.ThreadReselectionRequested = true;
  670. }
  671. private void AdjustSchedulingForNewPriority(int oldPriority)
  672. {
  673. if (SchedFlags != ThreadSchedState.Running)
  674. {
  675. return;
  676. }
  677. // Remove thread from the old priority queues.
  678. if (CurrentCore >= 0)
  679. {
  680. _schedulingData.Unschedule(oldPriority, CurrentCore, this);
  681. }
  682. for (int core = 0; core < KScheduler.CpuCoresCount; core++)
  683. {
  684. if (core != CurrentCore && ((AffinityMask >> core) & 1) != 0)
  685. {
  686. _schedulingData.Unsuggest(oldPriority, core, this);
  687. }
  688. }
  689. // Add thread to the new priority queues.
  690. KThread currentThread = _scheduler.GetCurrentThread();
  691. if (CurrentCore >= 0)
  692. {
  693. if (currentThread == this)
  694. {
  695. _schedulingData.SchedulePrepend(DynamicPriority, CurrentCore, this);
  696. }
  697. else
  698. {
  699. _schedulingData.Schedule(DynamicPriority, CurrentCore, this);
  700. }
  701. }
  702. for (int core = 0; core < KScheduler.CpuCoresCount; core++)
  703. {
  704. if (core != CurrentCore && ((AffinityMask >> core) & 1) != 0)
  705. {
  706. _schedulingData.Suggest(DynamicPriority, core, this);
  707. }
  708. }
  709. _scheduler.ThreadReselectionRequested = true;
  710. }
  711. private void AdjustSchedulingForNewAffinity(long oldAffinityMask, int oldCore)
  712. {
  713. if (SchedFlags != ThreadSchedState.Running || DynamicPriority >= KScheduler.PrioritiesCount)
  714. {
  715. return;
  716. }
  717. // Remove thread from the old priority queues.
  718. for (int core = 0; core < KScheduler.CpuCoresCount; core++)
  719. {
  720. if (((oldAffinityMask >> core) & 1) != 0)
  721. {
  722. if (core == oldCore)
  723. {
  724. _schedulingData.Unschedule(DynamicPriority, core, this);
  725. }
  726. else
  727. {
  728. _schedulingData.Unsuggest(DynamicPriority, core, this);
  729. }
  730. }
  731. }
  732. // Add thread to the new priority queues.
  733. for (int core = 0; core < KScheduler.CpuCoresCount; core++)
  734. {
  735. if (((AffinityMask >> core) & 1) != 0)
  736. {
  737. if (core == CurrentCore)
  738. {
  739. _schedulingData.Schedule(DynamicPriority, core, this);
  740. }
  741. else
  742. {
  743. _schedulingData.Suggest(DynamicPriority, core, this);
  744. }
  745. }
  746. }
  747. _scheduler.ThreadReselectionRequested = true;
  748. }
  749. public void SetEntryArguments(long argsPtr, int threadHandle)
  750. {
  751. Context.SetX(0, (ulong)argsPtr);
  752. Context.SetX(1, (ulong)threadHandle);
  753. }
  754. public void TimeUp()
  755. {
  756. ReleaseAndResume();
  757. }
  758. public string GetGuestStackTrace()
  759. {
  760. return Owner.Debugger.GetGuestStackTrace(Context);
  761. }
  762. public void PrintGuestStackTrace()
  763. {
  764. StringBuilder trace = new StringBuilder();
  765. trace.AppendLine("Guest stack trace:");
  766. trace.AppendLine(GetGuestStackTrace());
  767. Logger.PrintInfo(LogClass.Cpu, trace.ToString());
  768. }
  769. public void Execute()
  770. {
  771. if (Interlocked.CompareExchange(ref _hostThreadRunning, 1, 0) == 0)
  772. {
  773. HostThread.Start();
  774. }
  775. }
  776. private void ThreadStart(ulong entrypoint)
  777. {
  778. Owner.Translator.Execute(Context, entrypoint);
  779. ThreadExit();
  780. }
  781. private void ThreadExit()
  782. {
  783. System.Scheduler.ExitThread(this);
  784. System.Scheduler.RemoveThread(this);
  785. }
  786. public bool IsCurrentHostThread()
  787. {
  788. return Thread.CurrentThread == HostThread;
  789. }
  790. public override bool IsSignaled()
  791. {
  792. return _hasExited;
  793. }
  794. protected override void Destroy()
  795. {
  796. if (_hasBeenInitialized)
  797. {
  798. FreeResources();
  799. bool released = Owner != null || _hasBeenReleased;
  800. if (Owner != null)
  801. {
  802. Owner.ResourceLimit?.Release(LimitableResource.Thread, 1, released ? 0 : 1);
  803. Owner.DecrementReferenceCount();
  804. }
  805. else
  806. {
  807. System.ResourceLimit.Release(LimitableResource.Thread, 1, released ? 0 : 1);
  808. }
  809. }
  810. }
  811. private void FreeResources()
  812. {
  813. Owner?.RemoveThread(this);
  814. if (_tlsAddress != 0 && Owner.FreeThreadLocalStorage(_tlsAddress) != KernelResult.Success)
  815. {
  816. throw new InvalidOperationException("Unexpected failure freeing thread local storage.");
  817. }
  818. System.CriticalSection.Enter();
  819. // Wake up all threads that may be waiting for a mutex being held by this thread.
  820. foreach (KThread thread in _mutexWaiters)
  821. {
  822. thread.MutexOwner = null;
  823. thread._preferredCoreOverride = 0;
  824. thread.ObjSyncResult = KernelResult.InvalidState;
  825. thread.ReleaseAndResume();
  826. }
  827. System.CriticalSection.Leave();
  828. Owner?.DecrementThreadCountAndTerminateIfZero();
  829. }
  830. }
  831. }