KThread.cs 32 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117
  1. using ARMeilleure.Memory;
  2. using ARMeilleure.State;
  3. using Ryujinx.Common.Logging;
  4. using Ryujinx.HLE.HOS.Kernel.Common;
  5. using Ryujinx.HLE.HOS.Kernel.Process;
  6. using System;
  7. using System.Collections.Generic;
  8. using System.Linq;
  9. using System.Text;
  10. using System.Threading;
  11. namespace Ryujinx.HLE.HOS.Kernel.Threading
  12. {
  13. class KThread : KSynchronizationObject, IKFutureSchedulerObject
  14. {
  15. private int _hostThreadRunning;
  16. public Thread HostThread { get; private set; }
  17. public IExecutionContext Context { get; private set; }
  18. public long AffinityMask { get; set; }
  19. public long ThreadUid { get; private set; }
  20. public long TotalTimeRunning { get; set; }
  21. public KSynchronizationObject SignaledObj { get; set; }
  22. public ulong CondVarAddress { get; set; }
  23. private ulong _entrypoint;
  24. public ulong MutexAddress { get; set; }
  25. public KProcess Owner { get; private set; }
  26. private ulong _tlsAddress;
  27. public ulong TlsAddress => _tlsAddress;
  28. public ulong TlsDramAddress { get; private set; }
  29. public long LastScheduledTime { get; set; }
  30. public LinkedListNode<KThread>[] SiblingsPerCore { get; private set; }
  31. public LinkedList<KThread> Withholder { get; set; }
  32. public LinkedListNode<KThread> WithholderNode { get; set; }
  33. public LinkedListNode<KThread> ProcessListNode { get; set; }
  34. private LinkedList<KThread> _mutexWaiters;
  35. private LinkedListNode<KThread> _mutexWaiterNode;
  36. public KThread MutexOwner { get; private set; }
  37. public int ThreadHandleForUserMutex { get; set; }
  38. private ThreadSchedState _forcePauseFlags;
  39. public KernelResult ObjSyncResult { get; set; }
  40. public int DynamicPriority { get; set; }
  41. public int CurrentCore { get; set; }
  42. public int BasePriority { get; set; }
  43. public int PreferredCore { get; set; }
  44. private long _affinityMaskOverride;
  45. private int _preferredCoreOverride;
  46. private int _affinityOverrideCount;
  47. public ThreadSchedState SchedFlags { get; private set; }
  48. public bool ShallBeTerminated { get; private set; }
  49. public bool SyncCancelled { get; set; }
  50. public bool WaitingSync { get; set; }
  51. private bool _hasExited;
  52. private bool _hasBeenInitialized;
  53. private bool _hasBeenReleased;
  54. public bool WaitingInArbitration { get; set; }
  55. private KScheduler _scheduler;
  56. private KSchedulingData _schedulingData;
  57. public long LastPc { get; set; }
  58. public KThread(Horizon system) : base(system)
  59. {
  60. _scheduler = system.Scheduler;
  61. _schedulingData = system.Scheduler.SchedulingData;
  62. SiblingsPerCore = new LinkedListNode<KThread>[KScheduler.CpuCoresCount];
  63. _mutexWaiters = new LinkedList<KThread>();
  64. }
  65. public KernelResult Initialize(
  66. ulong entrypoint,
  67. ulong argsPtr,
  68. ulong stackTop,
  69. int priority,
  70. int defaultCpuCore,
  71. KProcess owner,
  72. ThreadType type = ThreadType.User)
  73. {
  74. if ((uint)type > 3)
  75. {
  76. throw new ArgumentException($"Invalid thread type \"{type}\".");
  77. }
  78. PreferredCore = defaultCpuCore;
  79. AffinityMask |= 1L << defaultCpuCore;
  80. SchedFlags = type == ThreadType.Dummy
  81. ? ThreadSchedState.Running
  82. : ThreadSchedState.None;
  83. CurrentCore = PreferredCore;
  84. DynamicPriority = priority;
  85. BasePriority = priority;
  86. ObjSyncResult = KernelResult.ThreadNotStarted;
  87. _entrypoint = entrypoint;
  88. if (type == ThreadType.User)
  89. {
  90. if (owner.AllocateThreadLocalStorage(out _tlsAddress) != KernelResult.Success)
  91. {
  92. return KernelResult.OutOfMemory;
  93. }
  94. TlsDramAddress = owner.MemoryManager.GetDramAddressFromVa(_tlsAddress);
  95. MemoryHelper.FillWithZeros(owner.CpuMemory, (long)_tlsAddress, KTlsPageInfo.TlsEntrySize);
  96. }
  97. bool is64Bits;
  98. if (owner != null)
  99. {
  100. Owner = owner;
  101. owner.IncrementReferenceCount();
  102. owner.IncrementThreadCount();
  103. is64Bits = (owner.MmuFlags & 1) != 0;
  104. }
  105. else
  106. {
  107. is64Bits = true;
  108. }
  109. HostThread = new Thread(() => ThreadStart(entrypoint));
  110. if (System.UseLegacyJit)
  111. {
  112. Context = new ChocolArm64.State.CpuThreadState();
  113. }
  114. else
  115. {
  116. Context = new ARMeilleure.State.ExecutionContext();
  117. }
  118. bool isAarch32 = (Owner.MmuFlags & 1) == 0;
  119. Context.SetX(0, argsPtr);
  120. if (isAarch32)
  121. {
  122. Context.SetX(13, (uint)stackTop);
  123. }
  124. else
  125. {
  126. Context.SetX(31, stackTop);
  127. }
  128. Context.CntfrqEl0 = 19200000;
  129. Context.Tpidr = (long)_tlsAddress;
  130. owner.SubscribeThreadEventHandlers(Context);
  131. ThreadUid = System.GetThreadUid();
  132. _hasBeenInitialized = true;
  133. if (owner != null)
  134. {
  135. owner.AddThread(this);
  136. if (owner.IsPaused)
  137. {
  138. System.CriticalSection.Enter();
  139. if (ShallBeTerminated || SchedFlags == ThreadSchedState.TerminationPending)
  140. {
  141. System.CriticalSection.Leave();
  142. return KernelResult.Success;
  143. }
  144. _forcePauseFlags |= ThreadSchedState.ProcessPauseFlag;
  145. CombineForcePauseFlags();
  146. System.CriticalSection.Leave();
  147. }
  148. }
  149. return KernelResult.Success;
  150. }
  151. public KernelResult Start()
  152. {
  153. if (!System.KernelInitialized)
  154. {
  155. System.CriticalSection.Enter();
  156. if (!ShallBeTerminated && SchedFlags != ThreadSchedState.TerminationPending)
  157. {
  158. _forcePauseFlags |= ThreadSchedState.KernelInitPauseFlag;
  159. CombineForcePauseFlags();
  160. }
  161. System.CriticalSection.Leave();
  162. }
  163. KernelResult result = KernelResult.ThreadTerminating;
  164. System.CriticalSection.Enter();
  165. if (!ShallBeTerminated)
  166. {
  167. KThread currentThread = System.Scheduler.GetCurrentThread();
  168. while (SchedFlags != ThreadSchedState.TerminationPending &&
  169. currentThread.SchedFlags != ThreadSchedState.TerminationPending &&
  170. !currentThread.ShallBeTerminated)
  171. {
  172. if ((SchedFlags & ThreadSchedState.LowMask) != ThreadSchedState.None)
  173. {
  174. result = KernelResult.InvalidState;
  175. break;
  176. }
  177. if (currentThread._forcePauseFlags == ThreadSchedState.None)
  178. {
  179. if (Owner != null && _forcePauseFlags != ThreadSchedState.None)
  180. {
  181. CombineForcePauseFlags();
  182. }
  183. SetNewSchedFlags(ThreadSchedState.Running);
  184. result = KernelResult.Success;
  185. break;
  186. }
  187. else
  188. {
  189. currentThread.CombineForcePauseFlags();
  190. System.CriticalSection.Leave();
  191. System.CriticalSection.Enter();
  192. if (currentThread.ShallBeTerminated)
  193. {
  194. break;
  195. }
  196. }
  197. }
  198. }
  199. System.CriticalSection.Leave();
  200. return result;
  201. }
  202. public void Exit()
  203. {
  204. // TODO: Debug event.
  205. if (Owner != null)
  206. {
  207. Owner.ResourceLimit?.Release(LimitableResource.Thread, 0, 1);
  208. _hasBeenReleased = true;
  209. }
  210. System.CriticalSection.Enter();
  211. _forcePauseFlags &= ~ThreadSchedState.ForcePauseMask;
  212. ExitImpl();
  213. System.CriticalSection.Leave();
  214. DecrementReferenceCount();
  215. }
  216. private void ExitImpl()
  217. {
  218. System.CriticalSection.Enter();
  219. SetNewSchedFlags(ThreadSchedState.TerminationPending);
  220. _hasExited = true;
  221. Signal();
  222. System.CriticalSection.Leave();
  223. }
  224. public KernelResult Sleep(long timeout)
  225. {
  226. System.CriticalSection.Enter();
  227. if (ShallBeTerminated || SchedFlags == ThreadSchedState.TerminationPending)
  228. {
  229. System.CriticalSection.Leave();
  230. return KernelResult.ThreadTerminating;
  231. }
  232. SetNewSchedFlags(ThreadSchedState.Paused);
  233. if (timeout > 0)
  234. {
  235. System.TimeManager.ScheduleFutureInvocation(this, timeout);
  236. }
  237. System.CriticalSection.Leave();
  238. if (timeout > 0)
  239. {
  240. System.TimeManager.UnscheduleFutureInvocation(this);
  241. }
  242. return 0;
  243. }
  244. public void Yield()
  245. {
  246. System.CriticalSection.Enter();
  247. if (SchedFlags != ThreadSchedState.Running)
  248. {
  249. System.CriticalSection.Leave();
  250. System.Scheduler.ContextSwitch();
  251. return;
  252. }
  253. if (DynamicPriority < KScheduler.PrioritiesCount)
  254. {
  255. // Move current thread to the end of the queue.
  256. _schedulingData.Reschedule(DynamicPriority, CurrentCore, this);
  257. }
  258. _scheduler.ThreadReselectionRequested = true;
  259. System.CriticalSection.Leave();
  260. System.Scheduler.ContextSwitch();
  261. }
  262. public void YieldWithLoadBalancing()
  263. {
  264. System.CriticalSection.Enter();
  265. if (SchedFlags != ThreadSchedState.Running)
  266. {
  267. System.CriticalSection.Leave();
  268. System.Scheduler.ContextSwitch();
  269. return;
  270. }
  271. int prio = DynamicPriority;
  272. int core = CurrentCore;
  273. KThread nextThreadOnCurrentQueue = null;
  274. if (DynamicPriority < KScheduler.PrioritiesCount)
  275. {
  276. // Move current thread to the end of the queue.
  277. _schedulingData.Reschedule(prio, core, this);
  278. Func<KThread, bool> predicate = x => x.DynamicPriority == prio;
  279. nextThreadOnCurrentQueue = _schedulingData.ScheduledThreads(core).FirstOrDefault(predicate);
  280. }
  281. IEnumerable<KThread> SuitableCandidates()
  282. {
  283. foreach (KThread thread in _schedulingData.SuggestedThreads(core))
  284. {
  285. int srcCore = thread.CurrentCore;
  286. if (srcCore >= 0)
  287. {
  288. KThread selectedSrcCore = _scheduler.CoreContexts[srcCore].SelectedThread;
  289. if (selectedSrcCore == thread || ((selectedSrcCore?.DynamicPriority ?? 2) < 2))
  290. {
  291. continue;
  292. }
  293. }
  294. // If the candidate was scheduled after the current thread, then it's not worth it,
  295. // unless the priority is higher than the current one.
  296. if (nextThreadOnCurrentQueue.LastScheduledTime >= thread.LastScheduledTime ||
  297. nextThreadOnCurrentQueue.DynamicPriority < thread.DynamicPriority)
  298. {
  299. yield return thread;
  300. }
  301. }
  302. }
  303. KThread dst = SuitableCandidates().FirstOrDefault(x => x.DynamicPriority <= prio);
  304. if (dst != null)
  305. {
  306. _schedulingData.TransferToCore(dst.DynamicPriority, core, dst);
  307. _scheduler.ThreadReselectionRequested = true;
  308. }
  309. if (this != nextThreadOnCurrentQueue)
  310. {
  311. _scheduler.ThreadReselectionRequested = true;
  312. }
  313. System.CriticalSection.Leave();
  314. System.Scheduler.ContextSwitch();
  315. }
  316. public void YieldAndWaitForLoadBalancing()
  317. {
  318. System.CriticalSection.Enter();
  319. if (SchedFlags != ThreadSchedState.Running)
  320. {
  321. System.CriticalSection.Leave();
  322. System.Scheduler.ContextSwitch();
  323. return;
  324. }
  325. int core = CurrentCore;
  326. _schedulingData.TransferToCore(DynamicPriority, -1, this);
  327. KThread selectedThread = null;
  328. if (!_schedulingData.ScheduledThreads(core).Any())
  329. {
  330. foreach (KThread thread in _schedulingData.SuggestedThreads(core))
  331. {
  332. if (thread.CurrentCore < 0)
  333. {
  334. continue;
  335. }
  336. KThread firstCandidate = _schedulingData.ScheduledThreads(thread.CurrentCore).FirstOrDefault();
  337. if (firstCandidate == thread)
  338. {
  339. continue;
  340. }
  341. if (firstCandidate == null || firstCandidate.DynamicPriority >= 2)
  342. {
  343. _schedulingData.TransferToCore(thread.DynamicPriority, core, thread);
  344. selectedThread = thread;
  345. }
  346. break;
  347. }
  348. }
  349. if (selectedThread != this)
  350. {
  351. _scheduler.ThreadReselectionRequested = true;
  352. }
  353. System.CriticalSection.Leave();
  354. System.Scheduler.ContextSwitch();
  355. }
  356. public void SetPriority(int priority)
  357. {
  358. System.CriticalSection.Enter();
  359. BasePriority = priority;
  360. UpdatePriorityInheritance();
  361. System.CriticalSection.Leave();
  362. }
  363. public KernelResult SetActivity(bool pause)
  364. {
  365. KernelResult result = KernelResult.Success;
  366. System.CriticalSection.Enter();
  367. ThreadSchedState lowNibble = SchedFlags & ThreadSchedState.LowMask;
  368. if (lowNibble != ThreadSchedState.Paused && lowNibble != ThreadSchedState.Running)
  369. {
  370. System.CriticalSection.Leave();
  371. return KernelResult.InvalidState;
  372. }
  373. System.CriticalSection.Enter();
  374. if (!ShallBeTerminated && SchedFlags != ThreadSchedState.TerminationPending)
  375. {
  376. if (pause)
  377. {
  378. // Pause, the force pause flag should be clear (thread is NOT paused).
  379. if ((_forcePauseFlags & ThreadSchedState.ThreadPauseFlag) == 0)
  380. {
  381. _forcePauseFlags |= ThreadSchedState.ThreadPauseFlag;
  382. CombineForcePauseFlags();
  383. }
  384. else
  385. {
  386. result = KernelResult.InvalidState;
  387. }
  388. }
  389. else
  390. {
  391. // Unpause, the force pause flag should be set (thread is paused).
  392. if ((_forcePauseFlags & ThreadSchedState.ThreadPauseFlag) != 0)
  393. {
  394. ThreadSchedState oldForcePauseFlags = _forcePauseFlags;
  395. _forcePauseFlags &= ~ThreadSchedState.ThreadPauseFlag;
  396. if ((oldForcePauseFlags & ~ThreadSchedState.ThreadPauseFlag) == ThreadSchedState.None)
  397. {
  398. ThreadSchedState oldSchedFlags = SchedFlags;
  399. SchedFlags &= ThreadSchedState.LowMask;
  400. AdjustScheduling(oldSchedFlags);
  401. }
  402. }
  403. else
  404. {
  405. result = KernelResult.InvalidState;
  406. }
  407. }
  408. }
  409. System.CriticalSection.Leave();
  410. System.CriticalSection.Leave();
  411. return result;
  412. }
  413. public void CancelSynchronization()
  414. {
  415. System.CriticalSection.Enter();
  416. if ((SchedFlags & ThreadSchedState.LowMask) != ThreadSchedState.Paused || !WaitingSync)
  417. {
  418. SyncCancelled = true;
  419. }
  420. else if (Withholder != null)
  421. {
  422. Withholder.Remove(WithholderNode);
  423. SetNewSchedFlags(ThreadSchedState.Running);
  424. Withholder = null;
  425. SyncCancelled = true;
  426. }
  427. else
  428. {
  429. SignaledObj = null;
  430. ObjSyncResult = KernelResult.Cancelled;
  431. SetNewSchedFlags(ThreadSchedState.Running);
  432. SyncCancelled = false;
  433. }
  434. System.CriticalSection.Leave();
  435. }
  436. public KernelResult SetCoreAndAffinityMask(int newCore, long newAffinityMask)
  437. {
  438. System.CriticalSection.Enter();
  439. bool useOverride = _affinityOverrideCount != 0;
  440. // The value -3 is "do not change the preferred core".
  441. if (newCore == -3)
  442. {
  443. newCore = useOverride ? _preferredCoreOverride : PreferredCore;
  444. if ((newAffinityMask & (1 << newCore)) == 0)
  445. {
  446. System.CriticalSection.Leave();
  447. return KernelResult.InvalidCombination;
  448. }
  449. }
  450. if (useOverride)
  451. {
  452. _preferredCoreOverride = newCore;
  453. _affinityMaskOverride = newAffinityMask;
  454. }
  455. else
  456. {
  457. long oldAffinityMask = AffinityMask;
  458. PreferredCore = newCore;
  459. AffinityMask = newAffinityMask;
  460. if (oldAffinityMask != newAffinityMask)
  461. {
  462. int oldCore = CurrentCore;
  463. if (CurrentCore >= 0 && ((AffinityMask >> CurrentCore) & 1) == 0)
  464. {
  465. if (PreferredCore < 0)
  466. {
  467. CurrentCore = HighestSetCore(AffinityMask);
  468. }
  469. else
  470. {
  471. CurrentCore = PreferredCore;
  472. }
  473. }
  474. AdjustSchedulingForNewAffinity(oldAffinityMask, oldCore);
  475. }
  476. }
  477. System.CriticalSection.Leave();
  478. return KernelResult.Success;
  479. }
  480. private static int HighestSetCore(long mask)
  481. {
  482. for (int core = KScheduler.CpuCoresCount - 1; core >= 0; core--)
  483. {
  484. if (((mask >> core) & 1) != 0)
  485. {
  486. return core;
  487. }
  488. }
  489. return -1;
  490. }
  491. private void CombineForcePauseFlags()
  492. {
  493. ThreadSchedState oldFlags = SchedFlags;
  494. ThreadSchedState lowNibble = SchedFlags & ThreadSchedState.LowMask;
  495. SchedFlags = lowNibble | _forcePauseFlags;
  496. AdjustScheduling(oldFlags);
  497. }
  498. private void SetNewSchedFlags(ThreadSchedState newFlags)
  499. {
  500. System.CriticalSection.Enter();
  501. ThreadSchedState oldFlags = SchedFlags;
  502. SchedFlags = (oldFlags & ThreadSchedState.HighMask) | newFlags;
  503. if ((oldFlags & ThreadSchedState.LowMask) != newFlags)
  504. {
  505. AdjustScheduling(oldFlags);
  506. }
  507. System.CriticalSection.Leave();
  508. }
  509. public void ReleaseAndResume()
  510. {
  511. System.CriticalSection.Enter();
  512. if ((SchedFlags & ThreadSchedState.LowMask) == ThreadSchedState.Paused)
  513. {
  514. if (Withholder != null)
  515. {
  516. Withholder.Remove(WithholderNode);
  517. SetNewSchedFlags(ThreadSchedState.Running);
  518. Withholder = null;
  519. }
  520. else
  521. {
  522. SetNewSchedFlags(ThreadSchedState.Running);
  523. }
  524. }
  525. System.CriticalSection.Leave();
  526. }
  527. public void Reschedule(ThreadSchedState newFlags)
  528. {
  529. System.CriticalSection.Enter();
  530. ThreadSchedState oldFlags = SchedFlags;
  531. SchedFlags = (oldFlags & ThreadSchedState.HighMask) |
  532. (newFlags & ThreadSchedState.LowMask);
  533. AdjustScheduling(oldFlags);
  534. System.CriticalSection.Leave();
  535. }
  536. public void AddMutexWaiter(KThread requester)
  537. {
  538. AddToMutexWaitersList(requester);
  539. requester.MutexOwner = this;
  540. UpdatePriorityInheritance();
  541. }
  542. public void RemoveMutexWaiter(KThread thread)
  543. {
  544. if (thread._mutexWaiterNode?.List != null)
  545. {
  546. _mutexWaiters.Remove(thread._mutexWaiterNode);
  547. }
  548. thread.MutexOwner = null;
  549. UpdatePriorityInheritance();
  550. }
  551. public KThread RelinquishMutex(ulong mutexAddress, out int count)
  552. {
  553. count = 0;
  554. if (_mutexWaiters.First == null)
  555. {
  556. return null;
  557. }
  558. KThread newMutexOwner = null;
  559. LinkedListNode<KThread> currentNode = _mutexWaiters.First;
  560. do
  561. {
  562. // Skip all threads that are not waiting for this mutex.
  563. while (currentNode != null && currentNode.Value.MutexAddress != mutexAddress)
  564. {
  565. currentNode = currentNode.Next;
  566. }
  567. if (currentNode == null)
  568. {
  569. break;
  570. }
  571. LinkedListNode<KThread> nextNode = currentNode.Next;
  572. _mutexWaiters.Remove(currentNode);
  573. currentNode.Value.MutexOwner = newMutexOwner;
  574. if (newMutexOwner != null)
  575. {
  576. // New owner was already selected, re-insert on new owner list.
  577. newMutexOwner.AddToMutexWaitersList(currentNode.Value);
  578. }
  579. else
  580. {
  581. // New owner not selected yet, use current thread.
  582. newMutexOwner = currentNode.Value;
  583. }
  584. count++;
  585. currentNode = nextNode;
  586. }
  587. while (currentNode != null);
  588. if (newMutexOwner != null)
  589. {
  590. UpdatePriorityInheritance();
  591. newMutexOwner.UpdatePriorityInheritance();
  592. }
  593. return newMutexOwner;
  594. }
  595. private void UpdatePriorityInheritance()
  596. {
  597. // If any of the threads waiting for the mutex has
  598. // higher priority than the current thread, then
  599. // the current thread inherits that priority.
  600. int highestPriority = BasePriority;
  601. if (_mutexWaiters.First != null)
  602. {
  603. int waitingDynamicPriority = _mutexWaiters.First.Value.DynamicPriority;
  604. if (waitingDynamicPriority < highestPriority)
  605. {
  606. highestPriority = waitingDynamicPriority;
  607. }
  608. }
  609. if (highestPriority != DynamicPriority)
  610. {
  611. int oldPriority = DynamicPriority;
  612. DynamicPriority = highestPriority;
  613. AdjustSchedulingForNewPriority(oldPriority);
  614. if (MutexOwner != null)
  615. {
  616. // Remove and re-insert to ensure proper sorting based on new priority.
  617. MutexOwner._mutexWaiters.Remove(_mutexWaiterNode);
  618. MutexOwner.AddToMutexWaitersList(this);
  619. MutexOwner.UpdatePriorityInheritance();
  620. }
  621. }
  622. }
  623. private void AddToMutexWaitersList(KThread thread)
  624. {
  625. LinkedListNode<KThread> nextPrio = _mutexWaiters.First;
  626. int currentPriority = thread.DynamicPriority;
  627. while (nextPrio != null && nextPrio.Value.DynamicPriority <= currentPriority)
  628. {
  629. nextPrio = nextPrio.Next;
  630. }
  631. if (nextPrio != null)
  632. {
  633. thread._mutexWaiterNode = _mutexWaiters.AddBefore(nextPrio, thread);
  634. }
  635. else
  636. {
  637. thread._mutexWaiterNode = _mutexWaiters.AddLast(thread);
  638. }
  639. }
  640. private void AdjustScheduling(ThreadSchedState oldFlags)
  641. {
  642. if (oldFlags == SchedFlags)
  643. {
  644. return;
  645. }
  646. if (oldFlags == ThreadSchedState.Running)
  647. {
  648. // Was running, now it's stopped.
  649. if (CurrentCore >= 0)
  650. {
  651. _schedulingData.Unschedule(DynamicPriority, CurrentCore, this);
  652. }
  653. for (int core = 0; core < KScheduler.CpuCoresCount; core++)
  654. {
  655. if (core != CurrentCore && ((AffinityMask >> core) & 1) != 0)
  656. {
  657. _schedulingData.Unsuggest(DynamicPriority, core, this);
  658. }
  659. }
  660. }
  661. else if (SchedFlags == ThreadSchedState.Running)
  662. {
  663. // Was stopped, now it's running.
  664. if (CurrentCore >= 0)
  665. {
  666. _schedulingData.Schedule(DynamicPriority, CurrentCore, this);
  667. }
  668. for (int core = 0; core < KScheduler.CpuCoresCount; core++)
  669. {
  670. if (core != CurrentCore && ((AffinityMask >> core) & 1) != 0)
  671. {
  672. _schedulingData.Suggest(DynamicPriority, core, this);
  673. }
  674. }
  675. }
  676. _scheduler.ThreadReselectionRequested = true;
  677. }
  678. private void AdjustSchedulingForNewPriority(int oldPriority)
  679. {
  680. if (SchedFlags != ThreadSchedState.Running)
  681. {
  682. return;
  683. }
  684. // Remove thread from the old priority queues.
  685. if (CurrentCore >= 0)
  686. {
  687. _schedulingData.Unschedule(oldPriority, CurrentCore, this);
  688. }
  689. for (int core = 0; core < KScheduler.CpuCoresCount; core++)
  690. {
  691. if (core != CurrentCore && ((AffinityMask >> core) & 1) != 0)
  692. {
  693. _schedulingData.Unsuggest(oldPriority, core, this);
  694. }
  695. }
  696. // Add thread to the new priority queues.
  697. KThread currentThread = _scheduler.GetCurrentThread();
  698. if (CurrentCore >= 0)
  699. {
  700. if (currentThread == this)
  701. {
  702. _schedulingData.SchedulePrepend(DynamicPriority, CurrentCore, this);
  703. }
  704. else
  705. {
  706. _schedulingData.Schedule(DynamicPriority, CurrentCore, this);
  707. }
  708. }
  709. for (int core = 0; core < KScheduler.CpuCoresCount; core++)
  710. {
  711. if (core != CurrentCore && ((AffinityMask >> core) & 1) != 0)
  712. {
  713. _schedulingData.Suggest(DynamicPriority, core, this);
  714. }
  715. }
  716. _scheduler.ThreadReselectionRequested = true;
  717. }
  718. private void AdjustSchedulingForNewAffinity(long oldAffinityMask, int oldCore)
  719. {
  720. if (SchedFlags != ThreadSchedState.Running || DynamicPriority >= KScheduler.PrioritiesCount)
  721. {
  722. return;
  723. }
  724. // Remove thread from the old priority queues.
  725. for (int core = 0; core < KScheduler.CpuCoresCount; core++)
  726. {
  727. if (((oldAffinityMask >> core) & 1) != 0)
  728. {
  729. if (core == oldCore)
  730. {
  731. _schedulingData.Unschedule(DynamicPriority, core, this);
  732. }
  733. else
  734. {
  735. _schedulingData.Unsuggest(DynamicPriority, core, this);
  736. }
  737. }
  738. }
  739. // Add thread to the new priority queues.
  740. for (int core = 0; core < KScheduler.CpuCoresCount; core++)
  741. {
  742. if (((AffinityMask >> core) & 1) != 0)
  743. {
  744. if (core == CurrentCore)
  745. {
  746. _schedulingData.Schedule(DynamicPriority, core, this);
  747. }
  748. else
  749. {
  750. _schedulingData.Suggest(DynamicPriority, core, this);
  751. }
  752. }
  753. }
  754. _scheduler.ThreadReselectionRequested = true;
  755. }
  756. public void SetEntryArguments(long argsPtr, int threadHandle)
  757. {
  758. Context.SetX(0, (ulong)argsPtr);
  759. Context.SetX(1, (ulong)threadHandle);
  760. }
  761. public void TimeUp()
  762. {
  763. ReleaseAndResume();
  764. }
  765. public string GetGuestStackTrace()
  766. {
  767. return Owner.Debugger.GetGuestStackTrace(Context);
  768. }
  769. public void PrintGuestStackTrace()
  770. {
  771. StringBuilder trace = new StringBuilder();
  772. trace.AppendLine("Guest stack trace:");
  773. trace.AppendLine(GetGuestStackTrace());
  774. Logger.PrintInfo(LogClass.Cpu, trace.ToString());
  775. }
  776. public void Execute()
  777. {
  778. if (Interlocked.CompareExchange(ref _hostThreadRunning, 1, 0) == 0)
  779. {
  780. HostThread.Start();
  781. }
  782. }
  783. private void ThreadStart(ulong entrypoint)
  784. {
  785. Owner.Translator.Execute(Context, entrypoint);
  786. ThreadExit();
  787. }
  788. private void ThreadExit()
  789. {
  790. System.Scheduler.ExitThread(this);
  791. System.Scheduler.RemoveThread(this);
  792. }
  793. public bool IsCurrentHostThread()
  794. {
  795. return Thread.CurrentThread == HostThread;
  796. }
  797. public override bool IsSignaled()
  798. {
  799. return _hasExited;
  800. }
  801. protected override void Destroy()
  802. {
  803. if (_hasBeenInitialized)
  804. {
  805. FreeResources();
  806. bool released = Owner != null || _hasBeenReleased;
  807. if (Owner != null)
  808. {
  809. Owner.ResourceLimit?.Release(LimitableResource.Thread, 1, released ? 0 : 1);
  810. Owner.DecrementReferenceCount();
  811. }
  812. else
  813. {
  814. System.ResourceLimit.Release(LimitableResource.Thread, 1, released ? 0 : 1);
  815. }
  816. }
  817. }
  818. private void FreeResources()
  819. {
  820. Owner?.RemoveThread(this);
  821. if (_tlsAddress != 0 && Owner.FreeThreadLocalStorage(_tlsAddress) != KernelResult.Success)
  822. {
  823. throw new InvalidOperationException("Unexpected failure freeing thread local storage.");
  824. }
  825. System.CriticalSection.Enter();
  826. // Wake up all threads that may be waiting for a mutex being held by this thread.
  827. foreach (KThread thread in _mutexWaiters)
  828. {
  829. thread.MutexOwner = null;
  830. thread._preferredCoreOverride = 0;
  831. thread.ObjSyncResult = KernelResult.InvalidState;
  832. thread.ReleaseAndResume();
  833. }
  834. System.CriticalSection.Leave();
  835. Owner?.DecrementThreadCountAndTerminateIfZero();
  836. }
  837. }
  838. }