KThread.cs 32 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087
  1. using ChocolArm64;
  2. using ChocolArm64.Memory;
  3. using Ryujinx.Common.Logging;
  4. using Ryujinx.HLE.HOS.Kernel.Common;
  5. using Ryujinx.HLE.HOS.Kernel.Process;
  6. using System;
  7. using System.Collections.Generic;
  8. using System.Linq;
  9. using System.Text;
  10. namespace Ryujinx.HLE.HOS.Kernel.Threading
  11. {
  12. class KThread : KSynchronizationObject, IKFutureSchedulerObject
  13. {
  14. public CpuThread Context { get; private set; }
  15. public long AffinityMask { get; set; }
  16. public long ThreadUid { get; private set; }
  17. public long TotalTimeRunning { get; set; }
  18. public KSynchronizationObject SignaledObj { get; set; }
  19. public ulong CondVarAddress { get; set; }
  20. private ulong _entrypoint;
  21. public ulong MutexAddress { get; set; }
  22. public KProcess Owner { get; private set; }
  23. private ulong _tlsAddress;
  24. public ulong TlsAddress => _tlsAddress;
  25. public ulong TlsDramAddress { get; private set; }
  26. public long LastScheduledTime { get; set; }
  27. public LinkedListNode<KThread>[] SiblingsPerCore { get; private set; }
  28. public LinkedList<KThread> Withholder { get; set; }
  29. public LinkedListNode<KThread> WithholderNode { get; set; }
  30. public LinkedListNode<KThread> ProcessListNode { get; set; }
  31. private LinkedList<KThread> _mutexWaiters;
  32. private LinkedListNode<KThread> _mutexWaiterNode;
  33. public KThread MutexOwner { get; private set; }
  34. public int ThreadHandleForUserMutex { get; set; }
  35. private ThreadSchedState _forcePauseFlags;
  36. public KernelResult ObjSyncResult { get; set; }
  37. public int DynamicPriority { get; set; }
  38. public int CurrentCore { get; set; }
  39. public int BasePriority { get; set; }
  40. public int PreferredCore { get; set; }
  41. private long _affinityMaskOverride;
  42. private int _preferredCoreOverride;
  43. private int _affinityOverrideCount;
  44. public ThreadSchedState SchedFlags { get; private set; }
  45. public bool ShallBeTerminated { get; private set; }
  46. public bool SyncCancelled { get; set; }
  47. public bool WaitingSync { get; set; }
  48. private bool _hasExited;
  49. private bool _hasBeenInitialized;
  50. private bool _hasBeenReleased;
  51. public bool WaitingInArbitration { get; set; }
  52. private KScheduler _scheduler;
  53. private KSchedulingData _schedulingData;
  54. public long LastPc { get; set; }
  55. public KThread(Horizon system) : base(system)
  56. {
  57. _scheduler = system.Scheduler;
  58. _schedulingData = system.Scheduler.SchedulingData;
  59. SiblingsPerCore = new LinkedListNode<KThread>[KScheduler.CpuCoresCount];
  60. _mutexWaiters = new LinkedList<KThread>();
  61. }
  62. public KernelResult Initialize(
  63. ulong entrypoint,
  64. ulong argsPtr,
  65. ulong stackTop,
  66. int priority,
  67. int defaultCpuCore,
  68. KProcess owner,
  69. ThreadType type = ThreadType.User)
  70. {
  71. if ((uint)type > 3)
  72. {
  73. throw new ArgumentException($"Invalid thread type \"{type}\".");
  74. }
  75. PreferredCore = defaultCpuCore;
  76. AffinityMask |= 1L << defaultCpuCore;
  77. SchedFlags = type == ThreadType.Dummy
  78. ? ThreadSchedState.Running
  79. : ThreadSchedState.None;
  80. CurrentCore = PreferredCore;
  81. DynamicPriority = priority;
  82. BasePriority = priority;
  83. ObjSyncResult = KernelResult.ThreadNotStarted;
  84. _entrypoint = entrypoint;
  85. if (type == ThreadType.User)
  86. {
  87. if (owner.AllocateThreadLocalStorage(out _tlsAddress) != KernelResult.Success)
  88. {
  89. return KernelResult.OutOfMemory;
  90. }
  91. TlsDramAddress = owner.MemoryManager.GetDramAddressFromVa(_tlsAddress);
  92. MemoryHelper.FillWithZeros(owner.CpuMemory, (long)_tlsAddress, KTlsPageInfo.TlsEntrySize);
  93. }
  94. bool is64Bits;
  95. if (owner != null)
  96. {
  97. Owner = owner;
  98. owner.IncrementReferenceCount();
  99. owner.IncrementThreadCount();
  100. is64Bits = (owner.MmuFlags & 1) != 0;
  101. }
  102. else
  103. {
  104. is64Bits = true;
  105. }
  106. Context = new CpuThread(owner.Translator, owner.CpuMemory, (long)entrypoint);
  107. bool isAarch32 = (Owner.MmuFlags & 1) == 0;
  108. Context.ThreadState.Aarch32 = isAarch32;
  109. Context.ThreadState.X0 = argsPtr;
  110. if (isAarch32)
  111. {
  112. Context.ThreadState.X13 = (uint)stackTop;
  113. }
  114. else
  115. {
  116. Context.ThreadState.X31 = stackTop;
  117. }
  118. Context.ThreadState.CntfrqEl0 = 19200000;
  119. Context.ThreadState.Tpidr = (long)_tlsAddress;
  120. owner.SubscribeThreadEventHandlers(Context);
  121. Context.WorkFinished += ThreadFinishedHandler;
  122. ThreadUid = System.GetThreadUid();
  123. _hasBeenInitialized = true;
  124. if (owner != null)
  125. {
  126. owner.AddThread(this);
  127. if (owner.IsPaused)
  128. {
  129. System.CriticalSection.Enter();
  130. if (ShallBeTerminated || SchedFlags == ThreadSchedState.TerminationPending)
  131. {
  132. System.CriticalSection.Leave();
  133. return KernelResult.Success;
  134. }
  135. _forcePauseFlags |= ThreadSchedState.ProcessPauseFlag;
  136. CombineForcePauseFlags();
  137. System.CriticalSection.Leave();
  138. }
  139. }
  140. return KernelResult.Success;
  141. }
  142. public KernelResult Start()
  143. {
  144. if (!System.KernelInitialized)
  145. {
  146. System.CriticalSection.Enter();
  147. if (!ShallBeTerminated && SchedFlags != ThreadSchedState.TerminationPending)
  148. {
  149. _forcePauseFlags |= ThreadSchedState.KernelInitPauseFlag;
  150. CombineForcePauseFlags();
  151. }
  152. System.CriticalSection.Leave();
  153. }
  154. KernelResult result = KernelResult.ThreadTerminating;
  155. System.CriticalSection.Enter();
  156. if (!ShallBeTerminated)
  157. {
  158. KThread currentThread = System.Scheduler.GetCurrentThread();
  159. while (SchedFlags != ThreadSchedState.TerminationPending &&
  160. currentThread.SchedFlags != ThreadSchedState.TerminationPending &&
  161. !currentThread.ShallBeTerminated)
  162. {
  163. if ((SchedFlags & ThreadSchedState.LowMask) != ThreadSchedState.None)
  164. {
  165. result = KernelResult.InvalidState;
  166. break;
  167. }
  168. if (currentThread._forcePauseFlags == ThreadSchedState.None)
  169. {
  170. if (Owner != null && _forcePauseFlags != ThreadSchedState.None)
  171. {
  172. CombineForcePauseFlags();
  173. }
  174. SetNewSchedFlags(ThreadSchedState.Running);
  175. result = KernelResult.Success;
  176. break;
  177. }
  178. else
  179. {
  180. currentThread.CombineForcePauseFlags();
  181. System.CriticalSection.Leave();
  182. System.CriticalSection.Enter();
  183. if (currentThread.ShallBeTerminated)
  184. {
  185. break;
  186. }
  187. }
  188. }
  189. }
  190. System.CriticalSection.Leave();
  191. return result;
  192. }
  193. public void Exit()
  194. {
  195. //TODO: Debug event.
  196. if (Owner != null)
  197. {
  198. Owner.ResourceLimit?.Release(LimitableResource.Thread, 0, 1);
  199. _hasBeenReleased = true;
  200. }
  201. System.CriticalSection.Enter();
  202. _forcePauseFlags &= ~ThreadSchedState.ForcePauseMask;
  203. ExitImpl();
  204. System.CriticalSection.Leave();
  205. DecrementReferenceCount();
  206. }
  207. private void ExitImpl()
  208. {
  209. System.CriticalSection.Enter();
  210. SetNewSchedFlags(ThreadSchedState.TerminationPending);
  211. _hasExited = true;
  212. Signal();
  213. System.CriticalSection.Leave();
  214. }
  215. public KernelResult Sleep(long timeout)
  216. {
  217. System.CriticalSection.Enter();
  218. if (ShallBeTerminated || SchedFlags == ThreadSchedState.TerminationPending)
  219. {
  220. System.CriticalSection.Leave();
  221. return KernelResult.ThreadTerminating;
  222. }
  223. SetNewSchedFlags(ThreadSchedState.Paused);
  224. if (timeout > 0)
  225. {
  226. System.TimeManager.ScheduleFutureInvocation(this, timeout);
  227. }
  228. System.CriticalSection.Leave();
  229. if (timeout > 0)
  230. {
  231. System.TimeManager.UnscheduleFutureInvocation(this);
  232. }
  233. return 0;
  234. }
  235. public void Yield()
  236. {
  237. System.CriticalSection.Enter();
  238. if (SchedFlags != ThreadSchedState.Running)
  239. {
  240. System.CriticalSection.Leave();
  241. System.Scheduler.ContextSwitch();
  242. return;
  243. }
  244. if (DynamicPriority < KScheduler.PrioritiesCount)
  245. {
  246. //Move current thread to the end of the queue.
  247. _schedulingData.Reschedule(DynamicPriority, CurrentCore, this);
  248. }
  249. _scheduler.ThreadReselectionRequested = true;
  250. System.CriticalSection.Leave();
  251. System.Scheduler.ContextSwitch();
  252. }
  253. public void YieldWithLoadBalancing()
  254. {
  255. System.CriticalSection.Enter();
  256. if (SchedFlags != ThreadSchedState.Running)
  257. {
  258. System.CriticalSection.Leave();
  259. System.Scheduler.ContextSwitch();
  260. return;
  261. }
  262. int prio = DynamicPriority;
  263. int core = CurrentCore;
  264. KThread nextThreadOnCurrentQueue = null;
  265. if (DynamicPriority < KScheduler.PrioritiesCount)
  266. {
  267. //Move current thread to the end of the queue.
  268. _schedulingData.Reschedule(prio, core, this);
  269. Func<KThread, bool> predicate = x => x.DynamicPriority == prio;
  270. nextThreadOnCurrentQueue = _schedulingData.ScheduledThreads(core).FirstOrDefault(predicate);
  271. }
  272. IEnumerable<KThread> SuitableCandidates()
  273. {
  274. foreach (KThread thread in _schedulingData.SuggestedThreads(core))
  275. {
  276. int srcCore = thread.CurrentCore;
  277. if (srcCore >= 0)
  278. {
  279. KThread selectedSrcCore = _scheduler.CoreContexts[srcCore].SelectedThread;
  280. if (selectedSrcCore == thread || ((selectedSrcCore?.DynamicPriority ?? 2) < 2))
  281. {
  282. continue;
  283. }
  284. }
  285. //If the candidate was scheduled after the current thread, then it's not worth it,
  286. //unless the priority is higher than the current one.
  287. if (nextThreadOnCurrentQueue.LastScheduledTime >= thread.LastScheduledTime ||
  288. nextThreadOnCurrentQueue.DynamicPriority < thread.DynamicPriority)
  289. {
  290. yield return thread;
  291. }
  292. }
  293. }
  294. KThread dst = SuitableCandidates().FirstOrDefault(x => x.DynamicPriority <= prio);
  295. if (dst != null)
  296. {
  297. _schedulingData.TransferToCore(dst.DynamicPriority, core, dst);
  298. _scheduler.ThreadReselectionRequested = true;
  299. }
  300. if (this != nextThreadOnCurrentQueue)
  301. {
  302. _scheduler.ThreadReselectionRequested = true;
  303. }
  304. System.CriticalSection.Leave();
  305. System.Scheduler.ContextSwitch();
  306. }
  307. public void YieldAndWaitForLoadBalancing()
  308. {
  309. System.CriticalSection.Enter();
  310. if (SchedFlags != ThreadSchedState.Running)
  311. {
  312. System.CriticalSection.Leave();
  313. System.Scheduler.ContextSwitch();
  314. return;
  315. }
  316. int core = CurrentCore;
  317. _schedulingData.TransferToCore(DynamicPriority, -1, this);
  318. KThread selectedThread = null;
  319. if (!_schedulingData.ScheduledThreads(core).Any())
  320. {
  321. foreach (KThread thread in _schedulingData.SuggestedThreads(core))
  322. {
  323. if (thread.CurrentCore < 0)
  324. {
  325. continue;
  326. }
  327. KThread firstCandidate = _schedulingData.ScheduledThreads(thread.CurrentCore).FirstOrDefault();
  328. if (firstCandidate == thread)
  329. {
  330. continue;
  331. }
  332. if (firstCandidate == null || firstCandidate.DynamicPriority >= 2)
  333. {
  334. _schedulingData.TransferToCore(thread.DynamicPriority, core, thread);
  335. selectedThread = thread;
  336. }
  337. break;
  338. }
  339. }
  340. if (selectedThread != this)
  341. {
  342. _scheduler.ThreadReselectionRequested = true;
  343. }
  344. System.CriticalSection.Leave();
  345. System.Scheduler.ContextSwitch();
  346. }
  347. public void SetPriority(int priority)
  348. {
  349. System.CriticalSection.Enter();
  350. BasePriority = priority;
  351. UpdatePriorityInheritance();
  352. System.CriticalSection.Leave();
  353. }
  354. public KernelResult SetActivity(bool pause)
  355. {
  356. KernelResult result = KernelResult.Success;
  357. System.CriticalSection.Enter();
  358. ThreadSchedState lowNibble = SchedFlags & ThreadSchedState.LowMask;
  359. if (lowNibble != ThreadSchedState.Paused && lowNibble != ThreadSchedState.Running)
  360. {
  361. System.CriticalSection.Leave();
  362. return KernelResult.InvalidState;
  363. }
  364. System.CriticalSection.Enter();
  365. if (!ShallBeTerminated && SchedFlags != ThreadSchedState.TerminationPending)
  366. {
  367. if (pause)
  368. {
  369. //Pause, the force pause flag should be clear (thread is NOT paused).
  370. if ((_forcePauseFlags & ThreadSchedState.ThreadPauseFlag) == 0)
  371. {
  372. _forcePauseFlags |= ThreadSchedState.ThreadPauseFlag;
  373. CombineForcePauseFlags();
  374. }
  375. else
  376. {
  377. result = KernelResult.InvalidState;
  378. }
  379. }
  380. else
  381. {
  382. //Unpause, the force pause flag should be set (thread is paused).
  383. if ((_forcePauseFlags & ThreadSchedState.ThreadPauseFlag) != 0)
  384. {
  385. ThreadSchedState oldForcePauseFlags = _forcePauseFlags;
  386. _forcePauseFlags &= ~ThreadSchedState.ThreadPauseFlag;
  387. if ((oldForcePauseFlags & ~ThreadSchedState.ThreadPauseFlag) == ThreadSchedState.None)
  388. {
  389. ThreadSchedState oldSchedFlags = SchedFlags;
  390. SchedFlags &= ThreadSchedState.LowMask;
  391. AdjustScheduling(oldSchedFlags);
  392. }
  393. }
  394. else
  395. {
  396. result = KernelResult.InvalidState;
  397. }
  398. }
  399. }
  400. System.CriticalSection.Leave();
  401. System.CriticalSection.Leave();
  402. return result;
  403. }
  404. public void CancelSynchronization()
  405. {
  406. System.CriticalSection.Enter();
  407. if ((SchedFlags & ThreadSchedState.LowMask) != ThreadSchedState.Paused || !WaitingSync)
  408. {
  409. SyncCancelled = true;
  410. }
  411. else if (Withholder != null)
  412. {
  413. Withholder.Remove(WithholderNode);
  414. SetNewSchedFlags(ThreadSchedState.Running);
  415. Withholder = null;
  416. SyncCancelled = true;
  417. }
  418. else
  419. {
  420. SignaledObj = null;
  421. ObjSyncResult = KernelResult.Cancelled;
  422. SetNewSchedFlags(ThreadSchedState.Running);
  423. SyncCancelled = false;
  424. }
  425. System.CriticalSection.Leave();
  426. }
  427. public KernelResult SetCoreAndAffinityMask(int newCore, long newAffinityMask)
  428. {
  429. System.CriticalSection.Enter();
  430. bool useOverride = _affinityOverrideCount != 0;
  431. //The value -3 is "do not change the preferred core".
  432. if (newCore == -3)
  433. {
  434. newCore = useOverride ? _preferredCoreOverride : PreferredCore;
  435. if ((newAffinityMask & (1 << newCore)) == 0)
  436. {
  437. System.CriticalSection.Leave();
  438. return KernelResult.InvalidCombination;
  439. }
  440. }
  441. if (useOverride)
  442. {
  443. _preferredCoreOverride = newCore;
  444. _affinityMaskOverride = newAffinityMask;
  445. }
  446. else
  447. {
  448. long oldAffinityMask = AffinityMask;
  449. PreferredCore = newCore;
  450. AffinityMask = newAffinityMask;
  451. if (oldAffinityMask != newAffinityMask)
  452. {
  453. int oldCore = CurrentCore;
  454. if (CurrentCore >= 0 && ((AffinityMask >> CurrentCore) & 1) == 0)
  455. {
  456. if (PreferredCore < 0)
  457. {
  458. CurrentCore = HighestSetCore(AffinityMask);
  459. }
  460. else
  461. {
  462. CurrentCore = PreferredCore;
  463. }
  464. }
  465. AdjustSchedulingForNewAffinity(oldAffinityMask, oldCore);
  466. }
  467. }
  468. System.CriticalSection.Leave();
  469. return KernelResult.Success;
  470. }
  471. private static int HighestSetCore(long mask)
  472. {
  473. for (int core = KScheduler.CpuCoresCount - 1; core >= 0; core--)
  474. {
  475. if (((mask >> core) & 1) != 0)
  476. {
  477. return core;
  478. }
  479. }
  480. return -1;
  481. }
  482. private void CombineForcePauseFlags()
  483. {
  484. ThreadSchedState oldFlags = SchedFlags;
  485. ThreadSchedState lowNibble = SchedFlags & ThreadSchedState.LowMask;
  486. SchedFlags = lowNibble | _forcePauseFlags;
  487. AdjustScheduling(oldFlags);
  488. }
  489. private void SetNewSchedFlags(ThreadSchedState newFlags)
  490. {
  491. System.CriticalSection.Enter();
  492. ThreadSchedState oldFlags = SchedFlags;
  493. SchedFlags = (oldFlags & ThreadSchedState.HighMask) | newFlags;
  494. if ((oldFlags & ThreadSchedState.LowMask) != newFlags)
  495. {
  496. AdjustScheduling(oldFlags);
  497. }
  498. System.CriticalSection.Leave();
  499. }
  500. public void ReleaseAndResume()
  501. {
  502. System.CriticalSection.Enter();
  503. if ((SchedFlags & ThreadSchedState.LowMask) == ThreadSchedState.Paused)
  504. {
  505. if (Withholder != null)
  506. {
  507. Withholder.Remove(WithholderNode);
  508. SetNewSchedFlags(ThreadSchedState.Running);
  509. Withholder = null;
  510. }
  511. else
  512. {
  513. SetNewSchedFlags(ThreadSchedState.Running);
  514. }
  515. }
  516. System.CriticalSection.Leave();
  517. }
  518. public void Reschedule(ThreadSchedState newFlags)
  519. {
  520. System.CriticalSection.Enter();
  521. ThreadSchedState oldFlags = SchedFlags;
  522. SchedFlags = (oldFlags & ThreadSchedState.HighMask) |
  523. (newFlags & ThreadSchedState.LowMask);
  524. AdjustScheduling(oldFlags);
  525. System.CriticalSection.Leave();
  526. }
  527. public void AddMutexWaiter(KThread requester)
  528. {
  529. AddToMutexWaitersList(requester);
  530. requester.MutexOwner = this;
  531. UpdatePriorityInheritance();
  532. }
  533. public void RemoveMutexWaiter(KThread thread)
  534. {
  535. if (thread._mutexWaiterNode?.List != null)
  536. {
  537. _mutexWaiters.Remove(thread._mutexWaiterNode);
  538. }
  539. thread.MutexOwner = null;
  540. UpdatePriorityInheritance();
  541. }
  542. public KThread RelinquishMutex(ulong mutexAddress, out int count)
  543. {
  544. count = 0;
  545. if (_mutexWaiters.First == null)
  546. {
  547. return null;
  548. }
  549. KThread newMutexOwner = null;
  550. LinkedListNode<KThread> currentNode = _mutexWaiters.First;
  551. do
  552. {
  553. //Skip all threads that are not waiting for this mutex.
  554. while (currentNode != null && currentNode.Value.MutexAddress != mutexAddress)
  555. {
  556. currentNode = currentNode.Next;
  557. }
  558. if (currentNode == null)
  559. {
  560. break;
  561. }
  562. LinkedListNode<KThread> nextNode = currentNode.Next;
  563. _mutexWaiters.Remove(currentNode);
  564. currentNode.Value.MutexOwner = newMutexOwner;
  565. if (newMutexOwner != null)
  566. {
  567. //New owner was already selected, re-insert on new owner list.
  568. newMutexOwner.AddToMutexWaitersList(currentNode.Value);
  569. }
  570. else
  571. {
  572. //New owner not selected yet, use current thread.
  573. newMutexOwner = currentNode.Value;
  574. }
  575. count++;
  576. currentNode = nextNode;
  577. }
  578. while (currentNode != null);
  579. if (newMutexOwner != null)
  580. {
  581. UpdatePriorityInheritance();
  582. newMutexOwner.UpdatePriorityInheritance();
  583. }
  584. return newMutexOwner;
  585. }
  586. private void UpdatePriorityInheritance()
  587. {
  588. //If any of the threads waiting for the mutex has
  589. //higher priority than the current thread, then
  590. //the current thread inherits that priority.
  591. int highestPriority = BasePriority;
  592. if (_mutexWaiters.First != null)
  593. {
  594. int waitingDynamicPriority = _mutexWaiters.First.Value.DynamicPriority;
  595. if (waitingDynamicPriority < highestPriority)
  596. {
  597. highestPriority = waitingDynamicPriority;
  598. }
  599. }
  600. if (highestPriority != DynamicPriority)
  601. {
  602. int oldPriority = DynamicPriority;
  603. DynamicPriority = highestPriority;
  604. AdjustSchedulingForNewPriority(oldPriority);
  605. if (MutexOwner != null)
  606. {
  607. //Remove and re-insert to ensure proper sorting based on new priority.
  608. MutexOwner._mutexWaiters.Remove(_mutexWaiterNode);
  609. MutexOwner.AddToMutexWaitersList(this);
  610. MutexOwner.UpdatePriorityInheritance();
  611. }
  612. }
  613. }
  614. private void AddToMutexWaitersList(KThread thread)
  615. {
  616. LinkedListNode<KThread> nextPrio = _mutexWaiters.First;
  617. int currentPriority = thread.DynamicPriority;
  618. while (nextPrio != null && nextPrio.Value.DynamicPriority <= currentPriority)
  619. {
  620. nextPrio = nextPrio.Next;
  621. }
  622. if (nextPrio != null)
  623. {
  624. thread._mutexWaiterNode = _mutexWaiters.AddBefore(nextPrio, thread);
  625. }
  626. else
  627. {
  628. thread._mutexWaiterNode = _mutexWaiters.AddLast(thread);
  629. }
  630. }
  631. private void AdjustScheduling(ThreadSchedState oldFlags)
  632. {
  633. if (oldFlags == SchedFlags)
  634. {
  635. return;
  636. }
  637. if (oldFlags == ThreadSchedState.Running)
  638. {
  639. //Was running, now it's stopped.
  640. if (CurrentCore >= 0)
  641. {
  642. _schedulingData.Unschedule(DynamicPriority, CurrentCore, this);
  643. }
  644. for (int core = 0; core < KScheduler.CpuCoresCount; core++)
  645. {
  646. if (core != CurrentCore && ((AffinityMask >> core) & 1) != 0)
  647. {
  648. _schedulingData.Unsuggest(DynamicPriority, core, this);
  649. }
  650. }
  651. }
  652. else if (SchedFlags == ThreadSchedState.Running)
  653. {
  654. //Was stopped, now it's running.
  655. if (CurrentCore >= 0)
  656. {
  657. _schedulingData.Schedule(DynamicPriority, CurrentCore, this);
  658. }
  659. for (int core = 0; core < KScheduler.CpuCoresCount; core++)
  660. {
  661. if (core != CurrentCore && ((AffinityMask >> core) & 1) != 0)
  662. {
  663. _schedulingData.Suggest(DynamicPriority, core, this);
  664. }
  665. }
  666. }
  667. _scheduler.ThreadReselectionRequested = true;
  668. }
  669. private void AdjustSchedulingForNewPriority(int oldPriority)
  670. {
  671. if (SchedFlags != ThreadSchedState.Running)
  672. {
  673. return;
  674. }
  675. //Remove thread from the old priority queues.
  676. if (CurrentCore >= 0)
  677. {
  678. _schedulingData.Unschedule(oldPriority, CurrentCore, this);
  679. }
  680. for (int core = 0; core < KScheduler.CpuCoresCount; core++)
  681. {
  682. if (core != CurrentCore && ((AffinityMask >> core) & 1) != 0)
  683. {
  684. _schedulingData.Unsuggest(oldPriority, core, this);
  685. }
  686. }
  687. //Add thread to the new priority queues.
  688. KThread currentThread = _scheduler.GetCurrentThread();
  689. if (CurrentCore >= 0)
  690. {
  691. if (currentThread == this)
  692. {
  693. _schedulingData.SchedulePrepend(DynamicPriority, CurrentCore, this);
  694. }
  695. else
  696. {
  697. _schedulingData.Schedule(DynamicPriority, CurrentCore, this);
  698. }
  699. }
  700. for (int core = 0; core < KScheduler.CpuCoresCount; core++)
  701. {
  702. if (core != CurrentCore && ((AffinityMask >> core) & 1) != 0)
  703. {
  704. _schedulingData.Suggest(DynamicPriority, core, this);
  705. }
  706. }
  707. _scheduler.ThreadReselectionRequested = true;
  708. }
  709. private void AdjustSchedulingForNewAffinity(long oldAffinityMask, int oldCore)
  710. {
  711. if (SchedFlags != ThreadSchedState.Running || DynamicPriority >= KScheduler.PrioritiesCount)
  712. {
  713. return;
  714. }
  715. //Remove thread from the old priority queues.
  716. for (int core = 0; core < KScheduler.CpuCoresCount; core++)
  717. {
  718. if (((oldAffinityMask >> core) & 1) != 0)
  719. {
  720. if (core == oldCore)
  721. {
  722. _schedulingData.Unschedule(DynamicPriority, core, this);
  723. }
  724. else
  725. {
  726. _schedulingData.Unsuggest(DynamicPriority, core, this);
  727. }
  728. }
  729. }
  730. //Add thread to the new priority queues.
  731. for (int core = 0; core < KScheduler.CpuCoresCount; core++)
  732. {
  733. if (((AffinityMask >> core) & 1) != 0)
  734. {
  735. if (core == CurrentCore)
  736. {
  737. _schedulingData.Schedule(DynamicPriority, core, this);
  738. }
  739. else
  740. {
  741. _schedulingData.Suggest(DynamicPriority, core, this);
  742. }
  743. }
  744. }
  745. _scheduler.ThreadReselectionRequested = true;
  746. }
  747. public void SetEntryArguments(long argsPtr, int threadHandle)
  748. {
  749. Context.ThreadState.X0 = (ulong)argsPtr;
  750. Context.ThreadState.X1 = (ulong)threadHandle;
  751. }
  752. public void TimeUp()
  753. {
  754. ReleaseAndResume();
  755. }
  756. public string GetGuestStackTrace()
  757. {
  758. return Owner.Debugger.GetGuestStackTrace(Context.ThreadState);
  759. }
  760. public void PrintGuestStackTrace()
  761. {
  762. StringBuilder trace = new StringBuilder();
  763. trace.AppendLine("Guest stack trace:");
  764. trace.AppendLine(GetGuestStackTrace());
  765. Logger.PrintInfo(LogClass.Cpu, trace.ToString());
  766. }
  767. private void ThreadFinishedHandler(object sender, EventArgs e)
  768. {
  769. System.Scheduler.ExitThread(this);
  770. System.Scheduler.RemoveThread(this);
  771. }
  772. public override bool IsSignaled()
  773. {
  774. return _hasExited;
  775. }
  776. protected override void Destroy()
  777. {
  778. if (_hasBeenInitialized)
  779. {
  780. FreeResources();
  781. bool released = Owner != null || _hasBeenReleased;
  782. if (Owner != null)
  783. {
  784. Owner.ResourceLimit?.Release(LimitableResource.Thread, 1, released ? 0 : 1);
  785. Owner.DecrementReferenceCount();
  786. }
  787. else
  788. {
  789. System.ResourceLimit.Release(LimitableResource.Thread, 1, released ? 0 : 1);
  790. }
  791. }
  792. }
  793. private void FreeResources()
  794. {
  795. Owner?.RemoveThread(this);
  796. if (_tlsAddress != 0 && Owner.FreeThreadLocalStorage(_tlsAddress) != KernelResult.Success)
  797. {
  798. throw new InvalidOperationException("Unexpected failure freeing thread local storage.");
  799. }
  800. System.CriticalSection.Enter();
  801. //Wake up all threads that may be waiting for a mutex being held by this thread.
  802. foreach (KThread thread in _mutexWaiters)
  803. {
  804. thread.MutexOwner = null;
  805. thread._preferredCoreOverride = 0;
  806. thread.ObjSyncResult = KernelResult.InvalidState;
  807. thread.ReleaseAndResume();
  808. }
  809. System.CriticalSection.Leave();
  810. Owner?.DecrementThreadCountAndTerminateIfZero();
  811. }
  812. }
  813. }