KThread.cs 31 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075
  1. using ChocolArm64;
  2. using ChocolArm64.Memory;
  3. using Ryujinx.HLE.HOS.Kernel.Common;
  4. using Ryujinx.HLE.HOS.Kernel.Process;
  5. using System;
  6. using System.Collections.Generic;
  7. using System.Linq;
  8. namespace Ryujinx.HLE.HOS.Kernel.Threading
  9. {
  10. class KThread : KSynchronizationObject, IKFutureSchedulerObject
  11. {
  12. public CpuThread Context { get; private set; }
  13. public long AffinityMask { get; set; }
  14. public long ThreadUid { get; private set; }
  15. public long TotalTimeRunning { get; set; }
  16. public KSynchronizationObject SignaledObj { get; set; }
  17. public ulong CondVarAddress { get; set; }
  18. private ulong _entrypoint;
  19. public ulong MutexAddress { get; set; }
  20. public KProcess Owner { get; private set; }
  21. private ulong _tlsAddress;
  22. public ulong TlsAddress => _tlsAddress;
  23. public ulong TlsDramAddress { get; private set; }
  24. public long LastScheduledTime { get; set; }
  25. public LinkedListNode<KThread>[] SiblingsPerCore { get; private set; }
  26. public LinkedList<KThread> Withholder { get; set; }
  27. public LinkedListNode<KThread> WithholderNode { get; set; }
  28. public LinkedListNode<KThread> ProcessListNode { get; set; }
  29. private LinkedList<KThread> _mutexWaiters;
  30. private LinkedListNode<KThread> _mutexWaiterNode;
  31. public KThread MutexOwner { get; private set; }
  32. public int ThreadHandleForUserMutex { get; set; }
  33. private ThreadSchedState _forcePauseFlags;
  34. public KernelResult ObjSyncResult { get; set; }
  35. public int DynamicPriority { get; set; }
  36. public int CurrentCore { get; set; }
  37. public int BasePriority { get; set; }
  38. public int PreferredCore { get; set; }
  39. private long _affinityMaskOverride;
  40. private int _preferredCoreOverride;
  41. private int _affinityOverrideCount;
  42. public ThreadSchedState SchedFlags { get; private set; }
  43. public bool ShallBeTerminated { get; private set; }
  44. public bool SyncCancelled { get; set; }
  45. public bool WaitingSync { get; set; }
  46. private bool _hasExited;
  47. private bool _hasBeenInitialized;
  48. private bool _hasBeenReleased;
  49. public bool WaitingInArbitration { get; set; }
  50. private KScheduler _scheduler;
  51. private KSchedulingData _schedulingData;
  52. public long LastPc { get; set; }
  53. public KThread(Horizon system) : base(system)
  54. {
  55. _scheduler = system.Scheduler;
  56. _schedulingData = system.Scheduler.SchedulingData;
  57. SiblingsPerCore = new LinkedListNode<KThread>[KScheduler.CpuCoresCount];
  58. _mutexWaiters = new LinkedList<KThread>();
  59. }
  60. public KernelResult Initialize(
  61. ulong entrypoint,
  62. ulong argsPtr,
  63. ulong stackTop,
  64. int priority,
  65. int defaultCpuCore,
  66. KProcess owner,
  67. ThreadType type = ThreadType.User)
  68. {
  69. if ((uint)type > 3)
  70. {
  71. throw new ArgumentException($"Invalid thread type \"{type}\".");
  72. }
  73. PreferredCore = defaultCpuCore;
  74. AffinityMask |= 1L << defaultCpuCore;
  75. SchedFlags = type == ThreadType.Dummy
  76. ? ThreadSchedState.Running
  77. : ThreadSchedState.None;
  78. CurrentCore = PreferredCore;
  79. DynamicPriority = priority;
  80. BasePriority = priority;
  81. ObjSyncResult = KernelResult.ThreadNotStarted;
  82. _entrypoint = entrypoint;
  83. if (type == ThreadType.User)
  84. {
  85. if (owner.AllocateThreadLocalStorage(out _tlsAddress) != KernelResult.Success)
  86. {
  87. return KernelResult.OutOfMemory;
  88. }
  89. TlsDramAddress = owner.MemoryManager.GetDramAddressFromVa(_tlsAddress);
  90. MemoryHelper.FillWithZeros(owner.CpuMemory, (long)_tlsAddress, KTlsPageInfo.TlsEntrySize);
  91. }
  92. bool is64Bits;
  93. if (owner != null)
  94. {
  95. Owner = owner;
  96. owner.IncrementReferenceCount();
  97. owner.IncrementThreadCount();
  98. is64Bits = (owner.MmuFlags & 1) != 0;
  99. }
  100. else
  101. {
  102. is64Bits = true;
  103. }
  104. Context = new CpuThread(owner.Translator, owner.CpuMemory, (long)entrypoint);
  105. bool isAarch32 = (Owner.MmuFlags & 1) == 0;
  106. Context.ThreadState.Aarch32 = isAarch32;
  107. Context.ThreadState.X0 = argsPtr;
  108. if (isAarch32)
  109. {
  110. Context.ThreadState.X13 = (uint)stackTop;
  111. }
  112. else
  113. {
  114. Context.ThreadState.X31 = stackTop;
  115. }
  116. Context.ThreadState.CntfrqEl0 = 19200000;
  117. Context.ThreadState.Tpidr = (long)_tlsAddress;
  118. owner.SubscribeThreadEventHandlers(Context);
  119. Context.WorkFinished += ThreadFinishedHandler;
  120. ThreadUid = System.GetThreadUid();
  121. _hasBeenInitialized = true;
  122. if (owner != null)
  123. {
  124. owner.AddThread(this);
  125. if (owner.IsPaused)
  126. {
  127. System.CriticalSection.Enter();
  128. if (ShallBeTerminated || SchedFlags == ThreadSchedState.TerminationPending)
  129. {
  130. System.CriticalSection.Leave();
  131. return KernelResult.Success;
  132. }
  133. _forcePauseFlags |= ThreadSchedState.ProcessPauseFlag;
  134. CombineForcePauseFlags();
  135. System.CriticalSection.Leave();
  136. }
  137. }
  138. return KernelResult.Success;
  139. }
  140. public KernelResult Start()
  141. {
  142. if (!System.KernelInitialized)
  143. {
  144. System.CriticalSection.Enter();
  145. if (!ShallBeTerminated && SchedFlags != ThreadSchedState.TerminationPending)
  146. {
  147. _forcePauseFlags |= ThreadSchedState.KernelInitPauseFlag;
  148. CombineForcePauseFlags();
  149. }
  150. System.CriticalSection.Leave();
  151. }
  152. KernelResult result = KernelResult.ThreadTerminating;
  153. System.CriticalSection.Enter();
  154. if (!ShallBeTerminated)
  155. {
  156. KThread currentThread = System.Scheduler.GetCurrentThread();
  157. while (SchedFlags != ThreadSchedState.TerminationPending &&
  158. currentThread.SchedFlags != ThreadSchedState.TerminationPending &&
  159. !currentThread.ShallBeTerminated)
  160. {
  161. if ((SchedFlags & ThreadSchedState.LowMask) != ThreadSchedState.None)
  162. {
  163. result = KernelResult.InvalidState;
  164. break;
  165. }
  166. if (currentThread._forcePauseFlags == ThreadSchedState.None)
  167. {
  168. if (Owner != null && _forcePauseFlags != ThreadSchedState.None)
  169. {
  170. CombineForcePauseFlags();
  171. }
  172. SetNewSchedFlags(ThreadSchedState.Running);
  173. result = KernelResult.Success;
  174. break;
  175. }
  176. else
  177. {
  178. currentThread.CombineForcePauseFlags();
  179. System.CriticalSection.Leave();
  180. System.CriticalSection.Enter();
  181. if (currentThread.ShallBeTerminated)
  182. {
  183. break;
  184. }
  185. }
  186. }
  187. }
  188. System.CriticalSection.Leave();
  189. return result;
  190. }
  191. public void Exit()
  192. {
  193. //TODO: Debug event.
  194. if (Owner != null)
  195. {
  196. Owner.ResourceLimit?.Release(LimitableResource.Thread, 0, 1);
  197. _hasBeenReleased = true;
  198. }
  199. System.CriticalSection.Enter();
  200. _forcePauseFlags &= ~ThreadSchedState.ForcePauseMask;
  201. ExitImpl();
  202. System.CriticalSection.Leave();
  203. DecrementReferenceCount();
  204. }
  205. private void ExitImpl()
  206. {
  207. System.CriticalSection.Enter();
  208. SetNewSchedFlags(ThreadSchedState.TerminationPending);
  209. _hasExited = true;
  210. Signal();
  211. System.CriticalSection.Leave();
  212. }
  213. public KernelResult Sleep(long timeout)
  214. {
  215. System.CriticalSection.Enter();
  216. if (ShallBeTerminated || SchedFlags == ThreadSchedState.TerminationPending)
  217. {
  218. System.CriticalSection.Leave();
  219. return KernelResult.ThreadTerminating;
  220. }
  221. SetNewSchedFlags(ThreadSchedState.Paused);
  222. if (timeout > 0)
  223. {
  224. System.TimeManager.ScheduleFutureInvocation(this, timeout);
  225. }
  226. System.CriticalSection.Leave();
  227. if (timeout > 0)
  228. {
  229. System.TimeManager.UnscheduleFutureInvocation(this);
  230. }
  231. return 0;
  232. }
  233. public void Yield()
  234. {
  235. System.CriticalSection.Enter();
  236. if (SchedFlags != ThreadSchedState.Running)
  237. {
  238. System.CriticalSection.Leave();
  239. System.Scheduler.ContextSwitch();
  240. return;
  241. }
  242. if (DynamicPriority < KScheduler.PrioritiesCount)
  243. {
  244. //Move current thread to the end of the queue.
  245. _schedulingData.Reschedule(DynamicPriority, CurrentCore, this);
  246. }
  247. _scheduler.ThreadReselectionRequested = true;
  248. System.CriticalSection.Leave();
  249. System.Scheduler.ContextSwitch();
  250. }
  251. public void YieldWithLoadBalancing()
  252. {
  253. System.CriticalSection.Enter();
  254. if (SchedFlags != ThreadSchedState.Running)
  255. {
  256. System.CriticalSection.Leave();
  257. System.Scheduler.ContextSwitch();
  258. return;
  259. }
  260. int prio = DynamicPriority;
  261. int core = CurrentCore;
  262. KThread nextThreadOnCurrentQueue = null;
  263. if (DynamicPriority < KScheduler.PrioritiesCount)
  264. {
  265. //Move current thread to the end of the queue.
  266. _schedulingData.Reschedule(prio, core, this);
  267. Func<KThread, bool> predicate = x => x.DynamicPriority == prio;
  268. nextThreadOnCurrentQueue = _schedulingData.ScheduledThreads(core).FirstOrDefault(predicate);
  269. }
  270. IEnumerable<KThread> SuitableCandidates()
  271. {
  272. foreach (KThread thread in _schedulingData.SuggestedThreads(core))
  273. {
  274. int srcCore = thread.CurrentCore;
  275. if (srcCore >= 0)
  276. {
  277. KThread selectedSrcCore = _scheduler.CoreContexts[srcCore].SelectedThread;
  278. if (selectedSrcCore == thread || ((selectedSrcCore?.DynamicPriority ?? 2) < 2))
  279. {
  280. continue;
  281. }
  282. }
  283. //If the candidate was scheduled after the current thread, then it's not worth it,
  284. //unless the priority is higher than the current one.
  285. if (nextThreadOnCurrentQueue.LastScheduledTime >= thread.LastScheduledTime ||
  286. nextThreadOnCurrentQueue.DynamicPriority < thread.DynamicPriority)
  287. {
  288. yield return thread;
  289. }
  290. }
  291. }
  292. KThread dst = SuitableCandidates().FirstOrDefault(x => x.DynamicPriority <= prio);
  293. if (dst != null)
  294. {
  295. _schedulingData.TransferToCore(dst.DynamicPriority, core, dst);
  296. _scheduler.ThreadReselectionRequested = true;
  297. }
  298. if (this != nextThreadOnCurrentQueue)
  299. {
  300. _scheduler.ThreadReselectionRequested = true;
  301. }
  302. System.CriticalSection.Leave();
  303. System.Scheduler.ContextSwitch();
  304. }
  305. public void YieldAndWaitForLoadBalancing()
  306. {
  307. System.CriticalSection.Enter();
  308. if (SchedFlags != ThreadSchedState.Running)
  309. {
  310. System.CriticalSection.Leave();
  311. System.Scheduler.ContextSwitch();
  312. return;
  313. }
  314. int core = CurrentCore;
  315. _schedulingData.TransferToCore(DynamicPriority, -1, this);
  316. KThread selectedThread = null;
  317. if (!_schedulingData.ScheduledThreads(core).Any())
  318. {
  319. foreach (KThread thread in _schedulingData.SuggestedThreads(core))
  320. {
  321. if (thread.CurrentCore < 0)
  322. {
  323. continue;
  324. }
  325. KThread firstCandidate = _schedulingData.ScheduledThreads(thread.CurrentCore).FirstOrDefault();
  326. if (firstCandidate == thread)
  327. {
  328. continue;
  329. }
  330. if (firstCandidate == null || firstCandidate.DynamicPriority >= 2)
  331. {
  332. _schedulingData.TransferToCore(thread.DynamicPriority, core, thread);
  333. selectedThread = thread;
  334. }
  335. break;
  336. }
  337. }
  338. if (selectedThread != this)
  339. {
  340. _scheduler.ThreadReselectionRequested = true;
  341. }
  342. System.CriticalSection.Leave();
  343. System.Scheduler.ContextSwitch();
  344. }
  345. public void SetPriority(int priority)
  346. {
  347. System.CriticalSection.Enter();
  348. BasePriority = priority;
  349. UpdatePriorityInheritance();
  350. System.CriticalSection.Leave();
  351. }
  352. public KernelResult SetActivity(bool pause)
  353. {
  354. KernelResult result = KernelResult.Success;
  355. System.CriticalSection.Enter();
  356. ThreadSchedState lowNibble = SchedFlags & ThreadSchedState.LowMask;
  357. if (lowNibble != ThreadSchedState.Paused && lowNibble != ThreadSchedState.Running)
  358. {
  359. System.CriticalSection.Leave();
  360. return KernelResult.InvalidState;
  361. }
  362. System.CriticalSection.Enter();
  363. if (!ShallBeTerminated && SchedFlags != ThreadSchedState.TerminationPending)
  364. {
  365. if (pause)
  366. {
  367. //Pause, the force pause flag should be clear (thread is NOT paused).
  368. if ((_forcePauseFlags & ThreadSchedState.ThreadPauseFlag) == 0)
  369. {
  370. _forcePauseFlags |= ThreadSchedState.ThreadPauseFlag;
  371. CombineForcePauseFlags();
  372. }
  373. else
  374. {
  375. result = KernelResult.InvalidState;
  376. }
  377. }
  378. else
  379. {
  380. //Unpause, the force pause flag should be set (thread is paused).
  381. if ((_forcePauseFlags & ThreadSchedState.ThreadPauseFlag) != 0)
  382. {
  383. ThreadSchedState oldForcePauseFlags = _forcePauseFlags;
  384. _forcePauseFlags &= ~ThreadSchedState.ThreadPauseFlag;
  385. if ((oldForcePauseFlags & ~ThreadSchedState.ThreadPauseFlag) == ThreadSchedState.None)
  386. {
  387. ThreadSchedState oldSchedFlags = SchedFlags;
  388. SchedFlags &= ThreadSchedState.LowMask;
  389. AdjustScheduling(oldSchedFlags);
  390. }
  391. }
  392. else
  393. {
  394. result = KernelResult.InvalidState;
  395. }
  396. }
  397. }
  398. System.CriticalSection.Leave();
  399. System.CriticalSection.Leave();
  400. return result;
  401. }
  402. public void CancelSynchronization()
  403. {
  404. System.CriticalSection.Enter();
  405. if ((SchedFlags & ThreadSchedState.LowMask) != ThreadSchedState.Paused || !WaitingSync)
  406. {
  407. SyncCancelled = true;
  408. }
  409. else if (Withholder != null)
  410. {
  411. Withholder.Remove(WithholderNode);
  412. SetNewSchedFlags(ThreadSchedState.Running);
  413. Withholder = null;
  414. SyncCancelled = true;
  415. }
  416. else
  417. {
  418. SignaledObj = null;
  419. ObjSyncResult = KernelResult.Cancelled;
  420. SetNewSchedFlags(ThreadSchedState.Running);
  421. SyncCancelled = false;
  422. }
  423. System.CriticalSection.Leave();
  424. }
  425. public KernelResult SetCoreAndAffinityMask(int newCore, long newAffinityMask)
  426. {
  427. System.CriticalSection.Enter();
  428. bool useOverride = _affinityOverrideCount != 0;
  429. //The value -3 is "do not change the preferred core".
  430. if (newCore == -3)
  431. {
  432. newCore = useOverride ? _preferredCoreOverride : PreferredCore;
  433. if ((newAffinityMask & (1 << newCore)) == 0)
  434. {
  435. System.CriticalSection.Leave();
  436. return KernelResult.InvalidCombination;
  437. }
  438. }
  439. if (useOverride)
  440. {
  441. _preferredCoreOverride = newCore;
  442. _affinityMaskOverride = newAffinityMask;
  443. }
  444. else
  445. {
  446. long oldAffinityMask = AffinityMask;
  447. PreferredCore = newCore;
  448. AffinityMask = newAffinityMask;
  449. if (oldAffinityMask != newAffinityMask)
  450. {
  451. int oldCore = CurrentCore;
  452. if (CurrentCore >= 0 && ((AffinityMask >> CurrentCore) & 1) == 0)
  453. {
  454. if (PreferredCore < 0)
  455. {
  456. CurrentCore = HighestSetCore(AffinityMask);
  457. }
  458. else
  459. {
  460. CurrentCore = PreferredCore;
  461. }
  462. }
  463. AdjustSchedulingForNewAffinity(oldAffinityMask, oldCore);
  464. }
  465. }
  466. System.CriticalSection.Leave();
  467. return KernelResult.Success;
  468. }
  469. private static int HighestSetCore(long mask)
  470. {
  471. for (int core = KScheduler.CpuCoresCount - 1; core >= 0; core--)
  472. {
  473. if (((mask >> core) & 1) != 0)
  474. {
  475. return core;
  476. }
  477. }
  478. return -1;
  479. }
  480. private void CombineForcePauseFlags()
  481. {
  482. ThreadSchedState oldFlags = SchedFlags;
  483. ThreadSchedState lowNibble = SchedFlags & ThreadSchedState.LowMask;
  484. SchedFlags = lowNibble | _forcePauseFlags;
  485. AdjustScheduling(oldFlags);
  486. }
  487. private void SetNewSchedFlags(ThreadSchedState newFlags)
  488. {
  489. System.CriticalSection.Enter();
  490. ThreadSchedState oldFlags = SchedFlags;
  491. SchedFlags = (oldFlags & ThreadSchedState.HighMask) | newFlags;
  492. if ((oldFlags & ThreadSchedState.LowMask) != newFlags)
  493. {
  494. AdjustScheduling(oldFlags);
  495. }
  496. System.CriticalSection.Leave();
  497. }
  498. public void ReleaseAndResume()
  499. {
  500. System.CriticalSection.Enter();
  501. if ((SchedFlags & ThreadSchedState.LowMask) == ThreadSchedState.Paused)
  502. {
  503. if (Withholder != null)
  504. {
  505. Withholder.Remove(WithholderNode);
  506. SetNewSchedFlags(ThreadSchedState.Running);
  507. Withholder = null;
  508. }
  509. else
  510. {
  511. SetNewSchedFlags(ThreadSchedState.Running);
  512. }
  513. }
  514. System.CriticalSection.Leave();
  515. }
  516. public void Reschedule(ThreadSchedState newFlags)
  517. {
  518. System.CriticalSection.Enter();
  519. ThreadSchedState oldFlags = SchedFlags;
  520. SchedFlags = (oldFlags & ThreadSchedState.HighMask) |
  521. (newFlags & ThreadSchedState.LowMask);
  522. AdjustScheduling(oldFlags);
  523. System.CriticalSection.Leave();
  524. }
  525. public void AddMutexWaiter(KThread requester)
  526. {
  527. AddToMutexWaitersList(requester);
  528. requester.MutexOwner = this;
  529. UpdatePriorityInheritance();
  530. }
  531. public void RemoveMutexWaiter(KThread thread)
  532. {
  533. if (thread._mutexWaiterNode?.List != null)
  534. {
  535. _mutexWaiters.Remove(thread._mutexWaiterNode);
  536. }
  537. thread.MutexOwner = null;
  538. UpdatePriorityInheritance();
  539. }
  540. public KThread RelinquishMutex(ulong mutexAddress, out int count)
  541. {
  542. count = 0;
  543. if (_mutexWaiters.First == null)
  544. {
  545. return null;
  546. }
  547. KThread newMutexOwner = null;
  548. LinkedListNode<KThread> currentNode = _mutexWaiters.First;
  549. do
  550. {
  551. //Skip all threads that are not waiting for this mutex.
  552. while (currentNode != null && currentNode.Value.MutexAddress != mutexAddress)
  553. {
  554. currentNode = currentNode.Next;
  555. }
  556. if (currentNode == null)
  557. {
  558. break;
  559. }
  560. LinkedListNode<KThread> nextNode = currentNode.Next;
  561. _mutexWaiters.Remove(currentNode);
  562. currentNode.Value.MutexOwner = newMutexOwner;
  563. if (newMutexOwner != null)
  564. {
  565. //New owner was already selected, re-insert on new owner list.
  566. newMutexOwner.AddToMutexWaitersList(currentNode.Value);
  567. }
  568. else
  569. {
  570. //New owner not selected yet, use current thread.
  571. newMutexOwner = currentNode.Value;
  572. }
  573. count++;
  574. currentNode = nextNode;
  575. }
  576. while (currentNode != null);
  577. if (newMutexOwner != null)
  578. {
  579. UpdatePriorityInheritance();
  580. newMutexOwner.UpdatePriorityInheritance();
  581. }
  582. return newMutexOwner;
  583. }
  584. private void UpdatePriorityInheritance()
  585. {
  586. //If any of the threads waiting for the mutex has
  587. //higher priority than the current thread, then
  588. //the current thread inherits that priority.
  589. int highestPriority = BasePriority;
  590. if (_mutexWaiters.First != null)
  591. {
  592. int waitingDynamicPriority = _mutexWaiters.First.Value.DynamicPriority;
  593. if (waitingDynamicPriority < highestPriority)
  594. {
  595. highestPriority = waitingDynamicPriority;
  596. }
  597. }
  598. if (highestPriority != DynamicPriority)
  599. {
  600. int oldPriority = DynamicPriority;
  601. DynamicPriority = highestPriority;
  602. AdjustSchedulingForNewPriority(oldPriority);
  603. if (MutexOwner != null)
  604. {
  605. //Remove and re-insert to ensure proper sorting based on new priority.
  606. MutexOwner._mutexWaiters.Remove(_mutexWaiterNode);
  607. MutexOwner.AddToMutexWaitersList(this);
  608. MutexOwner.UpdatePriorityInheritance();
  609. }
  610. }
  611. }
  612. private void AddToMutexWaitersList(KThread thread)
  613. {
  614. LinkedListNode<KThread> nextPrio = _mutexWaiters.First;
  615. int currentPriority = thread.DynamicPriority;
  616. while (nextPrio != null && nextPrio.Value.DynamicPriority <= currentPriority)
  617. {
  618. nextPrio = nextPrio.Next;
  619. }
  620. if (nextPrio != null)
  621. {
  622. thread._mutexWaiterNode = _mutexWaiters.AddBefore(nextPrio, thread);
  623. }
  624. else
  625. {
  626. thread._mutexWaiterNode = _mutexWaiters.AddLast(thread);
  627. }
  628. }
  629. private void AdjustScheduling(ThreadSchedState oldFlags)
  630. {
  631. if (oldFlags == SchedFlags)
  632. {
  633. return;
  634. }
  635. if (oldFlags == ThreadSchedState.Running)
  636. {
  637. //Was running, now it's stopped.
  638. if (CurrentCore >= 0)
  639. {
  640. _schedulingData.Unschedule(DynamicPriority, CurrentCore, this);
  641. }
  642. for (int core = 0; core < KScheduler.CpuCoresCount; core++)
  643. {
  644. if (core != CurrentCore && ((AffinityMask >> core) & 1) != 0)
  645. {
  646. _schedulingData.Unsuggest(DynamicPriority, core, this);
  647. }
  648. }
  649. }
  650. else if (SchedFlags == ThreadSchedState.Running)
  651. {
  652. //Was stopped, now it's running.
  653. if (CurrentCore >= 0)
  654. {
  655. _schedulingData.Schedule(DynamicPriority, CurrentCore, this);
  656. }
  657. for (int core = 0; core < KScheduler.CpuCoresCount; core++)
  658. {
  659. if (core != CurrentCore && ((AffinityMask >> core) & 1) != 0)
  660. {
  661. _schedulingData.Suggest(DynamicPriority, core, this);
  662. }
  663. }
  664. }
  665. _scheduler.ThreadReselectionRequested = true;
  666. }
  667. private void AdjustSchedulingForNewPriority(int oldPriority)
  668. {
  669. if (SchedFlags != ThreadSchedState.Running)
  670. {
  671. return;
  672. }
  673. //Remove thread from the old priority queues.
  674. if (CurrentCore >= 0)
  675. {
  676. _schedulingData.Unschedule(oldPriority, CurrentCore, this);
  677. }
  678. for (int core = 0; core < KScheduler.CpuCoresCount; core++)
  679. {
  680. if (core != CurrentCore && ((AffinityMask >> core) & 1) != 0)
  681. {
  682. _schedulingData.Unsuggest(oldPriority, core, this);
  683. }
  684. }
  685. //Add thread to the new priority queues.
  686. KThread currentThread = _scheduler.GetCurrentThread();
  687. if (CurrentCore >= 0)
  688. {
  689. if (currentThread == this)
  690. {
  691. _schedulingData.SchedulePrepend(DynamicPriority, CurrentCore, this);
  692. }
  693. else
  694. {
  695. _schedulingData.Schedule(DynamicPriority, CurrentCore, this);
  696. }
  697. }
  698. for (int core = 0; core < KScheduler.CpuCoresCount; core++)
  699. {
  700. if (core != CurrentCore && ((AffinityMask >> core) & 1) != 0)
  701. {
  702. _schedulingData.Suggest(DynamicPriority, core, this);
  703. }
  704. }
  705. _scheduler.ThreadReselectionRequested = true;
  706. }
  707. private void AdjustSchedulingForNewAffinity(long oldAffinityMask, int oldCore)
  708. {
  709. if (SchedFlags != ThreadSchedState.Running || DynamicPriority >= KScheduler.PrioritiesCount)
  710. {
  711. return;
  712. }
  713. //Remove thread from the old priority queues.
  714. for (int core = 0; core < KScheduler.CpuCoresCount; core++)
  715. {
  716. if (((oldAffinityMask >> core) & 1) != 0)
  717. {
  718. if (core == oldCore)
  719. {
  720. _schedulingData.Unschedule(DynamicPriority, core, this);
  721. }
  722. else
  723. {
  724. _schedulingData.Unsuggest(DynamicPriority, core, this);
  725. }
  726. }
  727. }
  728. //Add thread to the new priority queues.
  729. for (int core = 0; core < KScheduler.CpuCoresCount; core++)
  730. {
  731. if (((AffinityMask >> core) & 1) != 0)
  732. {
  733. if (core == CurrentCore)
  734. {
  735. _schedulingData.Schedule(DynamicPriority, core, this);
  736. }
  737. else
  738. {
  739. _schedulingData.Suggest(DynamicPriority, core, this);
  740. }
  741. }
  742. }
  743. _scheduler.ThreadReselectionRequested = true;
  744. }
  745. public void SetEntryArguments(long argsPtr, int threadHandle)
  746. {
  747. Context.ThreadState.X0 = (ulong)argsPtr;
  748. Context.ThreadState.X1 = (ulong)threadHandle;
  749. }
  750. public void TimeUp()
  751. {
  752. ReleaseAndResume();
  753. }
  754. public void PrintGuestStackTrace()
  755. {
  756. Owner.Debugger.PrintGuestStackTrace(Context.ThreadState);
  757. }
  758. private void ThreadFinishedHandler(object sender, EventArgs e)
  759. {
  760. System.Scheduler.ExitThread(this);
  761. System.Scheduler.RemoveThread(this);
  762. }
  763. public override bool IsSignaled()
  764. {
  765. return _hasExited;
  766. }
  767. protected override void Destroy()
  768. {
  769. if (_hasBeenInitialized)
  770. {
  771. FreeResources();
  772. bool released = Owner != null || _hasBeenReleased;
  773. if (Owner != null)
  774. {
  775. Owner.ResourceLimit?.Release(LimitableResource.Thread, 1, released ? 0 : 1);
  776. Owner.DecrementReferenceCount();
  777. }
  778. else
  779. {
  780. System.ResourceLimit.Release(LimitableResource.Thread, 1, released ? 0 : 1);
  781. }
  782. }
  783. }
  784. private void FreeResources()
  785. {
  786. Owner?.RemoveThread(this);
  787. if (_tlsAddress != 0 && Owner.FreeThreadLocalStorage(_tlsAddress) != KernelResult.Success)
  788. {
  789. throw new InvalidOperationException("Unexpected failure freeing thread local storage.");
  790. }
  791. System.CriticalSection.Enter();
  792. //Wake up all threads that may be waiting for a mutex being held by this thread.
  793. foreach (KThread thread in _mutexWaiters)
  794. {
  795. thread.MutexOwner = null;
  796. thread._preferredCoreOverride = 0;
  797. thread.ObjSyncResult = KernelResult.InvalidState;
  798. thread.ReleaseAndResume();
  799. }
  800. System.CriticalSection.Leave();
  801. Owner?.DecrementThreadCountAndTerminateIfZero();
  802. }
  803. }
  804. }