KThread.cs 31 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080
  1. using ChocolArm64;
  2. using ChocolArm64.Memory;
  3. using Ryujinx.HLE.HOS.Kernel.Common;
  4. using Ryujinx.HLE.HOS.Kernel.Process;
  5. using System;
  6. using System.Collections.Generic;
  7. using System.Linq;
  8. namespace Ryujinx.HLE.HOS.Kernel.Threading
  9. {
  10. class KThread : KSynchronizationObject, IKFutureSchedulerObject
  11. {
  12. public CpuThread Context { get; private set; }
  13. public long AffinityMask { get; set; }
  14. public long ThreadUid { get; private set; }
  15. public long TotalTimeRunning { get; set; }
  16. public KSynchronizationObject SignaledObj { get; set; }
  17. public ulong CondVarAddress { get; set; }
  18. private ulong _entrypoint;
  19. public ulong MutexAddress { get; set; }
  20. public KProcess Owner { get; private set; }
  21. private ulong _tlsAddress;
  22. public ulong TlsAddress => _tlsAddress;
  23. public ulong TlsDramAddress { get; private set; }
  24. public long LastScheduledTime { get; set; }
  25. public LinkedListNode<KThread>[] SiblingsPerCore { get; private set; }
  26. public LinkedList<KThread> Withholder { get; set; }
  27. public LinkedListNode<KThread> WithholderNode { get; set; }
  28. public LinkedListNode<KThread> ProcessListNode { get; set; }
  29. private LinkedList<KThread> _mutexWaiters;
  30. private LinkedListNode<KThread> _mutexWaiterNode;
  31. public KThread MutexOwner { get; private set; }
  32. public int ThreadHandleForUserMutex { get; set; }
  33. private ThreadSchedState _forcePauseFlags;
  34. public KernelResult ObjSyncResult { get; set; }
  35. public int DynamicPriority { get; set; }
  36. public int CurrentCore { get; set; }
  37. public int BasePriority { get; set; }
  38. public int PreferredCore { get; set; }
  39. private long _affinityMaskOverride;
  40. private int _preferredCoreOverride;
  41. private int _affinityOverrideCount;
  42. public ThreadSchedState SchedFlags { get; private set; }
  43. public bool ShallBeTerminated { get; private set; }
  44. public bool SyncCancelled { get; set; }
  45. public bool WaitingSync { get; set; }
  46. private bool _hasExited;
  47. private bool _hasBeenInitialized;
  48. private bool _hasBeenReleased;
  49. public bool WaitingInArbitration { get; set; }
  50. private KScheduler _scheduler;
  51. private KSchedulingData _schedulingData;
  52. public long LastPc { get; set; }
  53. public KThread(Horizon system) : base(system)
  54. {
  55. _scheduler = system.Scheduler;
  56. _schedulingData = system.Scheduler.SchedulingData;
  57. SiblingsPerCore = new LinkedListNode<KThread>[KScheduler.CpuCoresCount];
  58. _mutexWaiters = new LinkedList<KThread>();
  59. }
  60. public KernelResult Initialize(
  61. ulong entrypoint,
  62. ulong argsPtr,
  63. ulong stackTop,
  64. int priority,
  65. int defaultCpuCore,
  66. KProcess owner,
  67. ThreadType type = ThreadType.User)
  68. {
  69. if ((uint)type > 3)
  70. {
  71. throw new ArgumentException($"Invalid thread type \"{type}\".");
  72. }
  73. PreferredCore = defaultCpuCore;
  74. AffinityMask |= 1L << defaultCpuCore;
  75. SchedFlags = type == ThreadType.Dummy
  76. ? ThreadSchedState.Running
  77. : ThreadSchedState.None;
  78. CurrentCore = PreferredCore;
  79. DynamicPriority = priority;
  80. BasePriority = priority;
  81. ObjSyncResult = KernelResult.ThreadNotStarted;
  82. _entrypoint = entrypoint;
  83. if (type == ThreadType.User)
  84. {
  85. if (owner.AllocateThreadLocalStorage(out _tlsAddress) != KernelResult.Success)
  86. {
  87. return KernelResult.OutOfMemory;
  88. }
  89. TlsDramAddress = owner.MemoryManager.GetDramAddressFromVa(_tlsAddress);
  90. MemoryHelper.FillWithZeros(owner.CpuMemory, (long)_tlsAddress, KTlsPageInfo.TlsEntrySize);
  91. }
  92. bool is64Bits;
  93. if (owner != null)
  94. {
  95. Owner = owner;
  96. owner.IncrementReferenceCount();
  97. owner.IncrementThreadCount();
  98. is64Bits = (owner.MmuFlags & 1) != 0;
  99. }
  100. else
  101. {
  102. is64Bits = true;
  103. }
  104. Context = new CpuThread(owner.Translator, owner.CpuMemory, (long)entrypoint);
  105. bool isAarch32 = (Owner.MmuFlags & 1) == 0;
  106. Context.ThreadState.Aarch32 = isAarch32;
  107. Context.ThreadState.X0 = argsPtr;
  108. if (isAarch32)
  109. {
  110. Context.ThreadState.X13 = (uint)stackTop;
  111. }
  112. else
  113. {
  114. Context.ThreadState.X31 = stackTop;
  115. }
  116. Context.ThreadState.CntfrqEl0 = 19200000;
  117. Context.ThreadState.Tpidr = (long)_tlsAddress;
  118. owner.SubscribeThreadEventHandlers(Context);
  119. Context.WorkFinished += ThreadFinishedHandler;
  120. ThreadUid = System.GetThreadUid();
  121. _hasBeenInitialized = true;
  122. if (owner != null)
  123. {
  124. owner.AddThread(this);
  125. if (owner.IsPaused)
  126. {
  127. System.CriticalSection.Enter();
  128. if (ShallBeTerminated || SchedFlags == ThreadSchedState.TerminationPending)
  129. {
  130. System.CriticalSection.Leave();
  131. return KernelResult.Success;
  132. }
  133. _forcePauseFlags |= ThreadSchedState.ProcessPauseFlag;
  134. CombineForcePauseFlags();
  135. System.CriticalSection.Leave();
  136. }
  137. }
  138. return KernelResult.Success;
  139. }
  140. public KernelResult Start()
  141. {
  142. if (!System.KernelInitialized)
  143. {
  144. System.CriticalSection.Enter();
  145. if (!ShallBeTerminated && SchedFlags != ThreadSchedState.TerminationPending)
  146. {
  147. _forcePauseFlags |= ThreadSchedState.KernelInitPauseFlag;
  148. CombineForcePauseFlags();
  149. }
  150. System.CriticalSection.Leave();
  151. }
  152. KernelResult result = KernelResult.ThreadTerminating;
  153. System.CriticalSection.Enter();
  154. if (!ShallBeTerminated)
  155. {
  156. KThread currentThread = System.Scheduler.GetCurrentThread();
  157. while (SchedFlags != ThreadSchedState.TerminationPending &&
  158. currentThread.SchedFlags != ThreadSchedState.TerminationPending &&
  159. !currentThread.ShallBeTerminated)
  160. {
  161. if ((SchedFlags & ThreadSchedState.LowMask) != ThreadSchedState.None)
  162. {
  163. result = KernelResult.InvalidState;
  164. break;
  165. }
  166. if (currentThread._forcePauseFlags == ThreadSchedState.None)
  167. {
  168. if (Owner != null && _forcePauseFlags != ThreadSchedState.None)
  169. {
  170. CombineForcePauseFlags();
  171. }
  172. SetNewSchedFlags(ThreadSchedState.Running);
  173. result = KernelResult.Success;
  174. break;
  175. }
  176. else
  177. {
  178. currentThread.CombineForcePauseFlags();
  179. System.CriticalSection.Leave();
  180. System.CriticalSection.Enter();
  181. if (currentThread.ShallBeTerminated)
  182. {
  183. break;
  184. }
  185. }
  186. }
  187. }
  188. System.CriticalSection.Leave();
  189. return result;
  190. }
  191. public void Exit()
  192. {
  193. //TODO: Debug event.
  194. if (Owner != null)
  195. {
  196. Owner.ResourceLimit?.Release(LimitableResource.Thread, 0, 1);
  197. _hasBeenReleased = true;
  198. }
  199. System.CriticalSection.Enter();
  200. _forcePauseFlags &= ~ThreadSchedState.ForcePauseMask;
  201. ExitImpl();
  202. System.CriticalSection.Leave();
  203. DecrementReferenceCount();
  204. }
  205. private void ExitImpl()
  206. {
  207. System.CriticalSection.Enter();
  208. SetNewSchedFlags(ThreadSchedState.TerminationPending);
  209. _hasExited = true;
  210. Signal();
  211. System.CriticalSection.Leave();
  212. }
  213. public KernelResult Sleep(long timeout)
  214. {
  215. System.CriticalSection.Enter();
  216. if (ShallBeTerminated || SchedFlags == ThreadSchedState.TerminationPending)
  217. {
  218. System.CriticalSection.Leave();
  219. return KernelResult.ThreadTerminating;
  220. }
  221. SetNewSchedFlags(ThreadSchedState.Paused);
  222. if (timeout > 0)
  223. {
  224. System.TimeManager.ScheduleFutureInvocation(this, timeout);
  225. }
  226. System.CriticalSection.Leave();
  227. if (timeout > 0)
  228. {
  229. System.TimeManager.UnscheduleFutureInvocation(this);
  230. }
  231. return 0;
  232. }
  233. public void Yield()
  234. {
  235. System.CriticalSection.Enter();
  236. if (SchedFlags != ThreadSchedState.Running)
  237. {
  238. System.CriticalSection.Leave();
  239. System.Scheduler.ContextSwitch();
  240. return;
  241. }
  242. if (DynamicPriority < KScheduler.PrioritiesCount)
  243. {
  244. //Move current thread to the end of the queue.
  245. _schedulingData.Reschedule(DynamicPriority, CurrentCore, this);
  246. }
  247. _scheduler.ThreadReselectionRequested = true;
  248. System.CriticalSection.Leave();
  249. System.Scheduler.ContextSwitch();
  250. }
  251. public void YieldWithLoadBalancing()
  252. {
  253. System.CriticalSection.Enter();
  254. if (SchedFlags != ThreadSchedState.Running)
  255. {
  256. System.CriticalSection.Leave();
  257. System.Scheduler.ContextSwitch();
  258. return;
  259. }
  260. int prio = DynamicPriority;
  261. int core = CurrentCore;
  262. KThread nextThreadOnCurrentQueue = null;
  263. if (DynamicPriority < KScheduler.PrioritiesCount)
  264. {
  265. //Move current thread to the end of the queue.
  266. _schedulingData.Reschedule(prio, core, this);
  267. Func<KThread, bool> predicate = x => x.DynamicPriority == prio;
  268. nextThreadOnCurrentQueue = _schedulingData.ScheduledThreads(core).FirstOrDefault(predicate);
  269. }
  270. IEnumerable<KThread> SuitableCandidates()
  271. {
  272. foreach (KThread thread in _schedulingData.SuggestedThreads(core))
  273. {
  274. int srcCore = thread.CurrentCore;
  275. if (srcCore >= 0)
  276. {
  277. KThread selectedSrcCore = _scheduler.CoreContexts[srcCore].SelectedThread;
  278. if (selectedSrcCore == thread || ((selectedSrcCore?.DynamicPriority ?? 2) < 2))
  279. {
  280. continue;
  281. }
  282. }
  283. //If the candidate was scheduled after the current thread, then it's not worth it,
  284. //unless the priority is higher than the current one.
  285. if (nextThreadOnCurrentQueue.LastScheduledTime >= thread.LastScheduledTime ||
  286. nextThreadOnCurrentQueue.DynamicPriority < thread.DynamicPriority)
  287. {
  288. yield return thread;
  289. }
  290. }
  291. }
  292. KThread dst = SuitableCandidates().FirstOrDefault(x => x.DynamicPriority <= prio);
  293. if (dst != null)
  294. {
  295. _schedulingData.TransferToCore(dst.DynamicPriority, core, dst);
  296. _scheduler.ThreadReselectionRequested = true;
  297. }
  298. if (this != nextThreadOnCurrentQueue)
  299. {
  300. _scheduler.ThreadReselectionRequested = true;
  301. }
  302. System.CriticalSection.Leave();
  303. System.Scheduler.ContextSwitch();
  304. }
  305. public void YieldAndWaitForLoadBalancing()
  306. {
  307. System.CriticalSection.Enter();
  308. if (SchedFlags != ThreadSchedState.Running)
  309. {
  310. System.CriticalSection.Leave();
  311. System.Scheduler.ContextSwitch();
  312. return;
  313. }
  314. int core = CurrentCore;
  315. _schedulingData.TransferToCore(DynamicPriority, -1, this);
  316. KThread selectedThread = null;
  317. if (!_schedulingData.ScheduledThreads(core).Any())
  318. {
  319. foreach (KThread thread in _schedulingData.SuggestedThreads(core))
  320. {
  321. if (thread.CurrentCore < 0)
  322. {
  323. continue;
  324. }
  325. KThread firstCandidate = _schedulingData.ScheduledThreads(thread.CurrentCore).FirstOrDefault();
  326. if (firstCandidate == thread)
  327. {
  328. continue;
  329. }
  330. if (firstCandidate == null || firstCandidate.DynamicPriority >= 2)
  331. {
  332. _schedulingData.TransferToCore(thread.DynamicPriority, core, thread);
  333. selectedThread = thread;
  334. }
  335. break;
  336. }
  337. }
  338. if (selectedThread != this)
  339. {
  340. _scheduler.ThreadReselectionRequested = true;
  341. }
  342. System.CriticalSection.Leave();
  343. System.Scheduler.ContextSwitch();
  344. }
  345. public void SetPriority(int priority)
  346. {
  347. System.CriticalSection.Enter();
  348. BasePriority = priority;
  349. UpdatePriorityInheritance();
  350. System.CriticalSection.Leave();
  351. }
  352. public KernelResult SetActivity(bool pause)
  353. {
  354. KernelResult result = KernelResult.Success;
  355. System.CriticalSection.Enter();
  356. ThreadSchedState lowNibble = SchedFlags & ThreadSchedState.LowMask;
  357. if (lowNibble != ThreadSchedState.Paused && lowNibble != ThreadSchedState.Running)
  358. {
  359. System.CriticalSection.Leave();
  360. return KernelResult.InvalidState;
  361. }
  362. System.CriticalSection.Enter();
  363. if (!ShallBeTerminated && SchedFlags != ThreadSchedState.TerminationPending)
  364. {
  365. if (pause)
  366. {
  367. //Pause, the force pause flag should be clear (thread is NOT paused).
  368. if ((_forcePauseFlags & ThreadSchedState.ThreadPauseFlag) == 0)
  369. {
  370. _forcePauseFlags |= ThreadSchedState.ThreadPauseFlag;
  371. CombineForcePauseFlags();
  372. }
  373. else
  374. {
  375. result = KernelResult.InvalidState;
  376. }
  377. }
  378. else
  379. {
  380. //Unpause, the force pause flag should be set (thread is paused).
  381. if ((_forcePauseFlags & ThreadSchedState.ThreadPauseFlag) != 0)
  382. {
  383. ThreadSchedState oldForcePauseFlags = _forcePauseFlags;
  384. _forcePauseFlags &= ~ThreadSchedState.ThreadPauseFlag;
  385. if ((oldForcePauseFlags & ~ThreadSchedState.ThreadPauseFlag) == ThreadSchedState.None)
  386. {
  387. ThreadSchedState oldSchedFlags = SchedFlags;
  388. SchedFlags &= ThreadSchedState.LowMask;
  389. AdjustScheduling(oldSchedFlags);
  390. }
  391. }
  392. else
  393. {
  394. result = KernelResult.InvalidState;
  395. }
  396. }
  397. }
  398. System.CriticalSection.Leave();
  399. System.CriticalSection.Leave();
  400. return result;
  401. }
  402. public void CancelSynchronization()
  403. {
  404. System.CriticalSection.Enter();
  405. if ((SchedFlags & ThreadSchedState.LowMask) != ThreadSchedState.Paused || !WaitingSync)
  406. {
  407. SyncCancelled = true;
  408. }
  409. else if (Withholder != null)
  410. {
  411. Withholder.Remove(WithholderNode);
  412. SetNewSchedFlags(ThreadSchedState.Running);
  413. Withholder = null;
  414. SyncCancelled = true;
  415. }
  416. else
  417. {
  418. SignaledObj = null;
  419. ObjSyncResult = KernelResult.Cancelled;
  420. SetNewSchedFlags(ThreadSchedState.Running);
  421. SyncCancelled = false;
  422. }
  423. System.CriticalSection.Leave();
  424. }
  425. public KernelResult SetCoreAndAffinityMask(int newCore, long newAffinityMask)
  426. {
  427. System.CriticalSection.Enter();
  428. bool useOverride = _affinityOverrideCount != 0;
  429. //The value -3 is "do not change the preferred core".
  430. if (newCore == -3)
  431. {
  432. newCore = useOverride ? _preferredCoreOverride : PreferredCore;
  433. if ((newAffinityMask & (1 << newCore)) == 0)
  434. {
  435. System.CriticalSection.Leave();
  436. return KernelResult.InvalidCombination;
  437. }
  438. }
  439. if (useOverride)
  440. {
  441. _preferredCoreOverride = newCore;
  442. _affinityMaskOverride = newAffinityMask;
  443. }
  444. else
  445. {
  446. long oldAffinityMask = AffinityMask;
  447. PreferredCore = newCore;
  448. AffinityMask = newAffinityMask;
  449. if (oldAffinityMask != newAffinityMask)
  450. {
  451. int oldCore = CurrentCore;
  452. if (CurrentCore >= 0 && ((AffinityMask >> CurrentCore) & 1) == 0)
  453. {
  454. if (PreferredCore < 0)
  455. {
  456. CurrentCore = HighestSetCore(AffinityMask);
  457. }
  458. else
  459. {
  460. CurrentCore = PreferredCore;
  461. }
  462. }
  463. AdjustSchedulingForNewAffinity(oldAffinityMask, oldCore);
  464. }
  465. }
  466. System.CriticalSection.Leave();
  467. return KernelResult.Success;
  468. }
  469. private static int HighestSetCore(long mask)
  470. {
  471. for (int core = KScheduler.CpuCoresCount - 1; core >= 0; core--)
  472. {
  473. if (((mask >> core) & 1) != 0)
  474. {
  475. return core;
  476. }
  477. }
  478. return -1;
  479. }
  480. private void CombineForcePauseFlags()
  481. {
  482. ThreadSchedState oldFlags = SchedFlags;
  483. ThreadSchedState lowNibble = SchedFlags & ThreadSchedState.LowMask;
  484. SchedFlags = lowNibble | _forcePauseFlags;
  485. AdjustScheduling(oldFlags);
  486. }
  487. private void SetNewSchedFlags(ThreadSchedState newFlags)
  488. {
  489. System.CriticalSection.Enter();
  490. ThreadSchedState oldFlags = SchedFlags;
  491. SchedFlags = (oldFlags & ThreadSchedState.HighMask) | newFlags;
  492. if ((oldFlags & ThreadSchedState.LowMask) != newFlags)
  493. {
  494. AdjustScheduling(oldFlags);
  495. }
  496. System.CriticalSection.Leave();
  497. }
  498. public void ReleaseAndResume()
  499. {
  500. System.CriticalSection.Enter();
  501. if ((SchedFlags & ThreadSchedState.LowMask) == ThreadSchedState.Paused)
  502. {
  503. if (Withholder != null)
  504. {
  505. Withholder.Remove(WithholderNode);
  506. SetNewSchedFlags(ThreadSchedState.Running);
  507. Withholder = null;
  508. }
  509. else
  510. {
  511. SetNewSchedFlags(ThreadSchedState.Running);
  512. }
  513. }
  514. System.CriticalSection.Leave();
  515. }
  516. public void Reschedule(ThreadSchedState newFlags)
  517. {
  518. System.CriticalSection.Enter();
  519. ThreadSchedState oldFlags = SchedFlags;
  520. SchedFlags = (oldFlags & ThreadSchedState.HighMask) |
  521. (newFlags & ThreadSchedState.LowMask);
  522. AdjustScheduling(oldFlags);
  523. System.CriticalSection.Leave();
  524. }
  525. public void AddMutexWaiter(KThread requester)
  526. {
  527. AddToMutexWaitersList(requester);
  528. requester.MutexOwner = this;
  529. UpdatePriorityInheritance();
  530. }
  531. public void RemoveMutexWaiter(KThread thread)
  532. {
  533. if (thread._mutexWaiterNode?.List != null)
  534. {
  535. _mutexWaiters.Remove(thread._mutexWaiterNode);
  536. }
  537. thread.MutexOwner = null;
  538. UpdatePriorityInheritance();
  539. }
  540. public KThread RelinquishMutex(ulong mutexAddress, out int count)
  541. {
  542. count = 0;
  543. if (_mutexWaiters.First == null)
  544. {
  545. return null;
  546. }
  547. KThread newMutexOwner = null;
  548. LinkedListNode<KThread> currentNode = _mutexWaiters.First;
  549. do
  550. {
  551. //Skip all threads that are not waiting for this mutex.
  552. while (currentNode != null && currentNode.Value.MutexAddress != mutexAddress)
  553. {
  554. currentNode = currentNode.Next;
  555. }
  556. if (currentNode == null)
  557. {
  558. break;
  559. }
  560. LinkedListNode<KThread> nextNode = currentNode.Next;
  561. _mutexWaiters.Remove(currentNode);
  562. currentNode.Value.MutexOwner = newMutexOwner;
  563. if (newMutexOwner != null)
  564. {
  565. //New owner was already selected, re-insert on new owner list.
  566. newMutexOwner.AddToMutexWaitersList(currentNode.Value);
  567. }
  568. else
  569. {
  570. //New owner not selected yet, use current thread.
  571. newMutexOwner = currentNode.Value;
  572. }
  573. count++;
  574. currentNode = nextNode;
  575. }
  576. while (currentNode != null);
  577. if (newMutexOwner != null)
  578. {
  579. UpdatePriorityInheritance();
  580. newMutexOwner.UpdatePriorityInheritance();
  581. }
  582. return newMutexOwner;
  583. }
  584. private void UpdatePriorityInheritance()
  585. {
  586. //If any of the threads waiting for the mutex has
  587. //higher priority than the current thread, then
  588. //the current thread inherits that priority.
  589. int highestPriority = BasePriority;
  590. if (_mutexWaiters.First != null)
  591. {
  592. int waitingDynamicPriority = _mutexWaiters.First.Value.DynamicPriority;
  593. if (waitingDynamicPriority < highestPriority)
  594. {
  595. highestPriority = waitingDynamicPriority;
  596. }
  597. }
  598. if (highestPriority != DynamicPriority)
  599. {
  600. int oldPriority = DynamicPriority;
  601. DynamicPriority = highestPriority;
  602. AdjustSchedulingForNewPriority(oldPriority);
  603. if (MutexOwner != null)
  604. {
  605. //Remove and re-insert to ensure proper sorting based on new priority.
  606. MutexOwner._mutexWaiters.Remove(_mutexWaiterNode);
  607. MutexOwner.AddToMutexWaitersList(this);
  608. MutexOwner.UpdatePriorityInheritance();
  609. }
  610. }
  611. }
  612. private void AddToMutexWaitersList(KThread thread)
  613. {
  614. LinkedListNode<KThread> nextPrio = _mutexWaiters.First;
  615. int currentPriority = thread.DynamicPriority;
  616. while (nextPrio != null && nextPrio.Value.DynamicPriority <= currentPriority)
  617. {
  618. nextPrio = nextPrio.Next;
  619. }
  620. if (nextPrio != null)
  621. {
  622. thread._mutexWaiterNode = _mutexWaiters.AddBefore(nextPrio, thread);
  623. }
  624. else
  625. {
  626. thread._mutexWaiterNode = _mutexWaiters.AddLast(thread);
  627. }
  628. }
  629. private void AdjustScheduling(ThreadSchedState oldFlags)
  630. {
  631. if (oldFlags == SchedFlags)
  632. {
  633. return;
  634. }
  635. if (oldFlags == ThreadSchedState.Running)
  636. {
  637. //Was running, now it's stopped.
  638. if (CurrentCore >= 0)
  639. {
  640. _schedulingData.Unschedule(DynamicPriority, CurrentCore, this);
  641. }
  642. for (int core = 0; core < KScheduler.CpuCoresCount; core++)
  643. {
  644. if (core != CurrentCore && ((AffinityMask >> core) & 1) != 0)
  645. {
  646. _schedulingData.Unsuggest(DynamicPriority, core, this);
  647. }
  648. }
  649. }
  650. else if (SchedFlags == ThreadSchedState.Running)
  651. {
  652. //Was stopped, now it's running.
  653. if (CurrentCore >= 0)
  654. {
  655. _schedulingData.Schedule(DynamicPriority, CurrentCore, this);
  656. }
  657. for (int core = 0; core < KScheduler.CpuCoresCount; core++)
  658. {
  659. if (core != CurrentCore && ((AffinityMask >> core) & 1) != 0)
  660. {
  661. _schedulingData.Suggest(DynamicPriority, core, this);
  662. }
  663. }
  664. }
  665. _scheduler.ThreadReselectionRequested = true;
  666. }
  667. private void AdjustSchedulingForNewPriority(int oldPriority)
  668. {
  669. if (SchedFlags != ThreadSchedState.Running)
  670. {
  671. return;
  672. }
  673. //Remove thread from the old priority queues.
  674. if (CurrentCore >= 0)
  675. {
  676. _schedulingData.Unschedule(oldPriority, CurrentCore, this);
  677. }
  678. for (int core = 0; core < KScheduler.CpuCoresCount; core++)
  679. {
  680. if (core != CurrentCore && ((AffinityMask >> core) & 1) != 0)
  681. {
  682. _schedulingData.Unsuggest(oldPriority, core, this);
  683. }
  684. }
  685. //Add thread to the new priority queues.
  686. KThread currentThread = _scheduler.GetCurrentThread();
  687. if (CurrentCore >= 0)
  688. {
  689. if (currentThread == this)
  690. {
  691. _schedulingData.SchedulePrepend(DynamicPriority, CurrentCore, this);
  692. }
  693. else
  694. {
  695. _schedulingData.Schedule(DynamicPriority, CurrentCore, this);
  696. }
  697. }
  698. for (int core = 0; core < KScheduler.CpuCoresCount; core++)
  699. {
  700. if (core != CurrentCore && ((AffinityMask >> core) & 1) != 0)
  701. {
  702. _schedulingData.Suggest(DynamicPriority, core, this);
  703. }
  704. }
  705. _scheduler.ThreadReselectionRequested = true;
  706. }
  707. private void AdjustSchedulingForNewAffinity(long oldAffinityMask, int oldCore)
  708. {
  709. if (SchedFlags != ThreadSchedState.Running || DynamicPriority >= KScheduler.PrioritiesCount)
  710. {
  711. return;
  712. }
  713. //Remove thread from the old priority queues.
  714. for (int core = 0; core < KScheduler.CpuCoresCount; core++)
  715. {
  716. if (((oldAffinityMask >> core) & 1) != 0)
  717. {
  718. if (core == oldCore)
  719. {
  720. _schedulingData.Unschedule(DynamicPriority, core, this);
  721. }
  722. else
  723. {
  724. _schedulingData.Unsuggest(DynamicPriority, core, this);
  725. }
  726. }
  727. }
  728. //Add thread to the new priority queues.
  729. for (int core = 0; core < KScheduler.CpuCoresCount; core++)
  730. {
  731. if (((AffinityMask >> core) & 1) != 0)
  732. {
  733. if (core == CurrentCore)
  734. {
  735. _schedulingData.Schedule(DynamicPriority, core, this);
  736. }
  737. else
  738. {
  739. _schedulingData.Suggest(DynamicPriority, core, this);
  740. }
  741. }
  742. }
  743. _scheduler.ThreadReselectionRequested = true;
  744. }
  745. public void SetEntryArguments(long argsPtr, int threadHandle)
  746. {
  747. Context.ThreadState.X0 = (ulong)argsPtr;
  748. Context.ThreadState.X1 = (ulong)threadHandle;
  749. }
  750. public void ClearExclusive()
  751. {
  752. Owner.CpuMemory.ClearExclusive(CurrentCore);
  753. }
  754. public void TimeUp()
  755. {
  756. ReleaseAndResume();
  757. }
  758. public void PrintGuestStackTrace()
  759. {
  760. Owner.Debugger.PrintGuestStackTrace(Context.ThreadState);
  761. }
  762. private void ThreadFinishedHandler(object sender, EventArgs e)
  763. {
  764. System.Scheduler.ExitThread(this);
  765. System.Scheduler.RemoveThread(this);
  766. }
  767. public override bool IsSignaled()
  768. {
  769. return _hasExited;
  770. }
  771. protected override void Destroy()
  772. {
  773. if (_hasBeenInitialized)
  774. {
  775. FreeResources();
  776. bool released = Owner != null || _hasBeenReleased;
  777. if (Owner != null)
  778. {
  779. Owner.ResourceLimit?.Release(LimitableResource.Thread, 1, released ? 0 : 1);
  780. Owner.DecrementReferenceCount();
  781. }
  782. else
  783. {
  784. System.ResourceLimit.Release(LimitableResource.Thread, 1, released ? 0 : 1);
  785. }
  786. }
  787. }
  788. private void FreeResources()
  789. {
  790. Owner?.RemoveThread(this);
  791. if (_tlsAddress != 0 && Owner.FreeThreadLocalStorage(_tlsAddress) != KernelResult.Success)
  792. {
  793. throw new InvalidOperationException("Unexpected failure freeing thread local storage.");
  794. }
  795. System.CriticalSection.Enter();
  796. //Wake up all threads that may be waiting for a mutex being held by this thread.
  797. foreach (KThread thread in _mutexWaiters)
  798. {
  799. thread.MutexOwner = null;
  800. thread._preferredCoreOverride = 0;
  801. thread.ObjSyncResult = KernelResult.InvalidState;
  802. thread.ReleaseAndResume();
  803. }
  804. System.CriticalSection.Leave();
  805. Owner?.DecrementThreadCountAndTerminateIfZero();
  806. }
  807. }
  808. }