KThread.cs 32 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109
  1. using ARMeilleure.Memory;
  2. using Ryujinx.Common.Logging;
  3. using Ryujinx.HLE.HOS.Kernel.Common;
  4. using Ryujinx.HLE.HOS.Kernel.Process;
  5. using System;
  6. using System.Collections.Generic;
  7. using System.Linq;
  8. using System.Text;
  9. using System.Threading;
  10. namespace Ryujinx.HLE.HOS.Kernel.Threading
  11. {
  12. class KThread : KSynchronizationObject, IKFutureSchedulerObject
  13. {
  14. private int _hostThreadRunning;
  15. public Thread HostThread { get; private set; }
  16. public ARMeilleure.State.ExecutionContext Context { get; private set; }
  17. public long AffinityMask { get; set; }
  18. public long ThreadUid { get; private set; }
  19. public long TotalTimeRunning { get; set; }
  20. public KSynchronizationObject SignaledObj { get; set; }
  21. public ulong CondVarAddress { get; set; }
  22. private ulong _entrypoint;
  23. public ulong MutexAddress { get; set; }
  24. public KProcess Owner { get; private set; }
  25. private ulong _tlsAddress;
  26. public ulong TlsAddress => _tlsAddress;
  27. public ulong TlsDramAddress { get; private set; }
  28. public long LastScheduledTime { get; set; }
  29. public LinkedListNode<KThread>[] SiblingsPerCore { get; private set; }
  30. public LinkedList<KThread> Withholder { get; set; }
  31. public LinkedListNode<KThread> WithholderNode { get; set; }
  32. public LinkedListNode<KThread> ProcessListNode { get; set; }
  33. private LinkedList<KThread> _mutexWaiters;
  34. private LinkedListNode<KThread> _mutexWaiterNode;
  35. public KThread MutexOwner { get; private set; }
  36. public int ThreadHandleForUserMutex { get; set; }
  37. private ThreadSchedState _forcePauseFlags;
  38. public KernelResult ObjSyncResult { get; set; }
  39. public int DynamicPriority { get; set; }
  40. public int CurrentCore { get; set; }
  41. public int BasePriority { get; set; }
  42. public int PreferredCore { get; set; }
  43. private long _affinityMaskOverride;
  44. private int _preferredCoreOverride;
  45. private int _affinityOverrideCount;
  46. public ThreadSchedState SchedFlags { get; private set; }
  47. public bool ShallBeTerminated { get; private set; }
  48. public bool SyncCancelled { get; set; }
  49. public bool WaitingSync { get; set; }
  50. private bool _hasExited;
  51. private bool _hasBeenInitialized;
  52. private bool _hasBeenReleased;
  53. public bool WaitingInArbitration { get; set; }
  54. private KScheduler _scheduler;
  55. private KSchedulingData _schedulingData;
  56. public long LastPc { get; set; }
  57. public KThread(Horizon system) : base(system)
  58. {
  59. _scheduler = system.Scheduler;
  60. _schedulingData = system.Scheduler.SchedulingData;
  61. SiblingsPerCore = new LinkedListNode<KThread>[KScheduler.CpuCoresCount];
  62. _mutexWaiters = new LinkedList<KThread>();
  63. }
  64. public KernelResult Initialize(
  65. ulong entrypoint,
  66. ulong argsPtr,
  67. ulong stackTop,
  68. int priority,
  69. int defaultCpuCore,
  70. KProcess owner,
  71. ThreadType type = ThreadType.User)
  72. {
  73. if ((uint)type > 3)
  74. {
  75. throw new ArgumentException($"Invalid thread type \"{type}\".");
  76. }
  77. PreferredCore = defaultCpuCore;
  78. AffinityMask |= 1L << defaultCpuCore;
  79. SchedFlags = type == ThreadType.Dummy
  80. ? ThreadSchedState.Running
  81. : ThreadSchedState.None;
  82. CurrentCore = PreferredCore;
  83. DynamicPriority = priority;
  84. BasePriority = priority;
  85. ObjSyncResult = KernelResult.ThreadNotStarted;
  86. _entrypoint = entrypoint;
  87. if (type == ThreadType.User)
  88. {
  89. if (owner.AllocateThreadLocalStorage(out _tlsAddress) != KernelResult.Success)
  90. {
  91. return KernelResult.OutOfMemory;
  92. }
  93. TlsDramAddress = owner.MemoryManager.GetDramAddressFromVa(_tlsAddress);
  94. MemoryHelper.FillWithZeros(owner.CpuMemory, (long)_tlsAddress, KTlsPageInfo.TlsEntrySize);
  95. }
  96. bool is64Bits;
  97. if (owner != null)
  98. {
  99. Owner = owner;
  100. owner.IncrementReferenceCount();
  101. owner.IncrementThreadCount();
  102. is64Bits = (owner.MmuFlags & 1) != 0;
  103. }
  104. else
  105. {
  106. is64Bits = true;
  107. }
  108. HostThread = new Thread(() => ThreadStart(entrypoint));
  109. Context = new ARMeilleure.State.ExecutionContext();
  110. bool isAarch32 = (Owner.MmuFlags & 1) == 0;
  111. Context.SetX(0, argsPtr);
  112. if (isAarch32)
  113. {
  114. Context.SetX(13, (uint)stackTop);
  115. }
  116. else
  117. {
  118. Context.SetX(31, stackTop);
  119. }
  120. Context.CntfrqEl0 = 19200000;
  121. Context.Tpidr = (long)_tlsAddress;
  122. owner.SubscribeThreadEventHandlers(Context);
  123. ThreadUid = System.GetThreadUid();
  124. _hasBeenInitialized = true;
  125. if (owner != null)
  126. {
  127. owner.AddThread(this);
  128. if (owner.IsPaused)
  129. {
  130. System.CriticalSection.Enter();
  131. if (ShallBeTerminated || SchedFlags == ThreadSchedState.TerminationPending)
  132. {
  133. System.CriticalSection.Leave();
  134. return KernelResult.Success;
  135. }
  136. _forcePauseFlags |= ThreadSchedState.ProcessPauseFlag;
  137. CombineForcePauseFlags();
  138. System.CriticalSection.Leave();
  139. }
  140. }
  141. return KernelResult.Success;
  142. }
  143. public KernelResult Start()
  144. {
  145. if (!System.KernelInitialized)
  146. {
  147. System.CriticalSection.Enter();
  148. if (!ShallBeTerminated && SchedFlags != ThreadSchedState.TerminationPending)
  149. {
  150. _forcePauseFlags |= ThreadSchedState.KernelInitPauseFlag;
  151. CombineForcePauseFlags();
  152. }
  153. System.CriticalSection.Leave();
  154. }
  155. KernelResult result = KernelResult.ThreadTerminating;
  156. System.CriticalSection.Enter();
  157. if (!ShallBeTerminated)
  158. {
  159. KThread currentThread = System.Scheduler.GetCurrentThread();
  160. while (SchedFlags != ThreadSchedState.TerminationPending &&
  161. currentThread.SchedFlags != ThreadSchedState.TerminationPending &&
  162. !currentThread.ShallBeTerminated)
  163. {
  164. if ((SchedFlags & ThreadSchedState.LowMask) != ThreadSchedState.None)
  165. {
  166. result = KernelResult.InvalidState;
  167. break;
  168. }
  169. if (currentThread._forcePauseFlags == ThreadSchedState.None)
  170. {
  171. if (Owner != null && _forcePauseFlags != ThreadSchedState.None)
  172. {
  173. CombineForcePauseFlags();
  174. }
  175. SetNewSchedFlags(ThreadSchedState.Running);
  176. result = KernelResult.Success;
  177. break;
  178. }
  179. else
  180. {
  181. currentThread.CombineForcePauseFlags();
  182. System.CriticalSection.Leave();
  183. System.CriticalSection.Enter();
  184. if (currentThread.ShallBeTerminated)
  185. {
  186. break;
  187. }
  188. }
  189. }
  190. }
  191. System.CriticalSection.Leave();
  192. return result;
  193. }
  194. public void Exit()
  195. {
  196. // TODO: Debug event.
  197. if (Owner != null)
  198. {
  199. Owner.ResourceLimit?.Release(LimitableResource.Thread, 0, 1);
  200. _hasBeenReleased = true;
  201. }
  202. System.CriticalSection.Enter();
  203. _forcePauseFlags &= ~ThreadSchedState.ForcePauseMask;
  204. ExitImpl();
  205. System.CriticalSection.Leave();
  206. DecrementReferenceCount();
  207. }
  208. private void ExitImpl()
  209. {
  210. System.CriticalSection.Enter();
  211. SetNewSchedFlags(ThreadSchedState.TerminationPending);
  212. _hasExited = true;
  213. Signal();
  214. System.CriticalSection.Leave();
  215. }
  216. public KernelResult Sleep(long timeout)
  217. {
  218. System.CriticalSection.Enter();
  219. if (ShallBeTerminated || SchedFlags == ThreadSchedState.TerminationPending)
  220. {
  221. System.CriticalSection.Leave();
  222. return KernelResult.ThreadTerminating;
  223. }
  224. SetNewSchedFlags(ThreadSchedState.Paused);
  225. if (timeout > 0)
  226. {
  227. System.TimeManager.ScheduleFutureInvocation(this, timeout);
  228. }
  229. System.CriticalSection.Leave();
  230. if (timeout > 0)
  231. {
  232. System.TimeManager.UnscheduleFutureInvocation(this);
  233. }
  234. return 0;
  235. }
  236. public void Yield()
  237. {
  238. System.CriticalSection.Enter();
  239. if (SchedFlags != ThreadSchedState.Running)
  240. {
  241. System.CriticalSection.Leave();
  242. System.Scheduler.ContextSwitch();
  243. return;
  244. }
  245. if (DynamicPriority < KScheduler.PrioritiesCount)
  246. {
  247. // Move current thread to the end of the queue.
  248. _schedulingData.Reschedule(DynamicPriority, CurrentCore, this);
  249. }
  250. _scheduler.ThreadReselectionRequested = true;
  251. System.CriticalSection.Leave();
  252. System.Scheduler.ContextSwitch();
  253. }
  254. public void YieldWithLoadBalancing()
  255. {
  256. System.CriticalSection.Enter();
  257. if (SchedFlags != ThreadSchedState.Running)
  258. {
  259. System.CriticalSection.Leave();
  260. System.Scheduler.ContextSwitch();
  261. return;
  262. }
  263. int prio = DynamicPriority;
  264. int core = CurrentCore;
  265. KThread nextThreadOnCurrentQueue = null;
  266. if (DynamicPriority < KScheduler.PrioritiesCount)
  267. {
  268. // Move current thread to the end of the queue.
  269. _schedulingData.Reschedule(prio, core, this);
  270. Func<KThread, bool> predicate = x => x.DynamicPriority == prio;
  271. nextThreadOnCurrentQueue = _schedulingData.ScheduledThreads(core).FirstOrDefault(predicate);
  272. }
  273. IEnumerable<KThread> SuitableCandidates()
  274. {
  275. foreach (KThread thread in _schedulingData.SuggestedThreads(core))
  276. {
  277. int srcCore = thread.CurrentCore;
  278. if (srcCore >= 0)
  279. {
  280. KThread selectedSrcCore = _scheduler.CoreContexts[srcCore].SelectedThread;
  281. if (selectedSrcCore == thread || ((selectedSrcCore?.DynamicPriority ?? 2) < 2))
  282. {
  283. continue;
  284. }
  285. }
  286. // If the candidate was scheduled after the current thread, then it's not worth it,
  287. // unless the priority is higher than the current one.
  288. if (nextThreadOnCurrentQueue.LastScheduledTime >= thread.LastScheduledTime ||
  289. nextThreadOnCurrentQueue.DynamicPriority < thread.DynamicPriority)
  290. {
  291. yield return thread;
  292. }
  293. }
  294. }
  295. KThread dst = SuitableCandidates().FirstOrDefault(x => x.DynamicPriority <= prio);
  296. if (dst != null)
  297. {
  298. _schedulingData.TransferToCore(dst.DynamicPriority, core, dst);
  299. _scheduler.ThreadReselectionRequested = true;
  300. }
  301. if (this != nextThreadOnCurrentQueue)
  302. {
  303. _scheduler.ThreadReselectionRequested = true;
  304. }
  305. System.CriticalSection.Leave();
  306. System.Scheduler.ContextSwitch();
  307. }
  308. public void YieldAndWaitForLoadBalancing()
  309. {
  310. System.CriticalSection.Enter();
  311. if (SchedFlags != ThreadSchedState.Running)
  312. {
  313. System.CriticalSection.Leave();
  314. System.Scheduler.ContextSwitch();
  315. return;
  316. }
  317. int core = CurrentCore;
  318. _schedulingData.TransferToCore(DynamicPriority, -1, this);
  319. KThread selectedThread = null;
  320. if (!_schedulingData.ScheduledThreads(core).Any())
  321. {
  322. foreach (KThread thread in _schedulingData.SuggestedThreads(core))
  323. {
  324. if (thread.CurrentCore < 0)
  325. {
  326. continue;
  327. }
  328. KThread firstCandidate = _schedulingData.ScheduledThreads(thread.CurrentCore).FirstOrDefault();
  329. if (firstCandidate == thread)
  330. {
  331. continue;
  332. }
  333. if (firstCandidate == null || firstCandidate.DynamicPriority >= 2)
  334. {
  335. _schedulingData.TransferToCore(thread.DynamicPriority, core, thread);
  336. selectedThread = thread;
  337. }
  338. break;
  339. }
  340. }
  341. if (selectedThread != this)
  342. {
  343. _scheduler.ThreadReselectionRequested = true;
  344. }
  345. System.CriticalSection.Leave();
  346. System.Scheduler.ContextSwitch();
  347. }
  348. public void SetPriority(int priority)
  349. {
  350. System.CriticalSection.Enter();
  351. BasePriority = priority;
  352. UpdatePriorityInheritance();
  353. System.CriticalSection.Leave();
  354. }
  355. public KernelResult SetActivity(bool pause)
  356. {
  357. KernelResult result = KernelResult.Success;
  358. System.CriticalSection.Enter();
  359. ThreadSchedState lowNibble = SchedFlags & ThreadSchedState.LowMask;
  360. if (lowNibble != ThreadSchedState.Paused && lowNibble != ThreadSchedState.Running)
  361. {
  362. System.CriticalSection.Leave();
  363. return KernelResult.InvalidState;
  364. }
  365. System.CriticalSection.Enter();
  366. if (!ShallBeTerminated && SchedFlags != ThreadSchedState.TerminationPending)
  367. {
  368. if (pause)
  369. {
  370. // Pause, the force pause flag should be clear (thread is NOT paused).
  371. if ((_forcePauseFlags & ThreadSchedState.ThreadPauseFlag) == 0)
  372. {
  373. _forcePauseFlags |= ThreadSchedState.ThreadPauseFlag;
  374. CombineForcePauseFlags();
  375. }
  376. else
  377. {
  378. result = KernelResult.InvalidState;
  379. }
  380. }
  381. else
  382. {
  383. // Unpause, the force pause flag should be set (thread is paused).
  384. if ((_forcePauseFlags & ThreadSchedState.ThreadPauseFlag) != 0)
  385. {
  386. ThreadSchedState oldForcePauseFlags = _forcePauseFlags;
  387. _forcePauseFlags &= ~ThreadSchedState.ThreadPauseFlag;
  388. if ((oldForcePauseFlags & ~ThreadSchedState.ThreadPauseFlag) == ThreadSchedState.None)
  389. {
  390. ThreadSchedState oldSchedFlags = SchedFlags;
  391. SchedFlags &= ThreadSchedState.LowMask;
  392. AdjustScheduling(oldSchedFlags);
  393. }
  394. }
  395. else
  396. {
  397. result = KernelResult.InvalidState;
  398. }
  399. }
  400. }
  401. System.CriticalSection.Leave();
  402. System.CriticalSection.Leave();
  403. return result;
  404. }
  405. public void CancelSynchronization()
  406. {
  407. System.CriticalSection.Enter();
  408. if ((SchedFlags & ThreadSchedState.LowMask) != ThreadSchedState.Paused || !WaitingSync)
  409. {
  410. SyncCancelled = true;
  411. }
  412. else if (Withholder != null)
  413. {
  414. Withholder.Remove(WithholderNode);
  415. SetNewSchedFlags(ThreadSchedState.Running);
  416. Withholder = null;
  417. SyncCancelled = true;
  418. }
  419. else
  420. {
  421. SignaledObj = null;
  422. ObjSyncResult = KernelResult.Cancelled;
  423. SetNewSchedFlags(ThreadSchedState.Running);
  424. SyncCancelled = false;
  425. }
  426. System.CriticalSection.Leave();
  427. }
  428. public KernelResult SetCoreAndAffinityMask(int newCore, long newAffinityMask)
  429. {
  430. System.CriticalSection.Enter();
  431. bool useOverride = _affinityOverrideCount != 0;
  432. // The value -3 is "do not change the preferred core".
  433. if (newCore == -3)
  434. {
  435. newCore = useOverride ? _preferredCoreOverride : PreferredCore;
  436. if ((newAffinityMask & (1 << newCore)) == 0)
  437. {
  438. System.CriticalSection.Leave();
  439. return KernelResult.InvalidCombination;
  440. }
  441. }
  442. if (useOverride)
  443. {
  444. _preferredCoreOverride = newCore;
  445. _affinityMaskOverride = newAffinityMask;
  446. }
  447. else
  448. {
  449. long oldAffinityMask = AffinityMask;
  450. PreferredCore = newCore;
  451. AffinityMask = newAffinityMask;
  452. if (oldAffinityMask != newAffinityMask)
  453. {
  454. int oldCore = CurrentCore;
  455. if (CurrentCore >= 0 && ((AffinityMask >> CurrentCore) & 1) == 0)
  456. {
  457. if (PreferredCore < 0)
  458. {
  459. CurrentCore = HighestSetCore(AffinityMask);
  460. }
  461. else
  462. {
  463. CurrentCore = PreferredCore;
  464. }
  465. }
  466. AdjustSchedulingForNewAffinity(oldAffinityMask, oldCore);
  467. }
  468. }
  469. System.CriticalSection.Leave();
  470. return KernelResult.Success;
  471. }
  472. private static int HighestSetCore(long mask)
  473. {
  474. for (int core = KScheduler.CpuCoresCount - 1; core >= 0; core--)
  475. {
  476. if (((mask >> core) & 1) != 0)
  477. {
  478. return core;
  479. }
  480. }
  481. return -1;
  482. }
  483. private void CombineForcePauseFlags()
  484. {
  485. ThreadSchedState oldFlags = SchedFlags;
  486. ThreadSchedState lowNibble = SchedFlags & ThreadSchedState.LowMask;
  487. SchedFlags = lowNibble | _forcePauseFlags;
  488. AdjustScheduling(oldFlags);
  489. }
  490. private void SetNewSchedFlags(ThreadSchedState newFlags)
  491. {
  492. System.CriticalSection.Enter();
  493. ThreadSchedState oldFlags = SchedFlags;
  494. SchedFlags = (oldFlags & ThreadSchedState.HighMask) | newFlags;
  495. if ((oldFlags & ThreadSchedState.LowMask) != newFlags)
  496. {
  497. AdjustScheduling(oldFlags);
  498. }
  499. System.CriticalSection.Leave();
  500. }
  501. public void ReleaseAndResume()
  502. {
  503. System.CriticalSection.Enter();
  504. if ((SchedFlags & ThreadSchedState.LowMask) == ThreadSchedState.Paused)
  505. {
  506. if (Withholder != null)
  507. {
  508. Withholder.Remove(WithholderNode);
  509. SetNewSchedFlags(ThreadSchedState.Running);
  510. Withholder = null;
  511. }
  512. else
  513. {
  514. SetNewSchedFlags(ThreadSchedState.Running);
  515. }
  516. }
  517. System.CriticalSection.Leave();
  518. }
  519. public void Reschedule(ThreadSchedState newFlags)
  520. {
  521. System.CriticalSection.Enter();
  522. ThreadSchedState oldFlags = SchedFlags;
  523. SchedFlags = (oldFlags & ThreadSchedState.HighMask) |
  524. (newFlags & ThreadSchedState.LowMask);
  525. AdjustScheduling(oldFlags);
  526. System.CriticalSection.Leave();
  527. }
  528. public void AddMutexWaiter(KThread requester)
  529. {
  530. AddToMutexWaitersList(requester);
  531. requester.MutexOwner = this;
  532. UpdatePriorityInheritance();
  533. }
  534. public void RemoveMutexWaiter(KThread thread)
  535. {
  536. if (thread._mutexWaiterNode?.List != null)
  537. {
  538. _mutexWaiters.Remove(thread._mutexWaiterNode);
  539. }
  540. thread.MutexOwner = null;
  541. UpdatePriorityInheritance();
  542. }
  543. public KThread RelinquishMutex(ulong mutexAddress, out int count)
  544. {
  545. count = 0;
  546. if (_mutexWaiters.First == null)
  547. {
  548. return null;
  549. }
  550. KThread newMutexOwner = null;
  551. LinkedListNode<KThread> currentNode = _mutexWaiters.First;
  552. do
  553. {
  554. // Skip all threads that are not waiting for this mutex.
  555. while (currentNode != null && currentNode.Value.MutexAddress != mutexAddress)
  556. {
  557. currentNode = currentNode.Next;
  558. }
  559. if (currentNode == null)
  560. {
  561. break;
  562. }
  563. LinkedListNode<KThread> nextNode = currentNode.Next;
  564. _mutexWaiters.Remove(currentNode);
  565. currentNode.Value.MutexOwner = newMutexOwner;
  566. if (newMutexOwner != null)
  567. {
  568. // New owner was already selected, re-insert on new owner list.
  569. newMutexOwner.AddToMutexWaitersList(currentNode.Value);
  570. }
  571. else
  572. {
  573. // New owner not selected yet, use current thread.
  574. newMutexOwner = currentNode.Value;
  575. }
  576. count++;
  577. currentNode = nextNode;
  578. }
  579. while (currentNode != null);
  580. if (newMutexOwner != null)
  581. {
  582. UpdatePriorityInheritance();
  583. newMutexOwner.UpdatePriorityInheritance();
  584. }
  585. return newMutexOwner;
  586. }
  587. private void UpdatePriorityInheritance()
  588. {
  589. // If any of the threads waiting for the mutex has
  590. // higher priority than the current thread, then
  591. // the current thread inherits that priority.
  592. int highestPriority = BasePriority;
  593. if (_mutexWaiters.First != null)
  594. {
  595. int waitingDynamicPriority = _mutexWaiters.First.Value.DynamicPriority;
  596. if (waitingDynamicPriority < highestPriority)
  597. {
  598. highestPriority = waitingDynamicPriority;
  599. }
  600. }
  601. if (highestPriority != DynamicPriority)
  602. {
  603. int oldPriority = DynamicPriority;
  604. DynamicPriority = highestPriority;
  605. AdjustSchedulingForNewPriority(oldPriority);
  606. if (MutexOwner != null)
  607. {
  608. // Remove and re-insert to ensure proper sorting based on new priority.
  609. MutexOwner._mutexWaiters.Remove(_mutexWaiterNode);
  610. MutexOwner.AddToMutexWaitersList(this);
  611. MutexOwner.UpdatePriorityInheritance();
  612. }
  613. }
  614. }
  615. private void AddToMutexWaitersList(KThread thread)
  616. {
  617. LinkedListNode<KThread> nextPrio = _mutexWaiters.First;
  618. int currentPriority = thread.DynamicPriority;
  619. while (nextPrio != null && nextPrio.Value.DynamicPriority <= currentPriority)
  620. {
  621. nextPrio = nextPrio.Next;
  622. }
  623. if (nextPrio != null)
  624. {
  625. thread._mutexWaiterNode = _mutexWaiters.AddBefore(nextPrio, thread);
  626. }
  627. else
  628. {
  629. thread._mutexWaiterNode = _mutexWaiters.AddLast(thread);
  630. }
  631. }
  632. private void AdjustScheduling(ThreadSchedState oldFlags)
  633. {
  634. if (oldFlags == SchedFlags)
  635. {
  636. return;
  637. }
  638. if (oldFlags == ThreadSchedState.Running)
  639. {
  640. // Was running, now it's stopped.
  641. if (CurrentCore >= 0)
  642. {
  643. _schedulingData.Unschedule(DynamicPriority, CurrentCore, this);
  644. }
  645. for (int core = 0; core < KScheduler.CpuCoresCount; core++)
  646. {
  647. if (core != CurrentCore && ((AffinityMask >> core) & 1) != 0)
  648. {
  649. _schedulingData.Unsuggest(DynamicPriority, core, this);
  650. }
  651. }
  652. }
  653. else if (SchedFlags == ThreadSchedState.Running)
  654. {
  655. // Was stopped, now it's running.
  656. if (CurrentCore >= 0)
  657. {
  658. _schedulingData.Schedule(DynamicPriority, CurrentCore, this);
  659. }
  660. for (int core = 0; core < KScheduler.CpuCoresCount; core++)
  661. {
  662. if (core != CurrentCore && ((AffinityMask >> core) & 1) != 0)
  663. {
  664. _schedulingData.Suggest(DynamicPriority, core, this);
  665. }
  666. }
  667. }
  668. _scheduler.ThreadReselectionRequested = true;
  669. }
  670. private void AdjustSchedulingForNewPriority(int oldPriority)
  671. {
  672. if (SchedFlags != ThreadSchedState.Running)
  673. {
  674. return;
  675. }
  676. // Remove thread from the old priority queues.
  677. if (CurrentCore >= 0)
  678. {
  679. _schedulingData.Unschedule(oldPriority, CurrentCore, this);
  680. }
  681. for (int core = 0; core < KScheduler.CpuCoresCount; core++)
  682. {
  683. if (core != CurrentCore && ((AffinityMask >> core) & 1) != 0)
  684. {
  685. _schedulingData.Unsuggest(oldPriority, core, this);
  686. }
  687. }
  688. // Add thread to the new priority queues.
  689. KThread currentThread = _scheduler.GetCurrentThread();
  690. if (CurrentCore >= 0)
  691. {
  692. if (currentThread == this)
  693. {
  694. _schedulingData.SchedulePrepend(DynamicPriority, CurrentCore, this);
  695. }
  696. else
  697. {
  698. _schedulingData.Schedule(DynamicPriority, CurrentCore, this);
  699. }
  700. }
  701. for (int core = 0; core < KScheduler.CpuCoresCount; core++)
  702. {
  703. if (core != CurrentCore && ((AffinityMask >> core) & 1) != 0)
  704. {
  705. _schedulingData.Suggest(DynamicPriority, core, this);
  706. }
  707. }
  708. _scheduler.ThreadReselectionRequested = true;
  709. }
  710. private void AdjustSchedulingForNewAffinity(long oldAffinityMask, int oldCore)
  711. {
  712. if (SchedFlags != ThreadSchedState.Running || DynamicPriority >= KScheduler.PrioritiesCount)
  713. {
  714. return;
  715. }
  716. // Remove thread from the old priority queues.
  717. for (int core = 0; core < KScheduler.CpuCoresCount; core++)
  718. {
  719. if (((oldAffinityMask >> core) & 1) != 0)
  720. {
  721. if (core == oldCore)
  722. {
  723. _schedulingData.Unschedule(DynamicPriority, core, this);
  724. }
  725. else
  726. {
  727. _schedulingData.Unsuggest(DynamicPriority, core, this);
  728. }
  729. }
  730. }
  731. // Add thread to the new priority queues.
  732. for (int core = 0; core < KScheduler.CpuCoresCount; core++)
  733. {
  734. if (((AffinityMask >> core) & 1) != 0)
  735. {
  736. if (core == CurrentCore)
  737. {
  738. _schedulingData.Schedule(DynamicPriority, core, this);
  739. }
  740. else
  741. {
  742. _schedulingData.Suggest(DynamicPriority, core, this);
  743. }
  744. }
  745. }
  746. _scheduler.ThreadReselectionRequested = true;
  747. }
  748. public void SetEntryArguments(long argsPtr, int threadHandle)
  749. {
  750. Context.SetX(0, (ulong)argsPtr);
  751. Context.SetX(1, (ulong)threadHandle);
  752. }
  753. public void TimeUp()
  754. {
  755. ReleaseAndResume();
  756. }
  757. public string GetGuestStackTrace()
  758. {
  759. return Owner.Debugger.GetGuestStackTrace(Context);
  760. }
  761. public void PrintGuestStackTrace()
  762. {
  763. StringBuilder trace = new StringBuilder();
  764. trace.AppendLine("Guest stack trace:");
  765. trace.AppendLine(GetGuestStackTrace());
  766. Logger.PrintInfo(LogClass.Cpu, trace.ToString());
  767. }
  768. public void Execute()
  769. {
  770. if (Interlocked.CompareExchange(ref _hostThreadRunning, 1, 0) == 0)
  771. {
  772. HostThread.Start();
  773. }
  774. }
  775. private void ThreadStart(ulong entrypoint)
  776. {
  777. Owner.Translator.Execute(Context, entrypoint);
  778. ThreadExit();
  779. }
  780. private void ThreadExit()
  781. {
  782. System.Scheduler.ExitThread(this);
  783. System.Scheduler.RemoveThread(this);
  784. }
  785. public bool IsCurrentHostThread()
  786. {
  787. return Thread.CurrentThread == HostThread;
  788. }
  789. public override bool IsSignaled()
  790. {
  791. return _hasExited;
  792. }
  793. protected override void Destroy()
  794. {
  795. if (_hasBeenInitialized)
  796. {
  797. FreeResources();
  798. bool released = Owner != null || _hasBeenReleased;
  799. if (Owner != null)
  800. {
  801. Owner.ResourceLimit?.Release(LimitableResource.Thread, 1, released ? 0 : 1);
  802. Owner.DecrementReferenceCount();
  803. }
  804. else
  805. {
  806. System.ResourceLimit.Release(LimitableResource.Thread, 1, released ? 0 : 1);
  807. }
  808. }
  809. }
  810. private void FreeResources()
  811. {
  812. Owner?.RemoveThread(this);
  813. if (_tlsAddress != 0 && Owner.FreeThreadLocalStorage(_tlsAddress) != KernelResult.Success)
  814. {
  815. throw new InvalidOperationException("Unexpected failure freeing thread local storage.");
  816. }
  817. System.CriticalSection.Enter();
  818. // Wake up all threads that may be waiting for a mutex being held by this thread.
  819. foreach (KThread thread in _mutexWaiters)
  820. {
  821. thread.MutexOwner = null;
  822. thread._preferredCoreOverride = 0;
  823. thread.ObjSyncResult = KernelResult.InvalidState;
  824. thread.ReleaseAndResume();
  825. }
  826. System.CriticalSection.Leave();
  827. Owner?.DecrementThreadCountAndTerminateIfZero();
  828. }
  829. }
  830. }