- 016C026DBE4230BD120C0FC4269E61BD8A44B82580289EFC90FED0792B5893A5727E069191FBFB0E32C3C40D2700B4A39A5ACB0BE1FDBFC475274C344368626A+ 19593ABC66AB9FF8A02FA39884524BBA012AAF3AFAB4C0A588272D07B9269BB59140F584519317BA0C2F412FC0B47D31CD4FEA28D3D6754A3F5BBF7BACCA3E78m/ram.asm(93 . 6)(93 . 16)
366 ;-----------------------------------------------------------------------------
367
368 ;-----------------------------------------------------------------------------
369 section .bss
370 align 32
371 TLB_TAG_BYTE_0_COPY resb 16 ; Byte-0 of each TLB entry Tag
372 TLB_TAG_BYTE_1_COPY resb 16 ; Byte-1 of each TLB entry Tag
373 TLB_TAG_BYTE_2_COPY resb 16 ; Byte-2 of each TLB entry Tag
374 TLB_ASID_COPY resb 16 ; ASID of each TLB entry
375 section .text
376 ;-----------------------------------------------------------------------------
377
378 ;-----------------------------------------------------------------------------
379 ; Virt2Phys Read : virtual address in eax; output (physical addr) in eax
380 ;-----------------------------------------------------------------------------
381 align GRAIN, db 0x90
(122 . 99)(132 . 147)
383 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
384 ;; Save ebx, ecx, edx, AUX, to xmm ;;
385 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
386 movd xmm0, ebx
387 movd xmm1, ecx
388 movd xmm2, edx
389 movd xmm3, AUX
390 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
391 mov ecx, eax ; ecx := eax (vAddr)
392 and ecx, 0xFFFFF000 ; ecx := ecx & 0xFFFFF000
393 shr ecx, 13 ; ecx := ecx >> 13 (get vAddr's Tag)
394 movd xmm0, ebx
395 movd xmm1, ecx
396 movd xmm2, edx
397 movd xmm3, AUX
398 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
399 ;; Find out whether we actually must do the lookup, or can use cached:
400 Flg_Get TLB_Rd_Cache_Valid ; Is Read TLB Cache valid?
401 jnc .Lookup_Must ; If Read TLB Cache invalid -- must!
402 bt eax, 12 ; Test odd/even junior bit
403 jc .Rd_Cached_Odd ; If odd, look at last Odd vAddr Tag
404 .Rd_Cached_Even: ; If even, look at last Even vAddr Tag
405 movd edx, Rd_E_Last_Tag ; edx := last Even vAddr's Tag
406 cmp ecx, edx ; is the current vAddr's Tag equal?
407 jne .Lookup_Must ; ... if not, must do the lookup dance;
408 ;; ... Otherwise, we have an Even cache hit:
409 movd ebx, Rd_E_Last_PFN ; ebx := last good Even PFN
410 jmp .Cache_Hit ; apply the PFN and wrap up.
411 .Rd_Cached_Odd:
412 movd edx, Rd_O_Last_Tag ; edx := last Odd vAddr's Tag
413 cmp ecx, edx ; is the current vAddr's Tag equal?
414 jne .Lookup_Must ; ... if not, must do the lookup dance;
415 ;; ... Otherwise, we have an Odd cache hit:
416 movd ebx, Rd_O_Last_PFN ; ebx := last good Odd PFN
417 jmp .Cache_Hit ; apply the PFN and wrap up.
418 mov ecx, eax ; ecx := eax (vAddr)
419 and ecx, 0xFFFFF000 ; ecx := ecx & 0xFFFFF000
420 shr ecx, 13 ; ecx := ecx >> 13 (get vAddr's Tag)
421 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
422 ;; Failing the above, we must actually walk the TLB:
423 ; Find out whether we actually must do the lookup, or can use cached:
424 Flg_Get TLB_Rd_Cache_Valid ; Is Read TLB Cache valid?
425 jnc .Lookup_Must ; If Read TLB Cache invalid -- must!
426 ; If cache is valid, lookup:
427 mov AUX, ecx ; AUX := tag
428 xor ecx, ecx ; ecx := 0
429 bt eax, 12 ; Test vAddr's odd/even junior bit
430 setc cl ; ecx := {1 if a-odd, 0 if a-even}
431 shl rcx, 6 ; rcx := {64 if a-odd, 0 if a-even}
432 ; get the last-good-Tags:
433 movq rbx, R_TLB_Last_Good_Tag ; Get last good R-Tag pair
434 shr rbx, cl ; if arity is odd, get top half
435 cmp ebx, AUX ; is current Tag == to last-good ?
436 jne .Lookup_Must ; ... if not, go to Lookup_Must
437 ; given Tag matched last-good. So get last-good PFN and wrap up:
438 movq rbx, R_TLB_Last_Good_PFN ; Get last good PFN pair
439 shr rbx, cl ; if arity is odd, get top half
440 jmp .PFN_And_Done ; use ebx as the PFN and wrap up.
441 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
442 .Lookup_Must:
443 movd xmm4, ecx ; xmm4 := current vAddr's Tag
444 ;; Get the active ASID:
445 mov edx, Sr(CP0_EntryHi) ; edx := CP0_EntryHi
446 and edx, 0xFF ; edx := edx & 0xFF (get current ASID)
447 ;; For each slot in table (0 .. 15), attempt lookup
448 xor AUX, AUX ; Start with the 0-th entry in table
449 .Lookup_TLB_E:
450 movd ecx, xmm4 ; ecx := current vAddr's Tag
451 mov ebx, TLB_E(AUX) ; ebx := current TLB entry
452 and ebx, TLB_VPN2_Mask ; get VPN2 of this entry
453 cmp ebx, ecx ; cmp(entry.VPN2, vAddr.tag)
454 jne .Lookup_TLB_E_Not_Here ; if entry.VPN2 != vAddr.tag: no match
455 mov ebx, TLB_E(AUX) ; ebx := current TLB entry
456 bt ebx, TLB_G ; is entry.G = 1?
457 jc .Lookup_TLB_E_Match ; then match.
458 shr ebx, TLB_ASID_Shift ; ebx := ebx >> TLB_ASID_Shift
459 and ebx, TLB_ASID_Mask ; ebx := entry.ASID
460 cmp ebx, edx ; entry.ASID = current ASID ?
461 jne .Lookup_TLB_E_Not_Here ; if neither G=1 nor ASID match.
462 mov ebx, TLB_E(AUX) ; ebx := current TLB entry
463 .Lookup_TLB_E_Match: ; TLB Match:
464 bt eax, 12 ; Test odd/even junior bit
465 jc .Lookup_TLB_E_Match_Odd ; If odd: test V1, D1
466 .Lookup_TLB_E_Match_Even: ; If even: test V0, D0
467 bt ebx, TLB_V0 ; Is entry.V0=1 ?
468 jnc .Lookup_TLB_E_Invalid ; If not, TLBRET_INVALID
469 lea ecx, TLB_PFN_E(AUX) ; prepare to load even PFN entry
470 mov ebx, dword [ecx] ; Actually load the current PFN entry
471 movd Rd_E_Last_PFN, ebx ; Save the current PFN as last Even
472 movd ecx, xmm4 ; ecx := the current Tag
473 movd Rd_E_Last_Tag, ecx ; Save the current Tag as last Even
474 jmp .Lookup_TLB_E_Match_Yes ; Since we're reading: go to Match Yes
475 .Lookup_TLB_E_Match_Odd: ; Odd bit:
476 bt ebx, TLB_V1 ; Is entry.V1=1 ?
477 jnc .Lookup_TLB_E_Invalid ; If not, TLBRET_INVALID
478 lea ecx, TLB_PFN_O(AUX) ; prepare to load odd PFN entry
479 mov ebx, dword [ecx] ; Actually load the current PFN entry
480 movd Rd_O_Last_PFN, ebx ; Save the current PFN as last Odd
481 movd ecx, xmm4 ; ecx := the current Tag
482 movd Rd_O_Last_Tag, ecx ; Save the current Tag as last Odd
483 .Lookup_TLB_E_Match_Yes: ; This is the 'success' case
484 Flg_On TLB_Rd_Cache_Valid
485 ; Upon next TLB lookup, if cache is valid, and Tag remains same
486 ; as before, we can use the same PFN as was obtained last time
487 ; for the respective 12th bit arity of the vAddr!
488 .Cache_Hit:
489 and eax, 0xFFF ; vAddr := vAddr & 0xFFF
490 or eax, ebx ; vAddr := vAddr | entry.PFN[lowbit]
491 jmp _Lookup_TLB_Done ; vAddr is now correct pAddr, done.
492 .Lookup_TLB_E_Not_Here: ; try next one in the table, if any
493 inc AUX ; index := index + 1
494 cmp AUX, TLB_ENTRIES_COUNT ; see if still in range 0 .. n-1
495 jb .Lookup_TLB_E ; if in range, go to next entry
496 ;; ... else:
497 add rsp, 16 ; squelch return to _Virt_xxx and its caller
498 push _Handle_Exception_TLB_NoMatch ; 'return' straight to handler.
499 jmp _Lookup_TLB_E_WriteExtr ; Wrap up
500 .Lookup_TLB_E_Invalid:
501 SetEXC EXC_TLBL ; Set the EXC_TLBL Exception
502 add rsp, 16 ; squelch return to _Virt_xxx and its caller
503 push _Handle_Exception_Other ; 'return' straight to handler.
504 jmp _Lookup_TLB_E_WriteExtr ; Go to the common epilogue.
505 movd xmm4, ecx ; ecx := copy of Tag
506 ;; Search for B0, B1, B2 of Tag, accumulate result in ebx ;;
507 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
508 ; Search for Byte 0 of Tag:
509 mov edx, ecx ; edx := ecx (wanted Tag)
510 and edx, 0xFF ; Byte 0 (lowest) of wanted Tag
511 ; Fill T0 with 16 copies of Tag Byte 0:
512 movd XMM_T0, edx
513 punpcklbw XMM_T0, XMM_T0
514 punpcklwd XMM_T0, XMM_T0
515 pshufd XMM_T0, XMM_T0, 0
516 ; Now SIMD-compare:
517 pcmpeqb XMM_T0, TLB_TAG_BYTE_0
518 ; Get the result mask of the compare:
519 pmovmskb ebx, XMM_T0 ; i-th bit in ebx = 1 where match B0
520 test ebx, ebx ; if Byte 0 of Tag not found:
521 jz .Not_Found ; ... then go straight to 'not found'
522 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
523 ; Search for Byte 1 of Tag:
524 mov edx, ecx ; edx := ecx (wanted Tag)
525 shr edx, 8 ; Byte 1 (middle) of wanted Tag
526 and edx, 0xFF
527 ; Fill T0 with 16 copies of Tag Byte 1:
528 movd XMM_T0, edx
529 punpcklbw XMM_T0, XMM_T0
530 punpcklwd XMM_T0, XMM_T0
531 pshufd XMM_T0, XMM_T0, 0
532 ; Now SIMD-compare:
533 pcmpeqb XMM_T0, TLB_TAG_BYTE_1
534 ; Get the result mask of the compare:
535 pmovmskb edx, XMM_T0 ; i-th bit in edx = 1 where match B1
536 and ebx, edx ; Keep only where B0 also matched
537 test ebx, ebx ; if Bytes 0+1 of Tag not found:
538 jz .Not_Found ; ... then go straight to 'not found'
539 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
540 ; Search for Byte 2 of Tag:
541 mov edx, ecx ; eax := edx (wanted Tag)
542 shr edx, 16 ; Byte 2 (top) of wanted Tag
543 and edx, 0xFF
544 ; Fill T0 with 16 copies of Tag Byte 2:
545 movd XMM_T0, edx
546 punpcklbw XMM_T0, XMM_T0
547 punpcklwd XMM_T0, XMM_T0
548 pshufd XMM_T0, XMM_T0, 0
549 ; Now SIMD-compare:
550 pcmpeqb XMM_T0, TLB_TAG_BYTE_2
551 ; Get the result mask of the compare:
552 pmovmskb edx, XMM_T0 ; i-th bit in edx = 1 where match B2
553 and ebx, edx ; Keep only where B0,B1 also matched
554 test ebx, ebx ; if Bytes 0+1+2 of Tag not found:
555 jz .Not_Found ; ... then go straight to 'not found'
556 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
557 ; If we're here, Tag WAS found; so get the TLB index where it lies :
558 bsf AUX, ebx ; AUX := index of found TLB entry
559 mov edx, AUX ; edx := AUX
560 ; Now see whether the corresponding G flag is set:
561 add edx, 16 ; G's start at bit 16 of Flag_Reg
562 bt Flag_Reg, edx ; See whether i-th G bit is set
563 jc .Match ; Now if G is set, we've found it!
564 ; G was not set, so get the requested ASID and test whether it matches:
565 ; Get the active ASID:
566 mov ebx, Sr(CP0_EntryHi); ebx := CP0_EntryHi
567 and ebx, 0xFF ; ebx := ebx & 0xFF (get current ASID)
568 ; Determine whether it matches the found Tag entry's :
569 mov ecx, AUX ; ecx := AUX (index of found entry)
570 lea rdx, [TLB_ASID_COPY]; Load address of ASID Copy
571 cmp byte [rdx + rcx], bl ; Compare stored ASID to current
572 jne .Not_Found ; ... if not equal, 'not found'
573 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
574 .Match:
575 ; If we're here, we have a Match. AUX is index of matching entry;
576 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
577 xor edx, edx ; edx := 0
578 ; Get arity of desired entry:
579 bt eax, 12 ; Test vAddr's odd/even junior bit
580 setc dl ; edx := {1 if a-odd, 0 if a-even}
581 shl edx, 5 ; edx := {32 if a-odd, 0 if a-even}
582 mov ecx, edx ; ebx := edx (copy of above)
583 ; Now we know which bit of TLB_Flags is this entry's V. Test it:
584 add edx, AUX ; add the corresponding index
585 bt TLB_Flags, rdx ; test if V(Index) is set
586 jnc .Invalid_R ; ... V == 0, then go to Invalid
587 ; Now let's load the PFN:
588 mov rbx, TLB_PFN(AUX64) ; load the PFN pair to rbx
589 ; ebx is now the PFN. Before wrapping up, update the TLB read cache :
590 movq R_TLB_Last_Good_PFN, rbx ; Set last good PFN to this PFN:
591 ; now leave only the correct half of PFN, at bottom of rbx:
592 shr rbx, cl ; if arity is odd, get upper 32bit
593 ; set correct half of R_TLB_Last_Good_Tag to the found Tag:
594 mov rdx, 0xFFFFFFFF00000000 ; rdx := 0xFFFFFFFF00000000
595 shr rdx, cl ; if arity is odd, keep bottom
596 movq AUX64, R_TLB_Last_Good_Tag ; get last good Tag
597 and AUX64, rdx ; zap correct half of last good tag
598 movq rdx, xmm4 ; get the Tag again :
599 shl rdx, cl ; if arity if odd, slide into pos:
600 or AUX64, rdx ; now or it into place
601 movq R_TLB_Last_Good_Tag, AUX64 ; update last good Tag.
602 .PFN_And_Done:
603 and eax, 0xFFF ; vAddr := vAddr & 0xFFF
604 or eax, ebx ; vAddr := vAddr | entry.PFN[lowbit]
605 jmp _Lookup_TLB_Done ; vAddr is now correct pAddr, done.
606 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
607 .Not_Found: ; Not Found in Table:
608 add rsp, 16 ; squelch return to _Virt_xxx and its caller
609 push _Handle_Exception_TLB_NoMatch ; 'return' straight to handler.
610 jmp _Lookup_TLB_E_WriteExtr ; Wrap up
611 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
612 .Invalid_R:
613 SetEXC EXC_TLBL ; Set the EXC_TLBL Exception
614 add rsp, 16 ; squelch return to _Virt_xxx and its caller
615 push _Handle_Exception_Other ; 'return' straight to handler.
616 jmp _Lookup_TLB_E_WriteExtr ; Go to the common epilogue.
617 ;-----------------------------------------------------------------------------
618
619 ;-----------------------------------------------------------------------------
(247 . 111)(305 . 158)
621 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
622 ;; Save ebx, ecx, edx, AUX, to xmm ;;
623 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
624 movd xmm0, ebx
625 movd xmm1, ecx
626 movd xmm2, edx
627 movd xmm3, AUX
628 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
629 mov ecx, eax ; ecx := eax (vAddr)
630 and ecx, 0xFFFFF000 ; ecx := ecx & 0xFFFFF000
631 shr ecx, 13 ; ecx := ecx >> 13 (get vAddr's Tag)
632 movd xmm0, ebx
633 movd xmm1, ecx
634 movd xmm2, edx
635 movd xmm3, AUX
636 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
637 ;; Find out whether we actually must do the lookup, or can use cached:
638 Flg_Get TLB_Wr_Cache_Valid ; Is Write TLB Cache valid?
639 jnc .Lookup_Must ; If Write TLB Cache invalid -- must!
640 bt eax, 12 ; Test odd/even junior bit
641 jc .Wr_Cached_Odd ; If odd, look at last Odd vAddr Tag
642 .Wr_Cached_Even: ; If even, look at last Even vAddr Tag
643 movd edx, Wr_E_Last_Tag ; edx := last Even vAddr's Tag
644 cmp ecx, edx ; is the current vAddr's Tag equal?
645 jne .Lookup_Must ; ... if not, must do the lookup dance;
646 ;; ... Otherwise, we have an Even cache hit:
647 movd ebx, Wr_E_Last_PFN ; ebx := last good Even PFN
648 jmp .Cache_Hit ; apply the PFN and wrap up.
649 .Wr_Cached_Odd:
650 movd edx, Wr_O_Last_Tag ; edx := last Odd vAddr's Tag
651 cmp ecx, edx ; is the current vAddr's Tag equal?
652 jne .Lookup_Must ; ... if not, must do the lookup dance;
653 ;; ... Otherwise, we have an Odd cache hit:
654 movd ebx, Wr_O_Last_PFN ; ebx := last good Odd PFN
655 jmp .Cache_Hit ; apply the PFN and wrap up.
656 mov ecx, eax ; ecx := eax (vAddr)
657 and ecx, 0xFFFFF000 ; ecx := ecx & 0xFFFFF000
658 shr ecx, 13 ; ecx := ecx >> 13 (get vAddr's Tag)
659 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
660 ;; Failing the above, we must actually walk the TLB:
661 ; Find out whether we actually must do the lookup, or can use cached:
662 Flg_Get TLB_Wr_Cache_Valid ; Is Write TLB Cache valid?
663 jnc .Lookup_Must ; If Write TLB Cache invalid -- must!
664 ; If cache is valid, lookup:
665 mov AUX, ecx ; AUX := tag
666 xor ecx, ecx ; ecx := 0
667 bt eax, 12 ; Test vAddr's odd/even junior bit
668 setc cl ; ecx := {1 if a-odd, 0 if a-even}
669 shl rcx, 6 ; rcx := {64 if a-odd, 0 if a-even}
670 ; get the last-good-Tags:
671 movq rbx, W_TLB_Last_Good_Tag ; Get last good W-Tag pair
672 shr rbx, cl ; if arity is odd, get top half
673 cmp ebx, AUX ; is current Tag == to last-good ?
674 jne .Lookup_Must ; ... if not, go to Lookup_Must
675 ; given Tag matched last-good. So get last-good PFN and wrap up:
676 movq rbx, W_TLB_Last_Good_PFN ; Get last good PFN pair
677 shr rbx, cl ; if arity is odd, get top half
678 jmp .PFN_And_Done ; use ebx as the PFN and wrap up.
679 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
680 .Lookup_Must:
681 movd xmm4, ecx ; xmm4 := current vAddr's Tag
682 ;; Get the active ASID:
683 mov edx, Sr(CP0_EntryHi) ; edx := CP0_EntryHi
684 and edx, 0xFF ; edx := edx & 0xFF (get current ASID)
685 ;; For each slot in table (0 .. 15), attempt lookup
686 xor AUX, AUX ; Start with the 0-th entry in table
687 .Lookup_TLB_E:
688 movd ecx, xmm4 ; ecx := current vAddr's Tag
689 mov ebx, TLB_E(AUX) ; ebx := current TLB entry
690 and ebx, TLB_VPN2_Mask ; get VPN2 of this entry
691 cmp ebx, ecx ; cmp(entry.VPN2, vAddr.tag)
692 jne .Lookup_TLB_E_Not_Here ; if entry.VPN2 != vAddr.tag: no match
693 mov ebx, TLB_E(AUX) ; ebx := current TLB entry
694 bt ebx, TLB_G ; is entry.G = 1?
695 jc .Lookup_TLB_E_Match ; then match.
696 shr ebx, TLB_ASID_Shift ; ebx := ebx >> TLB_ASID_Shift
697 and ebx, TLB_ASID_Mask ; ebx := entry.ASID
698 cmp ebx, edx ; entry.ASID = current ASID ?
699 jne .Lookup_TLB_E_Not_Here ; if neither G=1 nor ASID match.
700 mov ebx, TLB_E(AUX) ; ebx := current TLB entry
701 .Lookup_TLB_E_Match: ; TLB Match:
702 bt eax, 12 ; Test odd/even junior bit
703 jc .Lookup_TLB_E_Match_Odd ; If odd: test V1, D1
704 .Lookup_TLB_E_Match_Even: ; If even: test V0, D0
705 bt ebx, TLB_V0 ; Is entry.V0=1 ?
706 jnc .Lookup_TLB_E_Invalid ; If not, TLBRET_INVALID
707 bt ebx, TLB_D0 ; Is entry.D0=1 ?
708 jnc .Lookup_TLB_E_Dirty ; If not, go to 'Dirty'
709 ;; Not invalid or dirty:
710 lea ecx, TLB_PFN_E(AUX) ; prepare to load even PFN entry
711 mov ebx, dword [ecx] ; Actually load the current PFN entry
712 movd Wr_E_Last_PFN, ebx ; Save the current PFN as last Even
713 movd ecx, xmm4 ; ecx := the current Tag
714 movd Wr_E_Last_Tag, ecx ; Save the current Tag as last Even
715 jmp .Lookup_TLB_E_Match_Yes ; ;; Proceed to 'match' :
716 .Lookup_TLB_E_Match_Odd: ; Odd bit:
717 bt ebx, TLB_V1 ; Is entry.V1=1 ?
718 jnc .Lookup_TLB_E_Invalid ; If not, TLBRET_INVALID
719 bt ebx, TLB_D1 ; Is entry.D1=1 ?
720 jnc .Lookup_TLB_E_Dirty ; If not, go to 'Dirty'
721 ;; Not invalid or dirty:
722 lea ecx, TLB_PFN_O(AUX) ; prepare to load odd PFN entry
723 mov ebx, dword [ecx] ; Actually load the current PFN entry
724 movd Wr_O_Last_PFN, ebx ; Save the current PFN as last Odd
725 movd ecx, xmm4 ; ecx := the current Tag
726 movd Wr_O_Last_Tag, ecx ; Save the current Tag as last Odd
727 ;; Proceed to 'match' :
728 .Lookup_TLB_E_Match_Yes: ; This is the 'success' case
729 Flg_On TLB_Wr_Cache_Valid
730 ; Upon next TLB lookup, if cache is valid, and Tag remains same
731 ; as before, we can use the same PFN as was obtained last time
732 ; for the respective 12th bit arity of the vAddr!
733 .Cache_Hit:
734 and eax, 0xFFF ; vAddr := vAddr & 0xFFF
735 or eax, ebx ; vAddr := vAddr | entry.PFN[lowbit]
736 jmp _Lookup_TLB_Done ; vAddr is now correct pAddr, done.
737 .Lookup_TLB_E_Not_Here: ; try next one in the table, if any
738 inc AUX ; index := index + 1
739 cmp AUX, TLB_ENTRIES_COUNT ; see if still in range 0 .. n-1
740 jb .Lookup_TLB_E ; if in range, go to next entry
741 ;; ... else:
742 add rsp, 16 ; squelch return to _Virt_xxx and its caller
743 push _Handle_Exception_TLB_NoMatch ; 'return' straight to handler.
744 jmp _Lookup_TLB_E_WriteExtr ; Wrap up
745 .Lookup_TLB_E_Dirty: ; ... else, Dirty:
746 SetEXC EXC_Mod ; Set the EXC_Mod Exception
747 add rsp, 16 ; squelch return to _Virt_xxx and its caller
748 push _Handle_Exception_Other ; 'return' straight to handler.
749 jmp _Lookup_TLB_E_WriteExtr ; Write the 'extra data' and finish.
750 .Lookup_TLB_E_Invalid: ; Invalid Write:
751 SetEXC EXC_TLBS ; Set the EXC_TLBS Exception
752 add rsp, 16 ; squelch return to _Virt_xxx and its caller
753 push _Handle_Exception_Other ; 'return' straight to handler.
754 ;; then drop down to _Lookup_TLB_E_WriteExtr
755 movd xmm4, ecx ; ecx := copy of Tag
756 ;; Search for B0, B1, B2 of Tag, accumulate result in ebx ;;
757 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
758 ; Search for Byte 0 of Tag:
759 mov edx, ecx ; edx := ecx (wanted Tag)
760 and edx, 0xFF ; Byte 0 (lowest) of wanted Tag
761 ; Fill T0 with 16 copies of Tag Byte 0:
762 movd XMM_T0, edx
763 punpcklbw XMM_T0, XMM_T0
764 punpcklwd XMM_T0, XMM_T0
765 pshufd XMM_T0, XMM_T0, 0
766 ; Now SIMD-compare:
767 pcmpeqb XMM_T0, TLB_TAG_BYTE_0
768 ; Get the result mask of the compare:
769 pmovmskb ebx, XMM_T0 ; i-th bit in ebx = 1 where match B0
770 test ebx, ebx ; if Byte 0 of Tag not found:
771 jz .Not_Found ; ... then go straight to 'not found'
772 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
773 ; Search for Byte 1 of Tag:
774 mov edx, ecx ; edx := ecx (wanted Tag)
775 shr edx, 8 ; Byte 1 (middle) of wanted Tag
776 and edx, 0xFF
777 ; Fill T0 with 16 copies of Tag Byte 1:
778 movd XMM_T0, edx
779 punpcklbw XMM_T0, XMM_T0
780 punpcklwd XMM_T0, XMM_T0
781 pshufd XMM_T0, XMM_T0, 0
782 ; Now SIMD-compare:
783 pcmpeqb XMM_T0, TLB_TAG_BYTE_1
784 ; Get the result mask of the compare:
785 pmovmskb edx, XMM_T0 ; i-th bit in edx = 1 where match B1
786 and ebx, edx ; Keep only where B0 also matched
787 test ebx, ebx ; if Bytes 0+1 of Tag not found:
788 jz .Not_Found ; ... then go straight to 'not found'
789 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
790 ; Search for Byte 2 of Tag:
791 mov edx, ecx ; eax := edx (wanted Tag)
792 shr edx, 16 ; Byte 2 (top) of wanted Tag
793 and edx, 0xFF
794 ; Fill T0 with 16 copies of Tag Byte 2:
795 movd XMM_T0, edx
796 punpcklbw XMM_T0, XMM_T0
797 punpcklwd XMM_T0, XMM_T0
798 pshufd XMM_T0, XMM_T0, 0
799 ; Now SIMD-compare:
800 pcmpeqb XMM_T0, TLB_TAG_BYTE_2
801 ; Get the result mask of the compare:
802 pmovmskb edx, XMM_T0 ; i-th bit in edx = 1 where match B2
803 and ebx, edx ; Keep only where B0,B1 also matched
804 test ebx, ebx ; if Bytes 0+1+2 of Tag not found:
805 jz .Not_Found ; ... then go straight to 'not found'
806 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
807 ; If we're here, Tag WAS found; so get the TLB index where it lies :
808 bsf AUX, ebx ; AUX := index of found TLB entry
809 mov edx, AUX ; edx := AUX
810 ; Now see whether the corresponding G flag is set:
811 add edx, 16 ; G's start at bit 16 of Flag_Reg
812 bt Flag_Reg, edx ; See whether i-th G bit is set
813 jc .Match ; Now if G is set, we've found it!
814 ; G was not set, so get the requested ASID and test whether it matches:
815 ; Get the active ASID:
816 mov ebx, Sr(CP0_EntryHi); ebx := CP0_EntryHi
817 and ebx, 0xFF ; ebx := ebx & 0xFF (get current ASID)
818 ; Determine whether it matches the found Tag entry's :
819 mov ecx, AUX ; ecx := AUX (index of found entry)
820 lea rdx, [TLB_ASID_COPY]; Load address of ASID Copy
821 cmp byte [rdx + rcx], bl ; Compare stored ASID to current
822 jne .Not_Found ; ... if not equal, 'not found'
823 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
824 .Match:
825 ; If we're here, we have a Match. AUX is index of matching entry;
826 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
827 xor edx, edx ; edx := 0
828 ; Get arity of desired entry:
829 bt eax, 12 ; Test vAddr's odd/even junior bit
830 setc dl ; edx := {1 if a-odd, 0 if a-even}
831 shl edx, 5 ; edx := {32 if a-odd, 0 if a-even}
832 mov ecx, edx ; ebx := edx (copy of above)
833 ; Now we know which bit of TLB_Flags is this entry's V. Test it:
834 add edx, AUX ; add the corresponding index
835 bt TLB_Flags, rdx ; test if V(Index) is set
836 jnc .Invalid_W ; ... if V == 0, then go to Invalid
837 ; Now, since we're writing, test this entry's D, at pos(V) + 16:
838 add edx, 16
839 bt TLB_Flags, rdx ; test if D(Index) is set
840 jnc .Dirty_W ; ... if D == 0, then go to Dirty
841 ; Now let's load the correct odd or even PFN:
842 mov rbx, TLB_PFN(AUX64) ; load the PFN pair to rbx
843 ; ebx is now the PFN. Before wrapping up, update the TLB read cache :
844 movq W_TLB_Last_Good_PFN, rbx ; Set last good PFN to this PFN:
845 ; now leave only the correct half of PFN, at bottom of rbx:
846 shr rbx, cl ; if arity is odd, get upper 32bit
847 ; set correct half of R_TLB_Last_Good_Tag to the found Tag:
848 mov rdx, 0xFFFFFFFF00000000 ; rdx := 0xFFFFFFFF00000000
849 shr rdx, cl ; if arity is odd, keep bottom
850 movq AUX64, W_TLB_Last_Good_Tag ; get last good Tag
851 and AUX64, rdx ; zap correct half of last good tag
852 movq rdx, xmm4 ; get the Tag again :
853 shl rdx, cl ; if arity if odd, slide into pos:
854 or AUX64, rdx ; now or it into place
855 movq W_TLB_Last_Good_Tag, AUX64 ; update last good Tag.
856 .PFN_And_Done:
857 and eax, 0xFFF ; vAddr := vAddr & 0xFFF
858 or eax, ebx ; vAddr := vAddr | entry.PFN[lowbit]
859 jmp _Lookup_TLB_Done ; vAddr is now correct pAddr, done.
860 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
861 .Not_Found: ; Not Found in Table:
862 add rsp, 16 ; squelch return to _Virt_xxx and its caller
863 push _Handle_Exception_TLB_NoMatch ; 'return' straight to handler.
864 jmp _Lookup_TLB_E_WriteExtr ; Wrap up
865 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
866 .Invalid_W:
867 SetEXC EXC_TLBS ; Set the EXC_TLBS Exception
868 add rsp, 16 ; squelch return to _Virt_xxx and its caller
869 push _Handle_Exception_Other ; 'return' straight to handler.
870 jmp _Lookup_TLB_E_WriteExtr ; Go to the common epilogue.
871 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
872 .Dirty_W:
873 SetEXC EXC_Mod ; Set the EXC_Mod Exception
874 add rsp, 16 ; squelch return to _Virt_xxx and its caller
875 push _Handle_Exception_Other ; 'return' straight to handler.
876 ; jmp _Lookup_TLB_E_WriteExtr ; Write the 'extra data' and finish.
877 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
878
879 ;-----------------------------------------------------------------------------
880 ; Epilogue common to _Virt_To_Phys_Read and _Virt_To_Phys_Write:
(526 . 56)(631 . 93)
882 ; Kills eax, ebx, ecx, edx.
883 ;-----------------------------------------------------------------------------
884 _write_tlb_entry:
885 mov edx, Sr(CP0_EntryHi) ; edx := CP0_EntryHi
886 mov ecx, edx ; ecx := edx
887 shr ecx, 13 ; ecx := ecx >> 13 to get VPN2
888 and edx, TLB_ASID_Mask ; edx := edx & 0xFF to get ASID
889 shl edx, TLB_ASID_Shift ; edx := edx << 19 to put ASID in place
890 or ecx, edx ; now we have VPN2 and ASID in ecx
891 ;; done with edx, can reuse
892 mov edx, Sr(CP0_EntryLo0) ; edx := CP0_EntryLo0
893 mov ebx, Sr(CP0_EntryLo1) ; ebx := CP0_EntryLo1
894 ;; get G:
895 mov eax, edx ; eax := CP0_EntryLo0
896 and eax, ebx ; eax := eax & CP0_EntryLo1
897 and eax, 0x1 ; eax := eax & 1 to get G
898 shl eax, TLB_G ; move G bit into position
899 or ecx, eax ; ecx := ecx | eax to put in G bit
900 ;; now ecx contains VPN2, ASID, G
901 ;; Get V0 from CP0_EntryLo0 and put in ecx where belongs:
902 mov eax, edx ; eax := CP0_EntryLo0
903 and eax, 0x2 ; eax := eax & 0x2 to get V0 bit
904 shl eax, (TLB_V0 - 1) ; put V0 bit in position
905 or ecx, eax ; ecx := ecx | eax to put in V0 bit
906 ;; Get D0 from CP0_EntryLo0 and put in ecx where belongs:
907 mov eax, edx ; eax := CP0_EntryLo0
908 and eax, 0x4 ; eax := eax & 0x4 to get D0 bit
909 shl eax, (TLB_D0 - 2) ; put D0 bit in position
910 or ecx, eax ; ecx := ecx | eax to put in D0 bit
911 ;; Get V1 from CP0_EntryLo1 and put in ecx where belongs:
912 mov eax, ebx ; eax := CP0_EntryLo1
913 and eax, 0x2 ; eax := eax & 0x2 to get V1 bit
914 shl eax, (TLB_V1 - 1) ; put V1 bit in position
915 or ecx, eax ; ecx := ecx | eax to put in V1 bit
916 ;; Get D1 from CP0_EntryLo1 and put in ecx where belongs:
917 mov eax, ebx ; eax := CP0_EntryLo1
918 and eax, 0x4 ; eax := eax & 0x4 to get D1 bit
919 shl eax, (TLB_D1 - 2) ; put D1 bit in position
920 or ecx, eax ; ecx := ecx | eax to put in D1 bit
921 ;; Write the TLB entry to the given index (in AUX) :
922 and AUX, 0xF ; Index of TLB entry is bottom 4 bits
923 mov TLB_E(AUX), ecx ; Write TLB entry in ecx to n-th slot.
924 ;; Transform CP0_EntryLo0 (edx) into PFN0:
925 shr edx, 6
926 and edx, 0xFFFFF
927 shl edx, 12
928 ;; Transform CP0_EntryLo1 (ebx) into PFN1:
929 shr ebx, 6
930 and ebx, 0xFFFFF
931 shl ebx, 12
932 ;; Store PFN:
933 mov TLB_PFN_E(AUX), edx ; Store PFN0
934 mov TLB_PFN_O(AUX), ebx ; Store PFN1
935 Invalidate_TLB_Cache ; Invalidate both R and W TLB Caches
936 ret ; Done.
937 and AUX, 0xF ; constrain Index range (0 .. 15)
938 mov ecx, AUX ; ecx := Index
939 ; Clear old Flags: V0, D0, V1, D1 :
940 mov rax, ~(1 | (1 << 16) | (1 << 32) | (1 << 48)) ; zap mask
941 rol rax, cl ; Rotate into indexed position
942 and TLB_Flags, rax ; Clear old V0, D0, V1, D1.
943 ; Clear old G:
944 add ecx, 16 ; G starts at 16th bit;
945 btr Flag_Reg, ecx ; Clear old G.
946 ; Now, set the new values:
947 mov ecx, AUX ; ecx := Index
948 mov edx, Sr(CP0_EntryHi) ; edx := CP0_EntryHi
949 ; First, get this entry's Tag (VPN2) from CP0_EntryHi :
950 mov eax, edx ; eax := edx (CP0_EntryHi)
951 shr eax, 13 ; eax := ecx >> 13 to get the Tag
952 ; Now, save this entry's Tag to the selected Index:
953 ; Write Byte 0 of Tag to TLB_TAG_BYTE_0_COPY and TLB_TAG_BYTE_0 :
954 lea rbx, [TLB_TAG_BYTE_0_COPY] ; Load address of B0 Copy
955 mov byte [rbx + rcx], al ; Change the indexed byte
956 movdqa TLB_TAG_BYTE_0, [rbx] ; Update XMM reg with B0 Copy
957 ; Write Byte 1 of Tag to TLB_TAG_BYTE_1_COPY and TLB_TAG_BYTE_1 :
958 lea rbx, [TLB_TAG_BYTE_1_COPY] ; Load address of B1 Copy
959 mov byte [rbx + rcx], ah ; Change the indexed byte
960 movdqa TLB_TAG_BYTE_1, [rbx] ; Update XMM reg with B1 Copy
961 ; Write Byte 2 of Tag to TLB_TAG_BYTE_2_COPY and TLB_TAG_BYTE_2 :
962 shr eax, 16
963 lea rbx, [TLB_TAG_BYTE_2_COPY] ; Load address of B2 Copy
964 mov byte [rbx + rcx], al ; Change the indexed byte
965 movdqa TLB_TAG_BYTE_2, [rbx] ; Update XMM reg with B2 Copy
966 ; Done with Tag. Now, get this entry's ASID from CP0_EntryHi :
967 mov eax, edx ; eax := edx
968 and eax, 0xFF ; Get ASID from CP0_EntryHi
969 ; Store this entry's ASID to the selected Index:
970 lea rbx, [TLB_ASID_COPY] ; Load address of ASID Copy
971 mov byte [rbx + rcx], al ; Change the indexed byte
972 ; Done with contents of CP0_EntryHi. Now, get G, V0, D0, V1, D1 :
973 mov edx, Sr(CP0_EntryLo0) ; edx := CP0_EntryLo0
974 mov ebx, Sr(CP0_EntryLo1) ; ebx := CP0_EntryLo1
975 ; Get G using CP0_EntryLo0 and CP0_EntryLo1 :
976 mov eax, edx ; eax := CP0_EntryLo0
977 and eax, ebx ; eax := eax & CP0_EntryLo1
978 and eax, 0x1 ; eax := eax & 1 to get G
979 ; Write the new G(Index) to indexed pos of upper 16 bits of Flag_Reg :
980 add ecx, 16 ; Position of all G's in Flag_Reg
981 shl eax, cl ; Slide new G into final position
982 or Flag_Reg, eax ; Set the new value G(Index)
983 ; Get V0 from CP0_EntryLo0 and write to TLB_Flags :
984 mov ecx, AUX ; ecx := Index
985 mov eax, edx ; eax := CP0_EntryLo0
986 and eax, 0x2 ; eax := eax & 0x2 to get V0 bit
987 shr eax, 1 ; Put V0 into bottom-most pos
988 shl eax, cl ; Slide V0 into final position
989 or TLB_Flags, rax ; Put the new value in V0(Index)
990 ; Get D0 from CP0_EntryLo0 and write to TLB_Flags :
991 add ecx, 16 ; Position where D0 lives:
992 mov eax, edx ; eax := CP0_EntryLo0
993 and eax, 0x4 ; eax := eax & 0x4 to get D0 bit
994 shr eax, 2 ; Put D0 into bottom-most pos
995 shl eax, cl ; Slide D0 into final position
996 or TLB_Flags, rax ; Put the new value in D0(Index)
997 ; Get V1 from CP0_EntryLo1 and write to TLB_Flags :
998 add ecx, 16 ; V1 starts at 32-nd bit
999 mov eax, ebx ; eax := CP0_EntryLo1
1000 and eax, 0x2 ; eax := eax & 0x2 to get V1 bit
1001 shr eax, 1 ; Put V0 into bottom-most pos
1002 shl rax, cl ; Slide V1 into final position
1003 or TLB_Flags, rax ; Put the new value in V1(Index)
1004 ; Get D1 from CP0_EntryLo1 and write to TLB_Flags :
1005 add ecx, 16 ; D1 starts at 48-th bit
1006 mov eax, ebx ; eax := CP0_EntryLo1
1007 and eax, 0x4 ; eax := eax & 0x4 to get D1 bit
1008 shr eax, 2 ; Put D1 into bottom-most pos
1009 shl rax, cl ; Slide D1 into final position
1010 or TLB_Flags, rax ; Put the new value in D1(Index)
1011 ; Transform CP0_EntryLo0 (edx) into PFN0:
1012 shr edx, 6
1013 and edx, 0xFFFFF
1014 shl edx, 12
1015 ; Transform CP0_EntryLo1 (ebx) into PFN1:
1016 shr ebx, 6
1017 and ebx, 0xFFFFF
1018 shl ebx, 12
1019 ; Store PFN:
1020 shl rbx, 32 ; rbx := rbx << 32 (Odd)
1021 and rdx, 0xFFFFFFFF ; rdx := rdx & 0xFFFFFFFF (Even)
1022 or rbx, rdx ; rbx := rbx | rdx (combined PFN)
1023 mov TLB_PFN(AUX64), rbx ; Store PFN
1024 ; Fin.
1025 ret
1026 ;-----------------------------------------------------------------------------