Untitled diff
196 lines
LEAF_ENTRY JIT_ByRefWriteBarrier, _TEXT
LEAF_ENTRY JIT_ByRefWriteBarrierBatch, _TEXT
NextByref:
mov rcx, [rsi]
mov rcx, [rsi]
; If !WRITE_BARRIER_CHECK do the write first, otherwise we might have to do some ShadowGC stuff
; If !WRITE_BARRIER_CHECK do the write first, otherwise we might have to do some ShadowGC stuff
ifndef WRITE_BARRIER_CHECK
ifndef WRITE_BARRIER_CHECK
; rcx is [rsi]
; rcx is [rsi]
mov [rdi], rcx
mov [rdi], rcx
endif
endif
; When WRITE_BARRIER_CHECK is defined _NotInHeap will write the reference
; When WRITE_BARRIER_CHECK is defined _NotInHeap will write the reference
; but if it isn't then it will just return.
; but if it isn't then it will just return.
;
;
; See if this is in GCHeap
; See if this is in GCHeap
cmp rdi, [g_lowest_address]
cmp rdi, [g_lowest_address]
jb NotInHeap
jb NotInHeap
cmp rdi, [g_highest_address]
cmp rdi, [g_highest_address]
jnb NotInHeap
jnb NotInHeap
ifdef WRITE_BARRIER_CHECK
ifdef WRITE_BARRIER_CHECK
; we can only trash rcx in this function so in _DEBUG we need to save
; we can only trash rcx in this function so in _DEBUG we need to save
; some scratch registers.
; some scratch registers.
push r10
push r10
push r11
push r11
push rax
push rax
; **ALSO update the shadow GC heap if that is enabled**
; **ALSO update the shadow GC heap if that is enabled**
; Do not perform the work if g_GCShadow is 0
; Do not perform the work if g_GCShadow is 0
cmp g_GCShadow, 0
cmp g_GCShadow, 0
je NoShadow
je NoShadow
; If we end up outside of the heap don't corrupt random memory
; If we end up outside of the heap don't corrupt random memory
mov r10, rdi
mov r10, rdi
sub r10, [g_lowest_address]
sub r10, [g_lowest_address]
jb NoShadow
jb NoShadow
; Check that our adjusted destination is somewhere in the shadow gc
; Check that our adjusted destination is somewhere in the shadow gc
add r10, [g_GCShadow]
add r10, [g_GCShadow]
cmp r10, [g_GCShadowEnd]
cmp r10, [g_GCShadowEnd]
jnb NoShadow
jnb NoShadow
; Write ref into real GC
; Write ref into real GC
mov [rdi], rcx
mov [rdi], rcx
; Write ref into shadow GC
; Write ref into shadow GC
mov [r10], rcx
mov [r10], rcx
; Ensure that the write to the shadow heap occurs before the read from
; Ensure that the write to the shadow heap occurs before the read from
; the GC heap so that race conditions are caught by INVALIDGCVALUE
; the GC heap so that race conditions are caught by INVALIDGCVALUE
mfence
mfence
; Check that GC/ShadowGC values match
; Check that GC/ShadowGC values match
mov r11, [rdi]
mov r11, [rdi]
mov rax, [r10]
mov rax, [r10]
cmp rax, r11
cmp rax, r11
je DoneShadow
je DoneShadow
mov r11, INVALIDGCVALUE
mov r11, INVALIDGCVALUE
mov [r10], r11
mov [r10], r11
jmp DoneShadow
jmp DoneShadow
; If we don't have a shadow GC we won't have done the write yet
; If we don't have a shadow GC we won't have done the write yet
NoShadow:
NoShadow:
mov [rdi], rcx
mov [rdi], rcx
; If we had a shadow GC then we already wrote to the real GC at the same time
; If we had a shadow GC then we already wrote to the real GC at the same time
; as the shadow GC so we want to jump over the real write immediately above.
; as the shadow GC so we want to jump over the real write immediately above.
; Additionally we know for sure that we are inside the heap and therefore don't
; Additionally we know for sure that we are inside the heap and therefore don't
; need to replicate the above checks.
; need to replicate the above checks.
DoneShadow:
DoneShadow:
pop rax
pop rax
pop r11
pop r11
pop r10
pop r10
endif
endif
ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
; Update the write watch table if necessary
; Update the write watch table if necessary
cmp byte ptr [g_sw_ww_enabled_for_gc_heap], 0h
cmp byte ptr [g_sw_ww_enabled_for_gc_heap], 0h
je CheckCardTable
je CheckCardTable
mov rax, rdi
mov rax, rdi
shr rax, 0Ch ; SoftwareWriteWatch::AddressToTableByteIndexShift
shr rax, 0Ch ; SoftwareWriteWatch::AddressToTableByteIndexShift
add rax, qword ptr [g_sw_ww_table]
add rax, qword ptr [g_sw_ww_table]
cmp byte ptr [rax], 0h
cmp byte ptr [rax], 0h
jne CheckCardTable
jne CheckCardTable
mov byte ptr [rax], 0FFh
mov byte ptr [rax], 0FFh
endif
endif
; See if we can just quick out
; See if we can just quick out
CheckCardTable:
CheckCardTable:
cmp rcx, [g_ephemeral_low]
cmp rcx, [g_ephemeral_low]
jb Exit
jb Exit
cmp rcx, [g_ephemeral_high]
cmp rcx, [g_ephemeral_high]
jnb Exit
jnb Exit
; do the following checks only if we are allowed to trash rax
; do the following checks only if we are allowed to trash rax
; otherwise we don't have enough registers
; otherwise we don't have enough registers
ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
mov rax, rcx
mov rax, rcx
mov cl, [g_region_shr]
mov cl, [g_region_shr]
test cl, cl
test cl, cl
je SkipCheck
je SkipCheck
; check if the source is in gen 2 - then it's not an ephemeral pointer
; check if the source is in gen 2 - then it's not an ephemeral pointer
shr rax, cl
shr rax, cl
add rax, [g_region_to_generation_table]
add rax, [g_region_to_generation_table]
cmp byte ptr [rax], 82h
cmp byte ptr [rax], 82h
je Exit
je Exit
; check if the destination happens to be in gen 0
; check if the destination happens to be in gen 0
mov rax, rdi
mov rax, rdi
shr rax, cl
shr rax, cl
add rax, [g_region_to_generation_table]
add rax, [g_region_to_generation_table]
cmp byte ptr [rax], 0
cmp byte ptr [rax], 0
je Exit
je Exit
SkipCheck:
SkipCheck:
cmp [g_region_use_bitwise_write_barrier], 0
cmp [g_region_use_bitwise_write_barrier], 0
je CheckCardTableByte
je CheckCardTableByte
; compute card table bit
; compute card table bit
mov rcx, rdi
mov rcx, rdi
mov al, 1
mov al, 1
shr rcx, 8
shr rcx, 8
and cl, 7
and cl, 7
shl al, cl
shl al, cl
; move current rdi value into rcx and then increment the pointers
; move current rdi value into rcx and then increment the pointers
mov rcx, rdi
mov rcx, rdi
add rsi, 8h
add rsi, 8h
add rdi, 8h
add rdi, 8h
; Check if we need to update the card table
; Check if we need to update the card table
; Calc pCardByte
; Calc pCardByte
shr rcx, 0Bh
shr rcx, 0Bh
add rcx, [g_card_table]
add rcx, [g_card_table]
; Check if this card table bit is already set
; Check if this card table bit is already set
test byte ptr [rcx], al
test byte ptr [rcx], al
je SetCardTableBit
je SetCardTableBit
; Check if we have more in the batch and run again
dec r8d
jne NextByref
REPRET
REPRET
SetCardTableBit:
SetCardTableBit:
lock or byte ptr [rcx], al
lock or byte ptr [rcx], al
jmp CheckCardBundle
jmp CheckCardBundle
endif
endif
CheckCardTableByte:
CheckCardTableByte:
; move current rdi value into rcx and then increment the pointers
; move current rdi value into rcx and then increment the pointers
mov rcx, rdi
mov rcx, rdi
add rsi, 8h
add rsi, 8h
add rdi, 8h
add rdi, 8h
; Check if we need to update the card table
; Check if we need to update the card table
; Calc pCardByte
; Calc pCardByte
shr rcx, 0Bh
shr rcx, 0Bh
add rcx, [g_card_table]
add rcx, [g_card_table]
; Check if this card is dirty
; Check if this card is dirty
cmp byte ptr [rcx], 0FFh
cmp byte ptr [rcx], 0FFh
jne UpdateCardTable
jne UpdateCardTable
; Check if we have more in the batch and run again
dec r8d
jne NextByref
REPRET
REPRET
UpdateCardTable:
UpdateCardTable:
mov byte ptr [rcx], 0FFh
mov byte ptr [rcx], 0FFh
CheckCardBundle:
CheckCardBundle:
ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
; check if we need to update the card bundle table
; check if we need to update the card bundle table
; restore destination address from rdi - rdi has been incremented by 8 already
; restore destination address from rdi - rdi has been incremented by 8 already
lea rcx, [rdi-8]
lea rcx, [rdi-8]
shr rcx, 15h
shr rcx, 15h
add rcx, [g_card_bundle_table]
add rcx, [g_card_bundle_table]
cmp byte ptr [rcx], 0FFh
cmp byte ptr [rcx], 0FFh
jne UpdateCardBundleTable
jne UpdateCardBundleTable
; Check if we have more in the batch and run again
dec r8d
jne NextByref
REPRET
REPRET
UpdateCardBundleTable:
UpdateCardBundleTable:
mov byte ptr [rcx], 0FFh
mov byte ptr [rcx], 0FFh
endif
endif
; Check if we have more in the batch and run again
dec r8d
jne NextByref
ret
ret
align 16
align 16
NotInHeap:
NotInHeap:
; If WRITE_BARRIER_CHECK then we won't have already done the mov and should do it here
; If WRITE_BARRIER_CHECK then we won't have already done the mov and should do it here
; If !WRITE_BARRIER_CHECK we want _NotInHeap and _Leave to be the same and have both
; If !WRITE_BARRIER_CHECK we want _NotInHeap and _Leave to be the same and have both
; 16 byte aligned.
; 16 byte aligned.
ifdef WRITE_BARRIER_CHECK
ifdef WRITE_BARRIER_CHECK
; rcx is [rsi]
; rcx is [rsi]
mov [rdi], rcx
mov [rdi], rcx
endif
endif
; At least one write is already done, increment the pointers
add rdi, 8h
add rsi, 8h
dec r8d
je NotInHeapExit
; Now we can do the rest of the writes without checking the heap
NextByrefUnchecked:
mov rcx, [rsi]
mov [rdi], rcx
add rdi, 8h
add rsi, 8h
dec r8d
jne NextByrefUnchecked
NotInHeapExit:
ret
Exit:
Exit:
; Increment the pointers before leaving
; Increment the pointers before leaving
add rdi, 8h
add rdi, 8h
add rsi, 8h
add rsi, 8h
; Check if we have more in the batch and run again
dec r8d
jne NextByref
ret
ret
LEAF_END_MARKED JIT_ByRefWriteBarrier, _TEXT
LEAF_END_MARKED JIT_ByRefWriteBarrierBatch, _TEXT