Untitled diff
195 Zeilen
LEAF_ENTRY JIT_ByRefWriteBarrier, _TEXT
LEAF_ENTRY JIT_ByRefWriteBarrierBatch, _TEXT
LOCAL_LABEL(NextByref_ByRefWriteBarrierBatch):
mov rcx, [rsi]
mov rcx, [rsi]
// If !WRITE_BARRIER_CHECK do the write first, otherwise we might have to do some ShadowGC stuff
// If !WRITE_BARRIER_CHECK do the write first, otherwise we might have to do some ShadowGC stuff
#ifndef WRITE_BARRIER_CHECK
#ifndef WRITE_BARRIER_CHECK
// rcx is [rsi]
// rcx is [rsi]
mov [rdi], rcx
mov [rdi], rcx
#endif
#endif
// When WRITE_BARRIER_CHECK is defined _NotInHeap will write the reference
// When WRITE_BARRIER_CHECK is defined _NotInHeap will write the reference
// but if it isn't then it will just return.
// but if it isn't then it will just return.
//
//
// See if this is in GCHeap
// See if this is in GCHeap
PREPARE_EXTERNAL_VAR g_lowest_address, rax
PREPARE_EXTERNAL_VAR g_lowest_address, rax
cmp rdi, [rax]
cmp rdi, [rax]
jb LOCAL_LABEL(NotInHeap_ByRefWriteBarrier)
jb LOCAL_LABEL(NotInHeap_ByRefWriteBarrierBatch)
PREPARE_EXTERNAL_VAR g_highest_address, rax
PREPARE_EXTERNAL_VAR g_highest_address, rax
cmp rdi, [rax]
cmp rdi, [rax]
jnb LOCAL_LABEL(NotInHeap_ByRefWriteBarrier)
jnb LOCAL_LABEL(NotInHeap_ByRefWriteBarrierBatch)
#ifdef WRITE_BARRIER_CHECK
#ifdef WRITE_BARRIER_CHECK
// **ALSO update the shadow GC heap if that is enabled**
// **ALSO update the shadow GC heap if that is enabled**
// Do not perform the work if g_GCShadow is 0
// Do not perform the work if g_GCShadow is 0
PREPARE_EXTERNAL_VAR g_GCShadow, rax
PREPARE_EXTERNAL_VAR g_GCShadow, rax
cmp qword ptr [rax], 0
cmp qword ptr [rax], 0
je LOCAL_LABEL(NoShadow_ByRefWriteBarrier)
je LOCAL_LABEL(NoShadow_ByRefWriteBarrierBatch)
// If we end up outside of the heap don't corrupt random memory
// If we end up outside of the heap don't corrupt random memory
mov r10, rdi
mov r10, rdi
PREPARE_EXTERNAL_VAR g_lowest_address, rax
PREPARE_EXTERNAL_VAR g_lowest_address, rax
sub r10, [rax]
sub r10, [rax]
jb LOCAL_LABEL(NoShadow_ByRefWriteBarrier)
jb LOCAL_LABEL(NoShadow_ByRefWriteBarrierBatch)
// Check that our adjusted destination is somewhere in the shadow gc
// Check that our adjusted destination is somewhere in the shadow gc
PREPARE_EXTERNAL_VAR g_GCShadow, rax
PREPARE_EXTERNAL_VAR g_GCShadow, rax
add r10, [rax]
add r10, [rax]
PREPARE_EXTERNAL_VAR g_GCShadowEnd, rax
PREPARE_EXTERNAL_VAR g_GCShadowEnd, rax
cmp r10, [rax]
cmp r10, [rax]
jnb LOCAL_LABEL(NoShadow_ByRefWriteBarrier)
jnb LOCAL_LABEL(NoShadow_ByRefWriteBarrierBatch)
// Write ref into real GC
// Write ref into real GC
mov [rdi], rcx
mov [rdi], rcx
// Write ref into shadow GC
// Write ref into shadow GC
mov [r10], rcx
mov [r10], rcx
// Ensure that the write to the shadow heap occurs before the read from
// Ensure that the write to the shadow heap occurs before the read from
// the GC heap so that race conditions are caught by INVALIDGCVALUE
// the GC heap so that race conditions are caught by INVALIDGCVALUE
mfence
mfence
// Check that GC/ShadowGC values match
// Check that GC/ShadowGC values match
mov r11, [rdi]
mov r11, [rdi]
mov rax, [r10]
mov rax, [r10]
cmp rax, r11
cmp rax, r11
je LOCAL_LABEL(DoneShadow_ByRefWriteBarrier)
je LOCAL_LABEL(DoneShadow_ByRefWriteBarrierBatch)
movabs r11, INVALIDGCVALUE
movabs r11, INVALIDGCVALUE
mov [r10], r11
mov [r10], r11
jmp LOCAL_LABEL(DoneShadow_ByRefWriteBarrier)
jmp LOCAL_LABEL(DoneShadow_ByRefWriteBarrierBatch)
// If we don't have a shadow GC we won't have done the write yet
// If we don't have a shadow GC we won't have done the write yet
LOCAL_LABEL(NoShadow_ByRefWriteBarrier):
LOCAL_LABEL(NoShadow_ByRefWriteBarrierBatch):
mov [rdi], rcx
mov [rdi], rcx
// If we had a shadow GC then we already wrote to the real GC at the same time
// If we had a shadow GC then we already wrote to the real GC at the same time
// as the shadow GC so we want to jump over the real write immediately above.
// as the shadow GC so we want to jump over the real write immediately above.
// Additionally we know for sure that we are inside the heap and therefore don't
// Additionally we know for sure that we are inside the heap and therefore don't
// need to replicate the above checks.
// need to replicate the above checks.
LOCAL_LABEL(DoneShadow_ByRefWriteBarrier):
LOCAL_LABEL(DoneShadow_ByRefWriteBarrierBatch):
#endif
#endif
#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
// Update the write watch table if necessary
// Update the write watch table if necessary
PREPARE_EXTERNAL_VAR g_sw_ww_enabled_for_gc_heap, rax
PREPARE_EXTERNAL_VAR g_sw_ww_enabled_for_gc_heap, rax
cmp byte ptr [rax], 0x0
cmp byte ptr [rax], 0x0
je LOCAL_LABEL(CheckCardTable_ByRefWriteBarrier)
je LOCAL_LABEL(CheckCardTable_ByRefWriteBarrierBatch)
mov rax, rdi
mov rax, rdi
shr rax, 0xC // SoftwareWriteWatch::AddressToTableByteIndexShift
shr rax, 0xC // SoftwareWriteWatch::AddressToTableByteIndexShift
PREPARE_EXTERNAL_VAR g_sw_ww_table, r10
PREPARE_EXTERNAL_VAR g_sw_ww_table, r10
add rax, qword ptr [r10]
add rax, qword ptr [r10]
cmp byte ptr [rax], 0x0
cmp byte ptr [rax], 0x0
jne LOCAL_LABEL(CheckCardTable_ByRefWriteBarrier)
jne LOCAL_LABEL(CheckCardTable_ByRefWriteBarrierBatch)
mov byte ptr [rax], 0xFF
mov byte ptr [rax], 0xFF
#endif
#endif
LOCAL_LABEL(CheckCardTable_ByRefWriteBarrier):
LOCAL_LABEL(CheckCardTable_ByRefWriteBarrierBatch):
// See if we can just quick out
// See if we can just quick out
PREPARE_EXTERNAL_VAR g_ephemeral_low, rax
PREPARE_EXTERNAL_VAR g_ephemeral_low, rax
cmp rcx, [rax]
cmp rcx, [rax]
jb LOCAL_LABEL(Exit_ByRefWriteBarrier)
jb LOCAL_LABEL(Exit_ByRefWriteBarrierBatch)
PREPARE_EXTERNAL_VAR g_ephemeral_high, rax
PREPARE_EXTERNAL_VAR g_ephemeral_high, rax
cmp rcx, [rax]
cmp rcx, [rax]
jnb LOCAL_LABEL(Exit_ByRefWriteBarrier)
jnb LOCAL_LABEL(Exit_ByRefWriteBarrierBatch)
mov rax, rcx
mov rax, rcx
PREPARE_EXTERNAL_VAR g_region_shr, rcx
PREPARE_EXTERNAL_VAR g_region_shr, rcx
mov cl, [rcx]
mov cl, [rcx]
test cl, cl
test cl, cl
je LOCAL_LABEL(SkipCheck_ByRefWriteBarrier)
je LOCAL_LABEL(SkipCheck_ByRefWriteBarrierBatch)
// check if the source is in gen 2 - then it's not an ephemeral pointer
// check if the source is in gen 2 - then it's not an ephemeral pointer
shr rax, cl
shr rax, cl
PREPARE_EXTERNAL_VAR g_region_to_generation_table, r10
PREPARE_EXTERNAL_VAR g_region_to_generation_table, r10
mov r10, [r10]
mov r10, [r10]
cmp byte ptr [rax + r10], 0x82
cmp byte ptr [rax + r10], 0x82
je LOCAL_LABEL(Exit_ByRefWriteBarrier)
je LOCAL_LABEL(Exit_ByRefWriteBarrierBatch)
// check if the destination happens to be in gen 0
// check if the destination happens to be in gen 0
mov rax, rdi
mov rax, rdi
shr rax, cl
shr rax, cl
cmp byte ptr [rax + r10], 0
cmp byte ptr [rax + r10], 0
je LOCAL_LABEL(Exit_ByRefWriteBarrier)
je LOCAL_LABEL(Exit_ByRefWriteBarrierBatch)
LOCAL_LABEL(SkipCheck_ByRefWriteBarrier):
LOCAL_LABEL(SkipCheck_ByRefWriteBarrierBatch):
PREPARE_EXTERNAL_VAR g_card_table, r10
PREPARE_EXTERNAL_VAR g_card_table, r10
mov r10, [r10]
mov r10, [r10]
PREPARE_EXTERNAL_VAR g_region_use_bitwise_write_barrier, rax
PREPARE_EXTERNAL_VAR g_region_use_bitwise_write_barrier, rax
cmp byte ptr [rax], 0
cmp byte ptr [rax], 0
je LOCAL_LABEL(CheckCardTableByte_ByRefWriteBarrier)
je LOCAL_LABEL(CheckCardTableByte_ByRefWriteBarrierBatch)
// compute card table bit
// compute card table bit
mov ecx, edi
mov ecx, edi
mov al, 1
mov al, 1
shr ecx, 8
shr ecx, 8
and cl, 7
and cl, 7
shl al, cl
shl al, cl
// move current rdi value into rcx and then increment the pointers
// move current rdi value into rcx and then increment the pointers
mov rcx, rdi
mov rcx, rdi
add rsi, 0x8
add rsi, 0x8
add rdi, 0x8
add rdi, 0x8
// Check if we need to update the card table
// Check if we need to update the card table
// Calc pCardByte
// Calc pCardByte
shr rcx, 0xB
shr rcx, 0xB
// Check if this card table bit is already set
// Check if this card table bit is already set
test byte ptr [rcx + r10], al
test byte ptr [rcx + r10], al
je LOCAL_LABEL(SetCardTableBit_ByRefWriteBarrier)
je LOCAL_LABEL(SetCardTableBit_ByRefWriteBarrierBatch)
// Check if we have more in the batch and run again
dec r8d
jne LOCAL_LABEL(NextByref_ByRefWriteBarrierBatch)
REPRET
REPRET
LOCAL_LABEL(SetCardTableBit_ByRefWriteBarrier):
LOCAL_LABEL(SetCardTableBit_ByRefWriteBarrierBatch):
lock or byte ptr [rcx + r10], al
lock or byte ptr [rcx + r10], al
jmp LOCAL_LABEL(CheckCardBundle_ByRefWriteBarrier)
jmp LOCAL_LABEL(CheckCardBundle_ByRefWriteBarrierBatch)
LOCAL_LABEL(CheckCardTableByte_ByRefWriteBarrier):
LOCAL_LABEL(CheckCardTableByte_ByRefWriteBarrierBatch):
// move current rdi value into rcx and then increment the pointers
// move current rdi value into rcx and then increment the pointers
mov rcx, rdi
mov rcx, rdi
add rsi, 0x8
add rsi, 0x8
add rdi, 0x8
add rdi, 0x8
shr rcx, 0xB
shr rcx, 0xB
cmp byte ptr [rcx + r10], 0xFF
cmp byte ptr [rcx + r10], 0xFF
jne LOCAL_LABEL(SetCardTableByte_ByRefWriteBarrier)
jne LOCAL_LABEL(SetCardTableByte_ByRefWriteBarrierBatch)
// Check if we have more in the batch and run again
dec r8d
jne LOCAL_LABEL(NextByref_ByRefWriteBarrierBatch)
REPRET
REPRET
LOCAL_LABEL(SetCardTableByte_ByRefWriteBarrier):
LOCAL_LABEL(SetCardTableByte_ByRefWriteBarrierBatch):
mov byte ptr [rcx + r10], 0xFF
mov byte ptr [rcx + r10], 0xFF
LOCAL_LABEL(CheckCardBundle_ByRefWriteBarrier):
LOCAL_LABEL(CheckCardBundle_ByRefWriteBarrierBatch):
#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
// Shift rcx by 0x0A more to get the card bundle byte (we shifted by 0x0B already)
// Shift rcx by 0x0A more to get the card bundle byte (we shifted by 0x0B already)
shr rcx, 0x0A
shr rcx, 0x0A
PREPARE_EXTERNAL_VAR g_card_bundle_table, rax
PREPARE_EXTERNAL_VAR g_card_bundle_table, rax
add rcx, [rax]
add rcx, [rax]
// Check if this bundle byte is dirty
// Check if this bundle byte is dirty
cmp byte ptr [rcx], 0xFF
cmp byte ptr [rcx], 0xFF
jne LOCAL_LABEL(UpdateCardBundle_ByRefWriteBarrier)
jne LOCAL_LABEL(UpdateCardBundle_ByRefWriteBarrierBatch)
// Check if we have more in the batch and run again
dec r8d
jne LOCAL_LABEL(NextByref_ByRefWriteBarrierBatch)
REPRET
REPRET
LOCAL_LABEL(UpdateCardBundle_ByRefWriteBarrier):
LOCAL_LABEL(UpdateCardBundle_ByRefWriteBarrierBatch):
mov byte ptr [rcx], 0xFF
mov byte ptr [rcx], 0xFF
#endif
#endif
// Check if we have more in the batch and run again
dec r8d
jne LOCAL_LABEL(NextByref_ByRefWriteBarrierBatch)
ret
ret
.balign 16
.balign 16
LOCAL_LABEL(NotInHeap_ByRefWriteBarrier):
LOCAL_LABEL(NotInHeap_ByRefWriteBarrierBatch):
// If WRITE_BARRIER_CHECK then we won't have already done the mov and should do it here
// If WRITE_BARRIER_CHECK then we won't have already done the mov and should do it here
// If !WRITE_BARRIER_CHECK we want _NotInHeap and _Leave to be the same and have both
// If !WRITE_BARRIER_CHECK we want _NotInHeap and _Leave to be the same and have both
// 16 byte aligned.
// 16 byte aligned.
#ifdef WRITE_BARRIER_CHECK
#ifdef WRITE_BARRIER_CHECK
// rcx is [rsi]
// rcx is [rsi]
mov [rdi], rcx
mov [rdi], rcx
#endif
#endif
LOCAL_LABEL(Exit_ByRefWriteBarrier):
// At least one write is already done, increment the pointers
add rdi, 0x8
add rsi, 0x8
dec r8d
je LOCAL_LABEL(NotInHeapExit)
// Now we can do the rest of the writes without checking the heap
LOCAL_LABEL(NextByrefUnchecked):
mov rcx, [rsi]
mov [rdi], rcx
add rdi, 0x8
add rsi, 0x8
dec r8d
jne LOCAL_LABEL(NextByrefUnchecked)
LOCAL_LABEL(NotInHeapExit):
ret
LOCAL_LABEL(Exit_ByRefWriteBarrierBatch):
// Increment the pointers before leaving
// Increment the pointers before leaving
add rdi, 0x8
add rdi, 0x8
add rsi, 0x8
add rsi, 0x8
// Check if we have more in the batch and run again
dec r8d
jne LOCAL_LABEL(NextByref_ByRefWriteBarrierBatch)
ret
ret
LEAF_END_MARKED JIT_ByRefWriteBarrier, _TEXT
LEAF_END_MARKED JIT_ByRefWriteBarrierBatch, _TEXT