Skip to content

Commit 2e55a42

Browse files
author
Mike Pall
committed
Merge branch 'master' into v2.1
2 parents 03a7ebc + e8ec6fe commit 2e55a42

File tree

5 files changed

+41
-9
lines changed

5 files changed

+41
-9
lines changed

‎src/lj_asm_arm.h

+6-1
Original file line numberDiff line numberDiff line change
@@ -2033,6 +2033,9 @@ static void asm_stack_restore(ASMState *as, SnapShot *snap)
20332033

20342034
/* -- GC handling --------------------------------------------------------- */
20352035

2036+
/* Marker to prevent patching the GC check exit. */
2037+
#define ARM_NOPATCH_GC_CHECK (ARMI_BIC|ARMI_K12)
2038+
20362039
/* Check GC threshold and do one or more GC steps. */
20372040
static void asm_gc_check(ASMState *as)
20382041
{
@@ -2044,6 +2047,7 @@ static void asm_gc_check(ASMState *as)
20442047
l_end = emit_label(as);
20452048
/* Exit trace if in GCSatomic or GCSfinalize. Avoids syncing GC objects. */
20462049
asm_guardcc(as, CC_NE); /* Assumes asm_snap_prep() already done. */
2050+
*--as->mcp = ARM_NOPATCH_GC_CHECK;
20472051
emit_n(as, ARMI_CMP|ARMI_K12|0, RID_RET);
20482052
args[0] = ASMREF_TMP1; /* global_State *g */
20492053
args[1] = ASMREF_TMP2; /* MSize steps */
@@ -2212,7 +2216,8 @@ void lj_asm_patchexit(jit_State *J, GCtrace *T, ExitNo exitno, MCode *target)
22122216
/* Look for bl_cc exitstub, replace with b_cc target. */
22132217
uint32_t ins = *p;
22142218
if ((ins & 0x0f000000u) == 0x0b000000u && ins < 0xf0000000u &&
2215-
((ins ^ (px-p)) & 0x00ffffffu) == 0) {
2219+
((ins ^ (px-p)) & 0x00ffffffu) == 0 &&
2220+
p[-1] != ARM_NOPATCH_GC_CHECK) {
22162221
*p = (ins & 0xfe000000u) | (((target-p)-2) & 0x00ffffffu);
22172222
cend = p+1;
22182223
if (!cstart) cstart = p;

‎src/lj_asm_arm64.h

+11-2
Original file line numberDiff line numberDiff line change
@@ -1794,6 +1794,10 @@ static void asm_stack_restore(ASMState *as, SnapShot *snap)
17941794

17951795
/* -- GC handling --------------------------------------------------------- */
17961796

1797+
/* Marker to prevent patching the GC check exit. */
1798+
#define ARM64_NOPATCH_GC_CHECK \
1799+
(A64I_ORRx|A64F_D(RID_TMP)|A64F_M(RID_TMP)|A64F_N(RID_TMP))
1800+
17971801
/* Check GC threshold and do one or more GC steps. */
17981802
static void asm_gc_check(ASMState *as)
17991803
{
@@ -1805,6 +1809,7 @@ static void asm_gc_check(ASMState *as)
18051809
l_end = emit_label(as);
18061810
/* Exit trace if in GCSatomic or GCSfinalize. Avoids syncing GC objects. */
18071811
asm_guardcnb(as, A64I_CBNZ, RID_RET); /* Assumes asm_snap_prep() is done. */
1812+
*--as->mcp = ARM64_NOPATCH_GC_CHECK;
18081813
args[0] = ASMREF_TMP1; /* global_State *g */
18091814
args[1] = ASMREF_TMP2; /* MSize steps */
18101815
asm_gencall(as, ci, args);
@@ -1972,6 +1977,7 @@ void lj_asm_patchexit(jit_State *J, GCtrace *T, ExitNo exitno, MCode *target)
19721977
MCode *cstart = NULL;
19731978
MCode *mcarea = lj_mcode_patch(J, p, 0);
19741979
MCode *px = exitstub_trace_addr(T, exitno);
1980+
int patchlong = 1;
19751981
/* Note: this assumes a trace exit is only ever patched once. */
19761982
for (; p < pe; p++) {
19771983
/* Look for exitstub branch, replace with branch to target. */
@@ -1993,7 +1999,9 @@ void lj_asm_patchexit(jit_State *J, GCtrace *T, ExitNo exitno, MCode *target)
19931999
} else if ((ins & 0x7e000000u) == 0x34000000u &&
19942000
((ins ^ ((px-p)<<5)) & 0x00ffffe0u) == 0) {
19952001
/* Patch cbz/cbnz, if within range. */
1996-
if (A64F_S_OK(delta, 19)) {
2002+
if (p[-1] == ARM64_NOPATCH_GC_CHECK) {
2003+
patchlong = 0;
2004+
} else if (A64F_S_OK(delta, 19)) {
19972005
*p = A64I_LE((ins & 0xff00001fu) | A64F_S19(delta));
19982006
if (!cstart) cstart = p;
19992007
}
@@ -2006,7 +2014,8 @@ void lj_asm_patchexit(jit_State *J, GCtrace *T, ExitNo exitno, MCode *target)
20062014
}
20072015
}
20082016
}
2009-
{ /* Always patch long-range branch in exit stub itself. */
2017+
/* Always patch long-range branch in exit stub itself. Except, if we can't. */
2018+
if (patchlong) {
20102019
ptrdiff_t delta = target - px;
20112020
lj_assertJ(A64F_S_OK(delta, 26), "branch target out of range");
20122021
*px = A64I_B | A64F_S26(delta);

‎src/lj_asm_mips.h

+5-1
Original file line numberDiff line numberDiff line change
@@ -2528,6 +2528,9 @@ static void asm_stack_restore(ASMState *as, SnapShot *snap)
25282528

25292529
/* -- GC handling --------------------------------------------------------- */
25302530

2531+
/* Marker to prevent patching the GC check exit. */
2532+
#define MIPS_NOPATCH_GC_CHECK MIPSI_OR
2533+
25312534
/* Check GC threshold and do one or more GC steps. */
25322535
static void asm_gc_check(ASMState *as)
25332536
{
@@ -2543,6 +2546,7 @@ static void asm_gc_check(ASMState *as)
25432546
args[0] = ASMREF_TMP1; /* global_State *g */
25442547
args[1] = ASMREF_TMP2; /* MSize steps */
25452548
asm_gencall(as, ci, args);
2549+
l_end[-3] = MIPS_NOPATCH_GC_CHECK; /* Replace the nop after the call. */
25462550
emit_tsi(as, MIPSI_AADDIU, ra_releasetmp(as, ASMREF_TMP1), RID_JGL, -32768);
25472551
tmp = ra_releasetmp(as, ASMREF_TMP2);
25482552
emit_loadi(as, tmp, as->gcsteps);
@@ -2694,7 +2698,7 @@ void lj_asm_patchexit(jit_State *J, GCtrace *T, ExitNo exitno, MCode *target)
26942698
#else
26952699
(p[-1] & 0xff600000u) == MIPSI_BC1EQZ
26962700
#endif
2697-
)) {
2701+
) && p[-2] != MIPS_NOPATCH_GC_CHECK) {
26982702
ptrdiff_t delta = target - p;
26992703
if (((delta + 0x8000) >> 16) == 0) { /* Patch in-range branch. */
27002704
patchbranch:

‎src/lj_asm_ppc.h

+10-3
Original file line numberDiff line numberDiff line change
@@ -2067,6 +2067,9 @@ static void asm_stack_restore(ASMState *as, SnapShot *snap)
20672067

20682068
/* -- GC handling --------------------------------------------------------- */
20692069

2070+
/* Marker to prevent patching the GC check exit. */
2071+
#define PPC_NOPATCH_GC_CHECK PPCI_ORIS
2072+
20702073
/* Check GC threshold and do one or more GC steps. */
20712074
static void asm_gc_check(ASMState *as)
20722075
{
@@ -2078,6 +2081,7 @@ static void asm_gc_check(ASMState *as)
20782081
l_end = emit_label(as);
20792082
/* Exit trace if in GCSatomic or GCSfinalize. Avoids syncing GC objects. */
20802083
asm_guardcc(as, CC_NE); /* Assumes asm_snap_prep() already done. */
2084+
*--as->mcp = PPC_NOPATCH_GC_CHECK;
20812085
emit_ai(as, PPCI_CMPWI, RID_RET, 0);
20822086
args[0] = ASMREF_TMP1; /* global_State *g */
20832087
args[1] = ASMREF_TMP2; /* MSize steps */
@@ -2217,7 +2221,7 @@ void lj_asm_patchexit(jit_State *J, GCtrace *T, ExitNo exitno, MCode *target)
22172221
MCode *px = exitstub_trace_addr(T, exitno);
22182222
MCode *cstart = NULL;
22192223
MCode *mcarea = lj_mcode_patch(J, p, 0);
2220-
int clearso = 0;
2224+
int clearso = 0, patchlong = 1;
22212225
for (; p < pe; p++) {
22222226
/* Look for exitstub branch, try to replace with branch to target. */
22232227
uint32_t ins = *p;
@@ -2229,7 +2233,9 @@ void lj_asm_patchexit(jit_State *J, GCtrace *T, ExitNo exitno, MCode *target)
22292233
delta -= sizeof(MCode);
22302234
}
22312235
/* Many, but not all short-range branches can be patched directly. */
2232-
if (((delta + 0x8000) >> 16) == 0) {
2236+
if (p[-1] == PPC_NOPATCH_GC_CHECK) {
2237+
patchlong = 0;
2238+
} else if (((delta + 0x8000) >> 16) == 0) {
22332239
*p = (ins & 0xffdf0000u) | ((uint32_t)delta & 0xffffu) |
22342240
((delta & 0x8000) * (PPCF_Y/0x8000));
22352241
if (!cstart) cstart = p;
@@ -2243,7 +2249,8 @@ void lj_asm_patchexit(jit_State *J, GCtrace *T, ExitNo exitno, MCode *target)
22432249
if (!cstart) cstart = p;
22442250
}
22452251
}
2246-
{ /* Always patch long-range branch in exit stub itself. */
2252+
/* Always patch long-range branch in exit stub itself. Except, if we can't. */
2253+
if (patchlong) {
22472254
ptrdiff_t delta = (char *)target - (char *)px - clearso;
22482255
lj_assertJ(((delta + 0x02000000) >> 26) == 0,
22492256
"branch target out of range");

‎src/lj_asm_x86.h

+9-2
Original file line numberDiff line numberDiff line change
@@ -3072,6 +3072,7 @@ void lj_asm_patchexit(jit_State *J, GCtrace *T, ExitNo exitno, MCode *target)
30723072
MSize len = T->szmcode;
30733073
MCode *px = exitstub_addr(J, exitno) - 6;
30743074
MCode *pe = p+len-6;
3075+
MCode *pgc = NULL;
30753076
#if LJ_GC64
30763077
uint32_t statei = (uint32_t)(GG_OFS(g.vmstate) - GG_OFS(dispatch));
30773078
#else
@@ -3086,9 +3087,15 @@ void lj_asm_patchexit(jit_State *J, GCtrace *T, ExitNo exitno, MCode *target)
30863087
break;
30873088
}
30883089
lj_assertJ(p < pe, "instruction length decoder failed");
3089-
for (; p < pe; p += asm_x86_inslen(p))
3090-
if ((*(uint16_t *)p & 0xf0ff) == 0x800f && p + *(int32_t *)(p+2) == px)
3090+
for (; p < pe; p += asm_x86_inslen(p)) {
3091+
if ((*(uint16_t *)p & 0xf0ff) == 0x800f && p + *(int32_t *)(p+2) == px &&
3092+
p != pgc) {
30913093
*(int32_t *)(p+2) = jmprel(J, p+6, target);
3094+
} else if (*p == XI_CALL &&
3095+
(void *)(p+5+*(int32_t *)(p+1)) == (void *)lj_gc_step_jit) {
3096+
pgc = p+7; /* Do not patch GC check exit. */
3097+
}
3098+
}
30923099
lj_mcode_sync(T->mcode, T->mcode + T->szmcode);
30933100
lj_mcode_patch(J, mcarea, 1);
30943101
}

0 commit comments

Comments
 (0)