From c98dffa186d48c41e76fd3a60e0129a8da60310f Mon Sep 17 00:00:00 2001 From: Jatin Bhateja Date: Wed, 11 Jun 2025 13:48:56 +0000 Subject: [PATCH] 8357982: Fix several failing BMI tests with -XX:+UseAPX Reviewed-by: epeter, sviswanathan --- src/hotspot/cpu/x86/x86_64.ad | 14 ++++++++---- .../intrinsics/bmi/verifycode/AndnTestI.java | 15 ++++++++++++- .../intrinsics/bmi/verifycode/BlsiTestI.java | 17 ++++++++++++++ .../bmi/verifycode/BlsmskTestI.java | 19 +++++++++++++++- .../intrinsics/bmi/verifycode/BlsrTestI.java | 20 ++++++++++++++++- .../bmi/verifycode/BmiIntrinsicBase.java | 22 +++++++++++++++---- .../bmi/verifycode/BzhiTestI2L.java | 17 +++++++++++++- .../intrinsics/bmi/verifycode/LZcntTestI.java | 6 ++++- .../intrinsics/bmi/verifycode/TZcntTestI.java | 6 ++++- 9 files changed, 122 insertions(+), 14 deletions(-) diff --git a/src/hotspot/cpu/x86/x86_64.ad b/src/hotspot/cpu/x86/x86_64.ad index d17bdf5e2c9..7b545f0f5f7 100644 --- a/src/hotspot/cpu/x86/x86_64.ad +++ b/src/hotspot/cpu/x86/x86_64.ad @@ -10531,7 +10531,8 @@ instruct xorI_rReg_im1_ndd(rRegI dst, rRegI src, immI_M1 imm) // Xor Register with Immediate instruct xorI_rReg_imm(rRegI dst, immI src, rFlagsReg cr) %{ - predicate(!UseAPX); + // Strict predicate check to make selection of xorI_rReg_im1 cost agnostic if immI src is -1. + predicate(!UseAPX && n->in(2)->bottom_type()->is_int()->get_con() != -1); match(Set dst (XorI dst src)); effect(KILL cr); flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag); @@ -10545,7 +10546,8 @@ instruct xorI_rReg_imm(rRegI dst, immI src, rFlagsReg cr) instruct xorI_rReg_rReg_imm_ndd(rRegI dst, rRegI src1, immI src2, rFlagsReg cr) %{ - predicate(UseAPX); + // Strict predicate check to make selection of xorI_rReg_im1_ndd cost agnostic if immI src2 is -1. + predicate(UseAPX && n->in(2)->bottom_type()->is_int()->get_con() != -1); match(Set dst (XorI src1 src2)); effect(KILL cr); flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag); @@ -10563,6 +10565,7 @@ instruct xorI_rReg_mem_imm_ndd(rRegI dst, memory src1, immI src2, rFlagsReg cr) predicate(UseAPX); match(Set dst (XorI (LoadI src1) src2)); effect(KILL cr); + ins_cost(150); flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag); format %{ "exorl $dst, $src1, $src2\t# int ndd" %} @@ -11205,7 +11208,8 @@ instruct xorL_rReg_im1_ndd(rRegL dst,rRegL src, immL_M1 imm) // Xor Register with Immediate instruct xorL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr) %{ - predicate(!UseAPX); + // Strict predicate check to make selection of xorL_rReg_im1 cost agnostic if immL32 src is -1. + predicate(!UseAPX && n->in(2)->bottom_type()->is_long()->get_con() != -1L); match(Set dst (XorL dst src)); effect(KILL cr); flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag); @@ -11219,7 +11223,8 @@ instruct xorL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr) instruct xorL_rReg_rReg_imm(rRegL dst, rRegL src1, immL32 src2, rFlagsReg cr) %{ - predicate(UseAPX); + // Strict predicate check to make selection of xorL_rReg_im1_ndd cost agnostic if immL32 src2 is -1. + predicate(UseAPX && n->in(2)->bottom_type()->is_long()->get_con() != -1L); match(Set dst (XorL src1 src2)); effect(KILL cr); flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag); @@ -11238,6 +11243,7 @@ instruct xorL_rReg_mem_imm(rRegL dst, memory src1, immL32 src2, rFlagsReg cr) match(Set dst (XorL (LoadL src1) src2)); effect(KILL cr); flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag); + ins_cost(150); format %{ "exorq $dst, $src1, $src2\t# long ndd" %} ins_encode %{ diff --git a/test/hotspot/jtreg/compiler/intrinsics/bmi/verifycode/AndnTestI.java b/test/hotspot/jtreg/compiler/intrinsics/bmi/verifycode/AndnTestI.java index 4ff18b0744c..4145b0f5641 100644 --- a/test/hotspot/jtreg/compiler/intrinsics/bmi/verifycode/AndnTestI.java +++ b/test/hotspot/jtreg/compiler/intrinsics/bmi/verifycode/AndnTestI.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014, 2022, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2014, 2025, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -57,6 +57,19 @@ public class AndnTestI extends BmiIntrinsicBase.BmiTestCase { (byte) 0x02, // 00010 implied 0F 38 leading opcode bytes (byte) 0x00, (byte) 0xF2}; + // from intel apx specifications EVEX.128.NP.0F38.W0 F2 /r + instrMaskAPX = new byte[]{ + (byte) 0xFF, + (byte) 0x07, + (byte) 0x00, + (byte) 0x00, + (byte) 0xFF}; + instrPatternAPX = new byte[]{ + (byte) 0x62, // fixed prefix byte 0x62 for extended EVEX instruction + (byte) 0x02, // 00010 implied 0F 38 leading opcode bytes + (byte) 0x00, + (byte) 0x00, + (byte) 0xF2}; } public static void main(String[] args) throws Exception { diff --git a/test/hotspot/jtreg/compiler/intrinsics/bmi/verifycode/BlsiTestI.java b/test/hotspot/jtreg/compiler/intrinsics/bmi/verifycode/BlsiTestI.java index 17a11146f4e..346169f1aad 100644 --- a/test/hotspot/jtreg/compiler/intrinsics/bmi/verifycode/BlsiTestI.java +++ b/test/hotspot/jtreg/compiler/intrinsics/bmi/verifycode/BlsiTestI.java @@ -59,6 +59,23 @@ public class BlsiTestI extends BmiIntrinsicBase.BmiTestCase { (byte) 0x00, (byte) 0xF3, (byte) 0b0001_1000}; // bits 543 == 011 (3) + + // from intel apx specifications EVEX.128.NP.0F38.W0 F3 /3(opcode extension) + instrMaskAPX = new byte[]{ + (byte) 0xFF, + (byte) 0x07, + (byte) 0x00, + (byte) 0x00, + (byte) 0xFF, + (byte) 0x38}; + + instrPatternAPX = new byte[]{ + (byte) 0x62, // fixed prefix byte 0x62 for extended EVEX instruction + (byte) 0x02, // 00010 implied 0F 38 leading opcode bytes + (byte) 0x00, + (byte) 0x00, + (byte) 0xF3, + (byte) 0b0001_1000}; // bits 543 == 011 (3) } public static void main(String[] args) throws Exception { diff --git a/test/hotspot/jtreg/compiler/intrinsics/bmi/verifycode/BlsmskTestI.java b/test/hotspot/jtreg/compiler/intrinsics/bmi/verifycode/BlsmskTestI.java index 1fc0e99fe08..55d0368b84b 100644 --- a/test/hotspot/jtreg/compiler/intrinsics/bmi/verifycode/BlsmskTestI.java +++ b/test/hotspot/jtreg/compiler/intrinsics/bmi/verifycode/BlsmskTestI.java @@ -57,7 +57,24 @@ public class BlsmskTestI extends BmiIntrinsicBase.BmiTestCase { (byte) 0x02, // 00010 implied 0F 38 leading opcode bytes (byte) 0x00, (byte) 0xF3, - (byte) 0b0001_0000}; // bits 543 == 011 (3) + (byte) 0b0001_0000}; // bits 543 == 010 (2) + + // from intel apx specifications EVEX.128.NP.0F38.W1 F3 /2(opcode extension part of ModRM.REG) + instrMaskAPX = new byte[]{ + (byte) 0xFF, + (byte) 0x07, + (byte) 0x00, + (byte) 0x00, + (byte) 0xFF, + (byte) 0x38}; + + instrPatternAPX = new byte[]{ + (byte) 0x62, // fixed prefix byte 0x62 for extended EVEX instruction + (byte) 0x02, // 00010 implied 0F 38 leading opcode bytes + (byte) 0x00, + (byte) 0x00, + (byte) 0xF3, + (byte) 0b0001_0000}; // bits 543 == 010 (2) } public static void main(String[] args) throws Exception { diff --git a/test/hotspot/jtreg/compiler/intrinsics/bmi/verifycode/BlsrTestI.java b/test/hotspot/jtreg/compiler/intrinsics/bmi/verifycode/BlsrTestI.java index 57523f11086..3841a2f155c 100644 --- a/test/hotspot/jtreg/compiler/intrinsics/bmi/verifycode/BlsrTestI.java +++ b/test/hotspot/jtreg/compiler/intrinsics/bmi/verifycode/BlsrTestI.java @@ -58,7 +58,25 @@ public class BlsrTestI extends BmiIntrinsicBase.BmiTestCase { (byte) 0x02, // 00010 implied 0F 38 leading opcode bytes (byte) 0x00, (byte) 0xF3, - (byte) 0b0000_1000}; // bits 543 == 011 (3) + (byte) 0b0000_1000}; // bits 543 == 001 (1) + + // from intel apx specifications EVEX.128.NP.0F38.W1 F3 /1(opcode extension part of ModRM.REG) + instrMaskAPX = new byte[]{ + (byte) 0xFF, + (byte) 0x07, + (byte) 0x00, + (byte) 0x00, + (byte) 0xFF, + (byte) 0x38}; + + instrPatternAPX = new byte[]{ + (byte) 0x62, // fixed prefix byte 0x62 for extended EVEX instruction + (byte) 0x02, // 00010 implied 0F 38 leading opcode bytes + (byte) 0x00, + (byte) 0x00, + (byte) 0xF3, + (byte) 0b0000_1000}; // bits 543 == 001 (1) + } public static void main(String[] args) throws Exception { diff --git a/test/hotspot/jtreg/compiler/intrinsics/bmi/verifycode/BmiIntrinsicBase.java b/test/hotspot/jtreg/compiler/intrinsics/bmi/verifycode/BmiIntrinsicBase.java index 743e371d5c2..0e8c8fe9514 100644 --- a/test/hotspot/jtreg/compiler/intrinsics/bmi/verifycode/BmiIntrinsicBase.java +++ b/test/hotspot/jtreg/compiler/intrinsics/bmi/verifycode/BmiIntrinsicBase.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014, 2022, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2014, 2025, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -111,7 +111,8 @@ public class BmiIntrinsicBase extends CompilerWhiteBoxTest { protected void checkEmittedCode(Executable executable) { final byte[] nativeCode = NMethod.get(executable, false).insts; final byte[] matchInstrPattern = (((BmiTestCase) testCase).getTestCaseX64() && Platform.isX64()) ? ((BmiTestCase_x64) testCase).getInstrPattern_x64() : ((BmiTestCase) testCase).getInstrPattern(); - if (!((BmiTestCase) testCase).verifyPositive(nativeCode)) { + boolean use_apx = CPUInfo.hasFeature("apx_f"); + if (!((BmiTestCase) testCase).verifyPositive(nativeCode, use_apx)) { throw new AssertionError(testCase.name() + " " + "CPU instructions expected not found in nativeCode: " + Utils.toHexString(nativeCode) + " ---- Expected instrPattern: " + Utils.toHexString(matchInstrPattern)); } else { @@ -124,6 +125,8 @@ public class BmiIntrinsicBase extends CompilerWhiteBoxTest { private final Method method; protected byte[] instrMask; protected byte[] instrPattern; + protected byte[] instrMaskAPX; + protected byte[] instrPatternAPX; protected boolean isLongOperation; protected String cpuFlag = "bmi1"; protected String vmFlag = "UseBMI1Instructions"; @@ -160,6 +163,13 @@ public class BmiIntrinsicBase extends CompilerWhiteBoxTest { return countCpuInstructions(nativeCode, instrMask, instrPattern); } + protected int countCpuInstructionsAPX(byte[] nativeCode) { + if (instrMaskAPX == null || instrPatternAPX == null) { + return 0; + } + return countCpuInstructions(nativeCode, instrMaskAPX, instrPatternAPX); + } + public static int countCpuInstructions(byte[] nativeCode, byte[] instrMask, byte[] instrPattern) { int count = 0; int patternSize = Math.min(instrMask.length, instrPattern.length); @@ -181,8 +191,12 @@ public class BmiIntrinsicBase extends CompilerWhiteBoxTest { return count; } - public boolean verifyPositive(byte[] nativeCode) { - final int cnt = countCpuInstructions(nativeCode); + public boolean verifyPositive(byte[] nativeCode, boolean use_apx) { + int cnt = countCpuInstructions(nativeCode); + if (use_apx) { + System.out.println("CHECKING APX INST PATTERNS"); + cnt += countCpuInstructionsAPX(nativeCode); + } if (Platform.isX86()) { return cnt >= (isLongOperation ? 2 : 1); } else { diff --git a/test/hotspot/jtreg/compiler/intrinsics/bmi/verifycode/BzhiTestI2L.java b/test/hotspot/jtreg/compiler/intrinsics/bmi/verifycode/BzhiTestI2L.java index daba485be50..4cf94a0eb8b 100644 --- a/test/hotspot/jtreg/compiler/intrinsics/bmi/verifycode/BzhiTestI2L.java +++ b/test/hotspot/jtreg/compiler/intrinsics/bmi/verifycode/BzhiTestI2L.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2021, 2025, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -73,6 +73,21 @@ public class BzhiTestI2L extends BmiIntrinsicBase.BmiTestCase_x64 { (byte) 0x62, // 00010 implied 0F 38 leading opcode bytes (byte) 0xA8, (byte) 0xF5}; + + // from intel apx specifications EVEX.128.NP.0F38.W0 F5 /r + instrMaskAPX = new byte[]{ + (byte) 0xFF, + (byte) 0x07, + (byte) 0x00, + (byte) 0x00, + (byte) 0xFF}; + + instrPatternAPX = new byte[]{ + (byte) 0x62, // fixed prefix byte 0x62 for extended EVEX instruction + (byte) 0x02, // 00010 implied 0F 38 leading opcode bytes + (byte) 0x00, + (byte) 0x00, + (byte) 0xF5}; } public static void main(String[] args) throws Exception { diff --git a/test/hotspot/jtreg/compiler/intrinsics/bmi/verifycode/LZcntTestI.java b/test/hotspot/jtreg/compiler/intrinsics/bmi/verifycode/LZcntTestI.java index d1b88ffd9d0..c905fca34a2 100644 --- a/test/hotspot/jtreg/compiler/intrinsics/bmi/verifycode/LZcntTestI.java +++ b/test/hotspot/jtreg/compiler/intrinsics/bmi/verifycode/LZcntTestI.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014, 2022, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2014, 2025, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -51,6 +51,10 @@ public class LZcntTestI extends BmiIntrinsicBase.BmiTestCase_x64 { instrMask_x64 = new byte[]{(byte) 0xFF, (byte) 0x00, (byte) 0xFF, (byte) 0xFF}; instrPattern_x64 = new byte[]{(byte) 0xF3, (byte) 0x00, (byte) 0x0F, (byte) 0xBD}; + + // REX2 variant + instrMaskAPX = new byte[]{(byte) 0xFF, (byte) 0xFF, (byte)0x80, (byte) 0xFF}; + instrPatternAPX = new byte[]{(byte) 0xF3, (byte) 0xD5, (byte) 0x80, (byte) 0xBD}; } public static void main(String[] args) throws Exception { diff --git a/test/hotspot/jtreg/compiler/intrinsics/bmi/verifycode/TZcntTestI.java b/test/hotspot/jtreg/compiler/intrinsics/bmi/verifycode/TZcntTestI.java index 641fdb0b203..8a8ce4508fa 100644 --- a/test/hotspot/jtreg/compiler/intrinsics/bmi/verifycode/TZcntTestI.java +++ b/test/hotspot/jtreg/compiler/intrinsics/bmi/verifycode/TZcntTestI.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014, 2022, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2014, 2025, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -50,6 +50,10 @@ public class TZcntTestI extends BmiIntrinsicBase.BmiTestCase_x64 { instrMask_x64 = new byte[]{(byte) 0xFF, (byte) 0x00, (byte) 0xFF, (byte) 0xFF}; instrPattern_x64 = new byte[]{(byte) 0xF3, (byte) 0x00, (byte) 0x0F, (byte) 0xBC}; + + // REX2 variant + instrMaskAPX = new byte[]{(byte) 0xFF, (byte) 0xFF, (byte)0x80, (byte) 0xFF}; + instrPatternAPX = new byte[]{(byte) 0xF3, (byte) 0xD5, (byte) 0x80, (byte) 0xBC}; } public static void main(String[] args) throws Exception {