1berry/openjdk
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

8283667: [vectorapi] Vectorization for masked load with IOOBE with predicate feature

Browse Source 
Reviewed-by: sviswanathan, psandoz
This commit is contained in:
Xiaohong Gong 2022-06-07 07:41:58 +00:00
parent 42261d752a
commit 39fa52b5f7
44 changed files with 432 additions and 279 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
src/hotspot/share/classfile/vmIntrinsics.hpp
View File

@ -958,6 +958,7 @@ class methodHandle;
"Ljava/lang/Object;" \
"J" \
"Ljdk/internal/vm/vector/VectorSupport$VectorMask;" \
"I" \
"Ljava/lang/Object;" \
"J" \
"Ljdk/internal/vm/vector/VectorSupport$VectorSpecies;" \

53
src/hotspot/share/opto/vectorIntrinsics.cpp
View File

@ -1131,7 +1131,7 @@ bool LibraryCallKit::inline_vector_mem_operation(bool is_store) {
// S extends VectorSpecies<E>,
// M extends VectorMask<E>>
// V loadMasked(Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType,
// int length, Object base, long offset, M m,
// int length, Object base, long offset, M m, int offsetInRange,
// C container, long index, S s, // Arguments for default implementation
// LoadVectorMaskedOperation<C, V, S, M> defaultImpl) {
//
@ -1216,24 +1216,47 @@ bool LibraryCallKit::inline_vector_mem_masked_operation(bool is_store) {
int mem_num_elem = using_byte_array ? num_elem * type2aelembytes(elem_bt) : num_elem;
BasicType mem_elem_bt = using_byte_array ? T_BYTE : elem_bt;
bool use_predicate = arch_supports_vector(is_store ? Op_StoreVectorMasked : Op_LoadVectorMasked,
mem_num_elem, mem_elem_bt,
(VectorMaskUseType) (VecMaskUseLoad | VecMaskUsePred));
// Masked vector store operation needs the architecture predicate feature. We need to check
// whether the predicated vector operation is supported by backend.
if (is_store && !use_predicate) {
if (C->print_intrinsics()) {
tty->print_cr(" ** not supported: op=storeMasked vlen=%d etype=%s using_byte_array=%d",
num_elem, type2name(elem_bt), using_byte_array ? 1 : 0);
bool supports_predicate = arch_supports_vector(is_store ? Op_StoreVectorMasked : Op_LoadVectorMasked,
mem_num_elem, mem_elem_bt,
(VectorMaskUseType) (VecMaskUseLoad | VecMaskUsePred));
// If current arch does not support the predicated operations, we have to bail
// out when current case uses the predicate feature.
if (!supports_predicate) {
bool needs_predicate = false;
if (is_store) {
// Masked vector store always uses the predicated store.
needs_predicate = true;
} else {
// Masked vector load with IOOBE always uses the predicated load.
const TypeInt* offset_in_range = gvn().type(argument(8))->isa_int();
if (!offset_in_range->is_con()) {
if (C->print_intrinsics()) {
tty->print_cr(" ** missing constant: offsetInRange=%s",
NodeClassNames[argument(8)->Opcode()]);
}
set_map(old_map);
set_sp(old_sp);
return false;
}
needs_predicate = (offset_in_range->get_con() == 0);
}
if (needs_predicate) {
if (C->print_intrinsics()) {
tty->print_cr(" ** not supported: op=%s vlen=%d etype=%s using_byte_array=%d",
is_store ? "storeMasked" : "loadMasked",
num_elem, type2name(elem_bt), using_byte_array ? 1 : 0);
}
set_map(old_map);
set_sp(old_sp);
return false;
}
set_map(old_map);
set_sp(old_sp);
return false;
}
// This only happens for masked vector load. If predicate is not supported, then check whether
// the normal vector load and blend operations are supported by backend.
if (!use_predicate && (!arch_supports_vector(Op_LoadVector, mem_num_elem, mem_elem_bt, VecMaskNotUsed) ||
if (!supports_predicate && (!arch_supports_vector(Op_LoadVector, mem_num_elem, mem_elem_bt, VecMaskNotUsed) ||
!arch_supports_vector(Op_VectorBlend, mem_num_elem, mem_elem_bt, VecMaskUseLoad))) {
if (C->print_intrinsics()) {
tty->print_cr(" ** not supported: op=loadMasked vlen=%d etype=%s using_byte_array=%d",
@ -1330,7 +1353,7 @@ bool LibraryCallKit::inline_vector_mem_masked_operation(bool is_store) {
mask = gvn().transform(new VectorReinterpretNode(mask, from_mask_type, to_mask_type));
}
if (use_predicate) {
if (supports_predicate) {
// Generate masked load vector node if predicate feature is supported.
const TypeVect* vt = TypeVect::make(mem_elem_bt, mem_num_elem);
vload = gvn().transform(new LoadVectorMaskedNode(control(), memory(addr), addr, addr_type, vt, mask));

8
src/java.base/share/classes/jdk/internal/misc/X-ScopedMemoryAccess.java.template
View File

@ -363,7 +363,7 @@ public class ScopedMemoryAccess {
<V extends VectorSupport.Vector<E>, E, S extends VectorSupport.VectorSpecies<E>,
M extends VectorSupport.VectorMask<E>>
V loadFromMemorySegmentMasked(Class<? extends V> vmClass, Class<M> maskClass, Class<E> e,
int length, AbstractMemorySegmentImpl msp, long offset, M m, S s,
int length, AbstractMemorySegmentImpl msp, long offset, M m, S s, int offsetInRange,
VectorSupport.LoadVectorMaskedOperation<AbstractMemorySegmentImpl, V, S, M> defaultImpl) {
// @@@ Smarter alignment checking if accessing heap segment backing non-byte[] array
if (msp.maxAlignMask() > 1) {
@ -375,7 +375,7 @@ public class ScopedMemoryAccess {
msp.sessionImpl(),
vmClass, maskClass, e, length,
msp, offset, m,
s,
s, offsetInRange,
defaultImpl);
} catch (ScopedAccessError ex) {
throw new IllegalStateException("This segment is already closed");
@ -390,13 +390,13 @@ public class ScopedMemoryAccess {
V loadFromMemorySegmentMaskedScopedInternal(MemorySessionImpl session, Class<? extends V> vmClass,
Class<M> maskClass, Class<E> e, int length,
AbstractMemorySegmentImpl msp, long offset, M m,
S s,
S s, int offsetInRange,
VectorSupport.LoadVectorMaskedOperation<AbstractMemorySegmentImpl, V, S, M> defaultImpl) {
try {
session.checkValidState();
return VectorSupport.loadMasked(vmClass, maskClass, e, length,
msp.unsafeGetBase(), msp.unsafeGetOffset() + offset, m,
msp.unsafeGetBase(), msp.unsafeGetOffset() + offset, m, offsetInRange,
msp, offset, s,
defaultImpl);
} finally {

8
src/java.base/share/classes/jdk/internal/vm/vector/VectorSupport.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2020, 2021, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2020, 2022, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -416,9 +416,9 @@ public class VectorSupport {
S extends VectorSpecies<E>,
M extends VectorMask<E>>
V loadMasked(Class<? extends V> vClass, Class<M> mClass, Class<E> eClass,
int length,
Object base, long offset,
M m, C container, long index, S s,
int length, Object base, long offset,
M m, int offsetInRange,
C container, long index, S s,
LoadVectorMaskedOperation<C, V, S, M> defaultImpl) {
assert isNonCapturingLambda(defaultImpl) : defaultImpl;
return defaultImpl.load(container, index, s, m);

6
src/jdk.incubator.vector/share/classes/jdk/incubator/vector/AbstractVector.java
View File

@ -62,6 +62,12 @@ abstract class AbstractVector<E> extends Vector<E> {
/*package-private*/
static final ByteOrder REGISTER_ENDIAN = ByteOrder.LITTLE_ENDIAN;
/*package-private*/
static final int OFFSET_IN_RANGE = 1;
/*package-private*/
static final int OFFSET_OUT_OF_RANGE = 0;
/*package-private*/
AbstractVector(Object bits) {
super(bits);

12
src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte128Vector.java
View File

@ -878,8 +878,8 @@ final class Byte128Vector extends ByteVector {
@ForceInline
@Override
final
ByteVector fromArray0(byte[] a, int offset, VectorMask<Byte> m) {
return super.fromArray0Template(Byte128Mask.class, a, offset, (Byte128Mask) m); // specialize
ByteVector fromArray0(byte[] a, int offset, VectorMask<Byte> m, int offsetInRange) {
return super.fromArray0Template(Byte128Mask.class, a, offset, (Byte128Mask) m, offsetInRange); // specialize
}
@ -894,8 +894,8 @@ final class Byte128Vector extends ByteVector {
@ForceInline
@Override
final
ByteVector fromBooleanArray0(boolean[] a, int offset, VectorMask<Byte> m) {
return super.fromBooleanArray0Template(Byte128Mask.class, a, offset, (Byte128Mask) m); // specialize
ByteVector fromBooleanArray0(boolean[] a, int offset, VectorMask<Byte> m, int offsetInRange) {
return super.fromBooleanArray0Template(Byte128Mask.class, a, offset, (Byte128Mask) m, offsetInRange); // specialize
}
@ForceInline
@ -908,8 +908,8 @@ final class Byte128Vector extends ByteVector {
@ForceInline
@Override
final
ByteVector fromMemorySegment0(MemorySegment ms, long offset, VectorMask<Byte> m) {
return super.fromMemorySegment0Template(Byte128Mask.class, ms, offset, (Byte128Mask) m); // specialize
ByteVector fromMemorySegment0(MemorySegment ms, long offset, VectorMask<Byte> m, int offsetInRange) {
return super.fromMemorySegment0Template(Byte128Mask.class, ms, offset, (Byte128Mask) m, offsetInRange); // specialize
}
@ForceInline

12
src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte256Vector.java
View File

@ -910,8 +910,8 @@ final class Byte256Vector extends ByteVector {
@ForceInline
@Override
final
ByteVector fromArray0(byte[] a, int offset, VectorMask<Byte> m) {
return super.fromArray0Template(Byte256Mask.class, a, offset, (Byte256Mask) m); // specialize
ByteVector fromArray0(byte[] a, int offset, VectorMask<Byte> m, int offsetInRange) {
return super.fromArray0Template(Byte256Mask.class, a, offset, (Byte256Mask) m, offsetInRange); // specialize
}
@ -926,8 +926,8 @@ final class Byte256Vector extends ByteVector {
@ForceInline
@Override
final
ByteVector fromBooleanArray0(boolean[] a, int offset, VectorMask<Byte> m) {
return super.fromBooleanArray0Template(Byte256Mask.class, a, offset, (Byte256Mask) m); // specialize
ByteVector fromBooleanArray0(boolean[] a, int offset, VectorMask<Byte> m, int offsetInRange) {
return super.fromBooleanArray0Template(Byte256Mask.class, a, offset, (Byte256Mask) m, offsetInRange); // specialize
}
@ForceInline
@ -940,8 +940,8 @@ final class Byte256Vector extends ByteVector {
@ForceInline
@Override
final
ByteVector fromMemorySegment0(MemorySegment ms, long offset, VectorMask<Byte> m) {
return super.fromMemorySegment0Template(Byte256Mask.class, ms, offset, (Byte256Mask) m); // specialize
ByteVector fromMemorySegment0(MemorySegment ms, long offset, VectorMask<Byte> m, int offsetInRange) {
return super.fromMemorySegment0Template(Byte256Mask.class, ms, offset, (Byte256Mask) m, offsetInRange); // specialize
}
@ForceInline

12
src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte512Vector.java
View File

@ -974,8 +974,8 @@ final class Byte512Vector extends ByteVector {
@ForceInline
@Override
final
ByteVector fromArray0(byte[] a, int offset, VectorMask<Byte> m) {
return super.fromArray0Template(Byte512Mask.class, a, offset, (Byte512Mask) m); // specialize
ByteVector fromArray0(byte[] a, int offset, VectorMask<Byte> m, int offsetInRange) {
return super.fromArray0Template(Byte512Mask.class, a, offset, (Byte512Mask) m, offsetInRange); // specialize
}
@ -990,8 +990,8 @@ final class Byte512Vector extends ByteVector {
@ForceInline
@Override
final
ByteVector fromBooleanArray0(boolean[] a, int offset, VectorMask<Byte> m) {
return super.fromBooleanArray0Template(Byte512Mask.class, a, offset, (Byte512Mask) m); // specialize
ByteVector fromBooleanArray0(boolean[] a, int offset, VectorMask<Byte> m, int offsetInRange) {
return super.fromBooleanArray0Template(Byte512Mask.class, a, offset, (Byte512Mask) m, offsetInRange); // specialize
}
@ForceInline
@ -1004,8 +1004,8 @@ final class Byte512Vector extends ByteVector {
@ForceInline
@Override
final
ByteVector fromMemorySegment0(MemorySegment ms, long offset, VectorMask<Byte> m) {
return super.fromMemorySegment0Template(Byte512Mask.class, ms, offset, (Byte512Mask) m); // specialize
ByteVector fromMemorySegment0(MemorySegment ms, long offset, VectorMask<Byte> m, int offsetInRange) {
return super.fromMemorySegment0Template(Byte512Mask.class, ms, offset, (Byte512Mask) m, offsetInRange); // specialize
}
@ForceInline

12
src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte64Vector.java
View File

@ -862,8 +862,8 @@ final class Byte64Vector extends ByteVector {
@ForceInline
@Override
final
ByteVector fromArray0(byte[] a, int offset, VectorMask<Byte> m) {
return super.fromArray0Template(Byte64Mask.class, a, offset, (Byte64Mask) m); // specialize
ByteVector fromArray0(byte[] a, int offset, VectorMask<Byte> m, int offsetInRange) {
return super.fromArray0Template(Byte64Mask.class, a, offset, (Byte64Mask) m, offsetInRange); // specialize
}
@ -878,8 +878,8 @@ final class Byte64Vector extends ByteVector {
@ForceInline
@Override
final
ByteVector fromBooleanArray0(boolean[] a, int offset, VectorMask<Byte> m) {
return super.fromBooleanArray0Template(Byte64Mask.class, a, offset, (Byte64Mask) m); // specialize
ByteVector fromBooleanArray0(boolean[] a, int offset, VectorMask<Byte> m, int offsetInRange) {
return super.fromBooleanArray0Template(Byte64Mask.class, a, offset, (Byte64Mask) m, offsetInRange); // specialize
}
@ForceInline
@ -892,8 +892,8 @@ final class Byte64Vector extends ByteVector {
@ForceInline
@Override
final
ByteVector fromMemorySegment0(MemorySegment ms, long offset, VectorMask<Byte> m) {
return super.fromMemorySegment0Template(Byte64Mask.class, ms, offset, (Byte64Mask) m); // specialize
ByteVector fromMemorySegment0(MemorySegment ms, long offset, VectorMask<Byte> m, int offsetInRange) {
return super.fromMemorySegment0Template(Byte64Mask.class, ms, offset, (Byte64Mask) m, offsetInRange); // specialize
}
@ForceInline

12
src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ByteMaxVector.java
View File

@ -848,8 +848,8 @@ final class ByteMaxVector extends ByteVector {
@ForceInline
@Override
final
ByteVector fromArray0(byte[] a, int offset, VectorMask<Byte> m) {
return super.fromArray0Template(ByteMaxMask.class, a, offset, (ByteMaxMask) m); // specialize
ByteVector fromArray0(byte[] a, int offset, VectorMask<Byte> m, int offsetInRange) {
return super.fromArray0Template(ByteMaxMask.class, a, offset, (ByteMaxMask) m, offsetInRange); // specialize
}
@ -864,8 +864,8 @@ final class ByteMaxVector extends ByteVector {
@ForceInline
@Override
final
ByteVector fromBooleanArray0(boolean[] a, int offset, VectorMask<Byte> m) {
return super.fromBooleanArray0Template(ByteMaxMask.class, a, offset, (ByteMaxMask) m); // specialize
ByteVector fromBooleanArray0(boolean[] a, int offset, VectorMask<Byte> m, int offsetInRange) {
return super.fromBooleanArray0Template(ByteMaxMask.class, a, offset, (ByteMaxMask) m, offsetInRange); // specialize
}
@ForceInline
@ -878,8 +878,8 @@ final class ByteMaxVector extends ByteVector {
@ForceInline
@Override
final
ByteVector fromMemorySegment0(MemorySegment ms, long offset, VectorMask<Byte> m) {
return super.fromMemorySegment0Template(ByteMaxMask.class, ms, offset, (ByteMaxMask) m); // specialize
ByteVector fromMemorySegment0(MemorySegment ms, long offset, VectorMask<Byte> m, int offsetInRange) {
return super.fromMemorySegment0Template(ByteMaxMask.class, ms, offset, (ByteMaxMask) m, offsetInRange); // specialize
}
@ForceInline

33
src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ByteVector.java
View File

@ -3009,12 +3009,11 @@ public abstract class ByteVector extends AbstractVector<Byte> {
VectorMask<Byte> m) {
ByteSpecies vsp = (ByteSpecies) species;
if (VectorIntrinsics.indexInRange(offset, vsp.length(), a.length)) {
return vsp.dummyVector().fromArray0(a, offset, m);
return vsp.dummyVector().fromArray0(a, offset, m, OFFSET_IN_RANGE);
}
// FIXME: optimize
checkMaskFromIndexSize(offset, vsp, m, 1, a.length);
return vsp.vOp(m, i -> a[offset + i]);
return vsp.dummyVector().fromArray0(a, offset, m, OFFSET_OUT_OF_RANGE);
}
/**
@ -3166,12 +3165,11 @@ public abstract class ByteVector extends AbstractVector<Byte> {
ByteSpecies vsp = (ByteSpecies) species;
if (VectorIntrinsics.indexInRange(offset, vsp.length(), a.length)) {
ByteVector zero = vsp.zero();
return vsp.dummyVector().fromBooleanArray0(a, offset, m);
return vsp.dummyVector().fromBooleanArray0(a, offset, m, OFFSET_IN_RANGE);
}
// FIXME: optimize
checkMaskFromIndexSize(offset, vsp, m, 1, a.length);
return vsp.vOp(m, i -> (byte) (a[offset + i] ? 1 : 0));
return vsp.dummyVector().fromBooleanArray0(a, offset, m, OFFSET_OUT_OF_RANGE);
}
/**
@ -3353,12 +3351,11 @@ public abstract class ByteVector extends AbstractVector<Byte> {
VectorMask<Byte> m) {
ByteSpecies vsp = (ByteSpecies) species;
if (VectorIntrinsics.indexInRange(offset, vsp.vectorByteSize(), ms.byteSize())) {
return vsp.dummyVector().fromMemorySegment0(ms, offset, m).maybeSwap(bo);
return vsp.dummyVector().fromMemorySegment0(ms, offset, m, OFFSET_IN_RANGE).maybeSwap(bo);
}
// FIXME: optimize
checkMaskFromIndexSize(offset, vsp, m, 1, ms.byteSize());
return vsp.ldLongOp(ms, offset, m, ByteVector::memorySegmentGet);
return vsp.dummyVector().fromMemorySegment0(ms, offset, m, OFFSET_OUT_OF_RANGE).maybeSwap(bo);
}
// Memory store operations
@ -3756,16 +3753,16 @@ public abstract class ByteVector extends AbstractVector<Byte> {
/*package-private*/
abstract
ByteVector fromArray0(byte[] a, int offset, VectorMask<Byte> m);
ByteVector fromArray0(byte[] a, int offset, VectorMask<Byte> m, int offsetInRange);
@ForceInline
final
<M extends VectorMask<Byte>>
ByteVector fromArray0Template(Class<M> maskClass, byte[] a, int offset, M m) {
ByteVector fromArray0Template(Class<M> maskClass, byte[] a, int offset, M m, int offsetInRange) {
m.check(species());
ByteSpecies vsp = vspecies();
return VectorSupport.loadMasked(
vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
a, arrayAddress(a, offset), m,
a, arrayAddress(a, offset), m, offsetInRange,
a, offset, vsp,
(arr, off, s, vm) -> s.ldOp(arr, (int) off, vm,
(arr_, off_, i) -> arr_[off_ + i]));
@ -3790,16 +3787,16 @@ public abstract class ByteVector extends AbstractVector<Byte> {
/*package-private*/
abstract
ByteVector fromBooleanArray0(boolean[] a, int offset, VectorMask<Byte> m);
ByteVector fromBooleanArray0(boolean[] a, int offset, VectorMask<Byte> m, int offsetInRange);
@ForceInline
final
<M extends VectorMask<Byte>>
ByteVector fromBooleanArray0Template(Class<M> maskClass, boolean[] a, int offset, M m) {
ByteVector fromBooleanArray0Template(Class<M> maskClass, boolean[] a, int offset, M m, int offsetInRange) {
m.check(species());
ByteSpecies vsp = vspecies();
return VectorSupport.loadMasked(
vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
a, booleanArrayAddress(a, offset), m,
a, booleanArrayAddress(a, offset), m, offsetInRange,
a, offset, vsp,
(arr, off, s, vm) -> s.ldOp(arr, (int) off, vm,
(arr_, off_, i) -> (byte) (arr_[off_ + i] ? 1 : 0)));
@ -3820,16 +3817,16 @@ public abstract class ByteVector extends AbstractVector<Byte> {
}
abstract
ByteVector fromMemorySegment0(MemorySegment ms, long offset, VectorMask<Byte> m);
ByteVector fromMemorySegment0(MemorySegment ms, long offset, VectorMask<Byte> m, int offsetInRange);
@ForceInline
final
<M extends VectorMask<Byte>>
ByteVector fromMemorySegment0Template(Class<M> maskClass, MemorySegment ms, long offset, M m) {
ByteVector fromMemorySegment0Template(Class<M> maskClass, MemorySegment ms, long offset, M m, int offsetInRange) {
ByteSpecies vsp = vspecies();
m.check(vsp);
return ScopedMemoryAccess.loadFromMemorySegmentMasked(
vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
(AbstractMemorySegmentImpl) ms, offset, m, vsp,
(AbstractMemorySegmentImpl) ms, offset, m, vsp, offsetInRange,
(msp, off, s, vm) -> {
return s.ldLongOp((MemorySegment) msp, off, vm, ByteVector::memorySegmentGet);
});

8
src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double128Vector.java
View File

@ -839,8 +839,8 @@ final class Double128Vector extends DoubleVector {
@ForceInline
@Override
final
DoubleVector fromArray0(double[] a, int offset, VectorMask<Double> m) {
return super.fromArray0Template(Double128Mask.class, a, offset, (Double128Mask) m); // specialize
DoubleVector fromArray0(double[] a, int offset, VectorMask<Double> m, int offsetInRange) {
return super.fromArray0Template(Double128Mask.class, a, offset, (Double128Mask) m, offsetInRange); // specialize
}
@ForceInline
@ -862,8 +862,8 @@ final class Double128Vector extends DoubleVector {
@ForceInline
@Override
final
DoubleVector fromMemorySegment0(MemorySegment ms, long offset, VectorMask<Double> m) {
return super.fromMemorySegment0Template(Double128Mask.class, ms, offset, (Double128Mask) m); // specialize
DoubleVector fromMemorySegment0(MemorySegment ms, long offset, VectorMask<Double> m, int offsetInRange) {
return super.fromMemorySegment0Template(Double128Mask.class, ms, offset, (Double128Mask) m, offsetInRange); // specialize
}
@ForceInline

8
src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double256Vector.java
View File

@ -843,8 +843,8 @@ final class Double256Vector extends DoubleVector {
@ForceInline
@Override
final
DoubleVector fromArray0(double[] a, int offset, VectorMask<Double> m) {
return super.fromArray0Template(Double256Mask.class, a, offset, (Double256Mask) m); // specialize
DoubleVector fromArray0(double[] a, int offset, VectorMask<Double> m, int offsetInRange) {
return super.fromArray0Template(Double256Mask.class, a, offset, (Double256Mask) m, offsetInRange); // specialize
}
@ForceInline
@ -866,8 +866,8 @@ final class Double256Vector extends DoubleVector {
@ForceInline
@Override
final
DoubleVector fromMemorySegment0(MemorySegment ms, long offset, VectorMask<Double> m) {
return super.fromMemorySegment0Template(Double256Mask.class, ms, offset, (Double256Mask) m); // specialize
DoubleVector fromMemorySegment0(MemorySegment ms, long offset, VectorMask<Double> m, int offsetInRange) {
return super.fromMemorySegment0Template(Double256Mask.class, ms, offset, (Double256Mask) m, offsetInRange); // specialize
}
@ForceInline

8
src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double512Vector.java
View File

@ -851,8 +851,8 @@ final class Double512Vector extends DoubleVector {
@ForceInline
@Override
final
DoubleVector fromArray0(double[] a, int offset, VectorMask<Double> m) {
return super.fromArray0Template(Double512Mask.class, a, offset, (Double512Mask) m); // specialize
DoubleVector fromArray0(double[] a, int offset, VectorMask<Double> m, int offsetInRange) {
return super.fromArray0Template(Double512Mask.class, a, offset, (Double512Mask) m, offsetInRange); // specialize
}
@ForceInline
@ -874,8 +874,8 @@ final class Double512Vector extends DoubleVector {
@ForceInline
@Override
final
DoubleVector fromMemorySegment0(MemorySegment ms, long offset, VectorMask<Double> m) {
return super.fromMemorySegment0Template(Double512Mask.class, ms, offset, (Double512Mask) m); // specialize
DoubleVector fromMemorySegment0(MemorySegment ms, long offset, VectorMask<Double> m, int offsetInRange) {
return super.fromMemorySegment0Template(Double512Mask.class, ms, offset, (Double512Mask) m, offsetInRange); // specialize
}
@ForceInline

8
src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double64Vector.java
View File

@ -837,8 +837,8 @@ final class Double64Vector extends DoubleVector {
@ForceInline
@Override
final
DoubleVector fromArray0(double[] a, int offset, VectorMask<Double> m) {
return super.fromArray0Template(Double64Mask.class, a, offset, (Double64Mask) m); // specialize
DoubleVector fromArray0(double[] a, int offset, VectorMask<Double> m, int offsetInRange) {
return super.fromArray0Template(Double64Mask.class, a, offset, (Double64Mask) m, offsetInRange); // specialize
}
@ForceInline
@ -860,8 +860,8 @@ final class Double64Vector extends DoubleVector {
@ForceInline
@Override
final
DoubleVector fromMemorySegment0(MemorySegment ms, long offset, VectorMask<Double> m) {
return super.fromMemorySegment0Template(Double64Mask.class, ms, offset, (Double64Mask) m); // specialize
DoubleVector fromMemorySegment0(MemorySegment ms, long offset, VectorMask<Double> m, int offsetInRange) {
return super.fromMemorySegment0Template(Double64Mask.class, ms, offset, (Double64Mask) m, offsetInRange); // specialize
}
@ForceInline

8
src/jdk.incubator.vector/share/classes/jdk/incubator/vector/DoubleMaxVector.java
View File

@ -836,8 +836,8 @@ final class DoubleMaxVector extends DoubleVector {
@ForceInline
@Override
final
DoubleVector fromArray0(double[] a, int offset, VectorMask<Double> m) {
return super.fromArray0Template(DoubleMaxMask.class, a, offset, (DoubleMaxMask) m); // specialize
DoubleVector fromArray0(double[] a, int offset, VectorMask<Double> m, int offsetInRange) {
return super.fromArray0Template(DoubleMaxMask.class, a, offset, (DoubleMaxMask) m, offsetInRange); // specialize
}
@ForceInline
@ -859,8 +859,8 @@ final class DoubleMaxVector extends DoubleVector {
@ForceInline
@Override
final
DoubleVector fromMemorySegment0(MemorySegment ms, long offset, VectorMask<Double> m) {
return super.fromMemorySegment0Template(DoubleMaxMask.class, ms, offset, (DoubleMaxMask) m); // specialize
DoubleVector fromMemorySegment0(MemorySegment ms, long offset, VectorMask<Double> m, int offsetInRange) {
return super.fromMemorySegment0Template(DoubleMaxMask.class, ms, offset, (DoubleMaxMask) m, offsetInRange); // specialize
}
@ForceInline

22
src/jdk.incubator.vector/share/classes/jdk/incubator/vector/DoubleVector.java
View File

@ -2806,12 +2806,11 @@ public abstract class DoubleVector extends AbstractVector<Double> {
VectorMask<Double> m) {
DoubleSpecies vsp = (DoubleSpecies) species;
if (VectorIntrinsics.indexInRange(offset, vsp.length(), a.length)) {
return vsp.dummyVector().fromArray0(a, offset, m);
return vsp.dummyVector().fromArray0(a, offset, m, OFFSET_IN_RANGE);
}
// FIXME: optimize
checkMaskFromIndexSize(offset, vsp, m, 1, a.length);
return vsp.vOp(m, i -> a[offset + i]);
return vsp.dummyVector().fromArray0(a, offset, m, OFFSET_OUT_OF_RANGE);
}
/**
@ -3039,12 +3038,11 @@ public abstract class DoubleVector extends AbstractVector<Double> {
VectorMask<Double> m) {
DoubleSpecies vsp = (DoubleSpecies) species;
if (VectorIntrinsics.indexInRange(offset, vsp.vectorByteSize(), ms.byteSize())) {
return vsp.dummyVector().fromMemorySegment0(ms, offset, m).maybeSwap(bo);
return vsp.dummyVector().fromMemorySegment0(ms, offset, m, OFFSET_IN_RANGE).maybeSwap(bo);
}
// FIXME: optimize
checkMaskFromIndexSize(offset, vsp, m, 8, ms.byteSize());
return vsp.ldLongOp(ms, offset, m, DoubleVector::memorySegmentGet);
return vsp.dummyVector().fromMemorySegment0(ms, offset, m, OFFSET_OUT_OF_RANGE).maybeSwap(bo);
}
// Memory store operations
@ -3311,16 +3309,16 @@ public abstract class DoubleVector extends AbstractVector<Double> {
/*package-private*/
abstract
DoubleVector fromArray0(double[] a, int offset, VectorMask<Double> m);
DoubleVector fromArray0(double[] a, int offset, VectorMask<Double> m, int offsetInRange);
@ForceInline
final
<M extends VectorMask<Double>>
DoubleVector fromArray0Template(Class<M> maskClass, double[] a, int offset, M m) {
DoubleVector fromArray0Template(Class<M> maskClass, double[] a, int offset, M m, int offsetInRange) {
m.check(species());
DoubleSpecies vsp = vspecies();
return VectorSupport.loadMasked(
vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
a, arrayAddress(a, offset), m,
a, arrayAddress(a, offset), m, offsetInRange,
a, offset, vsp,
(arr, off, s, vm) -> s.ldOp(arr, (int) off, vm,
(arr_, off_, i) -> arr_[off_ + i]));
@ -3395,16 +3393,16 @@ public abstract class DoubleVector extends AbstractVector<Double> {
}
abstract
DoubleVector fromMemorySegment0(MemorySegment ms, long offset, VectorMask<Double> m);
DoubleVector fromMemorySegment0(MemorySegment ms, long offset, VectorMask<Double> m, int offsetInRange);
@ForceInline
final
<M extends VectorMask<Double>>
DoubleVector fromMemorySegment0Template(Class<M> maskClass, MemorySegment ms, long offset, M m) {
DoubleVector fromMemorySegment0Template(Class<M> maskClass, MemorySegment ms, long offset, M m, int offsetInRange) {
DoubleSpecies vsp = vspecies();
m.check(vsp);
return ScopedMemoryAccess.loadFromMemorySegmentMasked(
vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
(AbstractMemorySegmentImpl) ms, offset, m, vsp,
(AbstractMemorySegmentImpl) ms, offset, m, vsp, offsetInRange,
(msp, off, s, vm) -> {
return s.ldLongOp((MemorySegment) msp, off, vm, DoubleVector::memorySegmentGet);
});

8
src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float128Vector.java
View File

@ -843,8 +843,8 @@ final class Float128Vector extends FloatVector {
@ForceInline
@Override
final
FloatVector fromArray0(float[] a, int offset, VectorMask<Float> m) {
return super.fromArray0Template(Float128Mask.class, a, offset, (Float128Mask) m); // specialize
FloatVector fromArray0(float[] a, int offset, VectorMask<Float> m, int offsetInRange) {
return super.fromArray0Template(Float128Mask.class, a, offset, (Float128Mask) m, offsetInRange); // specialize
}
@ForceInline
@ -866,8 +866,8 @@ final class Float128Vector extends FloatVector {
@ForceInline
@Override
final
FloatVector fromMemorySegment0(MemorySegment ms, long offset, VectorMask<Float> m) {
return super.fromMemorySegment0Template(Float128Mask.class, ms, offset, (Float128Mask) m); // specialize
FloatVector fromMemorySegment0(MemorySegment ms, long offset, VectorMask<Float> m, int offsetInRange) {
return super.fromMemorySegment0Template(Float128Mask.class, ms, offset, (Float128Mask) m, offsetInRange); // specialize
}
@ForceInline

8
src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float256Vector.java
View File

@ -851,8 +851,8 @@ final class Float256Vector extends FloatVector {
@ForceInline
@Override
final
FloatVector fromArray0(float[] a, int offset, VectorMask<Float> m) {
return super.fromArray0Template(Float256Mask.class, a, offset, (Float256Mask) m); // specialize
FloatVector fromArray0(float[] a, int offset, VectorMask<Float> m, int offsetInRange) {
return super.fromArray0Template(Float256Mask.class, a, offset, (Float256Mask) m, offsetInRange); // specialize
}
@ForceInline
@ -874,8 +874,8 @@ final class Float256Vector extends FloatVector {
@ForceInline
@Override
final
FloatVector fromMemorySegment0(MemorySegment ms, long offset, VectorMask<Float> m) {
return super.fromMemorySegment0Template(Float256Mask.class, ms, offset, (Float256Mask) m); // specialize
FloatVector fromMemorySegment0(MemorySegment ms, long offset, VectorMask<Float> m, int offsetInRange) {
return super.fromMemorySegment0Template(Float256Mask.class, ms, offset, (Float256Mask) m, offsetInRange); // specialize
}
@ForceInline

8
src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float512Vector.java
View File

@ -867,8 +867,8 @@ final class Float512Vector extends FloatVector {
@ForceInline
@Override
final
FloatVector fromArray0(float[] a, int offset, VectorMask<Float> m) {
return super.fromArray0Template(Float512Mask.class, a, offset, (Float512Mask) m); // specialize
FloatVector fromArray0(float[] a, int offset, VectorMask<Float> m, int offsetInRange) {
return super.fromArray0Template(Float512Mask.class, a, offset, (Float512Mask) m, offsetInRange); // specialize
}
@ForceInline
@ -890,8 +890,8 @@ final class Float512Vector extends FloatVector {
@ForceInline
@Override
final
FloatVector fromMemorySegment0(MemorySegment ms, long offset, VectorMask<Float> m) {
return super.fromMemorySegment0Template(Float512Mask.class, ms, offset, (Float512Mask) m); // specialize
FloatVector fromMemorySegment0(MemorySegment ms, long offset, VectorMask<Float> m, int offsetInRange) {
return super.fromMemorySegment0Template(Float512Mask.class, ms, offset, (Float512Mask) m, offsetInRange); // specialize
}
@ForceInline

8
src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float64Vector.java
View File

@ -839,8 +839,8 @@ final class Float64Vector extends FloatVector {
@ForceInline
@Override
final
FloatVector fromArray0(float[] a, int offset, VectorMask<Float> m) {
return super.fromArray0Template(Float64Mask.class, a, offset, (Float64Mask) m); // specialize
FloatVector fromArray0(float[] a, int offset, VectorMask<Float> m, int offsetInRange) {
return super.fromArray0Template(Float64Mask.class, a, offset, (Float64Mask) m, offsetInRange); // specialize
}
@ForceInline
@ -862,8 +862,8 @@ final class Float64Vector extends FloatVector {
@ForceInline
@Override
final
FloatVector fromMemorySegment0(MemorySegment ms, long offset, VectorMask<Float> m) {
return super.fromMemorySegment0Template(Float64Mask.class, ms, offset, (Float64Mask) m); // specialize
FloatVector fromMemorySegment0(MemorySegment ms, long offset, VectorMask<Float> m, int offsetInRange) {
return super.fromMemorySegment0Template(Float64Mask.class, ms, offset, (Float64Mask) m, offsetInRange); // specialize
}
@ForceInline

8
src/jdk.incubator.vector/share/classes/jdk/incubator/vector/FloatMaxVector.java
View File

@ -836,8 +836,8 @@ final class FloatMaxVector extends FloatVector {
@ForceInline
@Override
final
FloatVector fromArray0(float[] a, int offset, VectorMask<Float> m) {
return super.fromArray0Template(FloatMaxMask.class, a, offset, (FloatMaxMask) m); // specialize
FloatVector fromArray0(float[] a, int offset, VectorMask<Float> m, int offsetInRange) {
return super.fromArray0Template(FloatMaxMask.class, a, offset, (FloatMaxMask) m, offsetInRange); // specialize
}
@ForceInline
@ -859,8 +859,8 @@ final class FloatMaxVector extends FloatVector {
@ForceInline
@Override
final
FloatVector fromMemorySegment0(MemorySegment ms, long offset, VectorMask<Float> m) {
return super.fromMemorySegment0Template(FloatMaxMask.class, ms, offset, (FloatMaxMask) m); // specialize
FloatVector fromMemorySegment0(MemorySegment ms, long offset, VectorMask<Float> m, int offsetInRange) {
return super.fromMemorySegment0Template(FloatMaxMask.class, ms, offset, (FloatMaxMask) m, offsetInRange); // specialize
}
@ForceInline

22
src/jdk.incubator.vector/share/classes/jdk/incubator/vector/FloatVector.java
View File

@ -2830,12 +2830,11 @@ public abstract class FloatVector extends AbstractVector<Float> {
VectorMask<Float> m) {
FloatSpecies vsp = (FloatSpecies) species;
if (VectorIntrinsics.indexInRange(offset, vsp.length(), a.length)) {
return vsp.dummyVector().fromArray0(a, offset, m);
return vsp.dummyVector().fromArray0(a, offset, m, OFFSET_IN_RANGE);
}
// FIXME: optimize
checkMaskFromIndexSize(offset, vsp, m, 1, a.length);
return vsp.vOp(m, i -> a[offset + i]);
return vsp.dummyVector().fromArray0(a, offset, m, OFFSET_OUT_OF_RANGE);
}
/**
@ -3045,12 +3044,11 @@ public abstract class FloatVector extends AbstractVector<Float> {
VectorMask<Float> m) {
FloatSpecies vsp = (FloatSpecies) species;
if (VectorIntrinsics.indexInRange(offset, vsp.vectorByteSize(), ms.byteSize())) {
return vsp.dummyVector().fromMemorySegment0(ms, offset, m).maybeSwap(bo);
return vsp.dummyVector().fromMemorySegment0(ms, offset, m, OFFSET_IN_RANGE).maybeSwap(bo);
}
// FIXME: optimize
checkMaskFromIndexSize(offset, vsp, m, 4, ms.byteSize());
return vsp.ldLongOp(ms, offset, m, FloatVector::memorySegmentGet);
return vsp.dummyVector().fromMemorySegment0(ms, offset, m, OFFSET_OUT_OF_RANGE).maybeSwap(bo);
}
// Memory store operations
@ -3298,16 +3296,16 @@ public abstract class FloatVector extends AbstractVector<Float> {
/*package-private*/
abstract
FloatVector fromArray0(float[] a, int offset, VectorMask<Float> m);
FloatVector fromArray0(float[] a, int offset, VectorMask<Float> m, int offsetInRange);
@ForceInline
final
<M extends VectorMask<Float>>
FloatVector fromArray0Template(Class<M> maskClass, float[] a, int offset, M m) {
FloatVector fromArray0Template(Class<M> maskClass, float[] a, int offset, M m, int offsetInRange) {
m.check(species());
FloatSpecies vsp = vspecies();
return VectorSupport.loadMasked(
vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
a, arrayAddress(a, offset), m,
a, arrayAddress(a, offset), m, offsetInRange,
a, offset, vsp,
(arr, off, s, vm) -> s.ldOp(arr, (int) off, vm,
(arr_, off_, i) -> arr_[off_ + i]));
@ -3364,16 +3362,16 @@ public abstract class FloatVector extends AbstractVector<Float> {
}
abstract
FloatVector fromMemorySegment0(MemorySegment ms, long offset, VectorMask<Float> m);
FloatVector fromMemorySegment0(MemorySegment ms, long offset, VectorMask<Float> m, int offsetInRange);
@ForceInline
final
<M extends VectorMask<Float>>
FloatVector fromMemorySegment0Template(Class<M> maskClass, MemorySegment ms, long offset, M m) {
FloatVector fromMemorySegment0Template(Class<M> maskClass, MemorySegment ms, long offset, M m, int offsetInRange) {
FloatSpecies vsp = vspecies();
m.check(vsp);
return ScopedMemoryAccess.loadFromMemorySegmentMasked(
vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
(AbstractMemorySegmentImpl) ms, offset, m, vsp,
(AbstractMemorySegmentImpl) ms, offset, m, vsp, offsetInRange,
(msp, off, s, vm) -> {
return s.ldLongOp((MemorySegment) msp, off, vm, FloatVector::memorySegmentGet);
});

8
src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int128Vector.java
View File

@ -854,8 +854,8 @@ final class Int128Vector extends IntVector {
@ForceInline
@Override
final
IntVector fromArray0(int[] a, int offset, VectorMask<Integer> m) {
return super.fromArray0Template(Int128Mask.class, a, offset, (Int128Mask) m); // specialize
IntVector fromArray0(int[] a, int offset, VectorMask<Integer> m, int offsetInRange) {
return super.fromArray0Template(Int128Mask.class, a, offset, (Int128Mask) m, offsetInRange); // specialize
}
@ForceInline
@ -877,8 +877,8 @@ final class Int128Vector extends IntVector {
@ForceInline
@Override
final
IntVector fromMemorySegment0(MemorySegment ms, long offset, VectorMask<Integer> m) {
return super.fromMemorySegment0Template(Int128Mask.class, ms, offset, (Int128Mask) m); // specialize
IntVector fromMemorySegment0(MemorySegment ms, long offset, VectorMask<Integer> m, int offsetInRange) {
return super.fromMemorySegment0Template(Int128Mask.class, ms, offset, (Int128Mask) m, offsetInRange); // specialize
}
@ForceInline

8
src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int256Vector.java
View File

@ -862,8 +862,8 @@ final class Int256Vector extends IntVector {
@ForceInline
@Override
final
IntVector fromArray0(int[] a, int offset, VectorMask<Integer> m) {
return super.fromArray0Template(Int256Mask.class, a, offset, (Int256Mask) m); // specialize
IntVector fromArray0(int[] a, int offset, VectorMask<Integer> m, int offsetInRange) {
return super.fromArray0Template(Int256Mask.class, a, offset, (Int256Mask) m, offsetInRange); // specialize
}
@ForceInline
@ -885,8 +885,8 @@ final class Int256Vector extends IntVector {
@ForceInline
@Override
final
IntVector fromMemorySegment0(MemorySegment ms, long offset, VectorMask<Integer> m) {
return super.fromMemorySegment0Template(Int256Mask.class, ms, offset, (Int256Mask) m); // specialize
IntVector fromMemorySegment0(MemorySegment ms, long offset, VectorMask<Integer> m, int offsetInRange) {
return super.fromMemorySegment0Template(Int256Mask.class, ms, offset, (Int256Mask) m, offsetInRange); // specialize
}
@ForceInline

8
src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int512Vector.java
View File

@ -878,8 +878,8 @@ final class Int512Vector extends IntVector {
@ForceInline
@Override
final
IntVector fromArray0(int[] a, int offset, VectorMask<Integer> m) {
return super.fromArray0Template(Int512Mask.class, a, offset, (Int512Mask) m); // specialize
IntVector fromArray0(int[] a, int offset, VectorMask<Integer> m, int offsetInRange) {
return super.fromArray0Template(Int512Mask.class, a, offset, (Int512Mask) m, offsetInRange); // specialize
}
@ForceInline
@ -901,8 +901,8 @@ final class Int512Vector extends IntVector {
@ForceInline
@Override
final
IntVector fromMemorySegment0(MemorySegment ms, long offset, VectorMask<Integer> m) {
return super.fromMemorySegment0Template(Int512Mask.class, ms, offset, (Int512Mask) m); // specialize
IntVector fromMemorySegment0(MemorySegment ms, long offset, VectorMask<Integer> m, int offsetInRange) {
return super.fromMemorySegment0Template(Int512Mask.class, ms, offset, (Int512Mask) m, offsetInRange); // specialize
}
@ForceInline

8
src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int64Vector.java
View File

@ -850,8 +850,8 @@ final class Int64Vector extends IntVector {
@ForceInline
@Override
final
IntVector fromArray0(int[] a, int offset, VectorMask<Integer> m) {
return super.fromArray0Template(Int64Mask.class, a, offset, (Int64Mask) m); // specialize
IntVector fromArray0(int[] a, int offset, VectorMask<Integer> m, int offsetInRange) {
return super.fromArray0Template(Int64Mask.class, a, offset, (Int64Mask) m, offsetInRange); // specialize
}
@ForceInline
@ -873,8 +873,8 @@ final class Int64Vector extends IntVector {
@ForceInline
@Override
final
IntVector fromMemorySegment0(MemorySegment ms, long offset, VectorMask<Integer> m) {
return super.fromMemorySegment0Template(Int64Mask.class, ms, offset, (Int64Mask) m); // specialize
IntVector fromMemorySegment0(MemorySegment ms, long offset, VectorMask<Integer> m, int offsetInRange) {
return super.fromMemorySegment0Template(Int64Mask.class, ms, offset, (Int64Mask) m, offsetInRange); // specialize
}
@ForceInline

8
src/jdk.incubator.vector/share/classes/jdk/incubator/vector/IntMaxVector.java
View File

@ -859,8 +859,8 @@ final class IntMaxVector extends IntVector {
@ForceInline
@Override
final
IntVector fromArray0(int[] a, int offset, VectorMask<Integer> m) {
return super.fromArray0Template(IntMaxMask.class, a, offset, (IntMaxMask) m); // specialize
IntVector fromArray0(int[] a, int offset, VectorMask<Integer> m, int offsetInRange) {
return super.fromArray0Template(IntMaxMask.class, a, offset, (IntMaxMask) m, offsetInRange); // specialize
}
@ForceInline
@ -882,8 +882,8 @@ final class IntMaxVector extends IntVector {
@ForceInline
@Override
final
IntVector fromMemorySegment0(MemorySegment ms, long offset, VectorMask<Integer> m) {
return super.fromMemorySegment0Template(IntMaxMask.class, ms, offset, (IntMaxMask) m); // specialize
IntVector fromMemorySegment0(MemorySegment ms, long offset, VectorMask<Integer> m, int offsetInRange) {
return super.fromMemorySegment0Template(IntMaxMask.class, ms, offset, (IntMaxMask) m, offsetInRange); // specialize
}
@ForceInline

22
src/jdk.incubator.vector/share/classes/jdk/incubator/vector/IntVector.java
View File

@ -2987,12 +2987,11 @@ public abstract class IntVector extends AbstractVector<Integer> {
VectorMask<Integer> m) {
IntSpecies vsp = (IntSpecies) species;
if (VectorIntrinsics.indexInRange(offset, vsp.length(), a.length)) {
return vsp.dummyVector().fromArray0(a, offset, m);
return vsp.dummyVector().fromArray0(a, offset, m, OFFSET_IN_RANGE);
}
// FIXME: optimize
checkMaskFromIndexSize(offset, vsp, m, 1, a.length);
return vsp.vOp(m, i -> a[offset + i]);
return vsp.dummyVector().fromArray0(a, offset, m, OFFSET_OUT_OF_RANGE);
}
/**
@ -3202,12 +3201,11 @@ public abstract class IntVector extends AbstractVector<Integer> {
VectorMask<Integer> m) {
IntSpecies vsp = (IntSpecies) species;
if (VectorIntrinsics.indexInRange(offset, vsp.vectorByteSize(), ms.byteSize())) {
return vsp.dummyVector().fromMemorySegment0(ms, offset, m).maybeSwap(bo);
return vsp.dummyVector().fromMemorySegment0(ms, offset, m, OFFSET_IN_RANGE).maybeSwap(bo);
}
// FIXME: optimize
checkMaskFromIndexSize(offset, vsp, m, 4, ms.byteSize());
return vsp.ldLongOp(ms, offset, m, IntVector::memorySegmentGet);
return vsp.dummyVector().fromMemorySegment0(ms, offset, m, OFFSET_OUT_OF_RANGE).maybeSwap(bo);
}
// Memory store operations
@ -3455,16 +3453,16 @@ public abstract class IntVector extends AbstractVector<Integer> {
/*package-private*/
abstract
IntVector fromArray0(int[] a, int offset, VectorMask<Integer> m);
IntVector fromArray0(int[] a, int offset, VectorMask<Integer> m, int offsetInRange);
@ForceInline
final
<M extends VectorMask<Integer>>
IntVector fromArray0Template(Class<M> maskClass, int[] a, int offset, M m) {
IntVector fromArray0Template(Class<M> maskClass, int[] a, int offset, M m, int offsetInRange) {
m.check(species());
IntSpecies vsp = vspecies();
return VectorSupport.loadMasked(
vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
a, arrayAddress(a, offset), m,
a, arrayAddress(a, offset), m, offsetInRange,
a, offset, vsp,
(arr, off, s, vm) -> s.ldOp(arr, (int) off, vm,
(arr_, off_, i) -> arr_[off_ + i]));
@ -3521,16 +3519,16 @@ public abstract class IntVector extends AbstractVector<Integer> {
}
abstract
IntVector fromMemorySegment0(MemorySegment ms, long offset, VectorMask<Integer> m);
IntVector fromMemorySegment0(MemorySegment ms, long offset, VectorMask<Integer> m, int offsetInRange);
@ForceInline
final
<M extends VectorMask<Integer>>
IntVector fromMemorySegment0Template(Class<M> maskClass, MemorySegment ms, long offset, M m) {
IntVector fromMemorySegment0Template(Class<M> maskClass, MemorySegment ms, long offset, M m, int offsetInRange) {
IntSpecies vsp = vspecies();
m.check(vsp);
return ScopedMemoryAccess.loadFromMemorySegmentMasked(
vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
(AbstractMemorySegmentImpl) ms, offset, m, vsp,
(AbstractMemorySegmentImpl) ms, offset, m, vsp, offsetInRange,
(msp, off, s, vm) -> {
return s.ldLongOp((MemorySegment) msp, off, vm, IntVector::memorySegmentGet);
});

8
src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long128Vector.java
View File

@ -840,8 +840,8 @@ final class Long128Vector extends LongVector {
@ForceInline
@Override
final
LongVector fromArray0(long[] a, int offset, VectorMask<Long> m) {
return super.fromArray0Template(Long128Mask.class, a, offset, (Long128Mask) m); // specialize
LongVector fromArray0(long[] a, int offset, VectorMask<Long> m, int offsetInRange) {
return super.fromArray0Template(Long128Mask.class, a, offset, (Long128Mask) m, offsetInRange); // specialize
}
@ForceInline
@ -863,8 +863,8 @@ final class Long128Vector extends LongVector {
@ForceInline
@Override
final
LongVector fromMemorySegment0(MemorySegment ms, long offset, VectorMask<Long> m) {
return super.fromMemorySegment0Template(Long128Mask.class, ms, offset, (Long128Mask) m); // specialize
LongVector fromMemorySegment0(MemorySegment ms, long offset, VectorMask<Long> m, int offsetInRange) {
return super.fromMemorySegment0Template(Long128Mask.class, ms, offset, (Long128Mask) m, offsetInRange); // specialize
}
@ForceInline

8
src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long256Vector.java
View File

@ -844,8 +844,8 @@ final class Long256Vector extends LongVector {
@ForceInline
@Override
final
LongVector fromArray0(long[] a, int offset, VectorMask<Long> m) {
return super.fromArray0Template(Long256Mask.class, a, offset, (Long256Mask) m); // specialize
LongVector fromArray0(long[] a, int offset, VectorMask<Long> m, int offsetInRange) {
return super.fromArray0Template(Long256Mask.class, a, offset, (Long256Mask) m, offsetInRange); // specialize
}
@ForceInline
@ -867,8 +867,8 @@ final class Long256Vector extends LongVector {
@ForceInline
@Override
final
LongVector fromMemorySegment0(MemorySegment ms, long offset, VectorMask<Long> m) {
return super.fromMemorySegment0Template(Long256Mask.class, ms, offset, (Long256Mask) m); // specialize
LongVector fromMemorySegment0(MemorySegment ms, long offset, VectorMask<Long> m, int offsetInRange) {
return super.fromMemorySegment0Template(Long256Mask.class, ms, offset, (Long256Mask) m, offsetInRange); // specialize
}
@ForceInline

8
src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long512Vector.java
View File

@ -852,8 +852,8 @@ final class Long512Vector extends LongVector {
@ForceInline
@Override
final
LongVector fromArray0(long[] a, int offset, VectorMask<Long> m) {
return super.fromArray0Template(Long512Mask.class, a, offset, (Long512Mask) m); // specialize
LongVector fromArray0(long[] a, int offset, VectorMask<Long> m, int offsetInRange) {
return super.fromArray0Template(Long512Mask.class, a, offset, (Long512Mask) m, offsetInRange); // specialize
}
@ForceInline
@ -875,8 +875,8 @@ final class Long512Vector extends LongVector {
@ForceInline
@Override
final
LongVector fromMemorySegment0(MemorySegment ms, long offset, VectorMask<Long> m) {
return super.fromMemorySegment0Template(Long512Mask.class, ms, offset, (Long512Mask) m); // specialize
LongVector fromMemorySegment0(MemorySegment ms, long offset, VectorMask<Long> m, int offsetInRange) {
return super.fromMemorySegment0Template(Long512Mask.class, ms, offset, (Long512Mask) m, offsetInRange); // specialize
}
@ForceInline

8
src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long64Vector.java
View File

@ -838,8 +838,8 @@ final class Long64Vector extends LongVector {
@ForceInline
@Override
final
LongVector fromArray0(long[] a, int offset, VectorMask<Long> m) {
return super.fromArray0Template(Long64Mask.class, a, offset, (Long64Mask) m); // specialize
LongVector fromArray0(long[] a, int offset, VectorMask<Long> m, int offsetInRange) {
return super.fromArray0Template(Long64Mask.class, a, offset, (Long64Mask) m, offsetInRange); // specialize
}
@ForceInline
@ -861,8 +861,8 @@ final class Long64Vector extends LongVector {
@ForceInline
@Override
final
LongVector fromMemorySegment0(MemorySegment ms, long offset, VectorMask<Long> m) {
return super.fromMemorySegment0Template(Long64Mask.class, ms, offset, (Long64Mask) m); // specialize
LongVector fromMemorySegment0(MemorySegment ms, long offset, VectorMask<Long> m, int offsetInRange) {
return super.fromMemorySegment0Template(Long64Mask.class, ms, offset, (Long64Mask) m, offsetInRange); // specialize
}
@ForceInline

8
src/jdk.incubator.vector/share/classes/jdk/incubator/vector/LongMaxVector.java
View File

@ -838,8 +838,8 @@ final class LongMaxVector extends LongVector {
@ForceInline
@Override
final
LongVector fromArray0(long[] a, int offset, VectorMask<Long> m) {
return super.fromArray0Template(LongMaxMask.class, a, offset, (LongMaxMask) m); // specialize
LongVector fromArray0(long[] a, int offset, VectorMask<Long> m, int offsetInRange) {
return super.fromArray0Template(LongMaxMask.class, a, offset, (LongMaxMask) m, offsetInRange); // specialize
}
@ForceInline
@ -861,8 +861,8 @@ final class LongMaxVector extends LongVector {
@ForceInline
@Override
final
LongVector fromMemorySegment0(MemorySegment ms, long offset, VectorMask<Long> m) {
return super.fromMemorySegment0Template(LongMaxMask.class, ms, offset, (LongMaxMask) m); // specialize
LongVector fromMemorySegment0(MemorySegment ms, long offset, VectorMask<Long> m, int offsetInRange) {
return super.fromMemorySegment0Template(LongMaxMask.class, ms, offset, (LongMaxMask) m, offsetInRange); // specialize
}
@ForceInline

22
src/jdk.incubator.vector/share/classes/jdk/incubator/vector/LongVector.java
View File

@ -2848,12 +2848,11 @@ public abstract class LongVector extends AbstractVector<Long> {
VectorMask<Long> m) {
LongSpecies vsp = (LongSpecies) species;
if (VectorIntrinsics.indexInRange(offset, vsp.length(), a.length)) {
return vsp.dummyVector().fromArray0(a, offset, m);
return vsp.dummyVector().fromArray0(a, offset, m, OFFSET_IN_RANGE);
}
// FIXME: optimize
checkMaskFromIndexSize(offset, vsp, m, 1, a.length);
return vsp.vOp(m, i -> a[offset + i]);
return vsp.dummyVector().fromArray0(a, offset, m, OFFSET_OUT_OF_RANGE);
}
/**
@ -3081,12 +3080,11 @@ public abstract class LongVector extends AbstractVector<Long> {
VectorMask<Long> m) {
LongSpecies vsp = (LongSpecies) species;
if (VectorIntrinsics.indexInRange(offset, vsp.vectorByteSize(), ms.byteSize())) {
return vsp.dummyVector().fromMemorySegment0(ms, offset, m).maybeSwap(bo);
return vsp.dummyVector().fromMemorySegment0(ms, offset, m, OFFSET_IN_RANGE).maybeSwap(bo);
}
// FIXME: optimize
checkMaskFromIndexSize(offset, vsp, m, 8, ms.byteSize());
return vsp.ldLongOp(ms, offset, m, LongVector::memorySegmentGet);
return vsp.dummyVector().fromMemorySegment0(ms, offset, m, OFFSET_OUT_OF_RANGE).maybeSwap(bo);
}
// Memory store operations
@ -3353,16 +3351,16 @@ public abstract class LongVector extends AbstractVector<Long> {
/*package-private*/
abstract
LongVector fromArray0(long[] a, int offset, VectorMask<Long> m);
LongVector fromArray0(long[] a, int offset, VectorMask<Long> m, int offsetInRange);
@ForceInline
final
<M extends VectorMask<Long>>
LongVector fromArray0Template(Class<M> maskClass, long[] a, int offset, M m) {
LongVector fromArray0Template(Class<M> maskClass, long[] a, int offset, M m, int offsetInRange) {
m.check(species());
LongSpecies vsp = vspecies();
return VectorSupport.loadMasked(
vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
a, arrayAddress(a, offset), m,
a, arrayAddress(a, offset), m, offsetInRange,
a, offset, vsp,
(arr, off, s, vm) -> s.ldOp(arr, (int) off, vm,
(arr_, off_, i) -> arr_[off_ + i]));
@ -3437,16 +3435,16 @@ public abstract class LongVector extends AbstractVector<Long> {
}
abstract
LongVector fromMemorySegment0(MemorySegment ms, long offset, VectorMask<Long> m);
LongVector fromMemorySegment0(MemorySegment ms, long offset, VectorMask<Long> m, int offsetInRange);
@ForceInline
final
<M extends VectorMask<Long>>
LongVector fromMemorySegment0Template(Class<M> maskClass, MemorySegment ms, long offset, M m) {
LongVector fromMemorySegment0Template(Class<M> maskClass, MemorySegment ms, long offset, M m, int offsetInRange) {
LongSpecies vsp = vspecies();
m.check(vsp);
return ScopedMemoryAccess.loadFromMemorySegmentMasked(
vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
(AbstractMemorySegmentImpl) ms, offset, m, vsp,
(AbstractMemorySegmentImpl) ms, offset, m, vsp, offsetInRange,
(msp, off, s, vm) -> {
return s.ldLongOp((MemorySegment) msp, off, vm, LongVector::memorySegmentGet);
});

12
src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short128Vector.java
View File

@ -862,8 +862,8 @@ final class Short128Vector extends ShortVector {
@ForceInline
@Override
final
ShortVector fromArray0(short[] a, int offset, VectorMask<Short> m) {
return super.fromArray0Template(Short128Mask.class, a, offset, (Short128Mask) m); // specialize
ShortVector fromArray0(short[] a, int offset, VectorMask<Short> m, int offsetInRange) {
return super.fromArray0Template(Short128Mask.class, a, offset, (Short128Mask) m, offsetInRange); // specialize
}
@ -877,8 +877,8 @@ final class Short128Vector extends ShortVector {
@ForceInline
@Override
final
ShortVector fromCharArray0(char[] a, int offset, VectorMask<Short> m) {
return super.fromCharArray0Template(Short128Mask.class, a, offset, (Short128Mask) m); // specialize
ShortVector fromCharArray0(char[] a, int offset, VectorMask<Short> m, int offsetInRange) {
return super.fromCharArray0Template(Short128Mask.class, a, offset, (Short128Mask) m, offsetInRange); // specialize
}
@ -892,8 +892,8 @@ final class Short128Vector extends ShortVector {
@ForceInline
@Override
final
ShortVector fromMemorySegment0(MemorySegment ms, long offset, VectorMask<Short> m) {
return super.fromMemorySegment0Template(Short128Mask.class, ms, offset, (Short128Mask) m); // specialize
ShortVector fromMemorySegment0(MemorySegment ms, long offset, VectorMask<Short> m, int offsetInRange) {
return super.fromMemorySegment0Template(Short128Mask.class, ms, offset, (Short128Mask) m, offsetInRange); // specialize
}
@ForceInline

12
src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short256Vector.java
View File

@ -878,8 +878,8 @@ final class Short256Vector extends ShortVector {
@ForceInline
@Override
final
ShortVector fromArray0(short[] a, int offset, VectorMask<Short> m) {
return super.fromArray0Template(Short256Mask.class, a, offset, (Short256Mask) m); // specialize
ShortVector fromArray0(short[] a, int offset, VectorMask<Short> m, int offsetInRange) {
return super.fromArray0Template(Short256Mask.class, a, offset, (Short256Mask) m, offsetInRange); // specialize
}
@ -893,8 +893,8 @@ final class Short256Vector extends ShortVector {
@ForceInline
@Override
final
ShortVector fromCharArray0(char[] a, int offset, VectorMask<Short> m) {
return super.fromCharArray0Template(Short256Mask.class, a, offset, (Short256Mask) m); // specialize
ShortVector fromCharArray0(char[] a, int offset, VectorMask<Short> m, int offsetInRange) {
return super.fromCharArray0Template(Short256Mask.class, a, offset, (Short256Mask) m, offsetInRange); // specialize
}
@ -908,8 +908,8 @@ final class Short256Vector extends ShortVector {
@ForceInline
@Override
final
ShortVector fromMemorySegment0(MemorySegment ms, long offset, VectorMask<Short> m) {
return super.fromMemorySegment0Template(Short256Mask.class, ms, offset, (Short256Mask) m); // specialize
ShortVector fromMemorySegment0(MemorySegment ms, long offset, VectorMask<Short> m, int offsetInRange) {
return super.fromMemorySegment0Template(Short256Mask.class, ms, offset, (Short256Mask) m, offsetInRange); // specialize
}
@ForceInline

12
src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short512Vector.java
View File

@ -910,8 +910,8 @@ final class Short512Vector extends ShortVector {
@ForceInline
@Override
final
ShortVector fromArray0(short[] a, int offset, VectorMask<Short> m) {
return super.fromArray0Template(Short512Mask.class, a, offset, (Short512Mask) m); // specialize
ShortVector fromArray0(short[] a, int offset, VectorMask<Short> m, int offsetInRange) {
return super.fromArray0Template(Short512Mask.class, a, offset, (Short512Mask) m, offsetInRange); // specialize
}
@ -925,8 +925,8 @@ final class Short512Vector extends ShortVector {
@ForceInline
@Override
final
ShortVector fromCharArray0(char[] a, int offset, VectorMask<Short> m) {
return super.fromCharArray0Template(Short512Mask.class, a, offset, (Short512Mask) m); // specialize
ShortVector fromCharArray0(char[] a, int offset, VectorMask<Short> m, int offsetInRange) {
return super.fromCharArray0Template(Short512Mask.class, a, offset, (Short512Mask) m, offsetInRange); // specialize
}
@ -940,8 +940,8 @@ final class Short512Vector extends ShortVector {
@ForceInline
@Override
final
ShortVector fromMemorySegment0(MemorySegment ms, long offset, VectorMask<Short> m) {
return super.fromMemorySegment0Template(Short512Mask.class, ms, offset, (Short512Mask) m); // specialize
ShortVector fromMemorySegment0(MemorySegment ms, long offset, VectorMask<Short> m, int offsetInRange) {
return super.fromMemorySegment0Template(Short512Mask.class, ms, offset, (Short512Mask) m, offsetInRange); // specialize
}
@ForceInline

12
src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short64Vector.java
View File

@ -854,8 +854,8 @@ final class Short64Vector extends ShortVector {
@ForceInline
@Override
final
ShortVector fromArray0(short[] a, int offset, VectorMask<Short> m) {
return super.fromArray0Template(Short64Mask.class, a, offset, (Short64Mask) m); // specialize
ShortVector fromArray0(short[] a, int offset, VectorMask<Short> m, int offsetInRange) {
return super.fromArray0Template(Short64Mask.class, a, offset, (Short64Mask) m, offsetInRange); // specialize
}
@ -869,8 +869,8 @@ final class Short64Vector extends ShortVector {
@ForceInline
@Override
final
ShortVector fromCharArray0(char[] a, int offset, VectorMask<Short> m) {
return super.fromCharArray0Template(Short64Mask.class, a, offset, (Short64Mask) m); // specialize
ShortVector fromCharArray0(char[] a, int offset, VectorMask<Short> m, int offsetInRange) {
return super.fromCharArray0Template(Short64Mask.class, a, offset, (Short64Mask) m, offsetInRange); // specialize
}
@ -884,8 +884,8 @@ final class Short64Vector extends ShortVector {
@ForceInline
@Override
final
ShortVector fromMemorySegment0(MemorySegment ms, long offset, VectorMask<Short> m) {
return super.fromMemorySegment0Template(Short64Mask.class, ms, offset, (Short64Mask) m); // specialize
ShortVector fromMemorySegment0(MemorySegment ms, long offset, VectorMask<Short> m, int offsetInRange) {
return super.fromMemorySegment0Template(Short64Mask.class, ms, offset, (Short64Mask) m, offsetInRange); // specialize
}
@ForceInline

12
src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ShortMaxVector.java
View File

@ -848,8 +848,8 @@ final class ShortMaxVector extends ShortVector {
@ForceInline
@Override
final
ShortVector fromArray0(short[] a, int offset, VectorMask<Short> m) {
return super.fromArray0Template(ShortMaxMask.class, a, offset, (ShortMaxMask) m); // specialize
ShortVector fromArray0(short[] a, int offset, VectorMask<Short> m, int offsetInRange) {
return super.fromArray0Template(ShortMaxMask.class, a, offset, (ShortMaxMask) m, offsetInRange); // specialize
}
@ -863,8 +863,8 @@ final class ShortMaxVector extends ShortVector {
@ForceInline
@Override
final
ShortVector fromCharArray0(char[] a, int offset, VectorMask<Short> m) {
return super.fromCharArray0Template(ShortMaxMask.class, a, offset, (ShortMaxMask) m); // specialize
ShortVector fromCharArray0(char[] a, int offset, VectorMask<Short> m, int offsetInRange) {
return super.fromCharArray0Template(ShortMaxMask.class, a, offset, (ShortMaxMask) m, offsetInRange); // specialize
}
@ -878,8 +878,8 @@ final class ShortMaxVector extends ShortVector {
@ForceInline
@Override
final
ShortVector fromMemorySegment0(MemorySegment ms, long offset, VectorMask<Short> m) {
return super.fromMemorySegment0Template(ShortMaxMask.class, ms, offset, (ShortMaxMask) m); // specialize
ShortVector fromMemorySegment0(MemorySegment ms, long offset, VectorMask<Short> m, int offsetInRange) {
return super.fromMemorySegment0Template(ShortMaxMask.class, ms, offset, (ShortMaxMask) m, offsetInRange); // specialize
}
@ForceInline

33
src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ShortVector.java
View File

@ -3010,12 +3010,11 @@ public abstract class ShortVector extends AbstractVector<Short> {
VectorMask<Short> m) {
ShortSpecies vsp = (ShortSpecies) species;
if (VectorIntrinsics.indexInRange(offset, vsp.length(), a.length)) {
return vsp.dummyVector().fromArray0(a, offset, m);
return vsp.dummyVector().fromArray0(a, offset, m, OFFSET_IN_RANGE);
}
// FIXME: optimize
checkMaskFromIndexSize(offset, vsp, m, 1, a.length);
return vsp.vOp(m, i -> a[offset + i]);
return vsp.dummyVector().fromArray0(a, offset, m, OFFSET_OUT_OF_RANGE);
}
/**
@ -3159,12 +3158,11 @@ public abstract class ShortVector extends AbstractVector<Short> {
VectorMask<Short> m) {
ShortSpecies vsp = (ShortSpecies) species;
if (VectorIntrinsics.indexInRange(offset, vsp.length(), a.length)) {
return vsp.dummyVector().fromCharArray0(a, offset, m);
return vsp.dummyVector().fromCharArray0(a, offset, m, OFFSET_IN_RANGE);
}
// FIXME: optimize
checkMaskFromIndexSize(offset, vsp, m, 1, a.length);
return vsp.vOp(m, i -> (short) a[offset + i]);
return vsp.dummyVector().fromCharArray0(a, offset, m, OFFSET_OUT_OF_RANGE);
}
/**
@ -3352,12 +3350,11 @@ public abstract class ShortVector extends AbstractVector<Short> {
VectorMask<Short> m) {
ShortSpecies vsp = (ShortSpecies) species;
if (VectorIntrinsics.indexInRange(offset, vsp.vectorByteSize(), ms.byteSize())) {
return vsp.dummyVector().fromMemorySegment0(ms, offset, m).maybeSwap(bo);
return vsp.dummyVector().fromMemorySegment0(ms, offset, m, OFFSET_IN_RANGE).maybeSwap(bo);
}
// FIXME: optimize
checkMaskFromIndexSize(offset, vsp, m, 2, ms.byteSize());
return vsp.ldLongOp(ms, offset, m, ShortVector::memorySegmentGet);
return vsp.dummyVector().fromMemorySegment0(ms, offset, m, OFFSET_OUT_OF_RANGE).maybeSwap(bo);
}
// Memory store operations
@ -3742,16 +3739,16 @@ public abstract class ShortVector extends AbstractVector<Short> {
/*package-private*/
abstract
ShortVector fromArray0(short[] a, int offset, VectorMask<Short> m);
ShortVector fromArray0(short[] a, int offset, VectorMask<Short> m, int offsetInRange);
@ForceInline
final
<M extends VectorMask<Short>>
ShortVector fromArray0Template(Class<M> maskClass, short[] a, int offset, M m) {
ShortVector fromArray0Template(Class<M> maskClass, short[] a, int offset, M m, int offsetInRange) {
m.check(species());
ShortSpecies vsp = vspecies();
return VectorSupport.loadMasked(
vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
a, arrayAddress(a, offset), m,
a, arrayAddress(a, offset), m, offsetInRange,
a, offset, vsp,
(arr, off, s, vm) -> s.ldOp(arr, (int) off, vm,
(arr_, off_, i) -> arr_[off_ + i]));
@ -3775,16 +3772,16 @@ public abstract class ShortVector extends AbstractVector<Short> {
/*package-private*/
abstract
ShortVector fromCharArray0(char[] a, int offset, VectorMask<Short> m);
ShortVector fromCharArray0(char[] a, int offset, VectorMask<Short> m, int offsetInRange);
@ForceInline
final
<M extends VectorMask<Short>>
ShortVector fromCharArray0Template(Class<M> maskClass, char[] a, int offset, M m) {
ShortVector fromCharArray0Template(Class<M> maskClass, char[] a, int offset, M m, int offsetInRange) {
m.check(species());
ShortSpecies vsp = vspecies();
return VectorSupport.loadMasked(
vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
a, charArrayAddress(a, offset), m,
a, charArrayAddress(a, offset), m, offsetInRange,
a, offset, vsp,
(arr, off, s, vm) -> s.ldOp(arr, (int) off, vm,
(arr_, off_, i) -> (short) arr_[off_ + i]));
@ -3806,16 +3803,16 @@ public abstract class ShortVector extends AbstractVector<Short> {
}
abstract
ShortVector fromMemorySegment0(MemorySegment ms, long offset, VectorMask<Short> m);
ShortVector fromMemorySegment0(MemorySegment ms, long offset, VectorMask<Short> m, int offsetInRange);
@ForceInline
final
<M extends VectorMask<Short>>
ShortVector fromMemorySegment0Template(Class<M> maskClass, MemorySegment ms, long offset, M m) {
ShortVector fromMemorySegment0Template(Class<M> maskClass, MemorySegment ms, long offset, M m, int offsetInRange) {
ShortSpecies vsp = vspecies();
m.check(vsp);
return ScopedMemoryAccess.loadFromMemorySegmentMasked(
vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
(AbstractMemorySegmentImpl) ms, offset, m, vsp,
(AbstractMemorySegmentImpl) ms, offset, m, vsp, offsetInRange,
(msp, off, s, vm) -> {
return s.ldLongOp((MemorySegment) msp, off, vm, ShortVector::memorySegmentGet);
});

44
src/jdk.incubator.vector/share/classes/jdk/incubator/vector/X-Vector.java.template
View File

@ -3584,12 +3584,11 @@ public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> {
VectorMask<$Boxtype$> m) {
$Type$Species vsp = ($Type$Species) species;
if (VectorIntrinsics.indexInRange(offset, vsp.length(), a.length)) {
return vsp.dummyVector().fromArray0(a, offset, m);
return vsp.dummyVector().fromArray0(a, offset, m, OFFSET_IN_RANGE);
}
// FIXME: optimize
checkMaskFromIndexSize(offset, vsp, m, 1, a.length);
return vsp.vOp(m, i -> a[offset + i]);
return vsp.dummyVector().fromArray0(a, offset, m, OFFSET_OUT_OF_RANGE);
}
/**
@ -3805,12 +3804,11 @@ public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> {
VectorMask<$Boxtype$> m) {
$Type$Species vsp = ($Type$Species) species;
if (VectorIntrinsics.indexInRange(offset, vsp.length(), a.length)) {
return vsp.dummyVector().fromCharArray0(a, offset, m);
return vsp.dummyVector().fromCharArray0(a, offset, m, OFFSET_IN_RANGE);
}
// FIXME: optimize
checkMaskFromIndexSize(offset, vsp, m, 1, a.length);
return vsp.vOp(m, i -> (short) a[offset + i]);
return vsp.dummyVector().fromCharArray0(a, offset, m, OFFSET_OUT_OF_RANGE);
}
/**
@ -3965,12 +3963,11 @@ public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> {
$Type$Species vsp = ($Type$Species) species;
if (VectorIntrinsics.indexInRange(offset, vsp.length(), a.length)) {
$abstractvectortype$ zero = vsp.zero();
return vsp.dummyVector().fromBooleanArray0(a, offset, m);
return vsp.dummyVector().fromBooleanArray0(a, offset, m, OFFSET_IN_RANGE);
}
// FIXME: optimize
checkMaskFromIndexSize(offset, vsp, m, 1, a.length);
return vsp.vOp(m, i -> (byte) (a[offset + i] ? 1 : 0));
return vsp.dummyVector().fromBooleanArray0(a, offset, m, OFFSET_OUT_OF_RANGE);
}
/**
@ -4162,12 +4159,11 @@ public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> {
VectorMask<$Boxtype$> m) {
$Type$Species vsp = ($Type$Species) species;
if (VectorIntrinsics.indexInRange(offset, vsp.vectorByteSize(), ms.byteSize())) {
return vsp.dummyVector().fromMemorySegment0(ms, offset, m).maybeSwap(bo);
return vsp.dummyVector().fromMemorySegment0(ms, offset, m, OFFSET_IN_RANGE).maybeSwap(bo);
}
// FIXME: optimize
checkMaskFromIndexSize(offset, vsp, m, $sizeInBytes$, ms.byteSize());
return vsp.ldLongOp(ms, offset, m, $abstractvectortype$::memorySegmentGet);
return vsp.dummyVector().fromMemorySegment0(ms, offset, m, OFFSET_OUT_OF_RANGE).maybeSwap(bo);
}
// Memory store operations
@ -4793,16 +4789,16 @@ public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> {
/*package-private*/
abstract
$abstractvectortype$ fromArray0($type$[] a, int offset, VectorMask<$Boxtype$> m);
$abstractvectortype$ fromArray0($type$[] a, int offset, VectorMask<$Boxtype$> m, int offsetInRange);
@ForceInline
final
<M extends VectorMask<$Boxtype$>>
$abstractvectortype$ fromArray0Template(Class<M> maskClass, $type$[] a, int offset, M m) {
$abstractvectortype$ fromArray0Template(Class<M> maskClass, $type$[] a, int offset, M m, int offsetInRange) {
m.check(species());
$Type$Species vsp = vspecies();
return VectorSupport.loadMasked(
vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
a, arrayAddress(a, offset), m,
a, arrayAddress(a, offset), m, offsetInRange,
a, offset, vsp,
(arr, off, s, vm) -> s.ldOp(arr, (int) off, vm,
(arr_, off_, i) -> arr_[off_ + i]));
@ -4887,16 +4883,16 @@ public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> {
/*package-private*/
abstract
$abstractvectortype$ fromCharArray0(char[] a, int offset, VectorMask<$Boxtype$> m);
$abstractvectortype$ fromCharArray0(char[] a, int offset, VectorMask<$Boxtype$> m, int offsetInRange);
@ForceInline
final
<M extends VectorMask<$Boxtype$>>
$abstractvectortype$ fromCharArray0Template(Class<M> maskClass, char[] a, int offset, M m) {
$abstractvectortype$ fromCharArray0Template(Class<M> maskClass, char[] a, int offset, M m, int offsetInRange) {
m.check(species());
$Type$Species vsp = vspecies();
return VectorSupport.loadMasked(
vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
a, charArrayAddress(a, offset), m,
a, charArrayAddress(a, offset), m, offsetInRange,
a, offset, vsp,
(arr, off, s, vm) -> s.ldOp(arr, (int) off, vm,
(arr_, off_, i) -> (short) arr_[off_ + i]));
@ -4921,16 +4917,16 @@ public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> {
/*package-private*/
abstract
$abstractvectortype$ fromBooleanArray0(boolean[] a, int offset, VectorMask<$Boxtype$> m);
$abstractvectortype$ fromBooleanArray0(boolean[] a, int offset, VectorMask<$Boxtype$> m, int offsetInRange);
@ForceInline
final
<M extends VectorMask<$Boxtype$>>
$abstractvectortype$ fromBooleanArray0Template(Class<M> maskClass, boolean[] a, int offset, M m) {
$abstractvectortype$ fromBooleanArray0Template(Class<M> maskClass, boolean[] a, int offset, M m, int offsetInRange) {
m.check(species());
$Type$Species vsp = vspecies();
return VectorSupport.loadMasked(
vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
a, booleanArrayAddress(a, offset), m,
a, booleanArrayAddress(a, offset), m, offsetInRange,
a, offset, vsp,
(arr, off, s, vm) -> s.ldOp(arr, (int) off, vm,
(arr_, off_, i) -> (byte) (arr_[off_ + i] ? 1 : 0)));
@ -4952,16 +4948,16 @@ public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> {
}
abstract
$abstractvectortype$ fromMemorySegment0(MemorySegment ms, long offset, VectorMask<$Boxtype$> m);
$abstractvectortype$ fromMemorySegment0(MemorySegment ms, long offset, VectorMask<$Boxtype$> m, int offsetInRange);
@ForceInline
final
<M extends VectorMask<$Boxtype$>>
$abstractvectortype$ fromMemorySegment0Template(Class<M> maskClass, MemorySegment ms, long offset, M m) {
$abstractvectortype$ fromMemorySegment0Template(Class<M> maskClass, MemorySegment ms, long offset, M m, int offsetInRange) {
$Type$Species vsp = vspecies();
m.check(vsp);
return ScopedMemoryAccess.loadFromMemorySegmentMasked(
vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
(AbstractMemorySegmentImpl) ms, offset, m, vsp,
(AbstractMemorySegmentImpl) ms, offset, m, vsp, offsetInRange,
(msp, off, s, vm) -> {
return s.ldLongOp((MemorySegment) msp, off, vm, $abstractvectortype$::memorySegmentGet);
});

16
src/jdk.incubator.vector/share/classes/jdk/incubator/vector/X-VectorBits.java.template
View File

@ -1136,8 +1136,8 @@ final class $vectortype$ extends $abstractvectortype$ {
@ForceInline
@Override
final
$abstractvectortype$ fromArray0($type$[] a, int offset, VectorMask<$Boxtype$> m) {
return super.fromArray0Template($masktype$.class, a, offset, ($masktype$) m); // specialize
$abstractvectortype$ fromArray0($type$[] a, int offset, VectorMask<$Boxtype$> m, int offsetInRange) {
return super.fromArray0Template($masktype$.class, a, offset, ($masktype$) m, offsetInRange); // specialize
}
#if[!byteOrShort]
@ -1160,8 +1160,8 @@ final class $vectortype$ extends $abstractvectortype$ {
@ForceInline
@Override
final
$abstractvectortype$ fromCharArray0(char[] a, int offset, VectorMask<$Boxtype$> m) {
return super.fromCharArray0Template($masktype$.class, a, offset, ($masktype$) m); // specialize
$abstractvectortype$ fromCharArray0(char[] a, int offset, VectorMask<$Boxtype$> m, int offsetInRange) {
return super.fromCharArray0Template($masktype$.class, a, offset, ($masktype$) m, offsetInRange); // specialize
}
#end[short]
@ -1176,8 +1176,8 @@ final class $vectortype$ extends $abstractvectortype$ {
@ForceInline
@Override
final
$abstractvectortype$ fromBooleanArray0(boolean[] a, int offset, VectorMask<$Boxtype$> m) {
return super.fromBooleanArray0Template($masktype$.class, a, offset, ($masktype$) m); // specialize
$abstractvectortype$ fromBooleanArray0(boolean[] a, int offset, VectorMask<$Boxtype$> m, int offsetInRange) {
return super.fromBooleanArray0Template($masktype$.class, a, offset, ($masktype$) m, offsetInRange); // specialize
}
#end[byte]
@ -1191,8 +1191,8 @@ final class $vectortype$ extends $abstractvectortype$ {
@ForceInline
@Override
final
$abstractvectortype$ fromMemorySegment0(MemorySegment ms, long offset, VectorMask<$Boxtype$> m) {
return super.fromMemorySegment0Template($masktype$.class, ms, offset, ($masktype$) m); // specialize
$abstractvectortype$ fromMemorySegment0(MemorySegment ms, long offset, VectorMask<$Boxtype$> m, int offsetInRange) {
return super.fromMemorySegment0Template($masktype$.class, ms, offset, ($masktype$) m, offsetInRange); // specialize
}
@ForceInline

141
test/micro/org/openjdk/bench/jdk/incubator/vector/LoadMaskedIOOBEBenchmark.java Normal file
View File

@ -0,0 +1,141 @@
//
// Copyright (c) 2022, Arm Limited. All rights reserved.
// DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
//
// This code is free software; you can redistribute it and/or modify it
// under the terms of the GNU General Public License version 2 only, as
// published by the Free Software Foundation.
//
// This code is distributed in the hope that it will be useful, but WITHOUT
// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
// version 2 for more details (a copy is included in the LICENSE file that
// accompanied this code).
//
// You should have received a copy of the GNU General Public License version
// 2 along with this work; if not, write to the Free Software Foundation,
// Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
//
// Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
// or visit www.oracle.com if you need additional information or have any
// questions.
//
//
package org.openjdk.bench.jdk.incubator.vector;
import java.util.concurrent.TimeUnit;
import jdk.incubator.vector.*;
import org.openjdk.jmh.annotations.*;
@BenchmarkMode(Mode.Throughput)
@OutputTimeUnit(TimeUnit.MILLISECONDS)
@State(Scope.Benchmark)
@Warmup(iterations = 3, time = 1)
@Measurement(iterations = 5, time = 1)
@Fork(value = 1, jvmArgsPrepend = {"--add-modules=jdk.incubator.vector"})
public class LoadMaskedIOOBEBenchmark {
@Param({"1026"})
private int inSize;
@Param({"1152"})
private int outSize;
private byte[] byteIn;
private byte[] byteOut;
private short[] shortIn;
private short[] shortOut;
private int[] intIn;
private int[] intOut;
private long[] longIn;
private long[] longOut;
private float[] floatIn;
private float[] floatOut;
private double[] doubleIn;
private double[] doubleOut;
private boolean[] m;
private static final VectorSpecies<Byte> bspecies = VectorSpecies.ofLargestShape(byte.class);
private static final VectorSpecies<Short> sspecies = VectorSpecies.ofLargestShape(short.class);
private static final VectorSpecies<Integer> ispecies = VectorSpecies.ofLargestShape(int.class);
private static final VectorSpecies<Long> lspecies = VectorSpecies.ofLargestShape(long.class);
private static final VectorSpecies<Float> fspecies = VectorSpecies.ofLargestShape(float.class);
private static final VectorSpecies<Double> dspecies = VectorSpecies.ofLargestShape(double.class);
@Setup(Level.Trial)
public void Setup() {
byteIn = new byte[inSize];
byteOut = new byte[outSize];
shortIn = new short[inSize];
shortOut = new short[outSize];
intIn = new int[inSize];
intOut = new int[outSize];
longIn = new long[inSize];
longOut = new long[outSize];
floatIn = new float[inSize];
floatOut = new float[outSize];
doubleIn = new double[inSize];
doubleOut = new double[outSize];
for (int i = 0; i < inSize; i++) {
byteIn[i] = (byte) i;
shortIn[i] = (short) i;
intIn[i] = i;
longIn[i] = i;
floatIn[i] = (float) i;
doubleIn[i] = (double) i;
}
m = new boolean[outSize];
for (int i = 0; i < inSize; i++) {
m[i] = i % 2 == 0;
}
}
@Benchmark
public void byteLoadArrayMaskIOOBE() {
for (int i = 0; i < inSize; i += bspecies.length()) {
VectorMask<Byte> mask = VectorMask.fromArray(bspecies, m, i);
ByteVector.fromArray(bspecies, byteIn, i, mask).intoArray(byteOut, i);
}
}
@Benchmark
public void shortLoadArrayMaskIOOBE() {
for (int i = 0; i < inSize; i += sspecies.length()) {
VectorMask<Short> mask = VectorMask.fromArray(sspecies, m, i);
ShortVector.fromArray(sspecies, shortIn, i, mask).intoArray(shortOut, i);
}
}
@Benchmark
public void intLoadArrayMaskIOOBE() {
for (int i = 0; i < inSize; i += ispecies.length()) {
VectorMask<Integer> mask = VectorMask.fromArray(ispecies, m, i);
IntVector.fromArray(ispecies, intIn, i, mask).intoArray(intOut, i);
}
}
@Benchmark
public void longLoadArrayMaskIOOBE() {
for (int i = 0; i < inSize; i += lspecies.length()) {
VectorMask<Long> mask = VectorMask.fromArray(lspecies, m, i);
LongVector.fromArray(lspecies, longIn, i, mask).intoArray(longOut, i);
}
}
@Benchmark
public void floatLoadArrayMaskIOOBE() {
for (int i = 0; i < inSize; i += fspecies.length()) {
VectorMask<Float> mask = VectorMask.fromArray(fspecies, m, i);
FloatVector.fromArray(fspecies, floatIn, i, mask).intoArray(floatOut, i);
}
}
@Benchmark
public void doubleLoadArrayMaskIOOBE() {
for (int i = 0; i < inSize; i += dspecies.length()) {
VectorMask<Double> mask = VectorMask.fromArray(dspecies, m, i);
DoubleVector.fromArray(dspecies, doubleIn, i, mask).intoArray(doubleOut, i);
}
}
}