497 lines
20 KiB
Plaintext
497 lines
20 KiB
Plaintext
/*
|
|
* Copyright (c) 2020, 2022, Oracle and/or its affiliates. All rights reserved.
|
|
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
|
|
*
|
|
* This code is free software; you can redistribute it and/or modify it
|
|
* under the terms of the GNU General Public License version 2 only, as
|
|
* published by the Free Software Foundation. Oracle designates this
|
|
* particular file as subject to the "Classpath" exception as provided
|
|
* by Oracle in the LICENSE file that accompanied this code.
|
|
*
|
|
* This code is distributed in the hope that it will be useful, but WITHOUT
|
|
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
|
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
|
|
* version 2 for more details (a copy is included in the LICENSE file that
|
|
* accompanied this code).
|
|
*
|
|
* You should have received a copy of the GNU General Public License version
|
|
* 2 along with this work; if not, write to the Free Software Foundation,
|
|
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
|
|
*
|
|
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
|
|
* or visit www.oracle.com if you need additional information or have any
|
|
* questions.
|
|
*/
|
|
|
|
package jdk.internal.misc;
|
|
|
|
import java.lang.annotation.ElementType;
|
|
import java.lang.annotation.Retention;
|
|
import java.lang.annotation.RetentionPolicy;
|
|
import java.lang.annotation.Target;
|
|
import java.lang.foreign.MemorySegment;
|
|
import java.lang.ref.Reference;
|
|
import java.io.FileDescriptor;
|
|
import java.util.function.Supplier;
|
|
|
|
import jdk.internal.access.JavaNioAccess;
|
|
import jdk.internal.access.SharedSecrets;
|
|
import jdk.internal.foreign.AbstractMemorySegmentImpl;
|
|
import jdk.internal.foreign.MemorySessionImpl;
|
|
import jdk.internal.util.ArraysSupport;
|
|
import jdk.internal.vm.annotation.ForceInline;
|
|
import jdk.internal.vm.vector.VectorSupport;
|
|
|
|
|
|
/**
|
|
* This class defines low-level methods to access on-heap and off-heap memory. The methods in this class
|
|
* can be thought of as thin wrappers around methods provided in the {@link Unsafe} class. All the methods in this
|
|
* class accept one or more {@link MemorySessionImpl} parameter, which is used to validate as to whether access to memory
|
|
* can be performed in a safe fashion - more specifically, to ensure that the memory being accessed has not
|
|
* already been released (which would result in a hard VM crash).
|
|
* <p>
|
|
* Accessing and releasing memory from a single thread is not problematic - after all, a given thread cannot,
|
|
* at the same time, access a memory region <em>and</em> free it. But ensuring correctness of memory access
|
|
* when multiple threads are involved is much trickier, as there can be cases where a thread is accessing
|
|
* a memory region while another thread is releasing it.
|
|
* <p>
|
|
* This class provides tools to manage races when multiple threads are accessing and/or releasing the same memory
|
|
* session concurrently. More specifically, when a thread wants to release a memory session, it should call the
|
|
* {@link ScopedMemoryAccess#closeScope(MemorySessionImpl)} method. This method initiates thread-local handshakes with all the other VM threads,
|
|
* which are then stopped one by one. If any thread is found accessing a resource associated to the very memory session
|
|
* being closed, the handshake fails, and the session will not be closed.
|
|
* <p>
|
|
* This synchronization strategy relies on the idea that accessing memory is atomic with respect to checking the
|
|
* validity of the session associated with that memory region - that is, a thread that wants to perform memory access will be
|
|
* suspended either <em>before</em> a liveness check or <em>after</em> the memory access. To ensure this atomicity,
|
|
* all methods in this class are marked with the special {@link Scoped} annotation, which is recognized by the VM,
|
|
* and used during the thread-local handshake to detect (and stop) threads performing potentially problematic memory access
|
|
* operations. Additionally, to make sure that the session object(s) of the memory being accessed is always
|
|
* reachable during an access operation, all the methods in this class add reachability fences around the underlying
|
|
* unsafe access.
|
|
* <p>
|
|
* This form of synchronization allows APIs to use plain memory access without any other form of synchronization
|
|
* which might be deemed to expensive; in other words, this approach prioritizes the performance of memory access over
|
|
* that of releasing a shared memory resource.
|
|
*/
|
|
public class ScopedMemoryAccess {
|
|
|
|
private static final Unsafe UNSAFE = Unsafe.getUnsafe();
|
|
|
|
private static native void registerNatives();
|
|
static {
|
|
registerNatives();
|
|
}
|
|
|
|
public void closeScope(MemorySessionImpl session, ScopedAccessError error) {
|
|
closeScope0(session, error);
|
|
}
|
|
|
|
native void closeScope0(MemorySessionImpl session, ScopedAccessError error);
|
|
|
|
private ScopedMemoryAccess() {}
|
|
|
|
private static final ScopedMemoryAccess theScopedMemoryAccess = new ScopedMemoryAccess();
|
|
|
|
public static ScopedMemoryAccess getScopedMemoryAccess() {
|
|
return theScopedMemoryAccess;
|
|
}
|
|
|
|
public static final class ScopedAccessError extends Error {
|
|
|
|
@SuppressWarnings("serial")
|
|
private final Supplier<RuntimeException> runtimeExceptionSupplier;
|
|
|
|
public ScopedAccessError(Supplier<RuntimeException> runtimeExceptionSupplier) {
|
|
super("Invalid memory access", null, false, false);
|
|
this.runtimeExceptionSupplier = runtimeExceptionSupplier;
|
|
}
|
|
|
|
static final long serialVersionUID = 1L;
|
|
|
|
public final RuntimeException newRuntimeException() {
|
|
return runtimeExceptionSupplier.get();
|
|
}
|
|
}
|
|
|
|
@Target({ElementType.METHOD, ElementType.CONSTRUCTOR})
|
|
@Retention(RetentionPolicy.RUNTIME)
|
|
@interface Scoped { }
|
|
|
|
// bulk ops
|
|
|
|
@ForceInline
|
|
public void copyMemory(MemorySessionImpl srcSession, MemorySessionImpl dstSession,
|
|
Object srcBase, long srcOffset,
|
|
Object destBase, long destOffset,
|
|
long bytes) {
|
|
try {
|
|
copyMemoryInternal(srcSession, dstSession, srcBase, srcOffset, destBase, destOffset, bytes);
|
|
} catch (ScopedAccessError ex) {
|
|
throw ex.newRuntimeException();
|
|
}
|
|
}
|
|
|
|
@ForceInline @Scoped
|
|
private void copyMemoryInternal(MemorySessionImpl srcSession, MemorySessionImpl dstSession,
|
|
Object srcBase, long srcOffset,
|
|
Object destBase, long destOffset,
|
|
long bytes) {
|
|
try {
|
|
if (srcSession != null) {
|
|
srcSession.checkValidStateRaw();
|
|
}
|
|
if (dstSession != null) {
|
|
dstSession.checkValidStateRaw();
|
|
}
|
|
UNSAFE.copyMemory(srcBase, srcOffset, destBase, destOffset, bytes);
|
|
} finally {
|
|
Reference.reachabilityFence(srcSession);
|
|
Reference.reachabilityFence(dstSession);
|
|
}
|
|
}
|
|
|
|
@ForceInline
|
|
public void copySwapMemory(MemorySessionImpl srcSession, MemorySessionImpl dstSession,
|
|
Object srcBase, long srcOffset,
|
|
Object destBase, long destOffset,
|
|
long bytes, long elemSize) {
|
|
try {
|
|
copySwapMemoryInternal(srcSession, dstSession, srcBase, srcOffset, destBase, destOffset, bytes, elemSize);
|
|
} catch (ScopedAccessError ex) {
|
|
throw ex.newRuntimeException();
|
|
}
|
|
}
|
|
|
|
@ForceInline @Scoped
|
|
private void copySwapMemoryInternal(MemorySessionImpl srcSession, MemorySessionImpl dstSession,
|
|
Object srcBase, long srcOffset,
|
|
Object destBase, long destOffset,
|
|
long bytes, long elemSize) {
|
|
try {
|
|
if (srcSession != null) {
|
|
srcSession.checkValidStateRaw();
|
|
}
|
|
if (dstSession != null) {
|
|
dstSession.checkValidStateRaw();
|
|
}
|
|
UNSAFE.copySwapMemory(srcBase, srcOffset, destBase, destOffset, bytes, elemSize);
|
|
} finally {
|
|
Reference.reachabilityFence(srcSession);
|
|
Reference.reachabilityFence(dstSession);
|
|
}
|
|
}
|
|
|
|
@ForceInline
|
|
public void setMemory(MemorySessionImpl session, Object o, long offset, long bytes, byte value) {
|
|
try {
|
|
setMemoryInternal(session, o, offset, bytes, value);
|
|
} catch (ScopedAccessError ex) {
|
|
throw ex.newRuntimeException();
|
|
}
|
|
}
|
|
|
|
@ForceInline @Scoped
|
|
private void setMemoryInternal(MemorySessionImpl session, Object o, long offset, long bytes, byte value) {
|
|
try {
|
|
if (session != null) {
|
|
session.checkValidStateRaw();
|
|
}
|
|
UNSAFE.setMemory(o, offset, bytes, value);
|
|
} finally {
|
|
Reference.reachabilityFence(session);
|
|
}
|
|
}
|
|
|
|
@ForceInline
|
|
public int vectorizedMismatch(MemorySessionImpl aSession, MemorySessionImpl bSession,
|
|
Object a, long aOffset,
|
|
Object b, long bOffset,
|
|
int length,
|
|
int log2ArrayIndexScale) {
|
|
try {
|
|
return vectorizedMismatchInternal(aSession, bSession, a, aOffset, b, bOffset, length, log2ArrayIndexScale);
|
|
} catch (ScopedAccessError ex) {
|
|
throw ex.newRuntimeException();
|
|
}
|
|
}
|
|
|
|
@ForceInline @Scoped
|
|
private int vectorizedMismatchInternal(MemorySessionImpl aSession, MemorySessionImpl bSession,
|
|
Object a, long aOffset,
|
|
Object b, long bOffset,
|
|
int length,
|
|
int log2ArrayIndexScale) {
|
|
try {
|
|
if (aSession != null) {
|
|
aSession.checkValidStateRaw();
|
|
}
|
|
if (bSession != null) {
|
|
bSession.checkValidStateRaw();
|
|
}
|
|
return ArraysSupport.vectorizedMismatch(a, aOffset, b, bOffset, length, log2ArrayIndexScale);
|
|
} finally {
|
|
Reference.reachabilityFence(aSession);
|
|
Reference.reachabilityFence(bSession);
|
|
}
|
|
}
|
|
|
|
@ForceInline
|
|
public boolean isLoaded(MemorySessionImpl session, long address, boolean isSync, long size) {
|
|
try {
|
|
return isLoadedInternal(session, address, isSync, size);
|
|
} catch (ScopedAccessError ex) {
|
|
throw ex.newRuntimeException();
|
|
}
|
|
}
|
|
|
|
@ForceInline @Scoped
|
|
public boolean isLoadedInternal(MemorySessionImpl session, long address, boolean isSync, long size) {
|
|
try {
|
|
if (session != null) {
|
|
session.checkValidStateRaw();
|
|
}
|
|
return SharedSecrets.getJavaNioAccess().isLoaded(address, isSync, size);
|
|
} finally {
|
|
Reference.reachabilityFence(session);
|
|
}
|
|
}
|
|
|
|
@ForceInline
|
|
public void load(MemorySessionImpl session, long address, boolean isSync, long size) {
|
|
try {
|
|
loadInternal(session, address, isSync, size);
|
|
} catch (ScopedAccessError ex) {
|
|
throw ex.newRuntimeException();
|
|
}
|
|
}
|
|
|
|
@ForceInline @Scoped
|
|
public void loadInternal(MemorySessionImpl session, long address, boolean isSync, long size) {
|
|
try {
|
|
if (session != null) {
|
|
session.checkValidStateRaw();
|
|
}
|
|
SharedSecrets.getJavaNioAccess().load(address, isSync, size);
|
|
} finally {
|
|
Reference.reachabilityFence(session);
|
|
}
|
|
}
|
|
|
|
@ForceInline
|
|
public void unload(MemorySessionImpl session, long address, boolean isSync, long size) {
|
|
try {
|
|
unloadInternal(session, address, isSync, size);
|
|
} catch (ScopedAccessError ex) {
|
|
throw ex.newRuntimeException();
|
|
}
|
|
}
|
|
|
|
@ForceInline @Scoped
|
|
public void unloadInternal(MemorySessionImpl session, long address, boolean isSync, long size) {
|
|
try {
|
|
if (session != null) {
|
|
session.checkValidStateRaw();
|
|
}
|
|
SharedSecrets.getJavaNioAccess().unload(address, isSync, size);
|
|
} finally {
|
|
Reference.reachabilityFence(session);
|
|
}
|
|
}
|
|
|
|
@ForceInline
|
|
public void force(MemorySessionImpl session, FileDescriptor fd, long address, boolean isSync, long index, long length) {
|
|
try {
|
|
forceInternal(session, fd, address, isSync, index, length);
|
|
} catch (ScopedAccessError ex) {
|
|
throw ex.newRuntimeException();
|
|
}
|
|
}
|
|
|
|
@ForceInline @Scoped
|
|
public void forceInternal(MemorySessionImpl session, FileDescriptor fd, long address, boolean isSync, long index, long length) {
|
|
try {
|
|
if (session != null) {
|
|
session.checkValidStateRaw();
|
|
}
|
|
SharedSecrets.getJavaNioAccess().force(fd, address, isSync, index, length);
|
|
} finally {
|
|
Reference.reachabilityFence(session);
|
|
}
|
|
}
|
|
|
|
// MemorySegment vector access ops
|
|
|
|
@ForceInline
|
|
public static
|
|
<V extends VectorSupport.Vector<E>, E, S extends VectorSupport.VectorSpecies<E>>
|
|
V loadFromMemorySegment(Class<? extends V> vmClass, Class<E> e, int length,
|
|
AbstractMemorySegmentImpl msp, long offset,
|
|
S s,
|
|
VectorSupport.LoadOperation<AbstractMemorySegmentImpl, V, S> defaultImpl) {
|
|
|
|
try {
|
|
return loadFromMemorySegmentScopedInternal(
|
|
msp.sessionImpl(),
|
|
vmClass, e, length,
|
|
msp, offset,
|
|
s,
|
|
defaultImpl);
|
|
} catch (ScopedAccessError ex) {
|
|
throw ex.newRuntimeException();
|
|
}
|
|
}
|
|
|
|
@Scoped
|
|
@ForceInline
|
|
private static
|
|
<V extends VectorSupport.Vector<E>, E, S extends VectorSupport.VectorSpecies<E>>
|
|
V loadFromMemorySegmentScopedInternal(MemorySessionImpl session,
|
|
Class<? extends V> vmClass, Class<E> e, int length,
|
|
AbstractMemorySegmentImpl msp, long offset,
|
|
S s,
|
|
VectorSupport.LoadOperation<AbstractMemorySegmentImpl, V, S> defaultImpl) {
|
|
try {
|
|
session.checkValidStateRaw();
|
|
|
|
return VectorSupport.load(vmClass, e, length,
|
|
msp.unsafeGetBase(), msp.unsafeGetOffset() + offset,
|
|
msp, offset, s,
|
|
defaultImpl);
|
|
} finally {
|
|
Reference.reachabilityFence(session);
|
|
}
|
|
}
|
|
|
|
@ForceInline
|
|
public static
|
|
<V extends VectorSupport.Vector<E>, E, S extends VectorSupport.VectorSpecies<E>,
|
|
M extends VectorSupport.VectorMask<E>>
|
|
V loadFromMemorySegmentMasked(Class<? extends V> vmClass, Class<M> maskClass, Class<E> e,
|
|
int length, AbstractMemorySegmentImpl msp, long offset, M m, S s, int offsetInRange,
|
|
VectorSupport.LoadVectorMaskedOperation<AbstractMemorySegmentImpl, V, S, M> defaultImpl) {
|
|
|
|
try {
|
|
return loadFromMemorySegmentMaskedScopedInternal(
|
|
msp.sessionImpl(),
|
|
vmClass, maskClass, e, length,
|
|
msp, offset, m,
|
|
s, offsetInRange,
|
|
defaultImpl);
|
|
} catch (ScopedAccessError ex) {
|
|
throw ex.newRuntimeException();
|
|
}
|
|
}
|
|
|
|
@Scoped
|
|
@ForceInline
|
|
private static
|
|
<V extends VectorSupport.Vector<E>, E, S extends VectorSupport.VectorSpecies<E>,
|
|
M extends VectorSupport.VectorMask<E>>
|
|
V loadFromMemorySegmentMaskedScopedInternal(MemorySessionImpl session, Class<? extends V> vmClass,
|
|
Class<M> maskClass, Class<E> e, int length,
|
|
AbstractMemorySegmentImpl msp, long offset, M m,
|
|
S s, int offsetInRange,
|
|
VectorSupport.LoadVectorMaskedOperation<AbstractMemorySegmentImpl, V, S, M> defaultImpl) {
|
|
try {
|
|
session.checkValidStateRaw();
|
|
|
|
return VectorSupport.loadMasked(vmClass, maskClass, e, length,
|
|
msp.unsafeGetBase(), msp.unsafeGetOffset() + offset, m, offsetInRange,
|
|
msp, offset, s,
|
|
defaultImpl);
|
|
} finally {
|
|
Reference.reachabilityFence(session);
|
|
}
|
|
}
|
|
|
|
@ForceInline
|
|
public static
|
|
<V extends VectorSupport.Vector<E>, E>
|
|
void storeIntoMemorySegment(Class<? extends V> vmClass, Class<E> e, int length,
|
|
V v,
|
|
AbstractMemorySegmentImpl msp, long offset,
|
|
VectorSupport.StoreVectorOperation<AbstractMemorySegmentImpl, V> defaultImpl) {
|
|
|
|
try {
|
|
storeIntoMemorySegmentScopedInternal(
|
|
msp.sessionImpl(),
|
|
vmClass, e, length,
|
|
v,
|
|
msp, offset,
|
|
defaultImpl);
|
|
} catch (ScopedAccessError ex) {
|
|
throw ex.newRuntimeException();
|
|
}
|
|
}
|
|
|
|
@Scoped
|
|
@ForceInline
|
|
private static
|
|
<V extends VectorSupport.Vector<E>, E>
|
|
void storeIntoMemorySegmentScopedInternal(MemorySessionImpl session,
|
|
Class<? extends V> vmClass, Class<E> e, int length,
|
|
V v,
|
|
AbstractMemorySegmentImpl msp, long offset,
|
|
VectorSupport.StoreVectorOperation<AbstractMemorySegmentImpl, V> defaultImpl) {
|
|
try {
|
|
session.checkValidStateRaw();
|
|
|
|
VectorSupport.store(vmClass, e, length,
|
|
msp.unsafeGetBase(), msp.unsafeGetOffset() + offset,
|
|
v,
|
|
msp, offset,
|
|
defaultImpl);
|
|
} finally {
|
|
Reference.reachabilityFence(session);
|
|
}
|
|
}
|
|
|
|
@ForceInline
|
|
public static
|
|
<V extends VectorSupport.Vector<E>, E, M extends VectorSupport.VectorMask<E>>
|
|
void storeIntoMemorySegmentMasked(Class<? extends V> vmClass, Class<M> maskClass, Class<E> e,
|
|
int length, V v, M m,
|
|
AbstractMemorySegmentImpl msp, long offset,
|
|
VectorSupport.StoreVectorMaskedOperation<AbstractMemorySegmentImpl, V, M> defaultImpl) {
|
|
|
|
try {
|
|
storeIntoMemorySegmentMaskedScopedInternal(
|
|
msp.sessionImpl(),
|
|
vmClass, maskClass, e, length,
|
|
v, m,
|
|
msp, offset,
|
|
defaultImpl);
|
|
} catch (ScopedAccessError ex) {
|
|
throw ex.newRuntimeException();
|
|
}
|
|
}
|
|
|
|
@Scoped
|
|
@ForceInline
|
|
private static
|
|
<V extends VectorSupport.Vector<E>, E, M extends VectorSupport.VectorMask<E>>
|
|
void storeIntoMemorySegmentMaskedScopedInternal(MemorySessionImpl session,
|
|
Class<? extends V> vmClass, Class<M> maskClass,
|
|
Class<E> e, int length, V v, M m,
|
|
AbstractMemorySegmentImpl msp, long offset,
|
|
VectorSupport.StoreVectorMaskedOperation<AbstractMemorySegmentImpl, V, M> defaultImpl) {
|
|
try {
|
|
session.checkValidStateRaw();
|
|
|
|
VectorSupport.storeMasked(vmClass, maskClass, e, length,
|
|
msp.unsafeGetBase(), msp.unsafeGetOffset() + offset,
|
|
v, m,
|
|
msp, offset,
|
|
defaultImpl);
|
|
} finally {
|
|
Reference.reachabilityFence(session);
|
|
}
|
|
}
|
|
|
|
// typed-ops here
|
|
|
|
// Note: all the accessor methods defined below take advantage of argument type profiling
|
|
// (see src/hotspot/share/oops/methodData.cpp) which greatly enhances performance when the same accessor
|
|
// method is used repeatedly with different 'base' objects.
|