From 43d60570770e4a103a1c25dc34d485c29e085b0f Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Mon, 30 Nov 2015 16:31:02 +1100 Subject: [PATCH] WIP: Supporting native (C) clients. --- .../scala/uvm/refimpl/mem/TypeSizes.scala | 9 +- .../uvm/refimpl/nat/NativeCallHelper.scala | 2 +- .../scala/uvm/refimpl/nat/NativeSupport.scala | 11 +- .../uvm/refimpl/nat/PlatformConstants.scala | 12 + .../uvm/refimpl/nat/nativeClientSupport.scala | 252 ++++++++++++++++++ .../refimpl/nat/NativeClientSupportTest.scala | 12 + 6 files changed, 290 insertions(+), 8 deletions(-) create mode 100644 src/main/scala/uvm/refimpl/nat/PlatformConstants.scala create mode 100644 src/main/scala/uvm/refimpl/nat/nativeClientSupport.scala create mode 100644 src/test/scala/uvm/refimpl/nat/NativeClientSupportTest.scala diff --git a/src/main/scala/uvm/refimpl/mem/TypeSizes.scala b/src/main/scala/uvm/refimpl/mem/TypeSizes.scala index 144cd9d..d1116d6 100644 --- a/src/main/scala/uvm/refimpl/mem/TypeSizes.scala +++ b/src/main/scala/uvm/refimpl/mem/TypeSizes.scala @@ -2,6 +2,7 @@ package uvm.refimpl.mem import uvm.types._ import uvm.refimpl.UvmRefImplException +import uvm.refimpl.nat.PlatformConstants /** * Responsible for object layout. @@ -45,11 +46,11 @@ import uvm.refimpl.UvmRefImplException * */ object TypeSizes { - type Word = Long + type Word = PlatformConstants.Word - val WORD_SIZE_LOG: Word = 6L - val WORD_SIZE_BITS: Word = 1L << WORD_SIZE_LOG - val WORD_SIZE_BYTES: Word = 1L << (WORD_SIZE_LOG - 3L) + val WORD_SIZE_LOG: Word = PlatformConstants.WORD_SIZE_LOG + val WORD_SIZE_BITS: Word = PlatformConstants.WORD_SIZE_BITS + val WORD_SIZE_BYTES: Word = PlatformConstants.WORD_SIZE_BYTES val GC_HEADER_SIZE_SCALAR: Word = 8L; val GC_HEADER_SIZE_HYBRID: Word = 16L; diff --git a/src/main/scala/uvm/refimpl/nat/NativeCallHelper.scala b/src/main/scala/uvm/refimpl/nat/NativeCallHelper.scala index 68590d0..bbc57c5 100644 --- a/src/main/scala/uvm/refimpl/nat/NativeCallHelper.scala +++ b/src/main/scala/uvm/refimpl/nat/NativeCallHelper.scala @@ -316,7 +316,7 @@ class NativeCallHelper { val jRetTy = getNativeReturnType(sig.retTys) val clos = new MuCallbackClosure(muFunc, cookie) - val handle = NativeSupport.closureManager.newClosure(clos, jRetTy, jParamTypes.toArray, CallingConvention.DEFAULT) + val handle = NativeSupport.jffiClosureManager.newClosure(clos, jRetTy, jParamTypes.toArray, CallingConvention.DEFAULT) val addr = handle.getAddress val efr = new ExpFuncRec(muFunc, cookie, clos, handle, isDynamic) diff --git a/src/main/scala/uvm/refimpl/nat/NativeSupport.scala b/src/main/scala/uvm/refimpl/nat/NativeSupport.scala index cb25bf2..a6fa709 100644 --- a/src/main/scala/uvm/refimpl/nat/NativeSupport.scala +++ b/src/main/scala/uvm/refimpl/nat/NativeSupport.scala @@ -2,12 +2,17 @@ package uvm.refimpl.nat import jnr.ffi.{ Runtime, Memory, Pointer } import com.kenai.jffi.ClosureManager +import jnr.ffi.provider.jffi.NativeRuntime /** * Holder of JNR-specific resources. */ object NativeSupport { - val jnrRuntime = Runtime.getSystemRuntime - val theMemory = Pointer.wrap(jnrRuntime, 0L) - val closureManager = ClosureManager.getInstance() + // Force using NativeRuntime (although that's default). + val jnrRuntime = NativeRuntime.getInstance + val jnrMemoryManager = jnrRuntime.getMemoryManager + val theMemory = jnrMemoryManager.newPointer(0L) + + // This is from JFFI, not JNR-FFI. + val jffiClosureManager = ClosureManager.getInstance() } \ No newline at end of file diff --git a/src/main/scala/uvm/refimpl/nat/PlatformConstants.scala b/src/main/scala/uvm/refimpl/nat/PlatformConstants.scala new file mode 100644 index 0000000..b8c17cd --- /dev/null +++ b/src/main/scala/uvm/refimpl/nat/PlatformConstants.scala @@ -0,0 +1,12 @@ +package uvm.refimpl.nat + +/** + * Constants specific to the platform. Currently only supports x86_64. + */ +object PlatformConstants { + type Word = Long + + val WORD_SIZE_LOG: Word = 6L + val WORD_SIZE_BITS: Word = 1L << WORD_SIZE_LOG + val WORD_SIZE_BYTES: Word = 1L << (WORD_SIZE_LOG - 3L) +} \ No newline at end of file diff --git a/src/main/scala/uvm/refimpl/nat/nativeClientSupport.scala b/src/main/scala/uvm/refimpl/nat/nativeClientSupport.scala new file mode 100644 index 0000000..03afe04 --- /dev/null +++ b/src/main/scala/uvm/refimpl/nat/nativeClientSupport.scala @@ -0,0 +1,252 @@ +package uvm.refimpl.nat + +import java.nio.charset.StandardCharsets + +import scala.collection.mutable.HashMap +import scala.reflect.ClassTag +import scala.reflect.runtime.universe +import scala.reflect.runtime.{ universe => ru } + +import org.slf4j.LoggerFactory + +import com.kenai.jffi.CallingConvention +import com.kenai.jffi.Closure +import com.kenai.jffi.Closure.Buffer +import com.kenai.jffi.{ Type => JType } +import com.typesafe.scalalogging.Logger + +import NativeSupport._ +import PlatformConstants.WORD_SIZE_BYTES +import PlatformConstants.Word +import jnr.ffi.ObjectReferenceManager +import jnr.ffi.Pointer +import uvm.refimpl._ + +/** + * This object calls a C function to handle the trap. + */ +class NativeTrapHandler(val funcAddr: Long, val userdata: Long) extends TrapHandler { + def handleTrap(ctx: MuCtx, thread: MuThreadRefValue, stack: MuStackRefValue, watchPointID: Int): TrapHandlerResult = { + ??? + } +} + +/** + * This object routes calls from native clients to the MicroVM object. + */ +object NativeMuVM { + // Syntax sugar for MuCtx and MuValue return values, which are pointers + type MuCtxPtr = Word + type MuValuePtr = Word + + def new_context(microVM: MicroVM): MuCtxPtr = { + val ctx = microVM.newContext() + val ptr = NativeClientSupport.muCtxs.add(ctx) + ptr.address() + } + def id_of(microVM: MicroVM, name: String): Int = microVM.idOf(name) + def name_of(microVM: MicroVM, id: Int): String = microVM.nameOf(id) + def set_trap_handler(microVM: MicroVM, trap_handler: Word, userdata: Word): Unit = { + microVM.setTrapHandler(new NativeTrapHandler(trap_handler, userdata)) + } +} + +object ClientAccessibleClassExposer { + val MAX_NAME_SIZE = 65536 + + val logger = Logger(LoggerFactory.getLogger(getClass.getName)) + + // Type objects for common types. + val TUnit = ru.typeTag[Unit].tpe + val TByte = ru.typeTag[Byte].tpe + val TShort = ru.typeTag[Short].tpe + val TInt = ru.typeTag[Int].tpe + val TLong = ru.typeTag[Long].tpe + val TFloat = ru.typeTag[Float].tpe + val TDouble = ru.typeTag[Double].tpe + val TString = ru.typeTag[String].tpe + val TMicroVM = ru.typeTag[MicroVM].tpe + val TMuCtx = ru.typeTag[MuCtx].tpe + val TMuValue = ru.typeTag[MuValue].tpe + + // com.kenai.jffi.Closure.Buffer param getters and return value setters. + // These are partially-applied functions, and will be called in closures (callback from C). + def paramByte(index: Int)(buffer: Buffer): Any = buffer.getByte(index) + def paramShort(index: Int)(buffer: Buffer): Any = buffer.getShort(index) + def paramInt(index: Int)(buffer: Buffer): Any = buffer.getInt(index) + def paramLong(index: Int)(buffer: Buffer): Any = buffer.getLong(index) + def paramFloat(index: Int)(buffer: Buffer): Any = buffer.getFloat(index) + def paramDouble(index: Int)(buffer: Buffer): Any = buffer.getDouble(index) + def paramString(index: Int)(buffer: Buffer): Any = getStr(buffer, index) + def paramMicroVM(index: Int)(buffer: Buffer): Any = getObj(buffer, index, NativeClientSupport.microVMs) + def paramMuCtx(index: Int)(buffer: Buffer): Any = getObj(buffer, index, NativeClientSupport.muCtxs) + def paramMuValue(index: Int)(buffer: Buffer): Any = getObj(buffer, index, NativeClientSupport.muValues) + + def retVoid(buffer: Buffer, v: Any): Unit = {} + def retByte(buffer: Buffer, v: Any): Unit = buffer.setByteReturn(v.asInstanceOf[Byte]) + def retShort(buffer: Buffer, v: Any): Unit = buffer.setShortReturn(v.asInstanceOf[Short]) + def retInt(buffer: Buffer, v: Any): Unit = buffer.setIntReturn(v.asInstanceOf[Int]) + def retLong(buffer: Buffer, v: Any): Unit = buffer.setLongReturn(v.asInstanceOf[Long]) + def retFloat(buffer: Buffer, v: Any): Unit = buffer.setFloatReturn(v.asInstanceOf[Float]) + def retDouble(buffer: Buffer, v: Any): Unit = buffer.setDoubleReturn(v.asInstanceOf[Double]) + def retString(buffer: Buffer, v: Any): Unit = buffer.setLongReturn(exposeStr(v.asInstanceOf[String])) + + private def getStr(buffer: Buffer, index: Int): String = { + val addr = buffer.getAddress(index) + val str = theMemory.getString(addr, MAX_NAME_SIZE, StandardCharsets.US_ASCII) + str + } + + private def exposeStr(str: String): Word = { + val ptr = NativeClientSupport.stringPool.getOrElseUpdate(str, { + val bytes = str.getBytes(StandardCharsets.US_ASCII) + val newPtr = jnrMemoryManager.allocateDirect(bytes.size + 1) + newPtr.put(0L, bytes, 0, 0) + newPtr + }) + ptr.address() + } + + private def getObj[T](buffer: Buffer, index: Int, orm: ObjectReferenceManager[T]): T = { + val addr = buffer.getLong(index) + orm.get(jnrMemoryManager.newPointer(addr)) + } + + // Reflection utils. + val mirror = ru.runtimeMirror(getClass.getClassLoader) + + // Convert to JFFI native types. + def toJFFIType(t: ru.Type): JType = t match { + case t if t =:= TUnit => JType.VOID + case t if t =:= TByte => JType.SINT8 + case t if t =:= TShort => JType.SINT16 + case t if t =:= TInt => JType.SINT32 + case t if t =:= TLong => JType.SINT64 + case t if t =:= TFloat => JType.FLOAT + case t if t =:= TDouble => JType.DOUBLE + case t if t =:= TString => JType.POINTER + case t if t =:= TMicroVM => JType.POINTER + case t if t =:= TMuCtx => JType.POINTER + case t if t =:= TMuValue => JType.POINTER + } + + /** + * Let a native program call a reflectively-exposed method. + */ + class ExposedMethodClosure(method: ru.MethodMirror, paramGetters: Seq[Buffer => Any], returnSetter: (Buffer, Any) => Unit) extends Closure { + def invoke(buffer: Buffer): Unit = { + val params = paramGetters.map(_(buffer)) + val rv = try { + method.apply(params: _*) + } catch { + case e: Throwable => { + logger.error("Exception thrown before returning to native. This is fatal", e) + throw e + } + } + returnSetter(buffer, rv) + } + } +} + +/** + * Expose a class (NativeMuVM and NativeMuCtx) as a function table. + */ +class ClientAccessibleClassExposer[T: ru.TypeTag: ClassTag](obj: T) { + import ClientAccessibleClassExposer._ + import NativeSupport._ + + /** + * A list of JFFI closure handles, one for each declared method in obj, in the declared order. + */ + val closureHandles: IndexedSeq[Closure.Handle] = { + val tt = ru.typeTag[T] + val iter = for (m <- tt.tpe.decls if m.isMethod && !m.isConstructor) yield { + val meth = m.asMethod + logger.debug("Exposing method: %s".format(meth)) + + val paramTypes = meth.paramLists(0).map(_.asTerm.typeSignature) + val returnType = meth.returnType + + val paramGetters = for ((ty, i) <- paramTypes.zipWithIndex) yield ty match { + case t if t =:= TByte => paramByte(i) _ + case t if t =:= TShort => paramShort(i) _ + case t if t =:= TInt => paramInt(i) _ + case t if t =:= TLong => paramLong(i) _ + case t if t =:= TFloat => paramFloat(i) _ + case t if t =:= TDouble => paramDouble(i) _ + case t if t =:= TString => paramString(i) _ + case t if t =:= TMicroVM => paramMicroVM(i) _ + case t if t =:= TMuCtx => paramMuCtx(i) _ + case t if t =:= TMuValue => paramMuValue(i) _ + } + + val returnSetter = returnType match { + case t if t =:= TUnit => retVoid _ + case t if t =:= TByte => retByte _ + case t if t =:= TShort => retShort _ + case t if t =:= TInt => retInt _ + case t if t =:= TLong => retLong _ + case t if t =:= TFloat => retFloat _ + case t if t =:= TDouble => retDouble _ + case t if t =:= TString => retString _ + } + + val paramNativeTypes = for (ty <- paramTypes) yield toJFFIType(ty) + val returnNativeType = toJFFIType(returnType) + + val objMirror = mirror.reflect(obj) + val methMirror = objMirror.reflectMethod(meth) + + val closure = new ExposedMethodClosure(methMirror, paramGetters, returnSetter) + + val closureHandle = jffiClosureManager.newClosure(closure, returnNativeType, paramNativeTypes.toArray, CallingConvention.DEFAULT) + + closureHandle + } + + iter.toIndexedSeq + } + + /** + * Allocate and populate a native function table, + * i.e. the in-memory structure of struct MuVM and struct MuCtx in the client interface. + */ + def makeFuncTable(instance: Word): Word = { + val nMethods = closureHandles.size + val structSizeWords = nMethods + 1 + val structSizeBytes = structSizeWords * WORD_SIZE_BYTES.toInt + val s = jnrMemoryManager.allocateDirect(structSizeBytes) + s.putLong(0L, instance) + for ((handle, i) <- closureHandles.zipWithIndex) { + val offset = (i + 1) * WORD_SIZE_BYTES + val funcAddr = handle.getAddress() + s.putLong(offset, funcAddr) + } + val addr = s.address() + NativeClientSupport.funcTableToPointer(addr) = s + addr + } +} + +/** + * Support for native clients (such as C programs). + */ +object NativeClientSupport { + // Give exposed objects a random "memory address" so native programs can pass them back to Mu as parameters. + val microVMs = jnrRuntime.newObjectReferenceManager[MicroVM]() + val muCtxs = jnrRuntime.newObjectReferenceManager[MuCtx]() + val muValues = jnrRuntime.newObjectReferenceManager[MuValue]() + + /** + * Map function table addresses to Pointer so they can be closed. JFFI uses the TrantientNativeMemory (wrapped + * in the Pointer object) to keep the allocated native memory alive. + */ + val funcTableToPointer = HashMap[Word, Pointer]() + + /** + * Map Java strings (currently only names of Identified objects in MU) to Pointers to keep the allocated strings + * alive so that the native program can access them. + */ + val stringPool = HashMap[String, Pointer]() +} \ No newline at end of file diff --git a/src/test/scala/uvm/refimpl/nat/NativeClientSupportTest.scala b/src/test/scala/uvm/refimpl/nat/NativeClientSupportTest.scala new file mode 100644 index 0000000..c537f10 --- /dev/null +++ b/src/test/scala/uvm/refimpl/nat/NativeClientSupportTest.scala @@ -0,0 +1,12 @@ +package uvm.refimpl.nat + +import scala.reflect.runtime.universe + +import org.scalatest.FlatSpec +import org.scalatest.Matchers + +class NativeClientSupportTest extends FlatSpec with Matchers { + "The ClientAccessibleClassExposer" should "enumerate declared methods" in { + val cace = new ClientAccessibleClassExposer(NativeMuVM) + } +} \ No newline at end of file -- 2.24.3