Commit 3e1ba586 authored by Kunshan Wang's avatar Kunshan Wang

boot image loading

parent 1d4f248e
......@@ -32,16 +32,29 @@ object MicroVM {
def apply(): MicroVM = {
val vmConf = new VMConf()
new MicroVM(vmConf)
MicroVM(vmConf)
}
def apply(confStr: String): MicroVM = {
val vmConf = VMConf(confStr)
new MicroVM(vmConf)
MicroVM(confStr)
}
def apply(confStr: String, appArgs: Seq[String]): MicroVM = {
val vmConf = VMConf(confStr)
MicroVM(vmConf, appArgs)
}
def apply(vmConf: VMConf): MicroVM = {
new MicroVM(vmConf, None)
}
def apply(vmConf: VMConf, appArgs: Seq[String]): MicroVM = {
new MicroVM(vmConf, Some(appArgs))
}
}
class MicroVM(val vmConf: VMConf) extends IRBuilderListener {
class MicroVM private (val vmConf: VMConf, val appArgs: Option[Seq[String]]) extends IRBuilderListener {
// implicitly injected resources
private implicit val microVM = this
......@@ -202,7 +215,7 @@ class MicroVM(val vmConf: VMConf) extends IRBuilderListener {
* Load from a boot image.
*/
def loadBootImage(file: String): Unit = {
tryWithResource(new BootImageLoader(file)) { bil =>
tryWithResource(new BootImageLoader(file, appArgs)) { bil =>
bil.load()
}
}
......
......@@ -54,60 +54,9 @@ case class FieldAndSymbol(objRef: Word, offset: Word, symbol: String) {
}
object BootImageWriter {
import BootImageFile._
val logger = Logger(LoggerFactory.getLogger(getClass.getName))
val FILEEXT_DATA = ".data"
val FILEEXT_ALLOC = ".alloc"
val FILEEXT_RELOC = ".reloc"
val UIRBUNDLE_FILE = "bundle.uir"
val IDNAMEMAP_FILE = "idnamemap"
val METAINFO_FILE = "metainfo"
val KEY_INITFUNC = "initfunc"
val KEY_THREADLOCAL = "threadlocal"
val KEY_EXTRALIBS = "extralibs"
val AS_REF = "R"
val AS_IREF = "I"
val AS_TAGREF = "T"
val AS_FUNCREF = "F"
val AS_UPTR = "P"
val TO_GLOBAL = "G"
val TO_HEAP = "H"
val TO_FUNC = "F"
val TO_SYM = "S"
val NO_OFFSET = 0L
val NO_SYM = ""
/**
* Allocation record. For global and heap.
*
* @param num An integer to identify the unit in a .data file. For global, it is the ID of the global cell; for heap
* object, it is a sequential number.
* @param fileOffset The offset from the beginning of the .data file.
* @param ty The type of the allocation unit.
* @param varLen The length of the variable part if it is a hybrid; otherwise 0.
*
* @param addr The address in the current micro VM instance. Not persisted.
*/
case class UnitAllocRecord(num: Long, fileOffset: Long, ty: Type, varLen: Long, addr: Word)
/**
* Relocation record. One for each field of ref<T>, iref<T>, weakref<T> or funcref<sig> types.
*
* @param num The source allocation unit number.
* @param fieldOffset The offset of the field from the beginning of the allocation unit.
* @param targetKind Whether the target is in the global memory, heap, or is a function.
* @param targetNum To identify the target. For memory, it is the beginning of the allocation unit; for function, it
* is the ID of the function.
* @param targetOffset If the source is an iref<T>, it is the offset from the beginning of the target allocation unit;
* otherwise 0.
*/
case class FieldRelocRecord(num: Long, fieldOffset: Long, fieldKind: String, targetKind: String, targetNum: Long, targetOffset: Long, targetString: String)
class FileGroup(baseName: String, dir: Path) {
val dataFile = dir.resolve(baseName + FILEEXT_DATA)
val allocFile = dir.resolve(baseName + FILEEXT_ALLOC)
......@@ -123,6 +72,7 @@ object BootImageWriter {
class BootImageWriter(tcb: TransitiveClosureBuilder, syms: Seq[FieldAndSymbol],
relocs: Seq[FieldAndSymbol], outputFile: String)(implicit microVM: MicroVM) {
import BootImageWriter._
import BootImageFile._
val bundleSerializer = new BundleSerializer(microVM.globalBundle, tcb.tls.set)
......
package uvm.refimpl.bootimg
import uvm.types.Type
import uvm.refimpl.Word
object BootImageFile {
val FILEEXT_DATA = ".data"
val FILEEXT_ALLOC = ".alloc"
val FILEEXT_RELOC = ".reloc"
val UIRBUNDLE_FILE = "bundle.uir"
val IDNAMEMAP_FILE = "idnamemap"
val METAINFO_FILE = "metainfo"
val KEY_INITFUNC = "initfunc"
val KEY_THREADLOCAL = "threadlocal"
val KEY_EXTRALIBS = "extralibs"
val AS_REF = "R"
val AS_IREF = "I"
val AS_TAGREF = "T"
val AS_FUNCREF = "F"
val AS_UPTR = "P"
val TO_GLOBAL = "G"
val TO_HEAP = "H"
val TO_FUNC = "F"
val TO_SYM = "S"
val NO_OFFSET = 0L
val NO_SYM = ""
/**
* Allocation record. For global and heap.
*
* @param num An integer to identify the unit in a .data file. For global, it is the ID of the global cell; for heap
* object, it is a sequential number.
* @param fileOffset The offset from the beginning of the .data file.
* @param ty The type of the allocation unit.
* @param varLen The length of the variable part if it is a hybrid; otherwise 0.
*
* @param addr The address in the current micro VM instance. Not persisted.
*/
case class UnitAllocRecord(num: Long, fileOffset: Long, ty: Type, varLen: Long, addr: Word)
/**
* Relocation record. One for each field of ref<T>, iref<T>, weakref<T> or funcref<sig> types.
*
* @param num The source allocation unit number.
* @param fieldOffset The offset of the field from the beginning of the allocation unit.
* @param targetKind Whether the target is in the global memory, heap, or is a function.
* @param targetNum To identify the target. For memory, it is the beginning of the allocation unit; for function, it
* is the ID of the function.
* @param targetOffset If the source is an iref<T>, it is the offset from the beginning of the target allocation unit;
* otherwise 0.
*/
case class FieldRelocRecord(num: Long, fieldOffset: Long, fieldKind: String, targetKind: String, targetNum: Long, targetOffset: Long, targetString: String)
}
\ No newline at end of file
......@@ -25,15 +25,26 @@ import uvm.utils.IOHelpers.forEachLine
import uvm.utils.MappedIDFactory
import uvm.utils.WithUtils.tryWithResource
import scala.collection.mutable.ArrayBuffer
import uvm.refimpl.itpr.HowToResume
import uvm.refimpl.cmdline.NativeArgv
import uvm.refimpl.itpr.BoxInt
import uvm.refimpl.InternalTypePool
import uvm.refimpl.InternalTypes
import uvm.refimpl.itpr.BoxPointer
import uvm.utils.IOHelpers
import com.typesafe.scalalogging.Logger
import org.slf4j.LoggerFactory
object BootImageLoader {
val MUTATOR_NAME = "bootimgldr"
val logger = Logger(LoggerFactory.getLogger(getClass.getName))
val HEAP_INIT_MUTATOR_NAME = "boot-image-loader-heap-init"
val PRIMORDIAL_INIT_MUTATOR_NAME = "boot-image-loader-primordial"
val EXTRALIBS_FILE = "extralibs"
}
class BootImageLoader(file: String)(implicit microVM: MicroVM) extends AutoCloseable {
class BootImageLoader(file: String, maybeAppArgs: Option[Seq[String]])(implicit microVM: MicroVM) extends AutoCloseable {
import BootImageLoader._
import BootImageWriter._
import BootImageFile._
private val globalBundle = microVM.globalBundle
private val memoryManager = microVM.memoryManager
......@@ -60,39 +71,30 @@ class BootImageLoader(file: String)(implicit microVM: MicroVM) extends AutoClose
private def zipFile(name: String): InputStream = zipFile(zip.getEntry(name))
private def zipFileText(name: String): Reader = new InputStreamReader(zipFile(name), StandardCharsets.UTF_8)
private def zipFileTextBuf(name: String): BufferedReader = new BufferedReader(zipFileText(name))
override def close(): Unit = {
zip.close()
}
def load(): Unit = {
val metaInfo = loadMetaInfo()
loadExtraLibs(metaInfo)
loadUIRBundle()
initializeMemory()
maybeCreatePrimordialThread(metaInfo)
}
private def loadMetaInfo(): Map[String, String] = {
val lines = tryWithResource(zipFileTextBuf(METAINFO_FILE)) { br =>
val ab = ArrayBuffer[String]()
var endReached = false
while(endReached) {
val maybeLine = Option(br.readLine())
maybeLine match {
case Some(line) => ab += line
case None => endReached = true
}
}
ab
IOHelpers.readLines(br)
}
val pairs = lines.map { line =>
val Array(k,v) = line.split("=", -1)
(k,v)
val Array(k, v) = line.split("=", -1)
(k, v)
}
pairs.toMap
}
......@@ -164,7 +166,7 @@ class BootImageLoader(file: String)(implicit microVM: MicroVM) extends AutoClose
ais.readToMemory(addr, size)
}
tryWithResource(heap.makeMutator(MUTATOR_NAME)) { mutator =>
tryWithResource(heap.makeMutator(HEAP_INIT_MUTATOR_NAME)) { mutator =>
forEachUnit("heap") { (ais, ar) =>
val UnitAllocRecord(num, fileOffset, ty, varLen, _) = ar
......@@ -239,7 +241,7 @@ class BootImageLoader(file: String)(implicit microVM: MicroVM) extends AutoClose
}
private def relocField(srcBase: Word, srcOffset: Word, fieldKind: String,
targetKind: String, targetNum: Long, targetOffset: Word, targetSymbol: String): Unit = {
targetKind: String, targetNum: Long, targetOffset: Word, targetSymbol: String): Unit = {
val srcFieldAddr = srcBase + srcOffset
fieldKind match {
case AS_UPTR => {
......@@ -247,7 +249,7 @@ class BootImageLoader(file: String)(implicit microVM: MicroVM) extends AutoClose
assert(targetKind != TO_HEAP, "BUG: uptr field referring to obj")
val targetAddr = targetKind match {
case TO_GLOBAL => getTargetMemBase(targetKind, targetNum) + targetOffset
case TO_SYM => getSymbolAddr(targetSymbol)
case TO_SYM => getSymbolAddr(targetSymbol)
}
memorySupport.storeLong(srcFieldAddr, targetAddr)
}
......@@ -294,25 +296,43 @@ class BootImageLoader(file: String)(implicit microVM: MicroVM) extends AutoClose
case TO_HEAP => objNumToAddr(targetNum)
}
}
private def getSymbolAddr(targetSymbol: String): Word = {
microVM.nativeLibraryHolder.getSymbolAddress(targetSymbol)
}
private def maybeCreatePrimordialThread(metaInfo: Map[String, String]): Unit = {
metaInfo.get(KEY_INITFUNC).map(_.toInt).foreach { initFuncID =>
val func = microVM.globalBundle.funcNs(initFuncID)
val threadLocalAddr = metaInfo.get(KEY_THREADLOCAL).map(_.toLong).map { addr =>
addr
}.getOrElse(0L)
tryWithResource(microVM.newContext("boot-image-loader")) { ctx =>
val func = ctx.handleFromFunc(initFuncID)
val stack = ctx.newStack(func)
val thread = ctx.newThread(stack, threadLocal, htr)
metaInfo.get(KEY_INITFUNC).map(_.toInt) match {
case Some(initFuncID) => {
val func = microVM.globalBundle.funcNs(initFuncID)
val appArgs = maybeAppArgs.getOrElse {
throw new BootImageLoaderException("Boot image has a primordial thread, but the MicroVM instance is not created " +
"with command line arguments. Please use the tools/runmu.sh command line utility to run boot images that has " +
"primordial threads. In realistic scenarios, the primordial thread in the boot image is supposed to be the " +
"entry point of a program, thus primordial threads should only exist in STAND-ALONE EXECUTABLE boot iamges.")
}
val threadLocalAddr = metaInfo.get(KEY_THREADLOCAL).map(_.toLong).map { addr =>
addr
}.getOrElse(0L)
tryWithResource(heap.makeMutator(PRIMORDIAL_INIT_MUTATOR_NAME)) { mutator =>
logger.info("Creating primordial thread using function %d (name: %s)...".format(initFuncID))
logger.info("Thread local obj address: %d 0x%x".format(threadLocalAddr, threadLocalAddr))
val func = microVM.globalBundle.funcNs(initFuncID)
val stack = microVM.threadStackManager.newStack(func, mutator)
val allArgs = Seq(file) ++ appArgs
val nativeArgv = new NativeArgv(allArgs)
val argcBox = BoxInt(nativeArgv.argc)
val argvBox = BoxPointer(nativeArgv.argv)
val htr = HowToResume.PassValues(Seq(argcBox, argvBox))
val thread = microVM.threadStackManager.newThread(stack, threadLocalAddr, htr)
}
}
case None => {
logger.info("Boot image does not have a primordial thread (not an error).")
}
}
}
}
......
package uvm.refimpl.cmdline
import uvm.refimpl.WORD_SIZE_BYTES
import uvm.refimpl.nat.NativeSupport
class NativeArgv(args: Seq[String]) {
val argc = args.size
private val argvSB = new StringBuilder()
private val argvOffs = new Array[Long](argc)
for ((arg, i) <- (args).zipWithIndex) {
argvOffs(i) = argvSB.length
argvSB ++= arg += '\0'
}
// These two buffers are allocated natively and are freed by their finalizers. Keep references to them while using.
val cArgv = NativeSupport.jnrMemoryManager.allocateDirect(argvOffs.length * WORD_SIZE_BYTES.toInt)
val cArgvBuf = NativeSupport.jnrMemoryManager.allocateDirect(argvSB.length)
for (i <- 0 until argvOffs.length) {
cArgv.putAddress(i * WORD_SIZE_BYTES, cArgvBuf.address() + argvOffs(i))
}
for (i <- 0 until argvSB.length) {
cArgvBuf.putByte(i, argvSB.charAt(i).toByte) // Assume single-byte encoding
}
val argv = cArgv.address()
}
\ No newline at end of file
......@@ -67,7 +67,7 @@ object RunMu {
props("bootImg") = bootImg
val microVM = new MicroVM(VMConf(props.toMap))
val microVM = MicroVM(VMConf(props.toMap))
val entryPoint: Either[MuID, MuName] = {
tryWithResource(new ZipFile(bootImg)) { zipFile =>
......@@ -86,24 +86,11 @@ object RunMu {
}
}
}
val argc = appArgs.size + 1
val argvSB = new StringBuilder()
val argvOffs = new Array[Long](argc)
for ((arg, i) <- (bootImg :: appArgs).zipWithIndex) {
argvOffs(i) = argvSB.length
argvSB ++= arg += '\0'
}
val cArgv = NativeSupport.jnrMemoryManager.allocateDirect(argvOffs.length * WORD_SIZE_BYTES.toInt)
val cArgvBuf = NativeSupport.jnrMemoryManager.allocateDirect(argvSB.length)
for (i <- 0 until argvOffs.length) {
cArgv.putAddress(i * WORD_SIZE_BYTES, cArgvBuf.address() + argvOffs(i))
}
for (i <- 0 until argvSB.length) {
cArgvBuf.putByte(i, argvSB.charAt(i).toByte) // Assume single-byte encoding
}
val allArgs = bootImg :: appArgs
val nativeArgv = new NativeArgv(allArgs)
val (argc, argv) = (nativeArgv.argc, nativeArgv.argv)
tryWithResource(microVM.newContext()) { ctx =>
val id = entryPoint match {
......@@ -113,7 +100,7 @@ object RunMu {
val func = ctx.handleFromFunc(id)
val stack = ctx.newStack(func)
val hArgc = ctx.handleFromInt(argc, 32)
val hArgv = ctx.handleFromPtr(InternalTypes.C_CHARPP.id, cArgv.address())
val hArgv = ctx.handleFromPtr(InternalTypes.C_CHARPP.id, argv)
val thread = ctx.newThread(stack, None, PassValues(Seq(hArgc, hArgv)))
}
......
......@@ -4,6 +4,7 @@ import java.io.BufferedReader
import java.io.InputStream
import java.io.OutputStream
import java.io.Reader
import scala.collection.mutable.ArrayBuffer
object IOHelpers {
def slurp(r: Reader): String = {
......@@ -50,4 +51,12 @@ object IOHelpers {
}
}
}
def readLines(br: BufferedReader): Seq[String] = {
val ab = ArrayBuffer[String]()
forEachLine(br) { line =>
ab += line
}
ab
}
}
\ No newline at end of file
......@@ -26,8 +26,6 @@ class MuCtxIRBuilderTest extends UvmBundleTesterBase with ExtraMatchers {
setLogLevels(ROOT_LOGGER_NAME -> INFO,
"uvm" -> DEBUG)
override def makeMicroVM() = new MicroVM(new VMConf())
behavior of "The IR Builder of MuCtx"
it should "create an empty bundle" in {
......
......@@ -8,8 +8,6 @@ import uvm.refimpl.UvmBundleTesterBase
import uvm.refimpl.VMConf
class BootImageEchoExample extends UvmBundleTesterBase with ExtraMatchers {
override def makeMicroVM = new MicroVM(new VMConf())
preloadBundles("tests/uvm-refimpl-test/boot-image-echo.uir")
preloadHails("tests/uvm-refimpl-test/boot-image-echo.hail")
......
......@@ -14,8 +14,6 @@ import uvm.refimpl.MuUPtrValue
import uvm.refimpl.MuUFPValue
class BootImageWriterTest extends UvmBundleTesterBase with ExtraMatchers {
override def makeMicroVM = new MicroVM(new VMConf())
preloadBundles("tests/uvm-refimpl-test/transitive-closure.uir")
preloadHails("tests/uvm-refimpl-test/transitive-closure.hail")
......@@ -63,7 +61,7 @@ class BootImageWriterTest extends UvmBundleTesterBase with ExtraMatchers {
ctx.makeBootImage(everything.map(_.id), Some(h_main), None, Some(h_tl), Seq(), Seq(), Seq(h_gs3_3), Seq("getchar"), filename)
}
val anotherMicroVM = MicroVM()
val anotherMicroVM = MicroVM("", Seq("prog", "Hello", "world"))
anotherMicroVM.loadBootImage(filename)
microVM.globalBundle.allNs("@gd").id shouldEqual anotherMicroVM.globalBundle.allNs("@gd").id
......@@ -86,6 +84,10 @@ class BootImageWriterTest extends UvmBundleTesterBase with ExtraMatchers {
val getchar_val = anotherMicroVM.nativeLibraryHolder.getSymbolAddress("getchar")
gs3_3_val shouldEqual getchar_val
val allThreads = anotherMicroVM.threadStackManager.threadRegistry.values.toSeq
allThreads.size shouldBe 1
allThreads(0).stack.get.frames.toStream(0).curFuncID shouldBe ctx.idOf("@main")
}
}
}
\ No newline at end of file
......@@ -12,8 +12,6 @@ import uvm.TopLevel
import uvm.refimpl.nat.NativeSupport
class TransitiveClosureTest extends UvmBundleTesterBase with ExtraMatchers {
override def makeMicroVM = new MicroVM(new VMConf())
preloadBundles("tests/uvm-refimpl-test/transitive-closure.uir")
preloadHails("tests/uvm-refimpl-test/transitive-closure.hail")
......
......@@ -22,7 +22,7 @@ class UvmInterpreterGCTests extends UvmBundleTesterBase {
preloadBundles("tests/uvm-refimpl-test/primitives.uir", "tests/uvm-refimpl-test/gc-tests.uir")
// 256KiB SOS, 512KiB LOS. This gives the GC a higher heap pressure so that it will trigger GC more often.
override def makeMicroVM = new MicroVM(new VMConf(sosSize = 256L * 1024L, losSize = 512L * 1024L))
override def makeMicroVM = MicroVM(new VMConf(sosSize = 256L * 1024L, losSize = 512L * 1024L))
def gc() = microVM.memoryManager.heap.mutatorTriggerAndWaitForGCEnd(false)
......
......@@ -18,8 +18,6 @@ class UvmInterpreterInt128Test extends UvmBundleTesterBase {
//"uvm.refimpl.mem.simpleimmix.SimpleImmixCollector$" -> DEBUG,
"uvm.refimpl.itpr" -> DEBUG)
override def makeMicroVM = MicroVM()
preloadBundles("tests/uvm-refimpl-test/primitives.uir",
"tests/uvm-refimpl-test/int128-test.uir")
......
......@@ -23,7 +23,7 @@ class UvmInterpreterSpec extends UvmBundleTesterBase {
//"uvm.refimpl.mem.simpleimmix.SimpleImmixCollector$" -> DEBUG,
"uvm.refimpl.itpr" -> DEBUG)
override def makeMicroVM = new MicroVM(new VMConf(sosSize = 4L * 1024L * 1024L, losSize = 4L * 1024L * 1024L))
override def makeMicroVM = MicroVM(new VMConf(sosSize = 4L * 1024L * 1024L, losSize = 4L * 1024L * 1024L))
preloadBundles("tests/uvm-refimpl-test/primitives.uir",
"tests/uvm-refimpl-test/basic-tests.uir")
......
......@@ -40,7 +40,7 @@ class UvmInterpreterStackGCTests extends UvmBundleTesterBase {
}
// With 1MiB space is LOS. It can accommodate 16 stacks.
override def makeMicroVM = new MicroVM(new VMConf(losSize = 1L * 1024L * 1024L, stackSize = 63L * 1024L))
override def makeMicroVM = MicroVM(new VMConf(losSize = 1L * 1024L * 1024L, stackSize = 63L * 1024L))
"The memory manager" should "collect unreachable stacks." in {
val ctx = microVM.newContext()
......
......@@ -20,7 +20,7 @@ class ObjectPinningTest extends UvmBundleTesterBase {
"uvm.refimpl.mem" -> INFO,
"uvm.refimpl.mem.simpleimmix.SimpleImmixCollector$" -> DEBUG)
override def makeMicroVM() = new MicroVM(new VMConf(sosSize = 256L*1024L, losSize = 256L * 1024L))
override def makeMicroVM() = MicroVM(new VMConf(sosSize = 256L*1024L, losSize = 256L * 1024L))
def gc() = microVM.memoryManager.heap.mutatorTriggerAndWaitForGCEnd(false)
......
......@@ -15,7 +15,7 @@ class UvmMemOperationsSpec extends UvmBundleTesterBase {
// The heap size is intentionally reduced to make GC more often
// The heap is divided in two halves. There is a 256KiB small object space (with 8 32KiB blocks) and a 256KiB large
// object space.
override def makeMicroVM() = new MicroVM(new VMConf(sosSize = 256L*1024L, losSize = 256L * 1024L))
override def makeMicroVM() = MicroVM(new VMConf(sosSize = 256L*1024L, losSize = 256L * 1024L))
microVM.memoryManager.heap.space.debugLogBlockStates()
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment