GitLab will continue to be upgraded from 11.4.5-ce.0 on November 25th 2019 at 4.00pm (AEDT) to 5.00pm (AEDT) due to Critical Security Patch Availability. During the update, GitLab and Mattermost services will not be available.

Commit 85e4a1bb authored by Kunshan Wang's avatar Kunshan Wang

Serialise alloc units. TODO: relocation entries

parent 5d333f12
......@@ -3,6 +3,13 @@ package uvm
import uvm.types._
import uvm.ssavariables._
import scala.collection.mutable.HashMap
import scala.collection.mutable.ArrayBuffer
/**
* Abstract type for all top-level definitions. Unlike the text form, FuncVer is not considered as top-level, but a sub-
* node of Function.
*/
trait TopLevel extends Identified
abstract class Bundle {
/*
......@@ -39,6 +46,16 @@ abstract class Bundle {
val instNs = allNs.makeSubSpace[Instruction]("instruction")
val sourceInfoRepo = new SourceInfoRepo()
def allTopLevels(): Seq[TopLevel] = {
val buf = new ArrayBuffer[TopLevel]()
buf ++= typeNs.all
buf ++= funcSigNs.all
buf ++= constantNs.all
buf ++= globalCellNs.all
buf ++= funcNs.all
buf ++= expFuncNs.all
}
}
/**
......
......@@ -5,7 +5,7 @@ import scala.collection.mutable.ArrayBuffer
import uvm.types._
import uvm.ssavariables._
case class FuncSig(var paramTys: Seq[Type], var retTys: Seq[Type]) extends IdentifiedSettable {
case class FuncSig(var paramTys: Seq[Type], var retTys: Seq[Type]) extends IdentifiedSettable with TopLevel {
override final def toString: String = FuncSig.prettyPrint(this)
}
......
package uvm.ir.textoutput
import uvm.GlobalBundle
import scala.collection.immutable.Set
import scala.collection.Set
import uvm.Identified
import java.io.Writer
import uvm._
......
......@@ -146,7 +146,17 @@ class MicroVM(vmConf: VMConf) {
* Make boot image.
*/
def makeBootImage(whiteList: Seq[Int], outputFile: String): Unit = {
bootImageBuilder.makeBootImage(whiteList, outputFile)
val whiteListObjs = whiteList.map { id =>
val obj = globalBundle.allNs(id)
obj match {
case tl: TopLevel => tl
case _ => throw new UvmRuntimeException(
"White list may only contain top-level definitions. Entity ID %d%s is a %s".format(
obj.id, obj.name.map(n => s" (${n})").getOrElse(""), obj.getClass.getName))
}
}
bootImageBuilder.makeBootImage(whiteListObjs, outputFile)
}
/**
......
......@@ -20,9 +20,11 @@ import uvm.refimpl.nat.NativeSupport
import uvm.refimpl.nat.PlatformConstants.Word
import uvm.types._
import uvm.utils.WithUtils.tryWithResource
import uvm.ir.textoutput.UnnamedEntityUtils
import uvm.ir.textoutput.BundleSerializer
class BootImageBuilder(implicit microVM: MicroVM) {
def makeBootImage(whiteList: Seq[Int], outputFile: String): Unit = {
def makeBootImage(whiteList: Seq[TopLevel], outputFile: String): Unit = {
implicit val tcb = new TransitiveClosureBuilder(whiteList)
tcb.doTransitiveClosure()
......@@ -39,6 +41,7 @@ object BootImageWriter {
val FILEEXT_ALLOC = ".alloc"
val FILEEXT_RELOC = ".reloc"
val UIRBUNDLE_FILE = "bundle.uir"
val IDNAMEMAP_FILE = "idnamemap"
val AS_REF = "R"
......@@ -94,6 +97,9 @@ class BootImageWriter(tcb: TransitiveClosureBuilder, outputFile: String)(implici
val globalGroup = new FileGroup("global", tempDir)
val heapGroup = new FileGroup("heap", tempDir)
val idNameMapPath = tempDir.resolve(IDNAMEMAP_FILE)
val uirBundlePath = tempDir.resolve(UIRBUNDLE_FILE)
private val globalMemory: Space = microVM.memoryManager.globalMemory
private val smallObjectSpace: Space = microVM.memoryManager.heap.space
......@@ -115,6 +121,8 @@ class BootImageWriter(tcb: TransitiveClosureBuilder, outputFile: String)(implici
writeAllocRecs(heapGroup)
writeIDNameMap()
writeUIRBundle()
}
private def writeAllocUnit(addr: Word, group: FileGroup, isGlobal: Boolean): Unit = {
......@@ -147,13 +155,24 @@ class BootImageWriter(tcb: TransitiveClosureBuilder, outputFile: String)(implici
private def writeAllocRecs(group: FileGroup): Unit = {
tryWithResource(Files.newBufferedWriter(group.allocFile, StandardCharsets.UTF_8)) { writer =>
for (ar <- group.allocRecs) {
writer.write("%d,%d,%d,%d,%s\n".format(ar.num, ar.fileOffset, ar.ty.id, ar.varLen, ar.ty.name))
writer.write("%d,%d,%d,%d,%s\n".format(ar.num, ar.fileOffset, ar.ty.id, ar.varLen, UnnamedEntityUtils.getName(ar.ty)))
}
}
}
private def writeIDNameMap(): Unit = {
tryWithResource(Files.newBufferedWriter(idNameMapPath, StandardCharsets.UTF_8)) { writer =>
for (obj <- tcb.tls.set) {
writer.write("%d,%s\n".format(obj.id,UnnamedEntityUtils.getName(obj)))
}
}
}
private def writeUIRBundle(): Unit = {
val bs = new BundleSerializer(microVM.globalBundle, Some(tcb.tls.set.map(_.id)))
tryWithResource(Files.newBufferedWriter(uirBundlePath, StandardCharsets.UTF_8)) { writer =>
bs.writeUIR(writer)
}
}
}
......@@ -184,6 +203,8 @@ class AlignedOutputStream(os: OutputStream) extends Closeable {
if (remain > 0) {
transfer(nextAddr, remain.toInt)
}
position += size
}
private def transfer(addr: Long, size: Int): Unit = {
......
......@@ -19,7 +19,6 @@ import uvm.refimpl.mem.scanning.RefFieldHandler
import uvm.ssavariables._
import uvm.types._
object TransitiveClosure {
def apply[T](initialElems: T*) = new TransitiveClosure(initialElems)
}
......@@ -64,19 +63,19 @@ object TransitiveClosureBuilder {
case class GlobalCellRec(begin: Word, end: Word, g: GlobalCell)
}
class TransitiveClosureBuilder(initialSet: Seq[Int])(implicit microVM: MicroVM) extends RefFieldHandler {
class TransitiveClosureBuilder(initialSet: Seq[TopLevel])(implicit microVM: MicroVM) extends RefFieldHandler {
import TransitiveClosureBuilder._
private val globalMemory: Space = microVM.memoryManager.globalMemory
private val smallObjectSpace: Space = microVM.memoryManager.heap.space
private val largeObjectSpace: Space = microVM.memoryManager.heap.los
private implicit val memorySupport = microVM.memoryManager.memorySupport
private implicit val memorySupport = microVM.memoryManager.memorySupport
/**
* Top-level definitions in the closure. Function versions are not considered top-level here. A function may have at
* most one function version. Only that version is serialised into the boot image.
*/
val tls = TransitiveClosure[Identified](initialSet.map(microVM.globalBundle.allNs.apply): _*)
val tls = TransitiveClosure[TopLevel](initialSet: _*)
/**
* Global cells + heap objects.
......@@ -101,15 +100,24 @@ class TransitiveClosureBuilder(initialSet: Seq[Int])(implicit microVM: MicroVM)
}
TreeMap[Word, GlobalCellRec](elems: _*)
}
def getGlobalCellRecExact(begin: Word): GlobalCellRec = {
require(microVM.memoryManager.globalMemory.isInSpace(begin),
"Address %d 0x%x is not in the global space".format(begin, begin))
val gcr = globalCellMap(begin)
gcr
}
def getGlobalCellRec(iref: Word): GlobalCellRec = {
require(microVM.memoryManager.globalMemory.isInSpace(iref),
"Address %d 0x%x is not in the global space".format(iref, iref))
"Address %d 0x%x is not in the global space".format(iref, iref))
val (begin, gcr) = globalCellMap.to(iref).last
assert(begin <= iref && iref < gcr.end,
"Address %d 0x%x is in the global space, but the previous global cell's address range is %d 0x%x - %d 0x%x".format(
iref, iref, begin, begin, gcr.end, gcr.end))
"Address %d 0x%x is in the global space, but the previous global cell's address range is %d 0x%x - %d 0x%x".format(
iref, iref, begin, begin, gcr.end, gcr.end))
gcr
}
......@@ -254,24 +262,31 @@ class TransitiveClosureBuilder(initialSet: Seq[Int])(implicit microVM: MicroVM)
case ThrowExc(e) =>
}
private def visitAllocUnit(addrOrObjRef: Word): Unit = {
logger.debug("Visiting allocation unit 0x%x".format(addrOrObjRef))
if (globalMemory.isInSpace(addrOrObjRef)) {
val GlobalCellRec(begin, end, g) = getGlobalCellRec(addrOrObjRef)
/**
* Visit an allocation unit (global cell or heap object). As a pre-condition, begin must refer to the beginning of
* the unit.
*
* @param begin The beginning of the object.
*/
private def visitAllocUnit(begin: Word): Unit = {
logger.debug("Visiting allocation unit 0x%x".format(begin))
if (globalMemory.isInSpace(begin)) {
val GlobalCellRec(begin2, end, g) = getGlobalCellRecExact(begin)
assert(begin2 == begin)
logger.debug(" It's global cell. begin=0x%x, end=0x%x, type=%s".format(begin, end, g.cellTy))
tls += g
visitAllocUnitFields(begin, g.cellTy)
} else if (smallObjectSpace.isInSpace(addrOrObjRef) || largeObjectSpace.isInSpace(addrOrObjRef)) {
val objRef = addrOrObjRef
} else if (smallObjectSpace.isInSpace(begin) || largeObjectSpace.isInSpace(begin)) {
val objRef = begin
val tag = HeaderUtils.getTag(objRef)
val ty = HeaderUtils.getType(microVM, tag)
tls += ty
visitAllocUnitFields(objRef, ty)
} else {
throw new BootImageBuilderException("Address %d 0x%x is not in the heap or global space".format(addrOrObjRef, addrOrObjRef))
throw new BootImageBuilderException("Address %d 0x%x is not in the heap or global space".format(begin, begin))
}
}
private def visitAllocUnitFields(begin: Word, ty: Type): Unit = {
MemoryDataScanner.scanAllocUnit(begin, this)
}
......@@ -291,11 +306,23 @@ class TransitiveClosureBuilder(initialSet: Seq[Int])(implicit microVM: MicroVM)
}
}
override def memToNonHeap(objRef: Word, iRef: Word, addr: Word): Unit = {
require(globalMemory.isInSpace(addr), {
val inLOS = largeObjectSpace.isInSpace(addr)
("Error: Non-heap internal reference not referring to global space. objRef: 0x%x, iref: 0x%x, target: 0x%x, inLOS: %s. " +
"If the address is in the large object space, it may refer to a stack cell (ALLOCA)").format(
objRef, iRef, addr, inLOS)
})
val gr = getGlobalCellRec(addr)
tls += gr.g
}
def memToStack(objRef: Word, iRef: Word, toStack: Option[InterpreterStack]): Option[InterpreterStack] = {
if (toStack.isDefined) {
throw new BootImageBuilderException(
"Error: Stack reachable during boot image building. From: obj 0x%x, iref 0x%x. To stack id: %d".format(
objRef, iRef, toStack.get.id))
"Error: Stack reachable during boot image building. From: obj 0x%x, iref 0x%x. To stack id: %d".format(
objRef, iRef, toStack.get.id))
}
None
}
......
......@@ -35,6 +35,8 @@ object MemoryDataScanner extends StrictLogging {
def scanField(ty: Type, objRef: Word, iRef: Word, handler: RefFieldHandler)(implicit microVM: MicroVM, memorySupport: MemorySupport) {
def logField(kind: String, toObj: Word): String = "%s field [0x%x + 0x%x] = 0x%x -> 0x%x".format(kind, objRef, iRef - objRef, iRef, toObj)
def logIRefField(kind: String, toObj: Word, offset: Word): String = "%s field [0x%x + 0x%x] = 0x%x -> 0x%x + 0x%x = 0x%x".format(
kind, objRef, iRef - objRef, iRef, toObj, offset, toObj + offset)
ty match {
case t: TypeRef => {
val toObj = memorySupport.loadLong(iRef)
......@@ -43,8 +45,13 @@ object MemoryDataScanner extends StrictLogging {
}
case t: TypeIRef => {
val toObj = memorySupport.loadLong(iRef)
logger.debug(logField("IRef", toObj))
handler.memToHeap(objRef, iRef, toObj, false, false)
val offset = memorySupport.loadLong(iRef + WORD_SIZE_BYTES)
logger.debug(logIRefField("IRef", toObj, offset))
if (toObj == 0L && offset != 0L) {
handler.memToNonHeap(objRef, iRef, offset)
} else {
handler.memToHeap(objRef, iRef, toObj, false, false)
}
}
case t: TypeWeakRef => {
val toObj = memorySupport.loadLong(iRef)
......@@ -119,6 +126,18 @@ object MemoryDataScanner extends StrictLogging {
}
handler.memToStack(objRef, iRef, maybeToStack)
}
case t: TypeFuncRef => {
val toFuncID = memorySupport.loadLong(iRef)
val maybeToFunc = if (toFuncID == 0) {
None
} else {
val toFunc = microVM.globalBundle.funcNs.get(toFuncID.toInt).getOrElse {
throw new UvmRefImplException("Memory location 0x%x referring to non-existing function %d".format(iRef, toFuncID))
}
Some(toFunc)
}
handler.memToFunc(objRef, iRef, maybeToFunc)
}
case _ => // Ignore non-reference fields.
}
}
......
......@@ -10,6 +10,7 @@ import uvm.refimpl.itpr.InterpreterStack
import uvm.refimpl.UvmRefImplException
import org.slf4j.LoggerFactory
import com.typesafe.scalalogging.Logger
import uvm.Function
/**
* Handle references in the memory, value boxes or other Micro VM structures such as threads and stacks.
......@@ -36,6 +37,18 @@ trait RefFieldHandler {
def threadToStack(thread: InterpreterThread, toStack: Option[InterpreterStack]): Option[InterpreterStack]
/** Pin set referring to the memory. Pinned objects cannot be moved. */
def pinSetToMem(toObj: Word): Option[Word]
/**
* A memory location referring to non-heap Mu memory. For GC, such fields do not need to be traced. But
* boot image writers need them to track other global cells.
*/
def memToNonHeap(objRef: Word, iRef: Word, addr: Word): Unit = ()
/**
* A memory location referring to a Mu function. GC does not need to care about function references, but
* boot image builders need to find them so that functions can be relocated.
*/
def memToFunc(objRef: Word, iRef: Word, toFunc: Option[Function]): Unit = ()
}
object RefFieldUpdater {
......
......@@ -10,7 +10,7 @@ abstract class SSAVariable extends IdentifiedSettable
// Global variables: Constants, Global Cells and Functions (Function is defined in controlFlow.scala)
abstract class GlobalVariable extends SSAVariable
abstract class GlobalVariable extends SSAVariable with TopLevel
abstract class Constant extends GlobalVariable {
var constTy: Type
......
......@@ -2,7 +2,7 @@ package uvm.types
import uvm._
abstract class Type extends IdentifiedSettable {
abstract class Type extends IdentifiedSettable with TopLevel {
override final def toString: String = Type.prettyPrint(this)
}
......
package uvm.refimpl.bootimg
import uvm.refimpl.MicroVM
import uvm.refimpl.VMConf
import uvm.refimpl.UvmBundleTesterBase
import uvm.ir.textinput.ExtraMatchers
import uvm.ir.textinput.TestingBundlesValidators.MagicalOur
import uvm.refimpl.mem.HeaderUtils
import uvm.types.Type
import uvm.refimpl.mem.TypeSizes.Word
class BootImageWriterTest extends UvmBundleTesterBase with ExtraMatchers {
override def makeMicroVM = new MicroVM(new VMConf())
preloadBundles("tests/uvm-refimpl-test/transitive-closure.uir")
preloadHails("tests/uvm-refimpl-test/transitive-closure.hail")
val our = new MagicalOur(microVM.globalBundle)
behavior of "BootImageWriter"
it should "write everything to files" in {
val everything = microVM.globalBundle.allTopLevels()
val tcb = new TransitiveClosureBuilder(everything)(microVM)
tcb.doTransitiveClosure()
val biw = new BootImageWriter(tcb, "just-test")(microVM)
biw.writeFile()
}
}
\ No newline at end of file
......@@ -8,6 +8,8 @@ import uvm.ir.textinput.TestingBundlesValidators.MagicalOur
import uvm.refimpl.mem.HeaderUtils
import uvm.types.Type
import uvm.refimpl.mem.TypeSizes.Word
import uvm.TopLevel
import uvm.refimpl.nat.NativeSupport
class TransitiveClosureTest extends UvmBundleTesterBase with ExtraMatchers {
override def makeMicroVM = new MicroVM(new VMConf())
......@@ -18,6 +20,11 @@ class TransitiveClosureTest extends UvmBundleTesterBase with ExtraMatchers {
val our = new MagicalOur(microVM.globalBundle)
behavior of "TransitiveClosureBuilder"
implicit def nameToTopLevel(name: String): TopLevel = {
microVM.globalBundle.allNs(name).asInstanceOf[TopLevel]
}
it should "compute transitive closure over types" in {
val tsb = new TransitiveClosureBuilder(Seq("@refi64"))(microVM)
......@@ -134,6 +141,13 @@ class TransitiveClosureTest extends UvmBundleTesterBase with ExtraMatchers {
}
it should "scan global cells which refer to each other" in {
val ctx = microVM.newContext()
val h_gr2 = ctx.handleFromGlobal("@gr2")
val addr = h_gr2.vb.asIRefAddr
val content0= NativeSupport.theMemory.getLong(addr)
val content1= NativeSupport.theMemory.getLong(addr+8)
ctx.closeContext()
val tsb = new TransitiveClosureBuilder(Seq("@gr2"))(microVM)
tsb.doTransitiveClosure()
......@@ -146,7 +160,7 @@ class TransitiveClosureTest extends UvmBundleTesterBase with ExtraMatchers {
for (obj <- objs) {
val ty = getType(obj)
Seq(ty) should contain oneOf (our ty "@i64", our ty "@refi64")
Seq(ty) should contain oneOf (our ty "@i64", our ty "@irefi64")
}
}
}
\ No newline at end of file
......@@ -283,6 +283,9 @@ class UvmHailBasicTest extends UvmHailTesterBase {
assertIREqual("@g_iri8_2", hBarIr_vp_2)
assertIREqual("@g_irvoid", hBarIr_vp_3)
val h_g_irvoid = mc.handleFromGlobal("@g_irvoid")
assertIREqual("@g_irirvoid", h_g_irvoid)
mc.closeContext()
}
......
......@@ -10,3 +10,5 @@
.init @g_iri8_2 = &$bar[2][2]
.init @g_irvoid = &$bar[2][3] // implicit refcast
.init @g_irvoid2 = &@g_big_struct[2] // implicit refcast
.init @g_irirvoid = &@g_irvoid // iref between globals
......@@ -79,6 +79,9 @@
.global @g_iri8_2 <@irefi8>
.global @g_irvoid <@irefvoid>
.global @g_irvoid2 <@irefvoid>
.global @g_irirvoid <@irefirefvoid>
.typedef @irefirefvoid = iref<@irefvoid>
// test-6
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment