GitLab will be upgraded on June 2nd 2020 at 2.00 pm (AEDT) to 3.00 pm (AEDT) due to Critical Security Patch Availability. During the update, GitLab and Mattermost services will not be available. If you have any concerns with this, please talk to local Gitlab admin team.

Commit 5d333f12 authored by Kunshan Wang's avatar Kunshan Wang

WIP: write heap/global to file

parent 1d41a37e
......@@ -10,7 +10,20 @@ import uvm.ssavariables._
import java.time.temporal.IsoFields.Unit
import uvm.ssavariables.NewStackAction
object UnnamedEntityUtils {
def getName(obj: Identified): String = {
obj.name.getOrElse("@uvm.unnamed" + obj.id.toString())
}
def getNames(objs: Seq[Identified]): String = {
objs.map(getName).mkString(" ")
}
}
class BundleSerializer(val bundle: GlobalBundle, val writeList: Option[Set[Int]]) {
import UnnamedEntityUtils._
import BundleSerializer._
def isMetaEntity(id: Int): Boolean = {
id < 65536
}
......@@ -30,14 +43,6 @@ class BundleSerializer(val bundle: GlobalBundle, val writeList: Option[Set[Int]]
filter(obj.id)
}
def getName(obj: Identified): String = {
obj.name.getOrElse("@uvm.unnamed" + obj.id.toString())
}
def getNames(objs: Seq[Identified]): String = {
objs.map(getName).mkString(" ")
}
def writeIDNameMap(output: Writer): Unit = {
val sb = new StringBuilder()
writeIDNameMap(sb)
......@@ -71,32 +76,32 @@ class BundleSerializer(val bundle: GlobalBundle, val writeList: Option[Set[Int]]
def writeType(ty: Type, output: StringBuilder): Unit = {
val ctor = ty match {
case TypeInt(l) => "int<%d>".format(l)
case TypeInt(l) => s"int<${l}>"
case TypeFloat() => "float"
case TypeDouble() => "double"
case TypeUPtr(ty) => "uptr<%s>".format(getName(ty))
case TypeUFuncPtr(sig) => "ufuncptr<%s>".format(getName(sig))
case TypeStruct(fieldTys) => "struct<%s>".format(getNames(fieldTys))
case TypeHybrid(fieldTys, varPart) => "hybrid<%s %s>".format(getNames(fieldTys), getName(varPart))
case TypeArray(elemTy, len) => "array<%s %d>".format(getName(elemTy), len)
case TypeVector(elemTy, len) => "vector<%s %d>".format(getName(elemTy), len)
case TypeUPtr(ty) => s"uptr<${ty.n}>"
case TypeUFuncPtr(sig) => s"ufuncptr<${sig.n}>"
case TypeStruct(fieldTys) => s"struct<${fieldTys.ns}>"
case TypeHybrid(fieldTys, varPart) => s"hybrid<${fieldTys.ns} ${varPart.n}>"
case TypeArray(elemTy, len) => s"array<${elemTy.n} ${len}>"
case TypeVector(elemTy, len) => s"vector<${elemTy.n} ${len}>"
case TypeVoid() => "void"
case TypeRef(ty) => "ref<%s>".format(getName(ty))
case TypeIRef(ty) => "iref<%s>".format(getName(ty))
case TypeWeakRef(ty) => "weakref<%s>".format(getName(ty))
case TypeRef(ty) => s"ref<${ty.n}>"
case TypeIRef(ty) => s"iref<${ty.n}>"
case TypeWeakRef(ty) => s"weakref<${ty.n}>"
case TypeTagRef64() => "tagref64"
case TypeFuncRef(sig) => "funcref<%s>".format(getName(sig))
case TypeFuncRef(sig) => s"funcref<${sig.n}>"
case TypeThreadRef() => "threadref"
case TypeStackRef() => "stackref"
case TypeFrameCursorRef() => "framecursorref"
case TypeIRNodeRef() => "irnoderef"
case _ => throw new BundleSerializerException("unknown type " + ty.getClass.getName)
}
output.append(".typedef %s = %s\n".format(getName(ty), ctor))
output.append(s".typedef ${ty.n} = ${ctor}\n")
}
def writeSig(sig: FuncSig, output: StringBuilder): Unit = {
output.append(".funcsig %s = (%s) -> (%s)\n".format(getName(sig), getNames(sig.paramTys), getNames(sig.retTys)))
output.append(s".funcsig ${sig.n} = (${sig.paramTys.ns}) -> (${sig.retTys.ns})\n")
}
def writeConst(const: Constant, output: StringBuilder): Unit = {
......@@ -110,24 +115,23 @@ class BundleSerializer(val bundle: GlobalBundle, val writeList: Option[Set[Int]]
case ConstExtern(_, symbol) => "EXTERN \"%s\"".format(symbol)
case _ => throw new BundleSerializerException("unknown constant " + const.getClass.getName)
}
output.append(".const %s <%s> = %s\n".format(getName(const), getName(ty), ctor))
output.append(s".const ${const.n} <${ty.n}> = ${ctor}\n")
}
def writeGlobal(global: GlobalCell, output: StringBuilder): Unit = {
output.append(".global %s <%s>\n".format(getName(global), getName(global.cellTy)))
output.append(s".global ${global.n} <${global.cellTy.n}>\n")
}
def writeExpFunc(expFunc: ExposedFunc, output: StringBuilder): Unit = {
output.append(".expose %s = %s %s %s\n".format(
getName(expFunc), getName(expFunc.func), expFunc.callConv.name, getName(expFunc.cookie)))
output.append(s".expose ${expFunc.n} = ${expFunc.func.n} ${expFunc.callConv.name} ${expFunc.cookie.n}\n")
}
def writeFunc(func: Function, output: StringBuilder): Unit = {
bundle.funcToVers(func).headOption match {
case None =>
output.append(".funcdecl %s <%s>\n".format(getName(func), getName(func.sig)))
output.append(s".funcdecl ${func.n} <${func.sig.n}>\n")
case Some(ver) =>
output.append(".funcdef %s VERSION %s <%s> {\n".format(getName(func), getName(ver), getName(func.sig)))
output.append(s".funcdef ${func.n} VERSION ${ver.n} <${func.sig.n}> {\n")
for (bb <- ver.bbs) {
writeBB(bb, output)
}
......@@ -136,69 +140,26 @@ class BundleSerializer(val bundle: GlobalBundle, val writeList: Option[Set[Int]]
}
def norParamToString(p: NorParam): String = {
"<%s> %s ".format(getName(p.ty), getName(p))
s"<${p.ty.n}> ${p.n} "
}
def excParamToString(p: ExcParam): String = {
" [%s]".format(getName(p))
s" [${p.n}]"
}
def writeBB(bb: BasicBlock, output: StringBuilder): Unit = {
output.append(" %s(%s)%s:\n".format(
getName(bb), bb.norParams.map(norParamToString).mkString(" "), bb.excParam.map(excParamToString).getOrElse("")))
output.append {
val norParamList = bb.norParams.map(norParamToString).mkString(" ")
val maybeExcParam = bb.excParam.map(excParamToString).getOrElse("")
s" ${bb.n}(${norParamList})${maybeExcParam}:\n"
}
for (inst <- bb.insts) {
writeInst(inst, output)
}
}
def destToString(dest: DestClause): String = {
"%s(%s)".format(getName(dest.bb), getNames(dest.args))
}
def excClauseToString(maybeExc: Option[ExcClause]): String = maybeExc match {
case None => ""
case Some(ExcClause(nor, exc)) => "EXC(%s %s)".format(destToString(nor), destToString(exc))
}
def keepalivesToString(kas: Seq[LocalVariable]): String = {
"KEEPALIVE(%s)".format(getNames(kas))
}
def ptrToString(isPtr: Boolean): String = {
if (isPtr) "PTR" else ""
}
def maybeWPExc(me: Option[DestClause]): String = me match {
case None => ""
case Some(e) => s"WPEXC(${destToString(e)})"
}
def switchToString(sw: InstSwitch): String = {
val InstSwitch(opndTy, opnd, defDest, cases) = sw
"SWITCH <%s> %s %s { %s }".format(
getName(opndTy), getName(opnd), destToString(defDest), (cases.map {
case (v, d) => "%s %s".format(getName(v), destToString(d))
}).mkString("\n"))
}
def maybeThreadLocal(tl: Option[SSAVariable]): String = tl match {
case None => ""
case Some(v) => s"THREADLOCAL(${getName(v)})"
}
def newStackActionToString(nsa: NewStackAction): String = nsa match {
case PassValues(tys, vs) => s"PASS_VALUES <${getNames(tys)}> (${getNames(vs)})"
case ThrowExc(e) => s"THROW_EXC ${getName(e)}"
}
def curStackActionToString(csa: CurStackAction): String = csa match {
case RetWith(tys) => s"RET_WITH <${getNames(tys)}>"
case KillOld() => s"KILL_OLD"
}
def writeInst(inst: Instruction, output: StringBuilder): Unit = {
import BundleSerializer._
implicit val bs = this
val body = inst match {
......@@ -246,28 +207,75 @@ class BundleSerializer(val bundle: GlobalBundle, val writeList: Option[Set[Int]]
s"""COMMINST ${ci.name.get} [${flagList.map(_.name).mkString(" ")}] <${typeList.ns}> <[${funcSigList.ns}]> (${argList.ns}) ${excClause.e} ${keepalives.kas}"""
case _ => throw new BundleSerializerException("Unknown instruction: " + inst.getClass.getName)
}
output.append(" (%s) = [%s] %s\n".format(getNames(inst.results), getName(inst), body))
output.append(s" (${inst.results.ns}) = [${inst.n}] ${body}\n")
}
}
object BundleSerializer {
import UnnamedEntityUtils._
def destToString(dest: DestClause): String = {
s"${dest.bb.n}(${dest.args.ns})"
}
def excClauseToString(maybeExc: Option[ExcClause]): String = maybeExc match {
case None => ""
case Some(ExcClause(nor, exc)) => s"EXC(${nor.d} ${exc.d})"
}
def keepalivesToString(kas: Seq[LocalVariable]): String = {
s"KEEPALIVE(${kas.ns})"
}
def ptrToString(isPtr: Boolean): String = {
if (isPtr) "PTR" else ""
}
def maybeWPExc(me: Option[DestClause]): String = me match {
case None => ""
case Some(e) => s"WPEXC(${destToString(e)})"
}
def switchToString(sw: InstSwitch): String = {
val InstSwitch(opndTy, opnd, defDest, cases) = sw
val casesStr = (cases.map {
case (v, d) => s"${v.n} ${d.d}"
}).mkString("\n")
s"SWITCH <${opndTy.n}> ${opnd.n} ${defDest.d} { ${casesStr} }"
}
def maybeThreadLocal(tl: Option[SSAVariable]): String = tl match {
case None => ""
case Some(v) => s"THREADLOCAL(${getName(v)})"
}
def newStackActionToString(nsa: NewStackAction): String = nsa match {
case PassValues(tys, vs) => s"PASS_VALUES <${getNames(tys)}> (${getNames(vs)})"
case ThrowExc(e) => s"THROW_EXC ${getName(e)}"
}
def curStackActionToString(csa: CurStackAction): String = csa match {
case RetWith(tys) => s"RET_WITH <${getNames(tys)}>"
case KillOld() => s"KILL_OLD"
}
implicit class MagicalStringify(val thing: Any) extends AnyVal {
def s: String = thing.toString
}
implicit class MagicalNameLookup(val thing: Identified) extends AnyVal {
def n(implicit bs: BundleSerializer): String = bs.getName(thing)
def n: String = getName(thing)
}
implicit class MagicalNamesLookup(val things: Seq[Identified]) extends AnyVal {
def ns(implicit bs: BundleSerializer): String = bs.getNames(things)
def ns: String = getNames(things)
}
implicit class MagicalExcClause(val maybeExc: Option[ExcClause]) extends AnyVal {
def e(implicit bs: BundleSerializer): String = bs.excClauseToString(maybeExc)
def e: String = excClauseToString(maybeExc)
}
implicit class MagicalDestClause(val dest: DestClause) extends AnyVal {
def d(implicit bs: BundleSerializer): String = bs.destToString(dest)
def d: String = destToString(dest)
}
implicit class MagicalKeepalives(val lvs: Seq[LocalVariable]) extends AnyVal {
def kas(implicit bs: BundleSerializer): String = bs.keepalivesToString(lvs)
def kas: String = keepalivesToString(lvs)
}
implicit class MagicalBoolean(val value: Boolean) extends AnyVal {
def p: String = if (value) "PTR" else ""
......
package uvm.refimpl.bootimg
import uvm._
import uvm.types._
import uvm.ssavariables._
import java.io._
import java.io.OutputStream
import java.nio.charset.StandardCharsets
import java.nio.file._
import uvm.refimpl._
import uvm.refimpl.mem.TypeSizes.Word
import scala.collection.mutable.ArrayBuffer
import scala.collection.mutable.{ Set, Queue }
import uvm.refimpl.itpr.BoxIRef
import scala.collection.immutable.TreeMap
import uvm.refimpl.mem.TypeSizes
import uvm.refimpl.mem.Space
import uvm.refimpl.mem.HeaderUtils
import uvm.refimpl.mem.scanning.RefFieldHandler
import uvm.refimpl.itpr.InterpreterThread
import uvm.refimpl.itpr.BoxStack
import uvm.refimpl.itpr.HasObjRef
import uvm.refimpl.itpr.InterpreterStack
import org.slf4j.LoggerFactory
import com.typesafe.scalalogging.Logger
import uvm.refimpl.mem.scanning.MemoryDataScanner
import uvm._
import uvm.refimpl._
import uvm.refimpl.mem.HeaderUtils
import uvm.refimpl.mem.Space
import uvm.refimpl.mem.TypeSizes
import uvm.refimpl.nat.NativeSupport
import uvm.refimpl.nat.PlatformConstants.Word
import uvm.types._
import uvm.utils.WithUtils.tryWithResource
class BootImageBuilder(implicit microVM: MicroVM) {
def makeBootImage(whiteList: Seq[Int], outputFile: String): Unit = {
implicit val tc = new TransitiveClosureBuilder(whiteList)
implicit val tcb = new TransitiveClosureBuilder(whiteList)
tcb.doTransitiveClosure()
val biw = new BootImageWriter(tcb, outputFile)
biw.writeFile()
}
}
object BootImageWriter {
val logger = Logger(LoggerFactory.getLogger(getClass.getName))
val FILEEXT_DATA = ".data"
val FILEEXT_ALLOC = ".alloc"
val FILEEXT_RELOC = ".reloc"
val IDNAMEMAP_FILE = "idnamemap"
val AS_REF = "R"
val AS_IREF = "I"
val AS_FUNCREF = "F"
val IN_GLOBAL = "G"
val IN_HEAP = "H"
val TO_FUNC = "F"
/**
* Allocation record. For global and heap.
*
* @param num An integer to identify the unit in a .data file. For global, it is the ID of the global cell; for heap
* object, it is a sequential number.
* @param fileOffset The offset from the beginning of the .data file.
* @param ty The type of the allocation unit.
* @param varLen The length of the variable part if it is a hybrid; otherwise 0.
*/
case class UnitAllocRecord(num: Long, fileOffset: Long, ty: Type, varLen: Long)
/**
* Relocation record. One for each field of ref<T>, iref<T>, weakref<T> or funcref<sig> types.
*
* @param num The source allocation unit number.
* @param fieldOffset The offset of the field from the beginning of the allocation unit.
* @param targetKind Whether the target is in the global memory, heap, or is a function.
* @param targetNum To identify the target. For memory, it is the beginning of the allocation unit; for function, it
* is the ID of the function.
* @param targetOffset If the source is an iref<T>, it is the offset from the beginning of the target allocation unit;
* otherwise 0.
*/
case class FieldRelocRecord(num: Long, fieldOffset: Long, fieldKind: String, targetKind: String, targetNum: Long, targetOffset: Long)
class FileGroup(baseName: String, dir: Path) {
val dataFile = dir.resolve(baseName + FILEEXT_DATA)
val allocFile = dir.resolve(baseName + FILEEXT_ALLOC)
val relocFile = dir.resolve(baseName + FILEEXT_RELOC)
val dataOS = new AlignedOutputStream(new BufferedOutputStream(Files.newOutputStream(dataFile)))
val allocRecs = new ArrayBuffer[UnitAllocRecord]
val relocRecs = new ArrayBuffer[FieldRelocRecord]
}
}
class BootImageWriter(tcb: TransitiveClosureBuilder, outputFile: String)(implicit microVM: MicroVM) {
import BootImageWriter._
val tempDir = Files.createTempDirectory("mu-bootimg")
logger.info("Writing bootimg files to temp dir: %s".format(tempDir.toAbsolutePath().toString()))
val globalGroup = new FileGroup("global", tempDir)
val heapGroup = new FileGroup("heap", tempDir)
private val globalMemory: Space = microVM.memoryManager.globalMemory
private val smallObjectSpace: Space = microVM.memoryManager.heap.space
private val largeObjectSpace: Space = microVM.memoryManager.heap.los
private implicit val memorySupport = microVM.memoryManager.memorySupport
def writeFile(): Unit = {
for (addr <- tcb.allocs.set) {
if (globalMemory.isInSpace(addr)) {
writeAllocUnit(addr, globalGroup, isGlobal = true)
} else {
writeAllocUnit(addr, heapGroup, isGlobal = false)
}
}
globalGroup.dataOS.close()
heapGroup.dataOS.close()
writeAllocRecs(globalGroup)
writeAllocRecs(heapGroup)
writeIDNameMap()
}
private def writeAllocUnit(addr: Word, group: FileGroup, isGlobal: Boolean): Unit = {
val tag = HeaderUtils.getTag(addr)
val ty = HeaderUtils.getType(microVM, tag)
val (size, align, varLen) = ty match {
case t: TypeHybrid => {
val len = HeaderUtils.getVarLength(addr)
val sz = TypeSizes.hybridSizeOf(t, len)
val al = TypeSizes.hybridAlignOf(t, len)
(sz, al, len)
}
case t => {
val sz = TypeSizes.sizeOf(t)
val al = TypeSizes.alignOf(t)
(sz, al, 0L)
}
}
val num = if (isGlobal) ty.id else group.allocRecs.length
group.dataOS.alignUp(align)
val pos = group.dataOS.position
group.dataOS.copy(addr, size)
tc.doTransitiveClosure()
val rec = UnitAllocRecord(num, pos, ty, varLen)
group.allocRecs += rec
}
private def writeAllocRecs(group: FileGroup): Unit = {
tryWithResource(Files.newBufferedWriter(group.allocFile, StandardCharsets.UTF_8)) { writer =>
for (ar <- group.allocRecs) {
writer.write("%d,%d,%d,%d,%s\n".format(ar.num, ar.fileOffset, ar.ty.id, ar.varLen, ar.ty.name))
}
}
}
private def writeIDNameMap(): Unit = {
}
}
class AlignedOutputStream(os: OutputStream) extends Closeable {
var position: Long = 0L
private val BUFFER_SZ = 16384
private val buffer = new Array[Byte](BUFFER_SZ)
def alignUp(alignment: Long): Unit = {
val newPos = TypeSizes.alignUp(position, alignment)
position until newPos foreach { _ => os.write(0) }
position = newPos
}
def copy(addr: Long, size: Long): Unit = {
var nextAddr = addr
var remain = size
while (remain > BUFFER_SZ) {
transfer(nextAddr, BUFFER_SZ)
nextAddr += BUFFER_SZ
remain -= BUFFER_SZ
}
if (remain > 0) {
transfer(nextAddr, remain.toInt)
}
}
private def transfer(addr: Long, size: Int): Unit = {
NativeSupport.theMemory.get(addr, buffer, 0, size)
os.write(buffer, 0, size)
}
def close(): Unit = {
os.close()
}
}
\ No newline at end of file
package uvm.utils
/**
* Emulate Java 1.7 try-with-resource in Scala. Unlike Java, res cannot be null.
*/
object WithUtils {
def tryWithResource[T <: AutoCloseable, U](res: T)(f: T => U): U = {
var maybePrimaryExc: Option[Throwable] = None
try {
f(res)
} catch {
case t: Throwable =>
maybePrimaryExc = Some(t)
throw t
} finally {
maybePrimaryExc match {
case None => res.close()
case Some(primaryExc) => try {
res.close()
} catch {
case secondaryExc: Throwable =>
primaryExc.addSuppressed(secondaryExc)
}
}
}
}
}
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment