Commit e701e3d5 authored by Kunshan Wang's avatar Kunshan Wang

Support manual symbols for reloc.

Manual symbols can be added via the make_boot_image api function.

Note: curently manual symbols can only be used during boot image
building, and can be used for relocations, only.  At run time, the
manual symbols are invisible, because Holstein cannot actually generate
ELF (or MachO) images compatible with native programs.
parent 44ff66c1
......@@ -34,7 +34,7 @@ import uvm.utils.WithUtils.tryWithResource
class BootImageBuilder(implicit microVM: MicroVM) {
def makeBootImage(whiteList: Seq[TopLevel], primordial: PrimordialInfo,
syms: Seq[FieldAndSymbol], relocs: Seq[FieldAndSymbol], outputFile: String): Unit = {
syms: Seq[FieldAndSymbol], relocs: Seq[FieldAndSymbol], outputFile: String): Unit = {
implicit val tcb = new TransitiveClosureBuilder(whiteList, primordial)
tcb.doTransitiveClosure()
......@@ -82,22 +82,23 @@ class BootImageWriter(tcb: TransitiveClosureBuilder, syms: Seq[FieldAndSymbol],
val globalGroup = new FileGroup("global", tempDir)
val heapGroup = new FileGroup("heap", tempDir)
val symsMap = syms.map(_.toAddrSymPair).toMap
val symsRevMap = syms.map(_.toAddrSymPair.swap).toMap
val manualSymRecs = new ArrayBuffer[ManualSymRecord]
val relocsMap = relocs.map(_.toAddrSymPair).toMap
{
logger.debug("Relocation map: " + relocsMap.toString)
}
val idNameMapPath = tempDir.resolve(IDNAMEMAP_FILE)
val uirBundlePath = tempDir.resolve(UIRBUNDLE_FILE)
val manualSymsPath = tempDir.resolve(MANUALSYMS_FILE)
val metaInfoPath = tempDir.resolve(METAINFO_FILE)
val zipPath = Paths.get(outputFile)
val allZipContents = Seq(globalGroup.dataFile, heapGroup.dataFile, globalGroup.allocFile, heapGroup.allocFile,
globalGroup.relocFile, heapGroup.relocFile, idNameMapPath, uirBundlePath, metaInfoPath)
globalGroup.relocFile, heapGroup.relocFile, idNameMapPath, uirBundlePath, manualSymsPath, metaInfoPath)
private val globalMemory: Space = microVM.memoryManager.globalMemory
private val smallObjectSpace: Space = microVM.memoryManager.heap.space
......@@ -120,14 +121,17 @@ class BootImageWriter(tcb: TransitiveClosureBuilder, syms: Seq[FieldAndSymbol],
writeAllocRecs(globalGroup)
writeAllocRecs(heapGroup)
locateManualSyms()
writeManualSyms()
for (ar <- heapGroup.allocRecs) {
addrToHeapObjNum(ar.addr) = ar.num
}
scanForRelocs(globalGroup)
scanForRelocs(heapGroup)
addPointerRelocs()
writeRelocRecs(globalGroup)
......@@ -135,9 +139,9 @@ class BootImageWriter(tcb: TransitiveClosureBuilder, syms: Seq[FieldAndSymbol],
writeIDNameMap()
writeUIRBundle()
writeMetaInfo()
makeZipPackage()
}
......@@ -176,6 +180,37 @@ class BootImageWriter(tcb: TransitiveClosureBuilder, syms: Seq[FieldAndSymbol],
}
}
private def toManualSymRec(fns: FieldAndSymbol): ManualSymRecord = {
val iref = fns.objRef + fns.offset
if (fns.objRef != 0L || !globalMemory.isInSpace(iref)) {
throw new BootImageBuilderException("Symbol can only be in global cells. Symbol: '%s', objRef: 0x%x, offset: 0x%x, addr: 0x%x".format(
fns.symbol, fns.objRef, fns.offset, iref))
}
val gcr = tcb.getGlobalCellRec(iref)
val gid = gcr.g.id
val offset = fns.offset - gcr.begin
val msr = ManualSymRecord(fns.symbol, gid, offset)
msr
}
private def locateManualSyms(): Unit = {
for (fns <- syms) {
manualSymRecs += toManualSymRec(fns)
}
}
private def writeManualSyms(): Unit = {
tryWithResource(Files.newBufferedWriter(manualSymsPath, StandardCharsets.UTF_8)) { writer =>
for (msr <- manualSymRecs) {
writer.write("%s,%d,%d\n".format(
msr.sym, msr.targetNum, msr.targetOffset))
}
}
}
private def scanForRelocs(group: FileGroup): Unit = {
for (alloc <- group.allocRecs) {
val num = alloc.num
......@@ -204,7 +239,8 @@ class BootImageWriter(tcb: TransitiveClosureBuilder, syms: Seq[FieldAndSymbol],
group.relocRecs += reloc
logger.info("Relocation entry for uptr field automatically generated: %d 0x%x -> %d 0x%x".format(
iRef, iRef, toAddr, toAddr))
case None => }
case None =>
}
}
}
}
......@@ -262,7 +298,7 @@ class BootImageWriter(tcb: TransitiveClosureBuilder, syms: Seq[FieldAndSymbol],
})
}
}
private def addPointerRelocs(): Unit = {
// Seems uptrs are already handled when scanning fields. No further processing is needed at this moment.
}
......@@ -271,7 +307,7 @@ class BootImageWriter(tcb: TransitiveClosureBuilder, syms: Seq[FieldAndSymbol],
tryWithResource(Files.newBufferedWriter(group.relocFile, StandardCharsets.UTF_8)) { writer =>
for (rr <- group.relocRecs) {
writer.write("%d,%d,%s,%s,%d,%d,%s\n".format(
rr.num, rr.fieldOffset, rr.fieldKind, rr.targetKind, rr.targetNum, rr.targetOffset, rr.targetString))
rr.num, rr.fieldOffset, rr.fieldKind, rr.targetKind, rr.targetNum, rr.targetOffset, rr.targetString))
}
}
}
......@@ -287,7 +323,7 @@ class BootImageWriter(tcb: TransitiveClosureBuilder, syms: Seq[FieldAndSymbol],
bundleSerializer.writeUIR(writer)
}
}
private def writeMetaInfo(): Unit = {
tryWithResource(Files.newBufferedWriter(metaInfoPath, StandardCharsets.UTF_8)) { writer =>
val sb = new StringBuilder()
......
......@@ -10,6 +10,7 @@ object BootImageFile {
val UIRBUNDLE_FILE = "bundle.uir"
val IDNAMEMAP_FILE = "idnamemap"
val MANUALSYMS_FILE = "manualsyms"
val METAINFO_FILE = "metainfo"
val KEY_INITFUNC = "initfunc"
......@@ -43,6 +44,16 @@ object BootImageFile {
*/
case class UnitAllocRecord(num: Long, fileOffset: Long, ty: Type, varLen: Long, addr: Word)
/**
* Manual symbols. For global only. The heap cannot have any manual symbols, because the pinning of primordial
* objects is not supported.
*
* @param name The symbol name. Used for symbol lookup.
* @param targetNum The ID of the global cell.
* @param targetOffset Offset within the global cell.
*/
case class ManualSymRecord(sym: String, targetNum: Long, targetOffset: Word)
/**
* Relocation record. One for each field of ref<T>, iref<T>, weakref<T> or funcref<sig> types.
*
......
......@@ -67,6 +67,8 @@ class BootImageLoader(file: String, maybeAllArgs: Option[Seq[String]])(implicit
val g = globalBundle.globalCellNs(num.toInt)
globalMemory.addrForGlobalCell(g)
}
val extraSyms = new HashMap[String, ManualSymRecord]()
private def zipFile(entry: ZipEntry): InputStream = zip.getInputStream(entry)
private def zipFile(name: String): InputStream = zipFile(zip.getEntry(name))
......@@ -150,6 +152,7 @@ class BootImageLoader(file: String, maybeAllArgs: Option[Seq[String]])(implicit
heap.gcDisabled = true
allocateAndCopy()
loadManualSyms()
relocate()
// Re-enable GC when we are done.
......@@ -211,6 +214,25 @@ class BootImageLoader(file: String, maybeAllArgs: Option[Seq[String]])(implicit
}
}
}
private def loadManualSyms(): Unit = {
forEachManualSymRec { msr =>
extraSyms(msr.sym) = msr
}
}
private def forEachManualSymRec(f: ManualSymRecord => Unit): Unit = {
tryWithResource(zipFileTextBuf(MANUALSYMS_FILE)) { manualSymsIS =>
forEachLine(manualSymsIS) { line =>
val elems = line.split(",")
val sym = elems(0)
val targetNum = elems(1).toLong
val targetOffset = elems(2).toLong
val msr = ManualSymRecord(sym, targetNum, targetOffset)
f(msr)
}
}
}
private def relocate(): Unit = {
forEachReloc("global") { rr =>
......@@ -301,7 +323,12 @@ class BootImageLoader(file: String, maybeAllArgs: Option[Seq[String]])(implicit
}
private def getSymbolAddr(targetSymbol: String): Word = {
microVM.nativeLibraryHolder.getSymbolAddress(targetSymbol)
extraSyms.get(targetSymbol) match {
// If there is a manual symbol (specified in makeBootImage), use it;
case Some(msr) => globalNumToAddr(msr.targetNum) + msr.targetOffset
// Otherwise, resolve the symbol natively
case None => microVM.nativeLibraryHolder.getSymbolAddress(targetSymbol)
}
}
private def maybeCreatePrimordialThread(metaInfo: Map[String, String]): Unit = {
......
......@@ -49,8 +49,11 @@ class BootImageWriterTest extends UvmBundleTesterBase with ExtraMatchers {
tryWithResource(microVM.newContext()) { ctx =>
val h_gs3 = ctx.handleFromGlobal("@gs3")
val h_gs3_2 = ctx.getFieldIRef(h_gs3, 2)
val h_gs3_3 = ctx.getFieldIRef(h_gs3, 3)
val h_gt = ctx.handleFromGlobal("@gt")
val h_tl = ctx.newFixed("@i64")
val h_tl_i = ctx.getIRef(h_tl)
val h_fifty_two = ctx.handleFromInt(52, 64)
......@@ -58,7 +61,12 @@ class BootImageWriterTest extends UvmBundleTesterBase with ExtraMatchers {
val h_main = ctx.handleFromFunc("@main")
ctx.makeBootImage(everything.map(_.id), Some(h_main), None, Some(h_tl), Seq(), Seq(), Seq(h_gs3_3), Seq("getchar"), filename)
ctx.makeBootImage(everything.map(_.id), Some(h_main), None, Some(h_tl),
// manual syms
Seq(h_gs3_2), Seq("mycustom"),
// reloc recs
Seq(h_gs3_3, h_gt), Seq("getchar", "mycustom"),
filename)
}
val anotherMicroVM = MicroVM("", Seq("prog", "Hello", "world"))
......@@ -72,19 +80,26 @@ class BootImageWriterTest extends UvmBundleTesterBase with ExtraMatchers {
val h_gs_2_ptr = ctx.getAddr(h_gs_2)
val h_gs3 = ctx.handleFromGlobal("@gs3")
val h_gs3_2 = ctx.getFieldIRef(h_gs3, 2)
val h_gs3_2_ptr = ctx.getAddr(h_gs3_2)
val h_gs3_2_val = ctx.load(MemoryOrder.NOT_ATOMIC, h_gs3_2).asInstanceOf[MuUPtrValue]
val h_gs3_3 = ctx.getFieldIRef(h_gs3, 3)
val h_gs3_3_val = ctx.load(MemoryOrder.NOT_ATOMIC, h_gs3_3).asInstanceOf[MuUFPValue]
val h_gt = ctx.handleFromGlobal("@gt")
val h_gt_val = ctx.load(MemoryOrder.NOT_ATOMIC, h_gt).asInstanceOf[MuUPtrValue]
val gs_2_ptr = ctx.handleToPtr(h_gs_2_ptr)
val gs3_2_ptr = ctx.handleToPtr(h_gs3_2_ptr)
val gs3_2_val = ctx.handleToPtr(h_gs3_2_val)
val gs3_3_val = ctx.handleToFP(h_gs3_3_val)
val gt_val = ctx.handleToPtr(h_gt_val)
gs3_2_val shouldEqual gs_2_ptr
val getchar_val = anotherMicroVM.nativeLibraryHolder.getSymbolAddress("getchar")
gs3_3_val shouldEqual getchar_val
gt_val shouldEqual gs3_2_ptr
val allThreads = anotherMicroVM.threadStackManager.threadRegistry.values.toSeq
allThreads.size shouldBe 1
allThreads(0).stack.get.frames.toStream(0).curFuncID shouldBe ctx.idOf("@main")
......
......@@ -57,6 +57,9 @@
.global @gs3 <@s3>
.global @gs <@s>
.typedef @ppi32 = uptr<@pi32>
.global @gt <@ppi32>
.typedef @pi8 = uptr<@i8>
.typedef @ppi8 = uptr<@pi8>
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment