Commit 3f79b0e0 authored by John Zhang's avatar John Zhang

Merge branch 'master' into issuemir2-uptr-hack

parents fd9d9fb8 d8d384ea
......@@ -199,6 +199,11 @@ globalSize must be a multiple of 32768 bytes (32K).*
- **sourceInfo**: Provide line/column info in Mu IR when errors occur. May be
useful for debugging small Mu IR bundles, but will significantly slow down
parsing!!! Enable only if the bundle is small. default: false
- **automagicReloc**: Allow "automagic" relocation. If true, `uptr` and
`ufuncptr` fields will also be traced during boot image building. If a `uptr`
field points to a global cell field, it will still point to the same field
after boot image loading; if a `ufuncptr` points to a native function, it will
point to the same function after boot image loading. default: false
- **extraLibs**: Extra libraries to load when starting the micro VM. This is a
colon-separated list of libraries. Each library has the same syntax of the
`path` argument of the `dlopen` system function. By default, it does not load
......
......@@ -4,3 +4,4 @@ test_client2
classpath.txt
classpath.h
test_client2_bootimg.mu
*.dSYM
CFLAGS += -std=gnu11
ifndef JAVA_HOME
$(error JAVA_HOME is required. Invoke with 'make JAVA_HOME=/path/to/java/home')
endif
CFLAGS += -I $(JAVA_HOME)/include
CFLAGS += -std=gnu11 -g -I $(JAVA_HOME)/include
ifndef OS
uname := $(shell uname)
ifeq ($(uname),Darwin)
OS = OSX
else
ifeq ($(uname),Linux)
OS = LINUX
uname := $(shell uname)
ifeq ($(uname),Darwin)
OS = OSX
else ifeq ($(uname),Linux)
OS = LINUX
else
$(error Unrecognized operating system $(uname). I currently only worked on OSX and Linux.)
endif
$(error Unrecognized operating system $(uname). I currently only worked on OSX and Linux.)
endif
endif
ifeq ($(OS),OSX)
CFLAGS += -I $(JAVA_HOME)/include/darwin
LDFLAGS += -L $(JAVA_HOME)/jre/lib/server -l jvm -rpath $(JAVA_HOME)/jre/lib/server -install_name '@rpath/libmurefimpl2start.so'
CFLAGS += -I $(JAVA_HOME)/include/darwin
LDFLAGS += -L $(JAVA_HOME)/jre/lib/server -l jvm -rpath $(JAVA_HOME)/jre/lib/server -install_name '@rpath/libmurefimpl2start.so'
else ifeq ($(OS),LINUX)
CFLAGS += -I $(JAVA_HOME)/include/linux
LDFLAGS += -Wl,--no-as-needed -L $(JAVA_HOME)/jre/lib/amd64/server -l jvm -Wl,-rpath,$(JAVA_HOME)/jre/lib/amd64/server
endif
ifeq ($(OS),LINUX)
CFLAGS += -I $(JAVA_HOME)/include/linux
LDFLAGS += -Wl,--no-as-needed -L $(JAVA_HOME)/jre/lib/amd64/server -l jvm -Wl,-rpath,$(JAVA_HOME)/jre/lib/amd64/server
endif
.PHONY: all
all: libs tests
......@@ -56,10 +49,8 @@ test_client2: test_client2.c libmurefimpl2start.so
$(CC) `./refimpl2-config --istart --cflags --libs` -o $@ $<
.PHONY: clean veryclean
clean:
rm *.so test_client test_client2
rm -f *.so test_client test_client2
veryclean:
rm *.so test_client test_client2 classpath.txt classpath.h
rm ../classpath.txt
veryclean: clean
rm -f classpath.txt classpath.h ../classpath.txt
......@@ -36,8 +36,4 @@ if '--istart' in args:
whereami, whereami), end="")
else:
if '--cflags' in args:
print("-I {} ".format(whereami), end="")
if '--libs' in args:
pass
print("-I {} ".format(whereami), end="")
\ No newline at end of file
......@@ -21,7 +21,7 @@ const char *gc_conf =
"vmLog=DEBUG\n"
;
const char *boot_image_name = "test_client2_bootimg.mu";
char *boot_image_name = "test_client2_bootimg.mu";
int *muerrno;
......
......@@ -106,6 +106,12 @@ object VMConfParser {
parser = parseBoolean,
default = VMConf.DEFAULT_CONF.sourceInfo)
val automagicReloc = opt[Boolean](
name = "automagicReloc",
desc = """'Automagic' relocation. Affects boot image building, but not loading.""",
parser = parseBoolean,
default = VMConf.DEFAULT_CONF.automagicReloc)
val extraLibs = opt[Seq[String]](
name = "extraLibs",
desc = """Extra libraries to load when starting the micro VM. This is a colon-separated list of libraries.
......@@ -163,6 +169,7 @@ object VMConf {
dumpBundle = VMConfParser.dumpBundle(kvMap),
staticCheck = VMConfParser.staticCheck(kvMap),
sourceInfo = VMConfParser.sourceInfo(kvMap),
automagicReloc = VMConfParser.automagicReloc(kvMap),
extraLibs = VMConfParser.extraLibs(kvMap),
bootImg = VMConfParser.bootImg(kvMap)
)
......@@ -187,6 +194,7 @@ class VMConf(
val dumpBundle: Boolean = false,
val staticCheck: Boolean = true,
val sourceInfo: Boolean = false,
val automagicReloc: Boolean = false,
val extraLibs: Seq[String] = Seq(),
val bootImg: Option[String] = None)
......@@ -34,7 +34,7 @@ import uvm.utils.WithUtils.tryWithResource
class BootImageBuilder(implicit microVM: MicroVM) {
def makeBootImage(whiteList: Seq[TopLevel], primordial: PrimordialInfo,
syms: Seq[FieldAndSymbol], relocs: Seq[FieldAndSymbol], outputFile: String): Unit = {
syms: Seq[FieldAndSymbol], relocs: Seq[FieldAndSymbol], outputFile: String): Unit = {
implicit val tcb = new TransitiveClosureBuilder(whiteList, primordial)
tcb.doTransitiveClosure()
......@@ -74,6 +74,8 @@ class BootImageWriter(tcb: TransitiveClosureBuilder, syms: Seq[FieldAndSymbol],
import BootImageWriter._
import BootImageFile._
val automagicReloc = microVM.vmConf.automagicReloc
val bundleSerializer = new BundleSerializer(microVM.globalBundle, tcb.tls.set)
val tempDir = Files.createTempDirectory("mu-bootimg")
......@@ -82,22 +84,23 @@ class BootImageWriter(tcb: TransitiveClosureBuilder, syms: Seq[FieldAndSymbol],
val globalGroup = new FileGroup("global", tempDir)
val heapGroup = new FileGroup("heap", tempDir)
val symsMap = syms.map(_.toAddrSymPair).toMap
val symsRevMap = syms.map(_.toAddrSymPair.swap).toMap
val manualSymRecs = new ArrayBuffer[ManualSymRecord]
val relocsMap = relocs.map(_.toAddrSymPair).toMap
{
logger.debug("Relocation map: " + relocsMap.toString)
}
val idNameMapPath = tempDir.resolve(IDNAMEMAP_FILE)
val uirBundlePath = tempDir.resolve(UIRBUNDLE_FILE)
val manualSymsPath = tempDir.resolve(MANUALSYMS_FILE)
val metaInfoPath = tempDir.resolve(METAINFO_FILE)
val zipPath = Paths.get(outputFile)
val allZipContents = Seq(globalGroup.dataFile, heapGroup.dataFile, globalGroup.allocFile, heapGroup.allocFile,
globalGroup.relocFile, heapGroup.relocFile, idNameMapPath, uirBundlePath, metaInfoPath)
globalGroup.relocFile, heapGroup.relocFile, idNameMapPath, uirBundlePath, manualSymsPath, metaInfoPath)
private val globalMemory: Space = microVM.memoryManager.globalMemory
private val smallObjectSpace: Space = microVM.memoryManager.heap.space
......@@ -120,14 +123,17 @@ class BootImageWriter(tcb: TransitiveClosureBuilder, syms: Seq[FieldAndSymbol],
writeAllocRecs(globalGroup)
writeAllocRecs(heapGroup)
locateManualSyms()
writeManualSyms()
for (ar <- heapGroup.allocRecs) {
addrToHeapObjNum(ar.addr) = ar.num
}
scanForRelocs(globalGroup)
scanForRelocs(heapGroup)
addPointerRelocs()
writeRelocRecs(globalGroup)
......@@ -135,9 +141,9 @@ class BootImageWriter(tcb: TransitiveClosureBuilder, syms: Seq[FieldAndSymbol],
writeIDNameMap()
writeUIRBundle()
writeMetaInfo()
makeZipPackage()
}
......@@ -176,6 +182,37 @@ class BootImageWriter(tcb: TransitiveClosureBuilder, syms: Seq[FieldAndSymbol],
}
}
private def toManualSymRec(fns: FieldAndSymbol): ManualSymRecord = {
val iref = fns.objRef + fns.offset
if (fns.objRef != 0L || !globalMemory.isInSpace(iref)) {
throw new BootImageBuilderException("Symbol can only be in global cells. Symbol: '%s', objRef: 0x%x, offset: 0x%x, addr: 0x%x".format(
fns.symbol, fns.objRef, fns.offset, iref))
}
val gcr = tcb.getGlobalCellRec(iref)
val gid = gcr.g.id
val offset = fns.offset - gcr.begin
val msr = ManualSymRecord(fns.symbol, gid, offset)
msr
}
private def locateManualSyms(): Unit = {
for (fns <- syms) {
manualSymRecs += toManualSymRec(fns)
}
}
private def writeManualSyms(): Unit = {
tryWithResource(Files.newBufferedWriter(manualSymsPath, StandardCharsets.UTF_8)) { writer =>
for (msr <- manualSymRecs) {
writer.write("%s,%d,%d\n".format(
msr.sym, msr.targetNum, msr.targetOffset))
}
}
}
private def scanForRelocs(group: FileGroup): Unit = {
for (alloc <- group.allocRecs) {
val num = alloc.num
......@@ -192,29 +229,34 @@ class BootImageWriter(tcb: TransitiveClosureBuilder, syms: Seq[FieldAndSymbol],
logger.debug("External uptr relocation: %d 0x%x -> %s".format(iRef, iRef, symbol))
}
case None => {
// maybe a "reasonable" pointer to a mu global cell. Relocate it too.
val fieldOffset = iRef - objRef
val maybeGlobalCell = tcb.maybeGetGlobalCellRec(toAddr)
maybeGlobalCell match {
case Some(gcr) =>
val targetNum = gcr.g.id.toLong
val targetAddr = gcr.begin
val targetOffset = toAddr - targetAddr
val reloc = FieldRelocRecord(num, fieldOffset, AS_UPTR, TO_GLOBAL, targetNum, targetOffset, NO_SYM)
group.relocRecs += reloc
logger.info("Relocation entry for uptr field automatically generated: %d 0x%x -> %d 0x%x".format(
iRef, iRef, toAddr, toAddr))
case None => }
if (automagicReloc) {
// maybe a "reasonable" pointer to a mu global cell. Relocate it too.
val fieldOffset = iRef - objRef
val maybeGlobalCell = tcb.maybeGetGlobalCellRec(toAddr)
maybeGlobalCell match {
case Some(gcr) =>
val targetNum = gcr.g.id.toLong
val targetAddr = gcr.begin
val targetOffset = toAddr - targetAddr
val reloc = FieldRelocRecord(num, fieldOffset, AS_UPTR, TO_GLOBAL, targetNum, targetOffset, NO_SYM)
group.relocRecs += reloc
logger.info("Relocation entry for uptr field automatically generated: %d 0x%x -> %d 0x%x".format(
iRef, iRef, toAddr, toAddr))
case None =>
}
}
}
}
}
def visitUFPField(objRef: Word, iRef: Word, toAddr: Word): Unit = {
relocsMap.get(iRef).foreach { symbol =>
// by symbol
val fieldOffset = iRef - objRef
val reloc = FieldRelocRecord(num, fieldOffset, AS_UPTR, TO_SYM, 0, 0, symbol)
group.relocRecs += reloc
logger.debug("External ufuncptr relocation: %d 0x%x -> %s".format(iRef, iRef, symbol))
if (automagicReloc) {
relocsMap.get(iRef).foreach { symbol =>
// by symbol
val fieldOffset = iRef - objRef
val reloc = FieldRelocRecord(num, fieldOffset, AS_UPTR, TO_SYM, 0, 0, symbol)
group.relocRecs += reloc
logger.debug("External ufuncptr relocation: %d 0x%x -> %s".format(iRef, iRef, symbol))
}
}
}
......@@ -262,7 +304,7 @@ class BootImageWriter(tcb: TransitiveClosureBuilder, syms: Seq[FieldAndSymbol],
})
}
}
private def addPointerRelocs(): Unit = {
// Seems uptrs are already handled when scanning fields. No further processing is needed at this moment.
}
......@@ -271,7 +313,7 @@ class BootImageWriter(tcb: TransitiveClosureBuilder, syms: Seq[FieldAndSymbol],
tryWithResource(Files.newBufferedWriter(group.relocFile, StandardCharsets.UTF_8)) { writer =>
for (rr <- group.relocRecs) {
writer.write("%d,%d,%s,%s,%d,%d,%s\n".format(
rr.num, rr.fieldOffset, rr.fieldKind, rr.targetKind, rr.targetNum, rr.targetOffset, rr.targetString))
rr.num, rr.fieldOffset, rr.fieldKind, rr.targetKind, rr.targetNum, rr.targetOffset, rr.targetString))
}
}
}
......@@ -287,7 +329,7 @@ class BootImageWriter(tcb: TransitiveClosureBuilder, syms: Seq[FieldAndSymbol],
bundleSerializer.writeUIR(writer)
}
}
private def writeMetaInfo(): Unit = {
tryWithResource(Files.newBufferedWriter(metaInfoPath, StandardCharsets.UTF_8)) { writer =>
val sb = new StringBuilder()
......
......@@ -10,6 +10,7 @@ object BootImageFile {
val UIRBUNDLE_FILE = "bundle.uir"
val IDNAMEMAP_FILE = "idnamemap"
val MANUALSYMS_FILE = "manualsyms"
val METAINFO_FILE = "metainfo"
val KEY_INITFUNC = "initfunc"
......@@ -43,6 +44,16 @@ object BootImageFile {
*/
case class UnitAllocRecord(num: Long, fileOffset: Long, ty: Type, varLen: Long, addr: Word)
/**
* Manual symbols. For global only. The heap cannot have any manual symbols, because the pinning of primordial
* objects is not supported.
*
* @param name The symbol name. Used for symbol lookup.
* @param targetNum The ID of the global cell.
* @param targetOffset Offset within the global cell.
*/
case class ManualSymRecord(sym: String, targetNum: Long, targetOffset: Word)
/**
* Relocation record. One for each field of ref<T>, iref<T>, weakref<T> or funcref<sig> types.
*
......
......@@ -67,6 +67,8 @@ class BootImageLoader(file: String, maybeAllArgs: Option[Seq[String]])(implicit
val g = globalBundle.globalCellNs(num.toInt)
globalMemory.addrForGlobalCell(g)
}
val extraSyms = new HashMap[String, ManualSymRecord]()
private def zipFile(entry: ZipEntry): InputStream = zip.getInputStream(entry)
private def zipFile(name: String): InputStream = zipFile(zip.getEntry(name))
......@@ -150,6 +152,7 @@ class BootImageLoader(file: String, maybeAllArgs: Option[Seq[String]])(implicit
heap.gcDisabled = true
allocateAndCopy()
loadManualSyms()
relocate()
// Re-enable GC when we are done.
......@@ -211,6 +214,25 @@ class BootImageLoader(file: String, maybeAllArgs: Option[Seq[String]])(implicit
}
}
}
private def loadManualSyms(): Unit = {
forEachManualSymRec { msr =>
extraSyms(msr.sym) = msr
}
}
private def forEachManualSymRec(f: ManualSymRecord => Unit): Unit = {
tryWithResource(zipFileTextBuf(MANUALSYMS_FILE)) { manualSymsIS =>
forEachLine(manualSymsIS) { line =>
val elems = line.split(",")
val sym = elems(0)
val targetNum = elems(1).toLong
val targetOffset = elems(2).toLong
val msr = ManualSymRecord(sym, targetNum, targetOffset)
f(msr)
}
}
}
private def relocate(): Unit = {
forEachReloc("global") { rr =>
......@@ -301,7 +323,12 @@ class BootImageLoader(file: String, maybeAllArgs: Option[Seq[String]])(implicit
}
private def getSymbolAddr(targetSymbol: String): Word = {
microVM.nativeLibraryHolder.getSymbolAddress(targetSymbol)
extraSyms.get(targetSymbol) match {
// If there is a manual symbol (specified in makeBootImage), use it;
case Some(msr) => globalNumToAddr(msr.targetNum) + msr.targetOffset
// Otherwise, resolve the symbol natively
case None => microVM.nativeLibraryHolder.getSymbolAddress(targetSymbol)
}
}
private def maybeCreatePrimordialThread(metaInfo: Map[String, String]): Unit = {
......@@ -386,7 +413,10 @@ class AlignedInputStream(is: InputStream) extends Closeable {
}
private def transfer(addr: Long, size: Int): Unit = {
is.read(buffer, 0, size)
var actual = is.read(buffer, 0, size)
while (actual < size) {
actual += is.read(buffer, actual, size - actual)
}
NativeSupport.theMemory.put(addr, buffer, 0, size)
}
......
......@@ -16,6 +16,8 @@ import uvm.refimpl.MuUFPValue
class BootImageWriterTest extends UvmBundleTesterBase with ExtraMatchers {
preloadBundles("tests/uvm-refimpl-test/transitive-closure.uir")
preloadHails("tests/uvm-refimpl-test/transitive-closure.hail")
override def makeMicroVM = MicroVM(new VMConf(automagicReloc = true))
{
tryWithResource(microVM.newContext()) { ctx =>
......@@ -49,8 +51,12 @@ class BootImageWriterTest extends UvmBundleTesterBase with ExtraMatchers {
tryWithResource(microVM.newContext()) { ctx =>
val h_gs3 = ctx.handleFromGlobal("@gs3")
val h_gs3_1 = ctx.getFieldIRef(h_gs3, 1)
val h_gs3_2 = ctx.getFieldIRef(h_gs3, 2)
val h_gs3_3 = ctx.getFieldIRef(h_gs3, 3)
val h_gt = ctx.handleFromGlobal("@gt")
val h_tl = ctx.newFixed("@i64")
val h_tl_i = ctx.getIRef(h_tl)
val h_fifty_two = ctx.handleFromInt(52, 64)
......@@ -58,7 +64,12 @@ class BootImageWriterTest extends UvmBundleTesterBase with ExtraMatchers {
val h_main = ctx.handleFromFunc("@main")
ctx.makeBootImage(everything.map(_.id), Some(h_main), None, Some(h_tl), Seq(), Seq(), Seq(h_gs3_3), Seq("getchar"), filename)
ctx.makeBootImage(everything.map(_.id), Some(h_main), None, Some(h_tl),
// manual syms
Seq(h_gs3_2), Seq("mycustom"),
// reloc recs
Seq(h_gs3_3, h_gt), Seq("getchar", "mycustom"),
filename)
}
val anotherMicroVM = MicroVM("", Seq("prog", "Hello", "world"))
......@@ -67,24 +78,38 @@ class BootImageWriterTest extends UvmBundleTesterBase with ExtraMatchers {
microVM.globalBundle.allNs("@gd").id shouldEqual anotherMicroVM.globalBundle.allNs("@gd").id
tryWithResource(anotherMicroVM.newContext()) { ctx =>
//val h_gr1 = ctx.handleFromGlobal("@gr1")
//val h_gr1_ptr = ctx.getAddr(h_gr1)
val h_gs = ctx.handleFromGlobal("@gs")
val h_gs_2 = ctx.getFieldIRef(h_gs, 2)
val h_gs_2_ptr = ctx.getAddr(h_gs_2)
val h_gs3 = ctx.handleFromGlobal("@gs3")
val h_gs3_1 = ctx.getFieldIRef(h_gs3, 1)
val h_gs3_1_val = ctx.load(MemoryOrder.NOT_ATOMIC, h_gs3_1).asInstanceOf[MuUPtrValue]
val h_gs3_2 = ctx.getFieldIRef(h_gs3, 2)
val h_gs3_2_ptr = ctx.getAddr(h_gs3_2)
val h_gs3_2_val = ctx.load(MemoryOrder.NOT_ATOMIC, h_gs3_2).asInstanceOf[MuUPtrValue]
val h_gs3_3 = ctx.getFieldIRef(h_gs3, 3)
val h_gs3_3_val = ctx.load(MemoryOrder.NOT_ATOMIC, h_gs3_3).asInstanceOf[MuUFPValue]
val h_gt = ctx.handleFromGlobal("@gt")
val h_gt_val = ctx.load(MemoryOrder.NOT_ATOMIC, h_gt).asInstanceOf[MuUPtrValue]
//val gr1_ptr = ctx.handleToPtr(h_gr1_ptr)
val gs_2_ptr = ctx.handleToPtr(h_gs_2_ptr)
val gs3_1_val = ctx.handleToPtr(h_gs3_1_val)
val gs3_2_ptr = ctx.handleToPtr(h_gs3_2_ptr)
val gs3_2_val = ctx.handleToPtr(h_gs3_2_val)
val gs3_3_val = ctx.handleToFP(h_gs3_3_val)
val gt_val = ctx.handleToPtr(h_gt_val)
gs3_2_val shouldEqual gs_2_ptr
val getchar_val = anotherMicroVM.nativeLibraryHolder.getSymbolAddress("getchar")
gs3_3_val shouldEqual getchar_val
gt_val shouldEqual gs3_2_ptr
val allThreads = anotherMicroVM.threadStackManager.threadRegistry.values.toSeq
allThreads.size shouldBe 1
allThreads(0).stack.get.frames.toStream(0).curFuncID shouldBe ctx.idOf("@main")
......
......@@ -13,4 +13,3 @@
.init @gr2 = &@gr1
.init @gs3[0] = 43
.init @gs3[1] = 44
......@@ -57,6 +57,9 @@
.global @gs3 <@s3>
.global @gs <@s>
.typedef @ppi32 = uptr<@pi32>
.global @gt <@ppi32>
.typedef @pi8 = uptr<@i8>
.typedef @ppi8 = uptr<@pi8>
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment