Commit 3f3bf15e authored by John Zhang's avatar John Zhang

point url to GitHub; handle different raw/lfs url in GitHub

parent 51c85e1d
Pipeline #2946 passed with stage
in 83 minutes and 44 seconds
......@@ -438,9 +438,10 @@ See ${log.build.report} for a summary of benchmark build status.
<target name="update-externdata-info" depends="set-build-version-info">
<get src="${dacapo-dl.url.raw}/${dacapo-dl.commit}/META-INF/huge-data-md5s.list" dest="${dacapo.build.dir}"/>
<get src="${dacapo-dl.url.raw}/${dacapo-dl.commit}/META-INF/dlfiles.list" dest="${dacapo.build.dir}"/>
<echo message="setting DaCapo-DL-URL-RAW and DaCapo-DL-Commit in manifest"/>
<echo message="setting DaCapo-DL-URL-LFS, DaCapo-DL-URL-RAW DaCapo-DL-Commit in manifest"/>
<jar destfile="${build.target-jar}" update="true">
<manifest>
<attribute name="DaCapo-DL-URL-LFS" value="${dacapo-dl.url.lfs}"/>
<attribute name="DaCapo-DL-URL-RAW" value="${dacapo-dl.url.raw}"/>
<attribute name="DaCapo-DL-Commit" value="${dacapo-dl.commit}"/>
</manifest>
......
......@@ -3,9 +3,11 @@ dacapo.version=9.12-MR1
dacapo.nickname=uncalibrated
# External data
dacapo-dl.url.raw=https://gitlab.anu.edu.au/dacapo/anu-dev/dacapo-dl/raw
# dacapo-dl.url.raw=https://raw.githubusercontent.com/dacapobench/dacapo-dl/
dacapo-dl.commit=HEAD
# dacapo-dl.url.raw=https://gitlab.anu.edu.au/dacapo/anu-dev/dacapo-dl/raw
# dacapo-dl.url.lfs=https://gitlab.anu.edu.au/dacapo/anu-dev/dacapo-dl/raw
dacapo-dl.url.raw=https://raw.githubusercontent.com/john5f35/dacapo-dl
dacapo-dl.url.lfs=https://media.githubusercontent.com/media/john5f35/dacapo-dl
dacapo-dl.commit=cb479d7
#
# Common properties for the dacapo benchmarks
#
......
......@@ -41,6 +41,7 @@ public class ExternData {
*/
public static final Path DEFAULT_LOCAL_DACAPO_CONFIG = Paths.get(System.getProperty("user.home"), ".dacapo-config.properties");
public static final String CONFIG_KEY_EXTERN_DATA_LOC = "Extern-Data-Location";
public static final String DACAPO_DL_URL_LFS = "DaCapo-DL-URL-LFS";
public static final String DACAPO_DL_URL_RAW = "DaCapo-DL-URL-RAW";
public static final String DACAPO_DL_COMMIT = "DaCapo-DL-Commit";
......@@ -147,13 +148,14 @@ public class ExternData {
path.mkdirs();
// download
URL urlDLRawRoot = appendURL(new URL(TestHarness.getManifestAttribute(DACAPO_DL_URL_RAW)),
TestHarness.getManifestAttribute(DACAPO_DL_COMMIT));
String dlurlRaw = TestHarness.getManifestAttribute(DACAPO_DL_URL_RAW);
String dlurlLFS = TestHarness.getManifestAttribute(DACAPO_DL_URL_LFS);
String commit = TestHarness.getManifestAttribute(DACAPO_DL_COMMIT);
BufferedReader dllistReader = new BufferedReader(new InputStreamReader(
ClassLoader.getSystemResourceAsStream("META-INF/dlfiles.list")));
dllistReader.lines().forEach(s -> {
try {
downloadAndExtractItem(s, urlDLRawRoot, path);
downloadAndExtractItem(s, dlurlRaw, dlurlLFS, commit, path);
} catch (Exception e) {
System.err.println("Download external data failed.");
System.err.println("Please try again.");
......@@ -168,12 +170,9 @@ public class ExternData {
}
}
private static URL appendURL(URL url, String relPath) throws MalformedURLException{
return new URL(url.toString() + "/" + relPath);
}
private static void downloadAndExtractItem(String itemRelPath, URL urlDLRawRoot, File localDataPath) throws Exception {
URL urlItem = appendURL(urlDLRawRoot, itemRelPath);
private static void downloadAndExtractItem(String itemRelPath, String dlurlRaw, String dlurlLFS, String commit, File localDataPath) throws Exception {
URL urlItem = new URL(String.join("/", dlurlLFS, commit, itemRelPath));
URL urlMD5 = new URL(String.join("/", dlurlRaw, commit, itemRelPath + ".MD5"));
File fileLocalItem = new File(localDataPath, itemRelPath);
if (!fileLocalItem.getParentFile().exists())
......@@ -188,7 +187,7 @@ public class ExternData {
System.out.println("Done.");
System.out.printf("Checking %s MD5...", fileLocalItem.toString());
ReadableByteChannel rbcMD5 = Channels.newChannel(appendURL(urlDLRawRoot, itemRelPath + ".MD5").openStream());
ReadableByteChannel rbcMD5 = Channels.newChannel(urlMD5.openStream());
ByteBuffer buf = ByteBuffer.allocate(32); // MD5 is always 32 characters long
rbcMD5.read(buf);
String md5Expect = new String(buf.array(), Charset.forName("ASCII")).toLowerCase();
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment