Commit 3f3bf15e authored by John Zhang's avatar John Zhang

point url to GitHub; handle different raw/lfs url in GitHub

parent 51c85e1d
Pipeline #2946 passed with stage
in 83 minutes and 44 seconds
...@@ -438,9 +438,10 @@ See ${log.build.report} for a summary of benchmark build status. ...@@ -438,9 +438,10 @@ See ${log.build.report} for a summary of benchmark build status.
<target name="update-externdata-info" depends="set-build-version-info"> <target name="update-externdata-info" depends="set-build-version-info">
<get src="${dacapo-dl.url.raw}/${dacapo-dl.commit}/META-INF/huge-data-md5s.list" dest="${dacapo.build.dir}"/> <get src="${dacapo-dl.url.raw}/${dacapo-dl.commit}/META-INF/huge-data-md5s.list" dest="${dacapo.build.dir}"/>
<get src="${dacapo-dl.url.raw}/${dacapo-dl.commit}/META-INF/dlfiles.list" dest="${dacapo.build.dir}"/> <get src="${dacapo-dl.url.raw}/${dacapo-dl.commit}/META-INF/dlfiles.list" dest="${dacapo.build.dir}"/>
<echo message="setting DaCapo-DL-URL-RAW and DaCapo-DL-Commit in manifest"/> <echo message="setting DaCapo-DL-URL-LFS, DaCapo-DL-URL-RAW DaCapo-DL-Commit in manifest"/>
<jar destfile="${build.target-jar}" update="true"> <jar destfile="${build.target-jar}" update="true">
<manifest> <manifest>
<attribute name="DaCapo-DL-URL-LFS" value="${dacapo-dl.url.lfs}"/>
<attribute name="DaCapo-DL-URL-RAW" value="${dacapo-dl.url.raw}"/> <attribute name="DaCapo-DL-URL-RAW" value="${dacapo-dl.url.raw}"/>
<attribute name="DaCapo-DL-Commit" value="${dacapo-dl.commit}"/> <attribute name="DaCapo-DL-Commit" value="${dacapo-dl.commit}"/>
</manifest> </manifest>
......
...@@ -3,9 +3,11 @@ dacapo.version=9.12-MR1 ...@@ -3,9 +3,11 @@ dacapo.version=9.12-MR1
dacapo.nickname=uncalibrated dacapo.nickname=uncalibrated
# External data # External data
dacapo-dl.url.raw=https://gitlab.anu.edu.au/dacapo/anu-dev/dacapo-dl/raw # dacapo-dl.url.raw=https://gitlab.anu.edu.au/dacapo/anu-dev/dacapo-dl/raw
# dacapo-dl.url.raw=https://raw.githubusercontent.com/dacapobench/dacapo-dl/ # dacapo-dl.url.lfs=https://gitlab.anu.edu.au/dacapo/anu-dev/dacapo-dl/raw
dacapo-dl.commit=HEAD dacapo-dl.url.raw=https://raw.githubusercontent.com/john5f35/dacapo-dl
dacapo-dl.url.lfs=https://media.githubusercontent.com/media/john5f35/dacapo-dl
dacapo-dl.commit=cb479d7
# #
# Common properties for the dacapo benchmarks # Common properties for the dacapo benchmarks
# #
......
...@@ -41,6 +41,7 @@ public class ExternData { ...@@ -41,6 +41,7 @@ public class ExternData {
*/ */
public static final Path DEFAULT_LOCAL_DACAPO_CONFIG = Paths.get(System.getProperty("user.home"), ".dacapo-config.properties"); public static final Path DEFAULT_LOCAL_DACAPO_CONFIG = Paths.get(System.getProperty("user.home"), ".dacapo-config.properties");
public static final String CONFIG_KEY_EXTERN_DATA_LOC = "Extern-Data-Location"; public static final String CONFIG_KEY_EXTERN_DATA_LOC = "Extern-Data-Location";
public static final String DACAPO_DL_URL_LFS = "DaCapo-DL-URL-LFS";
public static final String DACAPO_DL_URL_RAW = "DaCapo-DL-URL-RAW"; public static final String DACAPO_DL_URL_RAW = "DaCapo-DL-URL-RAW";
public static final String DACAPO_DL_COMMIT = "DaCapo-DL-Commit"; public static final String DACAPO_DL_COMMIT = "DaCapo-DL-Commit";
...@@ -147,13 +148,14 @@ public class ExternData { ...@@ -147,13 +148,14 @@ public class ExternData {
path.mkdirs(); path.mkdirs();
// download // download
URL urlDLRawRoot = appendURL(new URL(TestHarness.getManifestAttribute(DACAPO_DL_URL_RAW)), String dlurlRaw = TestHarness.getManifestAttribute(DACAPO_DL_URL_RAW);
TestHarness.getManifestAttribute(DACAPO_DL_COMMIT)); String dlurlLFS = TestHarness.getManifestAttribute(DACAPO_DL_URL_LFS);
String commit = TestHarness.getManifestAttribute(DACAPO_DL_COMMIT);
BufferedReader dllistReader = new BufferedReader(new InputStreamReader( BufferedReader dllistReader = new BufferedReader(new InputStreamReader(
ClassLoader.getSystemResourceAsStream("META-INF/dlfiles.list"))); ClassLoader.getSystemResourceAsStream("META-INF/dlfiles.list")));
dllistReader.lines().forEach(s -> { dllistReader.lines().forEach(s -> {
try { try {
downloadAndExtractItem(s, urlDLRawRoot, path); downloadAndExtractItem(s, dlurlRaw, dlurlLFS, commit, path);
} catch (Exception e) { } catch (Exception e) {
System.err.println("Download external data failed."); System.err.println("Download external data failed.");
System.err.println("Please try again."); System.err.println("Please try again.");
...@@ -168,12 +170,9 @@ public class ExternData { ...@@ -168,12 +170,9 @@ public class ExternData {
} }
} }
private static URL appendURL(URL url, String relPath) throws MalformedURLException{ private static void downloadAndExtractItem(String itemRelPath, String dlurlRaw, String dlurlLFS, String commit, File localDataPath) throws Exception {
return new URL(url.toString() + "/" + relPath); URL urlItem = new URL(String.join("/", dlurlLFS, commit, itemRelPath));
} URL urlMD5 = new URL(String.join("/", dlurlRaw, commit, itemRelPath + ".MD5"));
private static void downloadAndExtractItem(String itemRelPath, URL urlDLRawRoot, File localDataPath) throws Exception {
URL urlItem = appendURL(urlDLRawRoot, itemRelPath);
File fileLocalItem = new File(localDataPath, itemRelPath); File fileLocalItem = new File(localDataPath, itemRelPath);
if (!fileLocalItem.getParentFile().exists()) if (!fileLocalItem.getParentFile().exists())
...@@ -188,7 +187,7 @@ public class ExternData { ...@@ -188,7 +187,7 @@ public class ExternData {
System.out.println("Done."); System.out.println("Done.");
System.out.printf("Checking %s MD5...", fileLocalItem.toString()); System.out.printf("Checking %s MD5...", fileLocalItem.toString());
ReadableByteChannel rbcMD5 = Channels.newChannel(appendURL(urlDLRawRoot, itemRelPath + ".MD5").openStream()); ReadableByteChannel rbcMD5 = Channels.newChannel(urlMD5.openStream());
ByteBuffer buf = ByteBuffer.allocate(32); // MD5 is always 32 characters long ByteBuffer buf = ByteBuffer.allocate(32); // MD5 is always 32 characters long
rbcMD5.read(buf); rbcMD5.read(buf);
String md5Expect = new String(buf.array(), Charset.forName("ASCII")).toLowerCase(); String md5Expect = new String(buf.array(), Charset.forName("ASCII")).toLowerCase();
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment