point url to GitHub; handle different raw/lfs url in GitHub

benchmarks/build.xml

@@ -438,9 +438,10 @@ See ${log.build.report} for a summary of benchmark build status.
     <target name="update-externdata-info" depends="set-build-version-info">
         <get src="${dacapo-dl.url.raw}/${dacapo-dl.commit}/META-INF/huge-data-md5s.list" dest="${dacapo.build.dir}"/>
         <get src="${dacapo-dl.url.raw}/${dacapo-dl.commit}/META-INF/dlfiles.list" dest="${dacapo.build.dir}"/>
-        <echo message="setting DaCapo-DL-URL-RAW and DaCapo-DL-Commit in manifest"/>
+        <echo message="setting DaCapo-DL-URL-LFS, DaCapo-DL-URL-RAW DaCapo-DL-Commit in manifest"/>
         <jar destfile="${build.target-jar}" update="true">
             <manifest>
+                <attribute name="DaCapo-DL-URL-LFS" value="${dacapo-dl.url.lfs}"/>
                 <attribute name="DaCapo-DL-URL-RAW" value="${dacapo-dl.url.raw}"/>
                 <attribute name="DaCapo-DL-Commit" value="${dacapo-dl.commit}"/>
             </manifest>

benchmarks/dacapo.properties

@@ -3,9 +3,11 @@ dacapo.version=9.12-MR1
 dacapo.nickname=uncalibrated
 
 # External data
-dacapo-dl.url.raw=https://gitlab.anu.edu.au/dacapo/anu-dev/dacapo-dl/raw
-# dacapo-dl.url.raw=https://raw.githubusercontent.com/dacapobench/dacapo-dl/
-dacapo-dl.commit=HEAD
+# dacapo-dl.url.raw=https://gitlab.anu.edu.au/dacapo/anu-dev/dacapo-dl/raw
+# dacapo-dl.url.lfs=https://gitlab.anu.edu.au/dacapo/anu-dev/dacapo-dl/raw
+dacapo-dl.url.raw=https://raw.githubusercontent.com/john5f35/dacapo-dl
+dacapo-dl.url.lfs=https://media.githubusercontent.com/media/john5f35/dacapo-dl
+dacapo-dl.commit=cb479d7
 #
 # Common properties for the dacapo benchmarks
 #

benchmarks/harness/src/org/dacapo/harness/ExternData.java

@@ -41,6 +41,7 @@ public class ExternData {
    */
   public static final Path DEFAULT_LOCAL_DACAPO_CONFIG = Paths.get(System.getProperty("user.home"), ".dacapo-config.properties");
   public static final String CONFIG_KEY_EXTERN_DATA_LOC = "Extern-Data-Location";
+  public static final String DACAPO_DL_URL_LFS = "DaCapo-DL-URL-LFS";
   public static final String DACAPO_DL_URL_RAW = "DaCapo-DL-URL-RAW";
   public static final String DACAPO_DL_COMMIT = "DaCapo-DL-Commit";
 
@@ -147,13 +148,14 @@ public class ExternData {
         path.mkdirs();
 
       // download
-      URL urlDLRawRoot = appendURL(new URL(TestHarness.getManifestAttribute(DACAPO_DL_URL_RAW)),
-              TestHarness.getManifestAttribute(DACAPO_DL_COMMIT));
+      String dlurlRaw = TestHarness.getManifestAttribute(DACAPO_DL_URL_RAW);
+      String dlurlLFS = TestHarness.getManifestAttribute(DACAPO_DL_URL_LFS);
+      String commit = TestHarness.getManifestAttribute(DACAPO_DL_COMMIT);
       BufferedReader dllistReader = new BufferedReader(new InputStreamReader(
               ClassLoader.getSystemResourceAsStream("META-INF/dlfiles.list")));
       dllistReader.lines().forEach(s -> {
         try {
-          downloadAndExtractItem(s, urlDLRawRoot, path);
+          downloadAndExtractItem(s, dlurlRaw, dlurlLFS, commit, path);
         } catch (Exception e) {
           System.err.println("Download external data failed.");
           System.err.println("Please try again.");
@@ -168,12 +170,9 @@ public class ExternData {
     }
   }
 
-  private static URL appendURL(URL url, String relPath) throws MalformedURLException{
-    return new URL(url.toString() + "/" + relPath);
-  }
-
-  private static void downloadAndExtractItem(String itemRelPath, URL urlDLRawRoot, File localDataPath) throws Exception {
-    URL urlItem = appendURL(urlDLRawRoot, itemRelPath);
+  private static void downloadAndExtractItem(String itemRelPath, String dlurlRaw, String dlurlLFS, String commit, File localDataPath) throws Exception {
+    URL urlItem = new URL(String.join("/", dlurlLFS, commit, itemRelPath));
+    URL urlMD5 = new URL(String.join("/", dlurlRaw, commit, itemRelPath + ".MD5"));
     File fileLocalItem = new File(localDataPath, itemRelPath);
 
     if (!fileLocalItem.getParentFile().exists())
@@ -188,7 +187,7 @@ public class ExternData {
       System.out.println("Done.");
 
       System.out.printf("Checking %s MD5...", fileLocalItem.toString());
-      ReadableByteChannel rbcMD5 = Channels.newChannel(appendURL(urlDLRawRoot, itemRelPath + ".MD5").openStream());
+      ReadableByteChannel rbcMD5 = Channels.newChannel(urlMD5.openStream());
       ByteBuffer buf = ByteBuffer.allocate(32); // MD5 is always 32 characters long
       rbcMD5.read(buf);
       String md5Expect = new String(buf.array(), Charset.forName("ASCII")).toLowerCase();