Commit c09cf5d1 authored by Rui Chen's avatar Rui Chen

[dataset] move data set to sourceforge

parent 5c9d1aef
Pipeline #4103 passed with stage
in 393 minutes and 29 seconds
......@@ -518,7 +518,7 @@ See ${log.build.report} for a summary of benchmark build status.
<target name="jython" depends="init,harness,update-externdata-info">
<build.bm benchmark="jython"/>
</target>
<target name="kafka" depends="init,harness">
<target name="kafka" depends="init,harness,update-externdata-info">
<build.bm benchmark="kafka"/>
</target>
<target name="luindex" depends="init,harness,update-externdata-info">
......@@ -550,14 +550,13 @@ See ${log.build.report} for a summary of benchmark build status.
</target>
<target name="update-externdata-info" depends="set-build-version-info">
<get src="${dacapo-dl.url.raw}/${dacapo-dl.commit}/META-INF/huge-data-md5s.list" dest="${dacapo.build.dir}"/>
<get src="${dacapo-dl.url.raw}/${dacapo-dl.commit}/META-INF/dlfiles.list" dest="${dacapo.build.dir}"/>
<get src="${dacapo-dl.url.raw}/META-INF/huge-data-md5s.list" dest="${dacapo.build.dir}"/>
<get src="${dacapo-dl.url.raw}/META-INF/dlfiles.list" dest="${dacapo.build.dir}"/>
<echo message="setting DaCapo-DL-URL-LFS, DaCapo-DL-URL-RAW DaCapo-DL-Commit in manifest"/>
<jar destfile="${build.target-jar}" update="true">
<manifest>
<attribute name="DaCapo-DL-URL-LFS" value="${dacapo-dl.url.lfs}"/>
<attribute name="DaCapo-DL-URL-RAW" value="${dacapo-dl.url.raw}"/>
<attribute name="DaCapo-DL-Commit" value="${dacapo-dl.commit}"/>
</manifest>
<metainf dir="${dacapo.build.dir}">
<include name="huge-data-md5s.list"/>
......
......@@ -5,9 +5,8 @@ dacapo.nickname=uncalibrated
# External data
# dacapo-dl.url.raw=https://gitlab.anu.edu.au/dacapo/anu-dev/dacapo-dl/raw
# dacapo-dl.url.lfs=https://gitlab.anu.edu.au/dacapo/anu-dev/dacapo-dl/raw
dacapo-dl.url.raw=https://raw.githubusercontent.com/RuiChen08/dacapo-dl
dacapo-dl.url.lfs=https://media.githubusercontent.com/media/RuiChen08/dacapo-dl
dacapo-dl.commit=master
dacapo-dl.url.raw=https://sourceforge.net/projects/dacapo-dl/files
dacapo-dl.url.lfs=https://sourceforge.net/projects/dacapo-dl/files
#
# Common properties for the dacapo benchmarks
#
......
......@@ -9,7 +9,6 @@
package org.dacapo.harness;
import java.io.*;
import java.net.MalformedURLException;
import java.net.URL;
import java.nio.ByteBuffer;
import java.nio.file.Files;
......@@ -35,7 +34,6 @@ public class ExternData {
public static final String CONFIG_KEY_EXTERN_DATA_LOC = "Extern-Data-Location";
public static final String DACAPO_DL_URL_LFS = "DaCapo-DL-URL-LFS";
public static final String DACAPO_DL_URL_RAW = "DaCapo-DL-URL-RAW";
public static final String DACAPO_DL_COMMIT = "DaCapo-DL-Commit";
public static final String DACAPO_CHECKSUM_RE_PATH = "META-INF" + File.separator + "huge-data-md5s.list";
private static String getDefaultLocation() {
......@@ -161,14 +159,13 @@ public class ExternData {
// download
String dlurlRaw = TestHarness.getManifestAttribute(DACAPO_DL_URL_RAW);
String dlurlLFS = TestHarness.getManifestAttribute(DACAPO_DL_URL_LFS);
String commit = TestHarness.getManifestAttribute(DACAPO_DL_COMMIT);
BufferedReader dllistReader = new BufferedReader(new InputStreamReader(
ClassLoader.getSystemResourceAsStream("META-INF/dlfiles.list")));
dllistReader.lines().forEach(s -> {
try {
if(s.startsWith("dat/" + bench))
downloadAndExtractItem(s, dlurlRaw, dlurlLFS, commit, path);
downloadAndExtractItem(s, dlurlRaw, dlurlLFS, path);
} catch (IOException e) {
System.err.println("Download external data failed.");
System.err.printf("You may want to manually download %s to %s\n", e.getMessage(), path);
......@@ -224,13 +221,12 @@ public class ExternData {
// download
String dlurlRaw = TestHarness.getManifestAttribute(DACAPO_DL_URL_RAW);
String dlurlLFS = TestHarness.getManifestAttribute(DACAPO_DL_URL_LFS);
String commit = TestHarness.getManifestAttribute(DACAPO_DL_COMMIT);
downloadAndExtractItem(itemRelPath, dlurlRaw, dlurlRaw, commit, path);
downloadAndExtractItem(itemRelPath, dlurlRaw, dlurlRaw, path);
}
private static void downloadAndExtractItem(String itemRelPath, String dlurlRaw, String dlurlLFS, String commit, File localDataPath) throws Exception {
URL urlItem = new URL(String.join("/", dlurlLFS, commit, itemRelPath));
URL urlMD5 = new URL(String.join("/", dlurlRaw, commit, itemRelPath + ".MD5"));
private static void downloadAndExtractItem(String itemRelPath, String dlurlRaw, String dlurlLFS, File localDataPath) throws Exception {
URL urlItem = new URL(String.join("/", dlurlLFS, itemRelPath));
URL urlMD5 = new URL(String.join("/", dlurlRaw, itemRelPath + ".MD5"));
File fileLocalItem = new File(localDataPath, itemRelPath);
if (!fileLocalItem.getParentFile().exists())
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment