Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

delete unnecessary code and fix a minor problem #13

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 9 additions & 15 deletions src/main/java/edu/cmu/graphchi/shards/MemoryShard.java
Original file line number Diff line number Diff line change
Expand Up @@ -285,7 +285,7 @@ private void loadAdjChunk(int windowStart, int windowEnd, ChiVertex[] vertices,
}


private DataInput loadAdj() throws FileNotFoundException, IOException {
private DataInput loadAdj() throws IOException {
File compressedFile = new File(adjDataFilename + ".gz");
InputStream adjStreamRaw;
long fileSizeEstimate = 0;
Expand All @@ -300,31 +300,25 @@ private DataInput loadAdj() throws FileNotFoundException, IOException {

/* Load index */
index = new ShardIndex(new File(adjDataFilename)).sparserIndex(1204 * 1024);
BufferedInputStream adjStream = new BufferedInputStream(adjStreamRaw, (int) fileSizeEstimate /
4);

byte[] buf = new byte[(int)fileSizeEstimate];
// Hack for cases when the load is not divided into subwindows
TimerContext _timer = loadAdjTimer.time();

ByteArrayOutputStream adjDataStream = new ByteArrayOutputStream((int) fileSizeEstimate);
try {
byte[] buf = new byte[(int) fileSizeEstimate / 4]; // Read in 16 chunks
while (true) {
int read = adjStream.read(buf);
if (read > 0) {
adjDataStream.write(buf, 0, read);
} else break;
int nread = adjStreamRaw.read(buf, 0, buf.length);
if (nread < buf.length) {
// compressed
byte[] buf2 = new byte[nread];
System.arraycopy(buf, 0, buf2, 0, nread);
buf = buf2;
}
} catch (EOFException err) {
// Done
}

adjData = adjDataStream.toByteArray();
adjData = buf;
adjDataLength = adjData.length;

adjStream.close();
adjDataStream.close();

_timer.stop();
return null;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ public static <VertexDataType> void foreach(int numVertices, String baseFilename

int CHUNK = 1000000;
for(int i=0; i < numVertices; i += CHUNK) {
int en = i + CHUNK;
int en = i + CHUNK - 1;
if (en >= numVertices) en = numVertices - 1;
int blockId = vertexData.load(i, en);

Expand Down