Skip to content

Commit

Permalink
fix cache handling and allow as parameter
Browse files Browse the repository at this point in the history
  • Loading branch information
@sa501428
sa501428 committed Oct 21, 2020
1 parent 949a5eb commit e565b6d
Show file tree
Hide file tree
Showing 9 changed files with 33 additions and 109 deletions.
3 changes: 1 addition & 2 deletions src/javastraw/HiCGlobals.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,14 +30,13 @@
*/
public class HiCGlobals {

public static final String versionNum = "1.00.00";
public static final String versionNum = "1.01.00";

// min hic file version supported
public static final int minVersion = 6;
public static final int bufferSize = 2097152;

// implement Map scaling with this global variable
public static boolean useCache = true;
public static boolean allowDynamicBlockIndex = true;
public static boolean printVerboseComments = false;
}
10 changes: 6 additions & 4 deletions src/javastraw/reader/ContactRecordIterator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,6 @@

package javastraw.reader;

import javastraw.HiCGlobals;
import javastraw.reader.basics.Block;
import javastraw.reader.basics.ContactRecord;
import javastraw.type.NormalizationHandler;
Expand All @@ -45,16 +44,19 @@ class ContactRecordIterator implements Iterator<ContactRecord> {
private final DatasetReader reader;
private final MatrixZoomData zd;
private final LRUCache<String, Block> blockCache;

private final boolean useCache;

/**
* Initializes the iterator
*/
ContactRecordIterator(DatasetReader reader, MatrixZoomData zd, LRUCache<String, Block> blockCache) {
ContactRecordIterator(DatasetReader reader, MatrixZoomData zd, LRUCache<String, Block> blockCache,
boolean useCache) {
this.reader = reader;
this.zd = zd;
this.blockCache = blockCache;
this.blockIdx = -1;
this.blockNumbers = reader.getBlockNumbers(zd);
this.useCache = useCache;
}

/**
Expand All @@ -77,7 +79,7 @@ public boolean hasNext() {
// Optionally check the cache
String key = zd.getBlockKey(blockNumber, NormalizationHandler.NONE);
Block nextBlock;
if (HiCGlobals.useCache && blockCache.containsKey(key)) {
if (useCache && blockCache.containsKey(key)) {
nextBlock = blockCache.get(key);
} else {
nextBlock = reader.readNormalizedBlock(blockNumber, zd, NormalizationHandler.NONE);
Expand Down
4 changes: 2 additions & 2 deletions src/javastraw/reader/DatasetReaderFactory.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,12 +42,12 @@
*/
public class DatasetReaderFactory {

public static DatasetReaderV2 getReaderForFile(String file) throws IOException {
public static DatasetReaderV2 getReaderForFile(String file, boolean useCache) throws IOException {
String magicString = getMagicString(file);

if (magicString != null) {
if (magicString.equals("HIC")) {
return new DatasetReaderV2(file);
return new DatasetReaderV2(file, useCache);
} else {
System.err.println("This version is deprecated and is no longer supported.");
//reader = new DatasetReaderV1(file);
Expand Down
6 changes: 6 additions & 0 deletions src/javastraw/reader/DatasetReaderV2.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,7 @@ public class DatasetReaderV2 extends AbstractDatasetReader {
private boolean activeStatus = true;
private final AtomicBoolean useMainStream = new AtomicBoolean();
public static double[] globalTimeDiffThings = new double[5];
private boolean useCache = true;

@Override
public Dataset read() throws IOException {
Expand Down Expand Up @@ -226,6 +227,10 @@ public DatasetReaderV2(String path) throws IOException {
blockIndexMap = Collections.synchronizedMap(new HashMap<>());
}

public DatasetReaderV2(String path, boolean useCache) throws IOException {
this(path);
this.useCache = useCache;
}

public String readStats() throws IOException {
String statsFileName = path.substring(0, path.lastIndexOf('.')) + "_stats.html";
Expand Down Expand Up @@ -280,6 +285,7 @@ private Pair<MatrixZoomData, Long> readMatrixZoomData(Chromosome chr1, Chromosom

MatrixZoomData zd = new MatrixZoomData(chr1, chr2, zoom, blockBinCount, blockColumnCount, chr1Sites, chr2Sites,
this);
zd.setUseCache(useCache);

int nBlocks = dis.readInt();

Expand Down
8 changes: 4 additions & 4 deletions src/javastraw/reader/HiCFileTools.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@
*/
public class HiCFileTools {

public static Dataset extractDatasetForCLT(String filename, boolean allowPrinting) {
public static Dataset extractDatasetForCLT(String filename, boolean allowPrinting, boolean useCache) {
Dataset dataset = null;
String file = filename;
if (isDropboxURL(file)) {
Expand All @@ -55,7 +55,7 @@ public static Dataset extractDatasetForCLT(String filename, boolean allowPrintin
System.out.println("Reading file: " + file);
String magicString = DatasetReaderFactory.getMagicString(file);
if (magicString.equals("HIC")) {
reader = new DatasetReaderV2(file);
reader = new DatasetReaderV2(file, useCache);
} else {
System.err.println("This version of HIC is no longer supported");
System.exit(32);
Expand All @@ -77,7 +77,7 @@ private static void verifySupportedHiCFileVersion(int version) throws RuntimeExc
}
}

public static DatasetReader extractDatasetReaderForCLT(List<String> files, boolean allowPrinting) {
public static DatasetReader extractDatasetReaderForCLT(List<String> files, boolean allowPrinting, boolean useCache) {
DatasetReader reader = null;
String file = files.get(0);
if (isDropboxURL(file)) {
Expand All @@ -88,7 +88,7 @@ public static DatasetReader extractDatasetReaderForCLT(List<String> files, boole
System.out.println("Reading file: " + file);
String magicString = DatasetReaderFactory.getMagicString(file);
if (magicString.equals("HIC")) {
reader = new DatasetReaderV2(file);
reader = new DatasetReaderV2(file, useCache);
} else {
System.err.println("This version of HIC is no longer supported");
System.exit(32);
Expand Down
71 changes: 0 additions & 71 deletions src/javastraw/reader/Matrix.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,75 +93,6 @@ public static String generateKey(int chr1, int chr2) {
return "" + chr1 + "_" + chr2;
}

public static Matrix createAssemblyChromosomeMatrix(ChromosomeHandler handler,
final Map<String, Matrix> matrices, DatasetReader reader) {
Map<HiCZoom, MatrixZoomData> assemblyZDs = new HashMap<>();

Matrix matrix = null;
int numAttempts = 0;
while (matrix == null && numAttempts < 3) {
try {
matrix = reader.readMatrix("1_1");
} catch (Exception ignored) {
numAttempts++;
}
}

long length = handler.getChromosomeFromName("pseudoassembly").getLength(); // TODO: scaling; also maybe chromosome ends need to shift to start with new bin at every zoom?
for (MatrixZoomData zd : matrix.bpZoomData) {
// todo @dudcha is this done for resolutions where conversion will be lossy?
assemblyZDs.put(zd.getZoom(), new MatrixZoomData(handler.getChromosomeFromName("pseudoassembly"), handler.getChromosomeFromName("pseudoassembly"), zd.getZoom(),
(int) (length / zd.getBinSize()), (int) (length / zd.getBinSize()), null, null, reader));
}


//TODO: assumption that we are doing this before modifying the handler

// for (Chromosome i : handler.getChromosomeArrayWithoutAllByAll()) {
// for (Chromosome j : handler.getChromosomeArrayWithoutAllByAll()) {
//
// //System.out.println("from mtrx");
// String key = Matrix.generateKey(i, j);
// try {
// Matrix m = matrices.get(key);
// if (m == null) {
// // TODO sometimes this fails once or twice, but later succeeds -
// // TODO high priority, needs to be fixed??????
// int numAttempts = 0;
// while (m == null && numAttempts < 3) {
// try {
// m = reader.readMatrix(key);
// } catch (Exception ignored) {
// numAttempts++;
// }
// }
//
// for(MatrixZoomData tempMatrixZoomData : m.bpZoomData){
// tempMatrixZoomData.
// }
//
//
// // modify m for each zoom
//// matrices.put(key, m); //perhaps move it to the end
// }
// for (MatrixZoomData zd : m.bpZoomData) {
// updateCustomZoomDataRegions(newChr1, newChr2, handler, key, zd, assemblyZDs, reader);
// }
//// for (MatrixZoomData zd : m.fragZoomData) {
//// updateCustomZoomDataRegions(newChr1, newChr2, handler, key, zd, customZDs, reader);
//// }
// } catch (Exception ee) {
// System.err.println("Everything failed in creatingAssemblyChromosomeMatrix " + key);
// ee.printStackTrace();
// }
// }
// }

Matrix m = new Matrix(handler.size(), handler.size(), new ArrayList<>(assemblyZDs.values()));
matrices.put(generateKey(handler.size(), handler.size()), m);
return m;
}

public void createDynamicResolutionMZD(Pair<Integer, Integer> resPair, boolean addToSet) {
int newRes = resPair.getFirst();
int highRes = resPair.getSecond();
Expand All @@ -174,8 +105,6 @@ public void createDynamicResolutionMZD(Pair<Integer, Integer> resPair, boolean a
dynamicBPZoomData.add(newMZD);
}



public static String generateKey(Chromosome chr1, Chromosome chr2) {
int t1 = Math.min(chr1.getIndex(), chr2.getIndex());
int t2 = Math.max(chr1.getIndex(), chr2.getIndex());
Expand Down
28 changes: 9 additions & 19 deletions src/javastraw/reader/MatrixZoomData.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,7 @@ public class MatrixZoomData {
private double averageCount = -1;
private List<List<ContactRecord>> localCacheOfRecords = null;
private long numberOfContactRecords = 0;
private boolean useCache = true;

/**
* Constructor, sets the grid axes. Called when read from file.
Expand All @@ -86,21 +87,10 @@ public MatrixZoomData(Chromosome chr1, Chromosome chr2, HiCZoom zoom, int blockB
this.reader = reader;
this.blockBinCount = blockBinCount;
this.blockColumnCount = blockColumnCount;
}

long correctedBinCount = blockBinCount;
if (!(this instanceof DynamicMatrixZoomData)) {
if (reader.getVersion() < 8 && chr1.getLength() < chr2.getLength()) {
boolean isFrag = zoom.getUnit() == HiCZoom.HiCUnit.FRAG;
long len1 = chr1.getLength();
long len2 = chr2.getLength();
if (chr1Sites != null && chr2Sites != null && isFrag) {
len1 = chr1Sites.length + 1;
len2 = chr2Sites.length + 1;
}
long nBinsX = Math.max(len1, len2) / zoom.getBinSize() + 1;
correctedBinCount = nBinsX / blockColumnCount + 1;
}
}
public void setUseCache(boolean useCache) {
this.useCache = useCache;
}

public int getBinSize() {
Expand Down Expand Up @@ -178,7 +168,7 @@ private void populateBlocksToLoadV9(int positionAlongDiagonal, int depth, Normal
int blockNumber = getBlockNumberVersion9FromPADAndDepth(positionAlongDiagonal, depth);
String key = getBlockKey(blockNumber, no);
Block b;
if (HiCGlobals.useCache && blockCache.containsKey(key)) {
if (useCache && blockCache.containsKey(key)) {
b = blockCache.get(key);
blockList.add(b);
} else {
Expand Down Expand Up @@ -254,7 +244,7 @@ private void populateBlocksToLoad(int r, int c, NormalizationType no, List<Block
int blockNumber = r * getBlockColumnCount() + c;
String key = getBlockKey(blockNumber, no);
Block b;
if (HiCGlobals.useCache && blockCache.containsKey(key)) {
if (useCache && blockCache.containsKey(key)) {
b = blockCache.get(key);
blockList.add(b);
} else {
Expand Down Expand Up @@ -346,7 +336,7 @@ public void run() {
b = new Block(blockNumber, key); // An empty block
}
//Run out of memory if do it here
if (HiCGlobals.useCache) {
if (useCache) {
blockCache.put(key, b);
}
blockList.add(b);
Expand Down Expand Up @@ -397,7 +387,7 @@ public void run() {
b = new Block(blockNumber, key); // An empty block
}
//Run out of memory if do it here
if (HiCGlobals.useCache) {
if (useCache) {
blockCache.put(key, b);
}
blockList.add(b);
Expand Down Expand Up @@ -763,7 +753,7 @@ public List<List<ContactRecord>> getContactRecordList() {
List<ContactRecord> currentList = new ArrayList<>(1000000);
int localCounter = 0;
int maxAllowed = 9 * (Integer.MAX_VALUE / 10);
Iterator<ContactRecord> iterator = new ContactRecordIterator(reader, this, blockCache);
Iterator<ContactRecord> iterator = new ContactRecordIterator(reader, this, blockCache, useCache);
while (iterator.hasNext()) {
ContactRecord cr = iterator.next();
currentList.add(cr);
Expand Down
10 changes: 5 additions & 5 deletions src/javastraw/tools/HiCFileUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,16 +42,16 @@
*/

class HiCFileUtils {

private final Dataset dataset;

private HiCFileUtils(String hicfile) throws IOException {
DatasetReaderV2 reader = new DatasetReaderV2(hicfile);
private final Dataset dataset;

private HiCFileUtils(String hicfile, boolean useCache) throws IOException {
DatasetReaderV2 reader = new DatasetReaderV2(hicfile, useCache);
dataset = reader.read();
}

public static void main(String[] args) throws IOException {
HiCFileUtils utils = new HiCFileUtils(args[0]);
HiCFileUtils utils = new HiCFileUtils(args[0], false);
utils.dumpNormalizationVectors(NormalizationHandler.KR, "1", HiCZoom.HiCUnit.BP, 250000);
utils.dumpExpectedVectors(NormalizationHandler.KR, HiCZoom.HiCUnit.BP, 1000000);
}
Expand Down
2 changes: 0 additions & 2 deletions src/javastraw/tools/UNIXTools.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,4 @@ private static String executeComplexCommand(List<String> command) {
}
return output.toString();
}


}

0 comments on commit e565b6d

Please sign in to comment.