Merge pull request #27 from yhoogstrate/mem_red2

Huge reduction of memory footprint

Changelog

@@ -1,3 +1,11 @@
+2016-03-14  Youri Hoogstrate
+
+	* Version 2.12.0: Huge reduction of memory footprint - from
+	  exponential to linear scale as the number of samples increase.
+	  Also the pruning system works very efficient and the memory does
+	  not increase too much after a large number of iterations with
+	  many samples.
+
 2016-03-14  Youri Hoogstrate
 
 	* Version 2.11.8: Reduction of memort footprint

fuma/OverlapComplex.py

@@ -85,7 +85,7 @@ def overlay_fusions(self,sparse=True,export_dir=False,args=None):
 		for i in range(len(self.datasets)):
 			self.matrix_tmp[str(i+1)] = self.datasets[i]
 
-		comparisons = self.find_combination_table(n)
+		#comparisons = self.find_combination_table(n)
 
 		if(args.format=="list" and export_dir != False):
 			if args.long_gene_size > 0:
@@ -95,8 +95,9 @@ def overlay_fusions(self,sparse=True,export_dir=False,args=None):
 
 			export_dir.write("Left-genes\tRight-genes\t"+large_genes+"\t"+"\t".join(self.dataset_names)+"\n")
 
-		for ri in range(len(comparisons)):
-			r = comparisons[ri]
+		for r in self.find_combination_table(len(self.datasets)):
+			ri = len(self.datasets)-1
+
 			# First cleanup the memory - reduces space complexity from 0.5(n^2) => 2n. In addition, memory should decrease in time
 			dont_remove = []
 			matches_this_iteration = set([])
@@ -129,7 +130,7 @@ def overlay_fusions(self,sparse=True,export_dir=False,args=None):
 
 			if(args.format=="list"):# Write those that are not marked to go to the next iteration to a file
 				if(len(r[0]) > 2):
-					for export_key in comparisons[ri-1]:
+					for export_key in previous_comparisons:#comparisons[ri-1]:
 						export_key = '.'.join(export_key)
 
 						self.matrix_tmp[export_key].export_to_list(export_dir,self.dataset_names,matches_this_iteration,args)
@@ -138,6 +139,8 @@ def overlay_fusions(self,sparse=True,export_dir=False,args=None):
 					for export_key in [str(i+1) for i in range(len(self.datasets))]:
 						self.matrix_tmp[export_key].export_to_list(export_dir,self.dataset_names,matches_this_iteration,args)
 						#del(self.matrix_tmp[export_key]) ## if this was once in a list to be removed, remove...
+
+			previous_comparisons = r
 
 		if(args.format == "list" and export_dir != False):
 			export_key = '.'.join(r[0])
@@ -147,16 +150,13 @@ def overlay_fusions(self,sparse=True,export_dir=False,args=None):
 
 	def find_combination_table(self,n):
 		in_list = range(1,n+1)
-		table = []
 
 		for r in range(2,len(in_list)+1):
 			table_i_tmp = itertools.combinations(in_list,r)
 			table_i = []
 			for i in table_i_tmp:
 				table_i.append(list(i))
-			table.append(table_i)
-
-		return table
+			yield table_i
 
 	def set_annotation(self,arg_gene_annotation):
 		self.gene_annotation = arg_gene_annotation

fuma/__init__.py

@@ -21,7 +21,7 @@
  <http://epydoc.sourceforge.net/manual-fields.html#fields-synonyms>
 """
 
-__version_info__ = ('2', '11', '8')
+__version_info__ = ('2', '12', '0')
 __version__ = '.'.join(__version_info__) if (len(__version_info__) == 3) else '.'.join(__version_info__[0:3])+"-"+__version_info__[3]
 __author__ = 'Youri Hoogstrate'
 __homepage__ = 'https://github.com/yhoogstrate/fuma'