data.sh updated with some small improvments

.gitignore

@@ -32,4 +32,5 @@ api.php
 conf/
 ensembl/
 conf
-*.json
+*.json
+!scripts/data/data.sh

biobtree.go

@@ -80,6 +80,11 @@ func main() {
 			Hidden: true,
 			Usage:  "to change default config directory while developing",
 		},
+		cli.StringFlag{
+			Name:   "lmdb-alloc-size,lmdbsize",
+			Hidden: true,
+			Usage:  "specify lmdb alloc size",
+		},
 		cli.StringFlag{
 			Name:  "pre-build-data,p",
 			Usage: "pre build data provided for common datasets and genomes with the value set1,set2,set3 .",
@@ -390,10 +395,15 @@ func runGenerateCommand(c *cli.Context) error {
 
 	confdir := c.GlobalString("confdir")
 	outDir := c.GlobalString("out-dir")
+	lmdbAllocSize := c.GlobalString("lmdb-alloc-size")
 
 	config = &configs.Conf{}
 	config.Init(confdir, versionTag, outDir, "", true)
 
+	if len(lmdbAllocSize) > 0 {
+		config.Appconf["lmdbAllocSize"] = lmdbAllocSize
+	}
+
 	cpu := c.GlobalInt(" maxcpu")
 	if cpu > 1 {
 		runtime.GOMAXPROCS(cpu)

configs/configs.go

@@ -262,7 +262,7 @@ func (c *Conf) checkForNewVersion() {
 
 func (c *Conf) retrConfFiles(confVersion, preBuildSet, confDir string) {
 
-	log.Println("Pulling configuration and dataset files ...")
+	log.Println("Pulling configuration files ...")
 	confPath := "https://github.com/tamerh/biobtree-conf/archive/" + confVersion + ".zip"
 
 	err := c.unzip(confPath, confDir, confVersion)
@@ -272,6 +272,8 @@ func (c *Conf) retrConfFiles(confVersion, preBuildSet, confDir string) {
 
 	if len(preBuildSet) > 0 {
 
+		log.Println("Pulling pre-build data files ...")
+
 		preDataPath := "https://github.com/tamerh/biobtree-conf/releases/download/" + confVersion + "/biobtree-conf-" + confVersion + "-" + preBuildSet + ".zip"
 
 		err := c.unzip(preDataPath, confDir, confVersion)

scripts/data/data.sh

@@ -63,11 +63,11 @@ prepCache(){
 
     #EXCLUDES="--exclude=biobtree --exclude=data.sh  --exclude=LICENSE* --exclude=*website* --exclude=*ensembl* --exclude=*conf* --exclude=*.git* --exclude=*build.sh  --exclude=*notes.txt --exclude=*.zip --exclude=*.DS_Store*"
 
-    zip -9 -r biobtree-conf-${VERSION}-${1}r.zip out
+    tar -czvf biobtree-conf-${VERSION}-${1}r.tar.gz out/
 
-    ./biobtree generate
+    ./biobtree --lmdbsize ${2} generate
 
-    zip -9 -r biobtree-conf-${VERSION}-${1}d.zip out
+    tar -czvf biobtree-conf-${VERSION}-${1}d.tar.gz out/
 
     rm -rf out
 
@@ -95,20 +95,20 @@ clearConfs
 # ./biobtree -d hgnc --idx cacheset1 update
 # prepCache "set1"
 
-### CACHE 1 datasets with above ensembl genomes except mouse strains.
+### CACHE 1 datasets with above ensembl genomes except mouse strains. ~ 5.2 db size
 
 ./biobtree -d hgnc,hmdb,uniprot,taxonomy,go,efo,eco,chebi,interpro -tax 9606,10090,4932,3702,7227,6239,562,511145,83333,7955,9031,10116 -x --skip-ensembl -idx cacheset1 update
 ./biobtree -tax 9606,4932,3702,7227,6239,562,511145,83333,7955,9031,10116 -keep --ensembl-orthologs -idx cacheset12 update
 ./biobtree --genome mus_musculus -keep --ensembl-orthologs -idx cacheset13 update
-prepCache "set1"
+prepCache "set1" "5600000000"
 
 ### CACHE 2 datasets with ensembl human and all mouse strains genomes
 ./biobtree -d hgnc,hmdb,uniprot,taxonomy,go,efo,eco,chebi,interpro -tax 9606,10090 -x -idx cacheset2 update
-prepCache "set2"
+prepCache "set2" "4100000000"
 
-### CACHE 3 datasets with no esembl and full uniprot
+### CACHE 3 datasets with no esembl and full uniprot ~ 3.2 db size
 ./biobtree -d hgnc,hmdb,uniprot,taxonomy,go,efo,eco,chebi,interpro -x -idx cacheset3 update
-prepCache "set3"
+prepCache "set3" "3600000000"
 
 
 clearConfs

service/grpc.go

@@ -43,7 +43,6 @@ func (g *biobtreegrpc) Start() {
 		log.Fatalf("failed to serve: %s", err)
 	}
 	**/
-	log.Println("gRPC started at port->", port)
 
 }
 func (g *biobtreegrpc) Search(ctx context.Context, in *pbuf.SearchRequest) (*pbuf.SearchResponse, error) {

service/web.go

@@ -90,10 +90,9 @@ func (web *Web) Start(c *configs.Conf, nowebpopup bool) {
 
 	}
 
-	log.Println("REST started at port->", port)
 	uiURL := "localhost:" + port + "/ui"
-	log.Println("Web interface url->", uiURL)
-	fmt.Println("biobtree web is running...")
+	log.Println("Web UI url->", uiURL)
+	log.Println("biobtree web is running...")
 	log.Fatal(http.ListenAndServe(":"+port, nil))
 
 }