tools for preparing seq2seq data

scripts/prepSeqToSeq.ssc

@@ -0,0 +1,22 @@
+:silent
+import BIDMach.tools.{SeqToSeqData,SeqToSeqDict,printmat}
+
+// Example of how to use SeqToSeqData
+
+// Options:
+val opts = new SeqToSeqData.Options;
+opts.srcvocabmaxsize = 150000;                                  // Max vocabulary size.  If <= 0, no maxsize performed.  If >= 1, must provide dict name.
+opts.dstvocabmaxsize =  80000;                                  // Max vocabulary size.  If <= 0, no maxsize performed.  If >= 1, must provide dict name.
+opts.srcminlen = 1;                                             // Minimum sentence length, discard shorter sentences
+opts.srcmaxlen = 12;                                            // Maximum sentence length, truncate longer sentences
+opts.dstminlen = 1;                                             // Minimum sentence length, discard shorter sentences
+opts.dstmaxlen = 12;                                            // Maximum sentence length, truncate longer sentences
+opts.revsrc = true;                                             // Reverse the src sentences
+opts.revdst = false;                                            // Reverse the dst sentences
+
+
+// Make data
+val sd = new SeqToSeqData(opts);
+sd.prepSeqToSeqDataWildcard("/path/to/dir/",
+                            ("src_*.imat","dst_*.imat"),
+                            (("srcdict.sbmat","srcdict.imat"),("dstdict.sbmat","dstdict.imat")))

src/main/C/newparse/makefile.gcc

@@ -2,10 +2,10 @@
 .SUFFIXES: 
 .SUFFIXES: .c .cpp .o .exe .lxc .flex
 
-EXES=xmltweet.exe xmlwiki.exe trec.exe tparse.exe parsevw.exe tparse2.exe
+EXES=xmltweet.exe xmlwiki.exe trec.exe tparse.exe parsevw.exe tparse2.exe nnparse.exe
 OBJS=newparse.o utils.o gzstream.o
 
-.SECONDARY: xmltweet.lxc xmlwiki.lxc trec.lxc
+.SECONDARY: xmltweet.lxc xmlwiki.lxc trec.lxc nnparse.lxc
 
 all: $(EXES)
 

src/main/C/newparse/makefile.w32

@@ -2,10 +2,10 @@
 .SUFFIXES: 
 .SUFFIXES: .c .cpp .obj .exe .lxc .flex
 
-EXES=xmltweet.exe xmlwiki.exe trec.exe tparse.exe parsevw.exe tparse2.exe
+EXES=xmltweet.exe xmlwiki.exe trec.exe tparse.exe parsevw.exe tparse2.exe nnparse.exe
 OBJS=newparse.obj utils.obj gzstream.obj
 
-.SECONDARY: xmltweet.lxc xmlwiki.lxc trec.lxc
+.SECONDARY: xmltweet.lxc xmlwiki.lxc trec.lxc nnparse.lxc
 
 all: $(EXES)
 

src/main/C/newparse/newparse.cpp

@@ -10,6 +10,8 @@
 
 ivector wcount;
 ivector tokens;
+ivector paragraphids;
+ivector sentenceids;
 unhash unh;
 strhash htab;
 
@@ -18,9 +20,14 @@ extern int yylex(void);
 extern FILE*   yyin;
 
 int numlines=0;
+int doparagraphids=0;
+int paragraphid=0;
+int sentenceid=0;
 }
 
 int checkword(char *str) {
+  paragraphids.push_back(paragraphid);
+  sentenceids.push_back(sentenceid);
   return checkword(str, htab, wcount, tokens, unh);
 }
 
@@ -98,9 +105,17 @@ int main(int argc, char ** argv) {
     pos = rname.rfind('/');
     if (pos == string::npos) pos = rname.rfind('\\');
     if (pos != string::npos) rname = rname.substr(pos+1, rname.size());
-    writeIntVec(tokens, odname+rname+".imat"+suffix, membuf);
+    if (doparagraphids) {
+      writeIntVec3Cols(paragraphids, sentenceids, tokens, odname+rname+".imat"+suffix, membuf);
+    } else {
+      writeIntVec(tokens, odname+rname+".imat"+suffix, membuf);
+    }
     tokens.clear();
+    sentenceids.clear();
+    paragraphids.clear();
     numlines = 0;
+    sentenceid = 0;
+    paragraphid = 0;
     here = strtok(NULL, " ,");
   }
   fprintf(stderr, "\nWriting Dictionary\n");

src/main/C/newparse/nnparse.flex

@@ -0,0 +1,76 @@
+/* Scanner for neural net datasources */
+
+%{
+  extern int checkword(char *);
+  extern void addtok(int tok);
+  extern int parsedate(char * str);
+  extern int numlines;
+  extern int doparagraphids;
+  extern int sentenceid;
+  extern int paragraphid;
+
+  #define YY_USER_ACTION doparagraphids=1;  // Macro happens at initialization
+%}
+
+%option never-interactive
+%option noyywrap
+
+LETTER	   [a-zA-Z_]
+DIGIT	   [0-9]
+PUNCT	   [;:,.?!]
+
+%% 
+
+{LETTER}+    {
+  int iv = checkword(yytext);
+	}
+
+"<"{LETTER}+">"    {
+  int iv = checkword(yytext);
+	}
+
+"</"{LETTER}+">"    {
+  int iv = checkword(yytext);
+	}
+
+{PUNCT}	  {
+  int iv = checkword(yytext);
+	  }
+
+"..""."*  {
+  char ell[] = "...";
+  int iv  = checkword(ell);
+	  }
+
+". "  {
+  sentenceid++;
+  char ell[] = ".";
+  int iv  = checkword(ell);
+	  }
+
+"? "  {
+  sentenceid++;
+  char ell[] = "?";
+  int iv  = checkword(ell);
+	  }
+
+"! "  {
+  sentenceid++;
+  char ell[] = "!";
+  int iv  = checkword(ell);
+	  }
+
+[\n]	  {
+	  numlines++;
+	  paragraphid++;
+	  sentenceid = 0;
+	  if (numlines % 1000000 == 0) {
+	  fprintf(stderr, "\r%05d lines", numlines);
+      fflush(stderr);
+	  }	  
+	  }
+
+.         {}
+
+%%
+

src/main/C/newparse/utils.cpp

@@ -364,6 +364,44 @@ int writeIntVec(ivector & im, string fname, int buffsize) {
   return 0;
 }
 
+int writeIntVec2Cols(ivector & im1, ivector & im2, string fname, int buffsize) {
+  int fmt, nrows, nnz;
+  int ncols = 2;
+
+  ostream *ofstr = open_out_buf(fname.c_str(), buffsize);
+  fmt = 110;
+  nrows = im1.size();
+  nnz = nrows * ncols;
+  ofstr->write((const char *)&fmt, 4);
+  ofstr->write((const char *)&nrows, 4);
+  ofstr->write((const char *)&ncols, 4);
+  ofstr->write((const char *)&nnz, 4);
+  ofstr->write((const char *)&im1[0], 4 * nrows);
+  ofstr->write((const char *)&im2[0], 4 * nrows);
+  closeos(ofstr);
+  return 0;
+}
+
+int writeIntVec3Cols(ivector & im1, ivector & im2, ivector & im3, string fname, int buffsize) {
+  int fmt, nrows, nnz;
+  int ncols = 3;
+
+  ostream *ofstr = open_out_buf(fname.c_str(), buffsize);
+  fmt = 110;
+  nrows = im1.size();
+  nnz = nrows * ncols;
+  ofstr->write((const char *)&fmt, 4);
+  ofstr->write((const char *)&nrows, 4);
+  ofstr->write((const char *)&ncols, 4);
+  ofstr->write((const char *)&nnz, 4);
+  ofstr->write((const char *)&im1[0], 4 * nrows);
+  ofstr->write((const char *)&im2[0], 4 * nrows);
+  ofstr->write((const char *)&im3[0], 4 * nrows);
+  closeos(ofstr);
+  return 0;
+}
+
+
 int writeDIntVec(divector & im, string fname, int buffsize) {
   int fmt, nrows, ncols, nnz;
   ostream *ofstr = open_out_buf(fname.c_str(), buffsize);

src/main/C/newparse/utils.h

@@ -162,6 +162,10 @@ void closeos(ostream *ofs);
 
 int writeIntVec(ivector & im, string fname, int buffsize);
 
+int writeIntVec2Cols(ivector & im1, ivector & im2, string fname, int buffsize);
+
+int writeIntVec3Cols(ivector & im1, ivector & im2, ivector & im3, string fname, int buffsize);
+
 int writeDIntVec(divector & im, string fname, int buffsize);
 
 int writeQIntVec(qvector & im, string fname, int buffsize);