Implement lt-merge --unmerge

cf. HEAD^

lttoolbox/fst_processor.cc

@@ -2009,6 +2009,67 @@ FSTProcessor::quoteMerge(InputFile& input, UFILE *output)
 }
 
 
+void
+FSTProcessor::quoteUnmerge(InputFile &input, UFILE *output)
+{
+  StreamReader reader(&input);
+  reader.alpha = &alphabet;
+  reader.add_unknowns = true;
+
+  UString surface;
+  while (!reader.at_eof) {
+    reader.next();
+    bool unmerging = false;
+    for (StreamReader::Reading &it : reader.readings) {
+      // TODO: look up in it.symbols instead (but need to make an alphabet then)
+      if(it.content.find(u"<MERGED>") != std::string::npos) {
+        unmerging = true;
+      }
+    }
+    write(reader.blank, output);
+    write(reader.wblank, output);
+    if(unmerging) {
+      // Just output the last reading (surface form), removing one level of escaping
+      StreamReader::Reading &lastReading = reader.readings.back(); // (we know there's at least one because of the above loop)
+      UString surface;
+      bool escaping = false;
+      for(auto &c : lastReading.content) {
+        if(escaping) {
+          surface += c;
+          escaping = false;
+        }
+        else if(c == u'\\') {
+          escaping = true;
+        }
+        else {
+          surface += c;
+        }
+      }
+      write(surface, output);
+    }
+    else {
+      if(reader.readings.size() > 0) {
+        // NB. ^$ will produce a readings vector of length 1 where the single item is empty. EOF should give length 0.
+        // (We *want* to keep ^$ in stream, but not print extra ^$ when there was no ^$)
+        u_fputc('^', output);
+        bool seen_reading = false;
+        for (StreamReader::Reading &it : reader.readings) {
+          if (seen_reading) {
+            u_fputc('/', output);
+          }
+          write(it.content, output);
+          seen_reading = true;
+        }
+        u_fputc('$', output);
+      }
+    }
+    if(reader.at_null) {
+      u_fputc('\0', output);
+      u_fflush(output);
+    }
+  }
+}
+
 bool
 FSTProcessor::valid() const
 {

lttoolbox/fst_processor.h

@@ -506,6 +506,7 @@ class FSTProcessor
   UString biltransfull(UStringView input_word, bool with_delim = true);
   void bilingual(InputFile& input, UFILE *output, GenerationMode mode = gm_unknown);
   void quoteMerge(InputFile& input, UFILE *output);
+  void quoteUnmerge(InputFile& input, UFILE *output);
   std::pair<UString, int> biltransWithQueue(UStringView input_word, bool with_delim = true);
   UString biltransWithoutQueue(UStringView input_word, bool with_delim = true);
   void SAO(InputFile& input, UFILE *output);

lttoolbox/lt_merge.cc

@@ -42,7 +42,12 @@ int main(int argc, char *argv[])
   FSTProcessor fstp;
   fstp.setNullFlush(true); // cf. description of cli["null-flush"]
   fstp.initBiltrans();
-  fstp.quoteMerge(input, output);
+  if(unmerge) {
+    fstp.quoteUnmerge(input, output);
+  }
+  else {
+    fstp.quoteMerge(input, output);
+  }
 
   return 0;
 }

tests/lt_merge/__init__.py

@@ -40,3 +40,17 @@ class WordblankTest(MergeTest):
     # Using r'' to avoid doubling escapes even more:
     inputs = [r'^«/«<lquot><MERGE_BEG>$[[tf:i:a]]^ve\/ldig/v<adv>$[[/]]^»/»<rquot><MERGE_END>$']
     expectedOutputs = [r'^«\[\[tf:i:a\]\]ve\\\/ldig\[\[\/\]\]»/«\[\[tf:i:a\]\]ve\\\/ldig\[\[\/\]\]»<MERGED>$']
+
+
+class SimpleUnmergeTest(MergeTest):
+    procflags = ['--unmerge']
+    # Using r'' to avoid doubling escapes even more:
+    inputs = [r'^ikkje<adv>/ikkje$ ^«Se og Hør»<MERGED>/«Se og Hør»$ ^då<adv>/då$']
+    expectedOutputs = [r'^ikkje<adv>/ikkje$ «Se og Hør» ^då<adv>/då$']
+
+
+class EscapedUnmergeTest(MergeTest):
+    procflags = ['--unmerge']
+    # Using r'' to avoid doubling escapes even more:
+    inputs = [r'^ikkje<adv>/ikkje$ ^«\[\[tf:i:a\]\]s\\\^å\[\[\/\]\]»<MERGED>/«\[\[tf:i:a\]\]s\\\^å\[\[\/\]\]»$']
+    expectedOutputs = [r'^ikkje<adv>/ikkje$ «[[tf:i:a]]s\^å[[/]]»']

tests/run_tests.py

@@ -13,8 +13,6 @@
            'lt_paradigm', 'lt_expand', 'lt_apply_acx', 'lt_compose',
            'lt_tmxproc', 'lt_merge']
 
-# modules = ['lt_merge']
-
 
 if __name__ == "__main__":
     os.chdir(os.path.dirname(__file__))