Multi delimiter support

Readme.md

@@ -81,7 +81,7 @@ Default is `!!]`.
     pseudoloc.str('A test string with a %token%.')
     // [!!Á ţȇšŧ śťřīņğ ŵıţħ ą %token%.##]
 
-#### Delimiter, StartDelimiter, EndDelimiter
+#### Delimiter, StartDelimiter, EndDelimiter, Delimiters
 
 Specifies the token delimiter. Any characters between token delimiters will not be pseudolocalized. Tokens are used to replace data within localized strings. You can either specify a single delimiter or use startDelimiter and endDelimiter to specify the delimiters seperately.
 
@@ -96,6 +96,39 @@ Default is `%`.
     pseudoloc.str('A test string with a {{token}}.')
     // [!!Á ţȇšŧ śťřīņğ ŵıţħ ą {{token}}.!!]
 
+If you need to support multiple types of delimiters, you can pass an array of delimiters (single or pairs) to the `delimiters` option.
+
+The `delimiters` option takes an array of objects. Set properties on the objects as follows:
+
+  * `{ start, end }`: specifies a pair of start and end delimiters, just like using `startDelimiter` and `endDelimiter`:
+    ```
+    { start: '<', end: '>' }
+    ```
+
+  * `{ both }`: specifies a marker to use as both the start and end delimiters, just like using `delimiter`
+    ```
+    { both: '$$' }
+    ```
+
+  * `{ full }`: specifies the entire pattern for the delimiter. This is useful for cases where the token doesn't have a start marker, name, and end marker, for example with printf-style placeholders `%s`, `%d`, etc.
+    ```
+    { full: '%d' }
+    ```
+
+Under the hood these strings are combined into a pattern that eventually is compiled into a RegExp. That can affect you in a couple of ways:
+
+  1. You can use regular expression matchers in your delimiters
+  2. If your delimiter includes any characters that are special characters in regular expressions, they will need to be escaped
+
+For example, to match named sprintf-style placeholders (such as `%(name)s`), you need to escape the parentheses:
+
+    // Note the double-backslash, which becomes a `\(` in the string
+    pseudoloc.option.startDelimiter = '%\\(';
+    // Note the square brackets, so it matches `)s` or `)d`
+    pseudoloc.option.endDelimiter = '\\)[sd]';
+    pseudoloc.str('A test string with a %(token)s.');
+    // [!!Á ţȇšŧ śťřīņğ ŵıţħ ą %(token)s.!!]
+
 #### Extend
 
 Extends the width of the string by the specified percentage. Useful if you will be localizing into languages such as German which can be 30% longer than English.

pseudoloc.js

@@ -73,7 +73,23 @@ pseudoloc = function() {
     return pStr;
   };
   pseudoloc.str = function(str) {
-    var opts = pseudoloc.option, startdelim = opts.startDelimiter || opts.delimiter, enddelim = opts.endDelimiter || opts.delimiter, re = new RegExp(startdelim + "\\s*[\\w\\.\\s*]+\\s*" + enddelim, "g"), m, tokens = [], i = 0, tokenIdx = 0, result = "", c, pc;
+    function makeTokenRegex(delims, tokenNameDelim) {
+      var tokenMatchers = delims.reduce(function(result, delim) {
+        if (delim.hasOwnProperty("both")) {
+          result.push(delim.both + tokenNameDelim + delim.both);
+        } else if (delim.hasOwnProperty("start") && delim.hasOwnProperty("end")) {
+          result.push(delim.start + tokenNameDelim + delim.end);
+        } else if (delim.hasOwnProperty("full")) {
+          result.push(delim.full);
+        }
+        return result;
+      }, []);
+      return new RegExp(tokenMatchers.join("|"), "g");
+    }
+    var opts = pseudoloc.option, tokenNameDelim = "\\s*[\\w\\.\\s*]+\\s*", startdelim = opts.startDelimiter || opts.delimiter, enddelim = opts.endDelimiter || opts.delimiter, delims = opts.delimiters || [ {
+      start: startdelim,
+      end: enddelim
+    } ], re = makeTokenRegex(delims, tokenNameDelim), m, tokens = [], i = 0, tokenIdx = 0, result = "", c, pc;
     while (m = re.exec(str)) {
       tokens.push(m);
     }

src/core/str.js

@@ -10,10 +10,26 @@
 *     http://bunkat.github.com/pseudoloc
 */
 pseudoloc.str = function(str) {
+  function makeTokenRegex(delims, tokenNameDelim) {
+    var tokenMatchers = delims.reduce(function(result, delim) {
+      if (delim.hasOwnProperty('both')) {
+        result.push(delim.both + tokenNameDelim + delim.both);
+      } else if (delim.hasOwnProperty('start') && delim.hasOwnProperty('end')) {
+        result.push(delim.start + tokenNameDelim + delim.end);
+      } else if (delim.hasOwnProperty('full')) {
+        result.push(delim.full);
+      }
+      return result;
+    }, []);
+    return new RegExp(tokenMatchers.join('|'), 'g');
+  }
+
   var opts = pseudoloc.option,
+      tokenNameDelim = '\\s*[\\w\\.\\s*]+\\s*',
       startdelim = opts.startDelimiter || opts.delimiter,
       enddelim = opts.endDelimiter || opts.delimiter,
-      re = new RegExp(startdelim + '\\s*[\\w\\.\\s*]+\\s*' + enddelim, 'g'),
+      delims = opts.delimiters || [{ start: startdelim, end: enddelim }],
+      re = makeTokenRegex(delims, tokenNameDelim),
       m, tokens = [], i = 0, tokenIdx = 0, result = '', c, pc;
 
   while((m = re.exec(str))) {
@@ -41,4 +57,4 @@ pseudoloc.str = function(str) {
   }
 
   return opts.prepend + pseudoloc.pad(result, opts.extend) + opts.append;
-};
+};

test/str-test.js

@@ -61,6 +61,56 @@ describe('pseudoloc.str', function() {
 
         s1.indexOf('%%this%%').should.not.eql(-1);
     });
+
+    it('should support multiple delimiter pairs', function() {
+        pseudoloc.option.delimiters = [
+            { both: '%%' },
+            { full: '%[sd]' },
+            { start: '%\\(', end: '\\)[sd]' },
+            { start: '<\\/?', end: '>' },
+            { start: '<', end: '\\/>' },
+            { start: '{{', end: '}}' }
+        ];
+        var s1 = pseudoloc.str('%%value%%');
+        s1.indexOf('%%value%%').should.not.eql(-1);
+
+        var s2 = pseudoloc.str('%d files');
+        s2.indexOf('%d').should.not.eql(-1);
+        s2.indexOf('files').should.eql(-1);
+
+        var s3 = pseudoloc.str('%s files');
+        s3.indexOf('%s').should.not.eql(-1);
+        s3.indexOf('files').should.eql(-1);
+
+        var s4 = pseudoloc.str('Hello %(userName)s!');
+        s4.indexOf('%(userName)s').should.not.eql(-1);
+        s4.indexOf('Hello').should.eql(-1);
+
+        var s5 = pseudoloc.str('%(count)d tacos!');
+        s5.indexOf('%(count)d').should.not.eql(-1);
+        s5.indexOf('tacos').should.eql(-1);
+
+        var s6 = pseudoloc.str('this is <b>bold</b> text');
+        s6.indexOf('<b>').should.not.eql(-1);
+        s6.indexOf('</b>').should.not.eql(-1);
+        s6.indexOf('bold').should.eql(-1);
+
+        var s7 = pseudoloc.str('this is <MyCoolTag/> stuff');
+        s7.indexOf('<MyCoolTag/>').should.not.eql(-1);
+        s7.indexOf('stuff').should.eql(-1);
+
+        var s8 = pseudoloc.str('remove %(user)s from <ProjectPicker/> <span>on</span> <b>%(date)s</b>');
+        s8.indexOf('%(user)s').should.not.eql(-1);
+        s8.indexOf('<ProjectPicker/>').should.not.eql(-1);
+        s8.indexOf('<span>').should.not.eql(-1);
+        s8.indexOf('</span>').should.not.eql(-1);
+        s8.indexOf('<b>').should.not.eql(-1);
+        s8.indexOf('%(date)s').should.not.eql(-1);
+        s8.indexOf('</b>').should.not.eql(-1);
+        s8.indexOf('remove').should.eql(-1);
+        s8.indexOf('from').should.eql(-1);
+        s8.indexOf('on').should.eql(-1);
+    });
 
     it('should pad the string by the specified pad amount', function() {
         pseudoloc.option.extend = 0.2;
@@ -93,4 +143,4 @@ describe('pseudoloc.str', function() {
         s1.should.eql('_____________________');
     });
 
-});
+});