test

teamtnt · Jul 3, 2016 · 05ca28d · 05ca28d
1 parent 845d804
commit 05ca28d
Showing 1 changed file with 5 additions and 1 deletion.
diff --git a/tests/indexer/IndexerTest.php b/tests/indexer/IndexerTest.php
@@ -112,12 +112,16 @@ public function testSetTokenizer()
         $indexer->setTokenizer($someTokenizer);
 
         $this->assertInstanceOf(TokenizerInterface::class, $indexer->tokenizer);
+
+        $res  = $indexer->breakIntoTokens('Canon 70-200');
+        $this->assertContains("canon", $res);
+        $this->assertContains("70-200", $res);
     }
 }
 
 class SomeTokenizer implements TokenizerInterface {
 
     public function tokenize($text) {
-        return $text;
+        return preg_split("/[^\p{L}\p{N}-]+/u", strtolower($text), -1, PREG_SPLIT_NO_EMPTY);
     }
 }