Initial commit

.gitignore

@@ -0,0 +1,4 @@
+/dist-newstyle/
+/dist/
+*~
+.ghc.environment.*

ChangeLog.md

@@ -0,0 +1,5 @@
+# Revision history for `text-short`
+
+## 0.1
+
+* First version. Released on an unsuspecting world.

LICENSE

@@ -0,0 +1,30 @@
+Copyright (c) 2017, Herbert Valerio Riedel
+
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+    * Redistributions of source code must retain the above copyright
+      notice, this list of conditions and the following disclaimer.
+
+    * Redistributions in binary form must reproduce the above
+      copyright notice, this list of conditions and the following
+      disclaimer in the documentation and/or other materials provided
+      with the distribution.
+
+    * Neither the name of Herbert Valerio Riedel nor the names of other
+      contributors may be used to endorse or promote products derived
+      from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

Setup.hs

@@ -0,0 +1,2 @@
+import Distribution.Simple
+main = defaultMain

cabal.project

@@ -0,0 +1,2 @@
+packages: .
+tests: True

cbits/cbits.c

@@ -0,0 +1,141 @@
+/*
+ * Copyright (c) 2017, Herbert Valerio Riedel
+ *  
+ * All rights reserved.
+ *  
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *  
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *  
+ *     * Redistributions in binary form must reproduce the above
+ *       copyright notice, this list of conditions and the following
+ *       disclaimer in the documentation and/or other materials provided
+ *       with the distribution.
+ *  
+ *     * Neither the name of Herbert Valerio Riedel nor the names of other
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *  
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <stdint.h>
+#include <string.h>
+#include <assert.h>
+
+/* Count number of code-points in well-formed utf8 string */
+size_t
+hs_text_short_length(const uint8_t buf[], const size_t n)
+{
+  size_t j,l = 0;
+  for (j = 0; j < n; j++)
+    if ((buf[j] & 0xc0) != 0x80)
+      l++;
+
+  return l;
+}
+
+/* Validate UTF8 encoding
+
+ 7 bits | 0xxxxxxx
+11 bits | 110yyyyx  10xxxxxx 
+16 bits | 1110yyyy  10yxxxxx  10xxxxxx 
+21 bits | 11110yyy  10yyxxxx  10xxxxxx  10xxxxxx 
+
+Valid code-points:
+
+ [U+0000 .. U+D7FF] + [U+E000 .. U+10FFFF]
+
+Return values:
+
+ 0 -> ok
+ 1 -> invalid byte/code-point
+ 2 -> truncated
+
+*/
+
+int
+hs_text_short_is_valid_utf8(const uint8_t buf[], const size_t n)
+{
+  size_t j = 0;
+
+  while (j < n) {
+    const uint8_t b0 = buf[j++];
+
+    if (!(b0 & 0x80))
+      continue;
+
+    if ((b0 & 0xe0) == 0xc0) {
+      if (!(b0 & 0x1e)) return 1; /* denorm */
+      if (j >= n) return 2;
+
+      /* b1 */
+      if ((buf[j++] & 0xc0) != 0x80) return 1;
+      continue;
+    }
+
+    if ((b0 & 0xf0) == 0xe0) {
+      if ((j+1) >= n) return 2;
+
+      const uint8_t b1 = buf[j++];
+      if ((b1 & 0xc0) != 0x80) return 1;
+      if (!((b0 & 0x0f) | (b1 & 0x20))) return 1; /* denorm */
+      /* UTF16 Surrogate pairs [U+D800 .. U+DFFF] */
+      if ((b0 == 0xed) && (b1 & 0x20)) return 1;
+
+      /* b2 */
+      if ((buf[j++] & 0xc0) != 0x80) return 1;
+
+      continue;
+    }
+
+    if ((b0 & 0xf8) == 0xf0) {
+      if ((j+2) >= n) return 2;
+
+      const uint8_t b1 = buf[j++];
+      if ((b1 & 0xc0) != 0x80) return 1;
+      if (!((b0 & 0x07) | (b1 & 0x30))) return 1; /* denorm */
+      /* make sure we're below U+10FFFF */
+      if (b0 > 0xf4) return 1;
+      if ((b0 == 0xf4) && (b1 & 0x30)) return 1;
+
+      /* b2 */
+      if ((buf[j++] & 0xc0) != 0x80) return 1;
+      /* b3 */
+      if ((buf[j++] & 0xc0) != 0x80) return 1;
+
+      continue;
+    }
+
+    return 1;
+  }
+
+  assert(j == n);
+
+  return 0;
+}
+
+
+/* Test whether well-formed UTF8 string contains only ASCII code-points
+ * Returns length of longest ASCII-code-point prefix.
+ */
+size_t
+hs_text_short_is_ascii(const uint8_t buf[], const size_t n)
+{
+  size_t j;
+  for (j = 0; j < n; j++)
+    if (buf[j] & 0x80)
+      return j;
+}

src-test/Tests.hs

@@ -0,0 +1,46 @@
+{-# LANGUAGE OverloadedStrings #-}
+
+module Main(main) where
+
+import qualified Data.Text.Short as IUT
+import Test.Tasty
+import Test.Tasty.QuickCheck as QC
+import Test.Tasty.HUnit
+import Test.QuickCheck.Instances ()
+import qualified Data.Text as T
+import qualified Data.Text.Encoding as T
+import qualified Data.String as D.S
+import Data.Binary
+import Data.Char
+import Data.Monoid
+
+fromByteStringRef = either (const Nothing) (Just . IUT.fromText) . T.decodeUtf8'
+
+main :: IO ()
+main = defaultMain tests
+
+tests :: TestTree
+tests = testGroup "Tests" [unitTests,qcProps]
+
+qcProps :: TestTree
+qcProps = testGroup "Properties"
+  [ QC.testProperty "length/fromText"   $ \t -> IUT.length (IUT.fromText t) == T.length t
+  , QC.testProperty "length/fromString" $ \s -> IUT.length (IUT.fromString s) == length s
+  , QC.testProperty "toText.fromText"   $ \t -> (IUT.toText . IUT.fromText) t == t
+  , QC.testProperty "fromByteString"    $ \b -> IUT.fromByteString b == fromByteStringRef b
+  , QC.testProperty "fromByteString.toByteString" $ \t -> let ts = IUT.fromText t in (IUT.fromByteString . IUT.toByteString) ts == Just ts
+  , QC.testProperty "toString.fromString" $ \s -> (IUT.toString . IUT.fromString) s == s
+  , QC.testProperty "isAscii"  $ \s -> IUT.isAscii (IUT.fromString s) == all isAscii s
+  , QC.testProperty "isAscii2" $ \t -> IUT.isAscii (IUT.fromText t)   == T.all isAscii t
+  ]
+
+unitTests = testGroup "Unit-tests"
+  [ testCase "fromText mempty" $ IUT.fromText mempty @?= mempty
+  , testCase "fromShortByteString [0xc0,0x80]" $ IUT.fromShortByteString "\xc0\x80" @?= Nothing
+  , testCase "fromByteString [0xc0,0x80]" $ IUT.fromByteString "\xc0\x80" @?= Nothing
+  , testCase "IsString U+D800" $ "\xFFFD" @?= (IUT.fromString "\xD800")
+--  , testCase "IsString U+D800" $ (IUT.fromString "\xD800") @?= IUT.fromText ("\xD800" :: T.Text)
+
+  , testCase "Binary.encode" $ encode ("Hello \8364 & \171581!\NUL" :: IUT.ShortText) @?= "\NUL\NUL\NUL\NUL\NUL\NUL\NUL\DC2Hello \226\130\172 & \240\169\184\189!\NUL"
+  , testCase "Binary.decode" $ decode ("\NUL\NUL\NUL\NUL\NUL\NUL\NUL\DC2Hello \226\130\172 & \240\169\184\189!\NUL") @?= ("Hello \8364 & \171581!\NUL" :: IUT.ShortText)
+  ]