Merge pull request #11 from edma2/develop

Version 1.1.0

README.md

@@ -2,7 +2,7 @@
 
 clojure-msgpack is a lightweight and simple library for converting
 between native Clojure data structures and MessagePack byte formats.
-clojure-msgpack only depends on Clojure itself; it has zero third-party
+clojure-msgpack only depends on Clojure itself; it has no third-party
 dependencies.
 
 ## Installation
@@ -16,64 +16,98 @@ Get it from clojars: https://clojars.org/clojure-msgpack
 * ```pack```: Serialize object as a sequence of java.lang.Bytes.
 * ```unpack``` Deserialize bytes as a Clojure object.
 ```clojure
-user=> (require '[msgpack.core :refer :all])
-nil
+(require '[msgpack.core :as msg])
+(require 'msgpack.clojure-extensions)
 
-user=> (pack {:compact true :schema 0})
-(-126 -90 115 99 104 101 109 97 0 -89 99 111 109 112 97 99 116 -61)
+(msg/pack {:compact true :schema 0})
+; => #<byte[] [B@60280b2e>
 
-user=> (unpack (pack {:compact true :schema 0}))
-{"schema" 0, "compact" true}
+(msg/unpack (msg/pack {:compact true :schema 0}))
+; => {:schema 0, :compact true}
 `````
 
 ### Streaming:
 * ```unpack-stream```: Takes a [java.io.DataInput](http://docs.oracle.com/javase/7/docs/api/java/io/DataInput.html) as an argument. Usually you wrap this around an [InputStream](http://docs.oracle.com/javase/7/docs/api/java/io/InputStream.html)
 * ```pack-stream```: Takes a [java.io.DataOutput](http://docs.oracle.com/javase/7/docs/api/java/io/DataOutput.html) as an argument. Usually you wrap this around an [OutputStream](http://docs.oracle.com/javase/7/docs/api/java/io/OutputStream.html)
 ```clojure
-user=> (use 'clojure.java.io)
-nil
+(use 'clojure.java.io)
+(import '(java.io.DataOutputStream) '(java.io.DataInputStream))
 
-user=> (import '(java.io.DataOutputStream) '(java.io.DataInputStream))
-nil
+(with-open [o (output-stream "test.dat")]
+  (let [data-output (java.io.DataOutputStream. o)]
+    (msg/pack-stream {:compact true :schema 0} data-output)))
 
-user=> (with-open [o (output-stream "test.dat")]
-  (let [data-output (DataOutputStream. o)]
-    (pack-stream {:compact true :schema 0} data-output)))
-nil
+(with-open [i (input-stream "test.dat")]
+  (let [data-input (java.io.DataInputStream. i)]
+    (msg/unpack-stream data-input)))
+; => {:schema 0, :compact true}
+```
+
+### Core types:
+
+Clojure			    | MessagePack
+----------------------------|------------
+nil			    | nil
+java.lang.Boolean	    | Boolean
+java.lang.Float		    | Float
+java.lang.Double	    | Float
+java.math.BigDecimal	    | Float
+java.lang.Number	    | Integer
+java.lang.String	    | String
+clojure.lang.Sequential	    | Array
+clojure.lang.IPersistentMap | Map
+msgpack.core.Ext	    | Extended
+
+### Clojure Extended types:
+Some native Clojure types don't have an obvious MessagePack counterpart. We can
+serialize them as Extended types. To enable automatic conversion of these
+types, load the `clojure-extensions` library.
+
+Clojure			    | MessagePack
+----------------------------|------------
+clojure.lang.Keyword	    | Extended (type = 3)
+clojure.lang.Symbol	    | Extended (type = 4)
+java.lang.Character	    | Extended (type = 5)
+clojure.lang.Ratio	    | Extended (type = 6)
+clojure.lang.IPersistentSet | Extended (type = 7)
+
+With `msgpack.clojure-extensions`:
+```clojure
+(require 'msgpack.clojure-extensions)
+(msg/pack :hello)
+; => #<byte[] [B@a8c55bf>
+```
 
-user=> (with-open [i (input-stream "test.dat")]
-  (let [data-input (DataInputStream. i)]
-    (unpack-stream data-input)))
-{"schema" 0, "compact" true}
+Without `msgpack.clojure-extensions`:
+```clojure
+(msg/pack :hello)
+; => IllegalArgumentException No implementation of method: :pack-stream of
+; protocol: #'msgpack.core/Packable found for class: clojure.lang.Keyword
+; clojure.core/-cache-protocol-fn (core _deftype.clj:544)
 ```
 
-### User-defined extensions:
-A macro ```defext``` is provided to allow serialization of application-specific types. Currently this only works one-way; data serialized this way will always deserialize as a raw ```Extended``` type.
+### Application Extended types:
+You can also define your own Extended types with `extend-msgpack`.
+
 ```clojure
-user=> (require '[msgpack.macros :refer [defext]])
-nil
+(require '[msgpack.macros :refer [extend-msgpack]])
 
-user=> (defrecord Employee [name])
-user.Employee
+(defrecord Person [name])
 
-user=> (defext Employee 1 #(.getBytes (:name %)))
-nil
+(extend-msgpack
+  Person
+  100
+  [p] (.getBytes (:name p))
+  [bytes] (->Person (String. bytes)))
 
-user=> (let [bob (Employee. "Bob")
-             bytes (pack bob)]
-         (map #(format "0x%x" %) bytes))
-("0xc7" "0x3" "0x1" "0x42" "0x6f" "0x62")
+(msg/unpack (msg/pack [(->Person "bob") 5 "test"]))
+; => (#user.Person{:name "bob"} 5 "test")
 ```
 
-### Extras:
-* Symbols and Keywords are treated as MessagePack strings.
-* Sets are treated as MessagePack arrays.
-
 ## TODO
 * Error checking
 * Compatibility mode
 * Benchmarks
 
 ## License
-
 clojure-msgpack is MIT licensed. See the included LICENSE file for more details.

src/msgpack/clojure_extensions.clj

@@ -0,0 +1,41 @@
+(ns msgpack.clojure-extensions
+  "Extended types for Clojure-specific types"
+  (:require [msgpack.core :as msg]
+            [msgpack.macros :refer [extend-msgpack]]))
+
+(defn- keyword->str
+  "Convert keyword to string with namespace preserved.
+  Example: :A/A => \"A/A\""
+  [k]
+  (subs (str k) 1))
+
+(extend-msgpack
+ clojure.lang.Keyword
+ 3
+ [k] (msg/pack (keyword->str k))
+ [bytes] (keyword (msg/unpack bytes)))
+
+(extend-msgpack
+ clojure.lang.Symbol
+ 4
+ [s] (msg/pack (str s))
+ [bytes] (symbol (msg/unpack bytes)))
+
+(extend-msgpack
+ java.lang.Character
+ 5
+ [c] (msg/pack (str c))
+ [bytes] (first (char-array (msg/unpack bytes))))
+
+(extend-msgpack
+ clojure.lang.Ratio
+ 6
+ [r] (msg/pack [(numerator r) (denominator r)])
+ [bytes] (let [[n d] (msg/unpack bytes)]
+           (/ n d)))
+
+(extend-msgpack
+ clojure.lang.IPersistentSet
+ 7
+ [s] (msg/pack (seq s))
+ [bytes] (set (msg/unpack bytes)))

src/msgpack/core.clj

@@ -5,15 +5,17 @@
            java.io.ByteArrayInputStream
            java.nio.charset.Charset))
 
+(declare pack unpack unpack-stream)
+
 (defprotocol Packable
   "Objects that can be serialized as MessagePack types"
   (pack-stream [this data-output]))
 
 ;; MessagePack allows applications to define application-specific types using
-;; the Extension type. Extension type consists of an integer and a byte array
+;; the Extended type. Extended type consists of an integer and a byte array
 ;; where the integer represents a kind of types and the byte array represents
 ;; data.
-(defrecord Extension [type data])
+(defrecord Ext [type data])
 
 (defmacro cond-let [bindings & clauses]
   `(let ~bindings (cond ~@clauses)))
@@ -30,8 +32,8 @@
             (<= len 0xffffffff)
             (do (.writeByte s 0xc6) (.writeInt s len) (.write s bytes))))
 
-(defn- pack-number
-  "Pack n using the most compact representation"
+(defn- pack-int
+  "Pack integer using the most compact representation"
   [n ^java.io.DataOutput s]
   (cond
     ; +fixnum
@@ -56,7 +58,7 @@
     (<= -0x8000000000000000 n -1) (do (.writeByte s 0xd3) (.writeLong s n))))
 
 (defn- pack-float
-  "Pack f using the most compact representation"
+  "Pack float using the most compact representation"
   [f ^java.io.DataOutput s]
   (if (<= f Float/MAX_VALUE)
     (do (.writeByte s 0xca) (.writeFloat s f))
@@ -79,31 +81,17 @@
       (.writeByte s 0xc3)
       (.writeByte s 0xc2)))
 
-  Byte
-  (pack-stream [n ^java.io.DataOutput s] (pack-number n s))
-
-  Short
-  (pack-stream [n ^java.io.DataOutput s] (pack-number n s))
-
-  Integer
-  (pack-stream [n ^java.io.DataOutput s] (pack-number n s))
-
-  Long
-  (pack-stream [n ^java.io.DataOutput s] (pack-number n s))
-
-  clojure.lang.BigInt
-  (pack-stream [n ^java.io.DataOutput s] (pack-number n s))
-
   Float
   (pack-stream [f ^java.io.DataOutput s] (pack-float f s))
 
   Double
   (pack-stream [d ^java.io.DataOutput s] (pack-float d s))
 
-  clojure.lang.Ratio
-  (pack-stream
-    [r ^java.io.DataOutput s]
-    (pack-stream (double r) s))
+  java.math.BigDecimal
+  (pack-stream [d ^java.io.DataOutput s] (pack-float d s))
+
+  Number
+  (pack-stream [n ^java.io.DataOutput s] (pack-int n s))
 
   String
   (pack-stream
@@ -122,20 +110,11 @@
               (<= len 0xffffffff)
               (do (.writeByte s 0xdb) (.writeInt s len) (.write s bytes))))
 
-  Character
-  (pack-stream [c ^java.io.DataOutput s] (pack-stream (str c) s))
-
-  clojure.lang.Keyword
-  (pack-stream [kw ^java.io.DataOutput s] (pack-stream (name kw) s))
-
-  clojure.lang.Symbol
-  (pack-stream [sym ^java.io.DataOutput s] (pack-stream (name sym) s))
-
-  Extension
+  Ext
   (pack-stream
     [e ^java.io.DataOutput s]
     (let [type (:type e)
-          data (byte-array (:data e))
+          ^bytes data (:data e)
           len (count data)]
       (do
         (cond
@@ -173,10 +152,7 @@
               (do (.writeByte s 0xde) (.writeShort s len) (pack-coll pairs s))
 
               (<= len 0xffffffff)
-              (do (.writeByte s 0xdf) (.writeInt s len) (pack-coll pairs s))))
-
-  clojure.lang.IPersistentSet
-  (pack-stream [set ^java.io.DataOutput s] (pack-stream (sequence set) s)))
+              (do (.writeByte s 0xdf) (.writeInt s len) (pack-coll pairs s)))))
 
 ; Note: the extensions below are not in extend-protocol above because of
 ; a Clojure bug. See http://dev.clojure.org/jira/browse/CLJ-1381
@@ -196,7 +172,7 @@
         data-output (DataOutputStream. output-stream)]
     (do
       (pack-stream obj data-output)
-      (seq (.toByteArray output-stream)))))
+      (.toByteArray output-stream))))
 
 (defn- read-uint8
   [^java.io.DataInput data-input]
@@ -224,10 +200,15 @@
       (.readFully data-input bytes)
       bytes)))
 
-(defn- unpack-extension [n ^java.io.DataInput data-input]
-  (->Extension (.readByte data-input) (seq (read-bytes n data-input))))
+(defmulti refine-ext
+  "Refine Extended type to an application-specific type."
+  :type)
+
+(defmethod refine-ext :default [ext] ext)
 
-(declare unpack-stream)
+(defn- unpack-ext [n ^java.io.DataInput data-input]
+  (refine-ext
+   (->Ext (.readByte data-input) (read-bytes n data-input))))
 
 (defn- unpack-n [n ^java.io.DataInput data-input]
   (doall (for [_ (range n)] (unpack-stream data-input))))
@@ -289,20 +270,20 @@
             (read-bytes (read-uint32 data-input) data-input)
 
             ; ext format family
-            (= byte 0xd4) (unpack-extension 1 data-input)
-            (= byte 0xd5) (unpack-extension 2 data-input)
-            (= byte 0xd6) (unpack-extension 4 data-input)
-            (= byte 0xd7) (unpack-extension 8 data-input)
-            (= byte 0xd8) (unpack-extension 16 data-input)
+            (= byte 0xd4) (unpack-ext 1 data-input)
+            (= byte 0xd5) (unpack-ext 2 data-input)
+            (= byte 0xd6) (unpack-ext 4 data-input)
+            (= byte 0xd7) (unpack-ext 8 data-input)
+            (= byte 0xd8) (unpack-ext 16 data-input)
 
             (= byte 0xc7)
-            (unpack-extension (read-uint8 data-input) data-input)
+            (unpack-ext (read-uint8 data-input) data-input)
 
             (= byte 0xc8)
-            (unpack-extension (read-uint16 data-input) data-input)
+            (unpack-ext (read-uint16 data-input) data-input)
 
             (= byte 0xc9)
-            (unpack-extension (read-uint32 data-input) data-input)
+            (unpack-ext (read-uint32 data-input) data-input)
 
             ; array format family
             (= (bit-and 2r11110000 byte) 2r10010000)

src/msgpack/macros.clj

@@ -1,29 +1,17 @@
 (ns msgpack.macros
+  "Macros for extending MessagePack with Extended types.
+  See msgpack.clojure-extensions for examples."
   (:require [msgpack.core :refer :all]))
 
-(defmacro defext
-  "Treat an existing class as a MessagePack extended type.
-  As a side-effect, the class will extend the Packable protocol.
-  Provide a type (non-negative signed integer) and a function that
-  converts the class to a sequence of bytes.
-
-  (defrecord Employee [name])
-  (defext Employee 1 #(.getBytes (:name %)))
-
-  expands into:
-
-  (extend-protocol Packable
-    Employee
-    (pack-stream [obj stream]
-      (pack-stream (->Extension 1 (.getBytes (:name obj))) stream)))
-
-  and this will work:
-  (pack (Employee. name))"
-  [class type f]
+(defmacro extend-msgpack
+  [class type pack-args pack unpack-args unpack]
   `(let [type# ~type]
      (assert (<= 0 type# 127)
              "[-1, -128]: reserved for future pre-defined extensions.")
-     (extend-protocol Packable
-       ~class
-       (pack-stream [obj# stream#]
-         (pack-stream (->Extension type# (~f obj#)) stream#)))))
+     (do
+       (extend-protocol Packable ~class
+                        (pack-stream [~@pack-args ^java.io.DataOutput s#]
+                          (pack-stream (->Ext type# ~pack) s#)))
+       (defmethod refine-ext type# [ext#]
+         (let [~@unpack-args (:data ext#)]
+           ~unpack)))))