diff --git a/lib/ruby/site_ruby/1.8/rbyaml.rb b/lib/ruby/site_ruby/1.8/rbyaml.rb new file mode 100644 index 00000000000..067783c059b --- /dev/null +++ b/lib/ruby/site_ruby/1.8/rbyaml.rb @@ -0,0 +1,378 @@ + +require 'rbyaml/yaml' + +module RbYAML + def self.dump(obj, io = nil) + _dump(obj,io) + end + + def self.load( io ) + _load(io) + end + + def self.load_file( filepath ) + File.open( filepath ) do |f| + load( f ) + end + end + + # this operation does not make sense in RbYAML (right now) + def self.parse( io ) + # yp = @@parser.new( :Model => :Generic ).load( io ) + end + + # this operation does not make sense in RbYAML (right now) + def self.parse_file( filepath ) + # File.open( filepath ) do |f| + # parse( f ) + # end + end + + def self.each_document( io, &block ) + _load_all(io,&block) + end + + def self.load_documents( io, &doc_proc ) + each_document( io, &doc_proc ) + end + + # this operation does not make sense in RbYAML (right now) + def self.each_node( io, &doc_proc ) + # yp = @@parser.new( :Model => :Generic ).load_documents( io, &doc_proc ) + end + + # this operation does not make sense in RbYAML (right now) + def self.parse_documents( io, &doc_proc ) + # YAML.each_node( io, &doc_proc ) + end + + def self.load_stream( io ) + d = nil + load_documents(io) { |doc| + d = Stream.new( nil ) if not d + d.add( doc ) + } + d + end + + def self.dump_stream( *objs ) + d = RbYAML::Stream.new + objs.each do |doc| + d.add( doc ) + end + d.emit + end + + # this operation does not make sense in RbYAML (right now) + def self.add_domain_type( domain, type_re, &transfer_proc ) + # @@loader.add_domain_type( domain, type_re, &transfer_proc ) + end + + # this operation does not make sense in RbYAML (right now) + def self.add_builtin_type( type_re, &transfer_proc ) + # @@loader.add_builtin_type( type_re, &transfer_proc ) + end + + # this operation does not make sense in RbYAML (right now) + def self.add_ruby_type( type_tag, &transfer_proc ) + # @@loader.add_ruby_type( type, &transfer_proc ) + end + + # this operation does not make sense in RbYAML (right now) + def self.add_private_type( type_re, &transfer_proc ) + # @@loader.add_private_type( type_re, &transfer_proc ) + end + + def self.detect_implicit( val ) + SimpleDetector.detect(val) + end + + # this operation does not make sense in RbYAML (right now) + def self.transfer( type_id, obj ) + # @@loader.transfer( type_id, obj ) + end + + # this operation does not make sense in RbYAML (right now) + def self.try_implicit( obj ) + # YAML.transfer( YAML.detect_implicit( obj ), obj ) + end + + def self.read_type_class( type, obj_class ) + scheme, domain, type, tclass = type.split( ':', 4 ) + tclass.split( "::" ).each { |c| obj_class = obj_class.const_get( c ) } if tclass + return [ type, obj_class ] + end + + def self.object_maker( obj_class, val ) + if Hash === val + o = obj_class.allocate + val.each_pair { |k,v| + o.instance_variable_set("@#{k}", v) + } + o + else + raise YAMLError, "Invalid object explicitly tagged !ruby/Object: " + val.inspect + end + end + + # this operation does not make sense in RbYAML (right now) + def self.quick_emit( oid, opts = {}, &e ) + end + + # A dictionary of taguris which map to + # Ruby classes. + @@tagged_classes = {} + + # + # Associates a taguri _tag_ with a Ruby class _cls_. The taguri is used to give types + # to classes when loading YAML. Taguris are of the form: + # + # tag:authorityName,date:specific + # + # The +authorityName+ is a domain name or email address. The +date+ is the date the type + # was issued in YYYY or YYYY-MM or YYYY-MM-DD format. The +specific+ is a name for + # the type being added. + # + # For example, built-in YAML types have 'yaml.org' as the +authorityName+ and '2002' as the + # +date+. The +specific+ is simply the name of the type: + # + # tag:yaml.org,2002:int + # tag:yaml.org,2002:float + # tag:yaml.org,2002:timestamp + # + # The domain must be owned by you on the +date+ declared. If you don't own any domains on the + # date you declare the type, you can simply use an e-mail address. + # + # tag:why@ruby-lang.org,2004:notes/personal + # + def self.tag_class( tag, cls ) + if @@tagged_classes.has_key? tag + warn "class #{ @@tagged_classes[tag] } held ownership of the #{ tag } tag" + end + @@tagged_classes[tag] = cls + end + + # Returns the complete dictionary of taguris, paired with classes. The key for + # the dictionary is the full taguri. The value for each key is the class constant + # associated to that taguri. + # + # YAML.tagged_classes["tag:yaml.org,2002:int"] => Integer + # + def self.tagged_classes + @@tagged_classes + end + + # + # RbYAML::Stream -- for emitting many documents + # + class Stream + include Enumerable + + attr_accessor :documents, :options + def initialize(opts = {}) + @options = opts + @documents = [] + end + + def [](i) + @documents[ i ] + end + + def add(doc) + @documents << doc + end + + def edit(doc_num,doc) + @documents[ doc_num ] = doc + end + + def each(&block) + @documents.each(&block) + end + + def emit +# TODO: implement + + opts = @options.dup + opts[:UseHeader] = true if @documents.length > 1 + ct = 0 + out = Emitter.new( opts ) + @documents.each { |v| + if ct > 0 + out << "\n--- " + end + v.to_yaml( :Emitter => out ) + ct += 1 + } + out.end_object + end + end +end + +if !Object.method_defined?(:to_yaml) + class Module # :nodoc: all + def yaml_as( tag, sc = true ) + class_eval <<-"end;", __FILE__, __LINE__+1 + attr_writer :taguri + def taguri + return @taguri if defined?(@taguri) and @taguri + tag = #{ tag.dump } + if self.class.yaml_tag_subclasses? and self.class != RbYAML::tagged_classes[tag] + tag = "\#{ tag }:\#{ self.class.yaml_tag_class_name }" + end + tag + end + def self.yaml_tag_subclasses?; #{ sc ? 'true' : 'false' }; end + end; + RbYAML::tag_class tag, self + end + # Transforms the subclass name into a name suitable for display + # in a subclassed tag. + def yaml_tag_class_name + self.name + end + # Transforms the subclass name found in the tag into a Ruby + # constant name. + def yaml_tag_read_class( name ) + name + end + end + + require 'date' + + class Class + def to_yaml( opts = {} ) + raise RbYAML::TypeError, "can't dump anonymous class %s" % self.class + end + end + + class Object + yaml_as "tag:ruby.yaml.org,2002:object" + def is_complex_yaml?; true; end + def to_yaml_style; end + def to_yaml_properties; instance_variables.sort; end + def to_yaml( opts = {} ) + RbYAML::_dump_ruby_object(self) + end + end + + class Hash + yaml_as "tag:ruby.yaml.org,2002:hash" + yaml_as "tag:yaml.org,2002:map" + def is_complex_yaml?; true; end + def yaml_initialize( tag, val ) + if Array === val + update Hash.[]( *val ) # Convert the map to a sequence + elsif Hash === val + update val + else + raise RbYAML::TypeError, "Invalid map explicitly tagged #{ tag }: " + val.inspect + end + end + end + + class Array + yaml_as "tag:ruby.yaml.org,2002:array" + yaml_as "tag:yaml.org,2002:seq" + def is_complex_yaml?; true; end + def yaml_initialize( tag, val ); concat( val.to_a ); end + end + + class Exception + yaml_as "tag:ruby.yaml.org,2002:exception" + def Exception.yaml_new( klass, tag, val ) + o = RbYAML.object_maker( klass, { 'mesg' => val.delete( 'message' ) } ) + val.each_pair do |k,v| + o.instance_variable_set("@#{k}", v) + end + o + end + end + + class String + yaml_as "tag:ruby.yaml.org,2002:string" + yaml_as "tag:yaml.org,2002:binary" + yaml_as "tag:yaml.org,2002:str" + def is_complex_yaml? + to_yaml_style or not to_yaml_properties.empty? or self =~ /\n.+/ + end + def is_binary_data? + ( self.count( "^ -~", "^\r\n" ) / self.size > 0.3 || self.count( "\x00" ) > 0 ) unless empty? + end + def String.yaml_new( klass, tag, val ) + val = val.unpack("m")[0] if tag == "tag:yaml.org,2002:binary" + val = { 'str' => val } if String === val + if Hash === val + s = klass.allocate + # Thank you, NaHi + String.instance_method(:initialize). + bind(s). + call( val.delete( 'str' ) ) + val.each { |k,v| s.instance_variable_set( k, v ) } + s + else + raise RbYAML::TypeError, "Invalid String: " + val.inspect + end + end + end + + class Symbol + yaml_as "tag:ruby.yaml.org,2002:symbol" + yaml_as "tag:ruby.yaml.org,2002:sym" + def is_complex_yaml?; false; end + def Symbol.yaml_new( klass, tag, val ) + if String === val + val.intern + else + raise RbYAML::TypeError, "Invalid Symbol: " + val.inspect + end + end + end + + class Time + yaml_as "tag:ruby.yaml.org,2002:time" + yaml_as "tag:yaml.org,2002:timestamp" + def is_complex_yaml?; false; end + def Time.yaml_new( klass, tag, val ) + if Hash === val + t = val.delete( 'at' ) + val.each { |k,v| t.instance_variable_set( k, v ) } + t + else + raise RbYAML::TypeError, "Invalid Time: " + val.inspect + end + end + end + + class Date + yaml_as "tag:yaml.org,2002:timestamp#ymd" + def is_complex_yaml?; false; end + end + + class Numeric + def is_complex_yaml?; false; end + end + + class Fixnum + yaml_as "tag:yaml.org,2002:int" + end + + class Float + yaml_as "tag:yaml.org,2002:float" + end + + class TrueClass + yaml_as "tag:yaml.org,2002:bool#yes" + def is_complex_yaml?; false; end + end + + class FalseClass + yaml_as "tag:yaml.org,2002:bool#no" + def is_complex_yaml?; false; end + end + + class NilClass + yaml_as "tag:yaml.org,2002:null" + def is_complex_yaml?; false; end + end +end diff --git a/lib/ruby/site_ruby/1.8/rbyaml/composer.rb b/lib/ruby/site_ruby/1.8/rbyaml/composer.rb new file mode 100644 index 00000000000..02273b7df86 --- /dev/null +++ b/lib/ruby/site_ruby/1.8/rbyaml/composer.rb @@ -0,0 +1,112 @@ +require 'rbyaml/error' +require 'rbyaml/events' +require 'rbyaml/nodes' + +module RbYAML + class ComposerError < MarkedYAMLError + end + + class Composer + def initialize(parser,resolver) + @parser = parser + @resolver = resolver + @anchors = {} + end + + def check_node + !@parser.peek_event.__is_stream_end + end + + def get_node + compose_document if check_node + end + + def each_node + yield compose_document while check_node + end + + def compose_document + # Drop the STREAM-START event. + @parser.get_event if @parser.peek_event.__is_stream_start + # Drop the DOCUMENT-START event. + @parser.get_event + # Compose the root node. + node = compose_node(nil,nil) + # Drop the DOCUMENT-END event. + @parser.get_event + @anchors = {} + node + end + + def compose_node(parent,index) + if @parser.peek_event.__is_alias + event = @parser.get_event + anchor = event.anchor + raise ComposerError.new(nil, nil, "found undefined alias #{anchor}", event.start_mark) if !@anchors.include?(anchor) + return @anchors[anchor] + end + event = @parser.peek_event + anchor = event.anchor + if !anchor.nil? + if @anchors.include?(anchor) + raise ComposerError.new("found duplicate anchor #{anchor}; first occurence", @anchors[anchor].start_mark,"second occurence", event.start_mark) + end + end + @resolver.descend_resolver(parent,index) + if @parser.peek_event.__is_scalar + node = compose_scalar_node(anchor) + elsif @parser.peek_event.__is_sequence_start + node = compose_sequence_node(anchor) + elsif @parser.peek_event.__is_mapping_start + node = compose_mapping_node(anchor) + end + @resolver.ascend_resolver + node + end + + def compose_scalar_node(anchor) + event = @parser.get_event + tag = event.tag + tag = @resolver.resolve(ScalarNode,event.value,event.implicit) if tag.nil? || tag == "!" + node = ScalarNode.new(tag, event.value,event.start_mark, event.end_mark, event.style) + @anchors[anchor] = node if !anchor.nil? + node + end + + def compose_sequence_node(anchor) + start_event = @parser.get_event + tag = start_event.tag + tag = @resolver.resolve(SequenceNode,nil,start_event.implicit) if tag.nil? || tag == "!" + node = SequenceNode.new(tag,[],start_event.start_mark,nil,start_event.flow_style) + @anchors[anchor] = node if !anchor.nil? + index = 0 + while !@parser.peek_event.__is_sequence_end + node.value << compose_node(node,index) + index += 1 + end + end_event = @parser.get_event + node.end_mark = end_event.end_mark + node + end + + def compose_mapping_node(anchor) + start_event = @parser.get_event + tag = start_event.tag + tag = @resolver.resolve(MappingNode,nil,start_event.implicit) if tag.nil? || tag == "!" + node = MappingNode.new(tag, {},start_event.start_mark,nil,start_event.flow_style) + @anchors[anchor] = node if !anchor.nil? + while !@parser.peek_event.__is_mapping_end + key_event = @parser.peek_event + item_key = compose_node(node,nil) + if node.value.include?(item_key) + raise ComposerError.new("while composing a mapping", start_event.start_mark,"found duplicate key", key_event.start_mark) + end + item_value = compose_node(node,item_key) + node.value[item_key] = item_value + end + end_event = @parser.get_event + node.end_mark = end_event.end_mark + node + end + end +end diff --git a/lib/ruby/site_ruby/1.8/rbyaml/constructor.rb b/lib/ruby/site_ruby/1.8/rbyaml/constructor.rb new file mode 100644 index 00000000000..0a2f75dad6c --- /dev/null +++ b/lib/ruby/site_ruby/1.8/rbyaml/constructor.rb @@ -0,0 +1,391 @@ +require 'base64' +require 'set' + +require 'rbyaml/error' +require 'rbyaml/nodes' +require 'rbyaml/composer' + +class Symbol + def __call(obj,*args) + obj.send(self,*args) + end +end + +class Proc + def __call(obj,*args) + call(*args) + end +end + +class Method + def __call(obj,*args) + call(*args) + end +end + + +module RbYAML + class ConstructorError < MarkedYAMLError + end + + class BaseConstructor + @@yaml_constructors = {} + @@yaml_multi_constructors = {} + @@yaml_multi_regexps = {} + + def initialize(composer) + @composer = composer + @constructed_objects = {} + @recursive_objects = {} + end + + def check_data + # If there are more documents available? + @composer.check_node + end + + def get_data + # Construct and return the next document. + construct_document(@composer.get_node) if @composer.check_node + end + + def each_document + # Iterator protocol. + while @composer.check_node + yield construct_document(@composer.get_node) + end + end + + def construct_document(node) + data = construct_object(node) + @constructed_objects = {} + @recursive_objects = {} + data + end + + def construct_object(node) + return @constructed_objects[node] if @constructed_objects.include?(node) + raise ConstructorError.new(nil,nil,"found recursive nod",node.start_mark) if @recursive_objects.include?(node) + @recursive_objects[node] = nil + constructor = @@yaml_constructors[node.tag] + if !constructor + ruby_cls = RbYAML::tagged_classes[node.tag] + if ruby_cls && (ruby_cls.method_defined?(:yaml_initialize) || ruby_cls.respond_to?(:yaml_new)) + constructor = lambda { |node| send(:construct_ruby_object,ruby_cls,node) } + else + through = true + for tag_prefix,reg in @@yaml_multi_regexps + if reg =~ node.tag + tag_suffix = node.tag[tag_prefix.length..-1] + constructor = lambda { |node| @@yaml_multi_constructors[tag_prefix].__call(self,tag_suffix, node) } + through = false + break + end + end + if through + ctor = @@yaml_multi_constructors[nil] || @@yaml_constructors[nil] + if ctor + constructor = lambda { |node| ctor.__call(self,node.tag,node) } + else + constructor = lambda { |node| construct_primitive(node) } + end + end + end + end + data = constructor.__call(self,node) + @constructed_objects[node] = data + @recursive_objects.delete(node) + data + end + + def construct_primitive(node) + if node.__is_scalar + construct_scalar(node) + elsif node.__is_sequence + construct_sequence(node) + elsif node.__is_mapping + construct_mapping(node) + else + puts node.tag + end + end + + def construct_scalar(node) + if !node.__is_scalar + if node.__is_mapping + for key_node in node.value.keys + if key_node.tag == "tag:yaml.org,2002:value" + return construct_scalar(node.value[key_node]) + end + end + end + raise ConstructorError.new(nil, nil,"expected a scalar node, but found #{node.tid}",node.start_mark) + end + node.value + end + + def construct_sequence(node) + raise ConstructorError.new(nil,nil,"expected a sequence node, but found #{node.tid}",node.start_mark) if !node.__is_sequence + node.value.map {|child| construct_object(child) } + end + + def construct_mapping(node) + raise ConstructorError.new(nil,nil,"expected a mapping node, but found #{node.tid}",node.start_mark) if !node.__is_mapping + mapping = {} + merge = nil + for key_node,value_node in node.value + if key_node.tag == "tag:yaml.org,2002:merge" + raise ConstructorError.new("while constructing a mapping", node.start_mark,"found duplicate merge key", key_node.start_mark) if !merge.nil? + if value_node.__is_mapping + merge = [construct_mapping(value_node)] + elsif value_node.__is_sequence + merge = [] + for subnode in value_node.value + if !subnode.__is_mapping + raise ConstructorError.new("while constructing a mapping",node.start_mark,"expected a mapping for merging, but found #{subnode.tid}", subnode.start_mark) + end + merge.unshift(construct_mapping(subnode)) + end + else + raise ConstructorError.new("while constructing a mapping", node.start_mark,"expected a mapping or list of mappings for merging, but found #{value_node.tid}", value_node.start_mark) + end + elsif key_node.tag == "tag:yaml.org,2002:value" + raise ConstructorError.new("while construction a mapping", node.start_mark,"found duplicate value key", key_node.start_mark) if mapping.include?("=") + value = construct_object(value_node) + mapping["="] = value + else + key = construct_object(key_node) +# raise ConstructorError.new("while constructing a mapping", node.start_mark,"found duplicate key", key_node.start_mark) if mapping.include?(key) + end + value = construct_object(value_node) + mapping[key] = value + end + if !merge.nil? + merge << mapping + mapping = {} + for submapping in merge + mapping.merge!(submapping) + end + end + mapping + end + + def construct_pairs(node) + raise ConstructorError.new(nil,nil,"expected a mapping node, but found #{node.tid}",node.start_mark) if !node.__is_mapping + node.value.collect {|key_node,value_node| [construct_object(key_node), construct_object(value_node)] } + end + + def self.add_constructor(tag, constructor) + @@yaml_constructors[tag] = constructor + end + + def self.add_multi_constructor(tag_prefix, multi_constructor) + @@yaml_multi_constructors[tag_prefix] = multi_constructor + @@yaml_multi_regexps[tag_prefix] = Regexp.new("^"+Regexp.escape(tag_prefix)) + end + end + + class SafeConstructor < BaseConstructor + def construct_yaml_null(node) + construct_scalar(node) + nil + end + + BOOL_VALUES = { + "y" => true, + "n" => false, + "yes" => true, + "no" => false, + "true" => true, + "false" => false, + "on" => true, + "off" => false + } + + def construct_yaml_bool(node) + value = construct_scalar(node) + BOOL_VALUES[value.downcase] + end + + def construct_yaml_int(node) + value = construct_scalar(node).to_s + value = value.gsub(/_/, '') + sign = +1 + first = value[0] + if first == ?- + sign = -1 + value.slice!(0) + elsif first == ?+ + value.slice!(0) + end + base = 10 + if value == "0" + return 0 + elsif value[0..1] == "0b" + value.slice!(0..1) + base = 2 + elsif value[0..1] == "0x" + value.slice!(0..1) + base = 16 + elsif value[0] == ?0 + value.slice!(0) + base = 8 + elsif value.include?(?:) + digits = (value.split(/:/).map {|val| val.to_i}).reverse + base = 1 + value = 0 + for digit in digits + value += digit*base + base *= 60 + end + return sign*value + else + return sign*value.to_i + end + return sign*value.to_i(base) + end + + INF_VALUE = +1.0/0.0 + NAN_VALUE = 0.0/0.0 + + def construct_yaml_float(node) + value = construct_scalar(node).to_s + value = value.gsub(/_/, '') + sign = +1 + first = value[0] + if first == ?- + sign = -1 + value.slice!(0) + elsif first == ?+ + value.slice!(0) + end + if value.downcase == ".inf" + return sign*INF_VALUE + elsif value.downcase == ".nan" + return NAN_VALUE + elsif value.include?(?:) + digits = (value.split(/:/).map {|val| val.to_f}).reverse + base = 1 + value = 0.0 + for digit in digits + value += digit*base + base *= 60 + end + return sign*value + else + return value.to_f + end + end + + def construct_yaml_binary(node) + value = construct_scalar(node) + Base64.decode64(value.to_s) + end + + TIMESTAMP_REGEXP = /^([0-9][0-9][0-9][0-9])-([0-9][0-9]?)-([0-9][0-9]?)(?:(?:[Tt]|[ \t]+)([0-9][0-9]?):([0-9][0-9]):([0-9][0-9])(?:\.([0-9]*))?(?:[ \t]*(?:Z|([-+][0-9][0-9]?)(?::([0-9][0-9])?)?))?)?$/ + + def construct_yaml_timestamp(node) + value = construct_scalar(node) + match = TIMESTAMP_REGEXP.match(node.value) + values = match.captures.map {|val| val.to_i} + fraction = values[6] + if fraction != 0 + fraction *= 10 while 10*fraction < 1000 + values[6] = fraction + end + stamp = Time.gm(values[0],values[1],values[2],values[3],values[4],values[5],values[6]) + + diff = values[7] * 3600 + values[8] * 60 + return stamp-diff + end + + def construct_yaml_omap(node) + # Note: we do not check for duplicate keys, because its too + # CPU-expensive. + raise ConstructorError.new("while constructing an ordered map", node.start_mark, + "expected a sequence, but found #{node.tid}", node.start_mark) if !node.__is_sequence + omap = [] + for subnode in node.value + raise ConstructorError.new("while constructing an ordered map", node.start_mark, + "expected a mapping of length 1, but found #{subnode.tid}",subnode.start_mark) if !subnode.__is_mapping + raise ConstructorError.new("while constructing an ordered map", node.start_mark, + "expected a single mapping item, but found #{subnode.value.length} items",subnode.start_mark) if subnode.value.length != 1 + key_node = subnode.value.keys[0] + key = construct_object(key_node) + value = construct_object(subnode.value[key_node]) + omap << [key, value] + end + omap + end + + def construct_yaml_pairs(node) + construct_yaml_omap(node) + end + + def construct_yaml_set(node) + Set.new(construct_mapping(node).keys) + end + + def construct_yaml_str(node) + construct_scalar(node).to_s + end + + def construct_yaml_seq(node) + construct_sequence(node) + end + + def construct_yaml_map(node) + construct_mapping(node) + end + + def construct_yaml_object(node, cls) + mapping = construct_mapping(node) + data = cls.new + mapping.each {|key,val| data.instance_variable_set("@#{key}",val)} + data + end + + def construct_undefined(node) + raise ConstructorError.new(nil,nil,"could not determine a constructor for the tag #{node.tag}",node.start_mark) + end + + def construct_ruby_object(cls,node) + val = construct_primitive(node) + if cls.respond_to?(:yaml_new) + obj = cls.yaml_new(cls,node.tag,val) + else + obj = cls.allocate + obj.yaml_initialize(node.tag,val) + end + obj + end + + def construct_ruby(tag,node) + obj_class = Object + tag.split( "::" ).each { |c| obj_class = obj_class.const_get( c ) } if tag + o = obj_class.allocate + mapping = construct_mapping(node) + mapping.each {|key,val| o.instance_variable_set("@#{key}",val)} + o + end + end + + SafeConstructor::add_constructor('tag:yaml.org,2002:null',:construct_yaml_null) + BaseConstructor::add_constructor('tag:yaml.org,2002:bool',:construct_yaml_bool) + BaseConstructor::add_constructor('tag:yaml.org,2002:int',:construct_yaml_int) + BaseConstructor::add_constructor('tag:yaml.org,2002:float',:construct_yaml_float) + BaseConstructor::add_constructor('tag:yaml.org,2002:binary',:construct_yaml_binary) + BaseConstructor::add_constructor('tag:yaml.org,2002:timestamp',:construct_yaml_timestamp) + BaseConstructor::add_constructor('tag:yaml.org,2002:omap',:construct_yaml_omap) + BaseConstructor::add_constructor('tag:yaml.org,2002:pairs',:construct_yaml_pairs) + BaseConstructor::add_constructor('tag:yaml.org,2002:set',:construct_yaml_set) + BaseConstructor::add_constructor('tag:yaml.org,2002:str',:construct_yaml_str) + BaseConstructor::add_constructor('tag:yaml.org,2002:seq',:construct_yaml_seq) + BaseConstructor::add_constructor('tag:yaml.org,2002:map',:construct_yaml_map) + BaseConstructor::add_constructor(nil,:construct_undefined) + + BaseConstructor::add_multi_constructor("!ruby/object:",:construct_ruby) + + class Constructor < SafeConstructor + end +end diff --git a/lib/ruby/site_ruby/1.8/rbyaml/dumper.rb b/lib/ruby/site_ruby/1.8/rbyaml/dumper.rb new file mode 100644 index 00000000000..6a941701783 --- /dev/null +++ b/lib/ruby/site_ruby/1.8/rbyaml/dumper.rb @@ -0,0 +1,36 @@ +require 'rbyaml/emitter' +require 'rbyaml/serializer' +require 'rbyaml/representer' +require 'rbyaml/resolver' + +module RbYAML + class CommonDumper + attr_accessor :emitter, :serializer, :representer, :resolver + def initialize(stream,default_style=nil,default_flow_style=nil,canonical=nil,indent=nil,width=nil,line_break=nil,explicit_start=nil,explicit_end=nil,version=nil,tags=nil,emitter=Emitter,serializer=Serializer,representer=Representer,resolver=Resolver) + super() + @emitter = emitter.new(stream,canonical,indent,width,line_break) + @resolver = resolver.new + @serializer = serializer.new(@emitter,@resolver,explicit_start,explicit_end,version,tags) + @representer = representer.new(@serializer,default_style,default_flow_style) + end + end + + class BaseDumper < CommonDumper + attr_accessor + def initialize(stream,default_style=nil,default_flow_style=nil,canonical=nil,indent=nil,width=nil,line_break=nil,explicit_start=nil,explicit_end=nil,version=nil,tags=nil,emitter=Emitter,serializer=Serializer,representer=BaseRepresenter,resolver=BaseResolver) + super + end + end + + class SafeDumper < CommonDumper + def initialize(stream,default_style=nil,default_flow_style=nil,canonical=nil,indent=nil,width=nil,line_break=nil,explicit_start=nil,explicit_end=nil,version=nil,tags=nil,emitter=Emitter,serializer=Serializer,representer=SafeRepresenter,resolver=Resolver) + super + end + end + + class Dumper < CommonDumper + def initialize(stream,default_style=nil,default_flow_style=nil,canonical=nil,indent=nil,width=nil,line_break=nil,explicit_start=nil,explicit_end=nil,version=nil,tags=nil,emitter=Emitter,serializer=Serializer,representer=Representer,resolver=Resolver) + super + end + end +end diff --git a/lib/ruby/site_ruby/1.8/rbyaml/emitter.rb b/lib/ruby/site_ruby/1.8/rbyaml/emitter.rb new file mode 100644 index 00000000000..4a314c67964 --- /dev/null +++ b/lib/ruby/site_ruby/1.8/rbyaml/emitter.rb @@ -0,0 +1,1103 @@ +# Emitter expects events obeying the following grammar: +# stream ::= STREAM-START document* STREAM-END +# document ::= DOCUMENT-START node DOCUMENT-END +# node ::= SCALAR | sequence | mapping +# sequence ::= SEQUENCE-START node* SEQUENCE-END +# mapping ::= MAPPING-START (node node)* MAPPING-END + +require 'rbyaml/error' +require 'rbyaml/events' + +module RbYAML + class EmitterError < YAMLError + end + + ScalarAnalysis = Struct.new(:scalar,:empty,:multiline,:allow_flow_plain,:allow_block_plain,:allow_single_quoted,:allow_double_quoted,:allow_block) + + class Emitter + DEFAULT_TAG_PREFIXES = { + "!" => "!", + "tag:yaml.org,2002:" => "!!" + } + + def initialize(stream, canonical=nil, indent=nil, width=nil,line_break=nil) + # The stream should have the methods `write` and possibly `flush`. + @stream = stream + + # Emitter is a state machine with a stack of states to handle nested + # structures. + @states = [] + @state = :expect_stream_start + + # Current event and the event queue. + @events = [] + @event = nil + + # The current indentation level and the stack of previous indents. + @indents = [] + @indent = nil + + # Flow level. + @flow_level = 0 + + # Contexts. + @root_context = false + @sequence_context = false + @mapping_context = false + @simple_key_context = false + + # Characteristics of the last emitted character: + # - current position. + # - is it a whitespace? + # - is it an indention character + # (indentation space, '-', '?', or ':')? + @line = 0 + @column = 0 + @whitespace = true + @indention = true + + # Formatting details. + @canonical = canonical + @best_indent = 2 + @best_indent = indent if indent && indent!=0 && 1 < indent && indent < 10 + + @best_width = 80 + @best_width = width if width && width != 0 && width > @best_indent*2 + + @best_line_break = "\n" + @best_line_break = line_break if ["\r", "\n", "\r\n"].include?(line_break) + + # Tag prefixes. + @tag_prefixes = nil + + # Prepared anchor and tag. + @prepared_anchor = nil + @prepared_tag = nil + + # Scalar analysis and style. + @analysis = nil + @style = nil + end + + def emit(event) + @events << event + while !need_more_events + @event = @events.shift + send(@state) + @event = nil + end + end + + # In some cases, we wait for a few next events before emitting. + + def need_more_events + return true if @events.empty? + event = @events.first + if DocumentStartEvent === event + need_events(1) + elsif SequenceStartEvent === event + need_events(2) + elsif MappingStartEvent === event + need_events(3) + else + false + end + end + + def need_events(count) + level = 0 + for event in @events[1..-1] + if DocumentStartEvent === event || CollectionStartEvent === event + level += 1 + elsif DocumentEndEvent === event || CollectionEndEvent === event + level -= 1 + elsif StreamEndEvent === event + level = -1 + end + if level < 0 + return false + end + end + @events.length < count+1 + end + + def increase_indent(flow=false, indentless=false) + @indents << @indent + if @indent.nil? + if flow + @indent = @best_indent + else + @indent = 0 + end + elsif !indentless + @indent += @best_indent + end + end + + # States. + + # Stream handlers. + + def expect_stream_start + if StreamStartEvent === @event + write_stream_start + @state = :expect_first_document_start + else + raise EmitterError.new("expected StreamStartEvent, but got #{@event}") + end + end + + def expect_nothing + raise EmitterError.new("expected nothing, but got #{@event}") + end + + # Document handlers. + + def expect_first_document_start + expect_document_start(true) + end + + def expect_document_start(first=false) + if DocumentStartEvent === @event + if @event.version + version_text = prepare_version(@event.version) + write_version_directive(version_text) + end + @tag_prefixes = Emitter::DEFAULT_TAG_PREFIXES.dup + if @event.tags + handles = @event.tags.keys + handles.sort! + for handle in handles + prefix = @event.tags[handle] + @tag_prefixes[prefix] = handle + handle_text = prepare_tag_handle(handle) + prefix_text = prepare_tag_prefix(prefix) + write_tag_directive(handle_text, prefix_text) + end + end + implicit = first && !@event.explicit && !@canonical && !@event.version && !@event.tags && !check_empty_document + if !implicit + write_indent + write_indicator("---",true) + if @canonical + write_indent + end + end + @state = :expect_document_root + elsif StreamEndEvent === @event + write_stream_end + @state = :expect_nothing + else + raise EmitterError.new("expected DocumentStartEvent, but got #{@event}") + end + end + + def expect_document_end + if DocumentEndEvent === @event + write_indent + if @event.explicit + write_indicator("...", true) + write_indent + end + flush_stream + @state = :expect_document_start + else + raise EmitterError.new("expected DocumentEndEvent, but got #{@event}") + end + end + + def expect_document_root + @states << :expect_document_end + expect_node(true) + end + + # Node handlers. + + def expect_node(root=false, sequence=false, mapping=false, simple_key=false) + @root_context = root + @sequence_context = sequence + @mapping_context = mapping + @simple_key_context = simple_key + if AliasEvent === @event + expect_alias + elsif ScalarEvent === @event || CollectionStartEvent === @event + process_anchor("&") + process_tag + if ScalarEvent === @event + expect_scalar + elsif SequenceStartEvent === @event + if @flow_level!=0 || @canonical || @event.flow_style || check_empty_sequence + expect_flow_sequence + else + expect_block_sequence + end + elsif MappingStartEvent === @event + if @flow_level!=0 || @canonical || @event.flow_style || check_empty_mapping + expect_flow_mapping + else + expect_block_mapping + end + end + else + raise EmitterError.new("expected NodeEvent, but got #{@event}") + end + end + + def expect_alias + raise EmitterError.new("anchor is not specified for alias") if @event.anchor.nil? + process_anchor("*") + @state = @states.pop + end + + def expect_scalar + increase_indent(true) + process_scalar + @indent = @indents.pop + @state = @states.pop + end + + # Flow sequence handlers. + + def expect_flow_sequence + write_indicator("[", true, true) + @flow_level += 1 + increase_indent(true) + @state = :expect_first_flow_sequence_item + end + + def expect_first_flow_sequence_item + if SequenceEndEvent === @event + @indent = @indents.pop + @flow_level -= 1 + write_indicator("]", false) + @state = @states.pop + else + write_indent if @canonical || @column > @best_width + @states << :expect_flow_sequence_item + expect_node(false,true) + end + end + + def expect_flow_sequence_item + if SequenceEndEvent === @event + @indent = @indents.pop + @flow_level -= 1 + if @canonical + write_indicator(",",false) + write_indent + end + write_indicator("]",false) + @state = @states.pop + else + write_indicator(",", false) + write_indent if @canonical or @column > @best_width + @states << :expect_flow_sequence_item + expect_node(false,true) + end + end + + # Flow mapping handlers. + + def expect_flow_mapping + write_indicator("{", true, true) + @flow_level += 1 + increase_indent(true) + @state = :expect_first_flow_mapping_key + end + + def expect_first_flow_mapping_key + if MappingEndEvent === @event + @indent = @indents.pop + @flow_level -= 1 + write_indicator("}", false) + @state = @states.pop + else + write_indent if @canonical || @column > @best_width + if !@canonical && check_simple_key + @states << :expect_flow_mapping_simple_value + expect_node(false,false,true,true) + else + write_indicator("?", true) + @states << :expect_flow_mapping_value + expect_node(false,false,true) + end + end + end + + def expect_flow_mapping_key + if MappingEndEvent === @event + @indent = @indents.pop + @flow_level -= 1 + if @canonical + write_indicator(",", false) + write_indent + end + write_indicator("}", false) + @state = @states.pop + else + write_indicator(",", false) + write_indent if @canonical || @column > @best_width + if !@canonical && check_simple_key + @states << :expect_flow_mapping_simple_value + expect_node(false,false,true,true) + else + write_indicator("?", true) + @states << :expect_flow_mapping_value + expect_node(false,false,true) + end + end + end + + def expect_flow_mapping_simple_value + write_indicator(":", false) + @states << :expect_flow_mapping_key + expect_node(false,false,true) + end + + def expect_flow_mapping_value + write_indent if @canonical || @column > @best_width + write_indicator(":", true) + @states << :expect_flow_mapping_key + expect_node(false,false,true) + end + + # Block sequence handlers. + + def expect_block_sequence + indentless = @mapping_context && !@indention + increase_indent(false,indentless) + @state = :expect_first_block_sequence_item + end + + def expect_first_block_sequence_item + expect_block_sequence_item(true) + end + + def expect_block_sequence_item(first=false) + if !first && SequenceEndEvent === @event + @indent = @indents.pop + @state = @states.pop + else + write_indent + write_indicator("-", true, false, true) + @states << :expect_block_sequence_item + expect_node(false,true) + end + end + + # Block mapping handlers. + + def expect_block_mapping + increase_indent(false) + @state = :expect_first_block_mapping_key + end + + def expect_first_block_mapping_key + expect_block_mapping_key(true) + end + + def expect_block_mapping_key(first=false) + if !first && MappingEndEvent === @event + @indent = @indents.pop + @state = @states.pop + else + write_indent + if check_simple_key + @states << :expect_block_mapping_simple_value + expect_node(false,false,true,true) + else + write_indicator("?", true, false, true) + @states << :expect_block_mapping_value + expect_node(false,false,true) + end + end + end + + def expect_block_mapping_simple_value + write_indicator(":", false) + @states << :expect_block_mapping_key + expect_node(false,false,true) + end + + def expect_block_mapping_value + write_indent + write_indicator(":",true,false,true) + @states << :expect_block_mapping_key + expect_node(false,false,true) + end + + # Checkers. + + def check_empty_sequence + @event.__is_sequence_start && !@events.empty? && @events.first.__is_sequence_end + end + + def check_empty_mapping + @event.__is_mapping_start && !@events.empty? && @events.first.__is_mapping_end + end + + def check_empty_document + return false if !@event.__is_document_start || @events.empty? + event = @events.first + event.__is_scalar && event.anchor.nil? && event.tag.nil? && event.implicit && event.value == "" + end + + def check_simple_key + length = 0 + if @event.__is_node && !@event.anchor.nil? + @prepared_anchor = prepare_anchor(@event.anchor) if @prepared_anchor.nil? + length += @prepared_anchor.length + end + if (@event.__is_scalar || @event.__is_collection_start) && !@event.tag.nil? + @prepared_tag = prepare_tag(@event.tag) if @prepared_tag.nil? + length += @prepared_tag.length + end + if @event.__is_scalar + @analysis = analyze_scalar(@event.value) if @analysis.nil? + length += @analysis.scalar.length + end + + (length < 128 && (@event.__is_alias || (@event.__is_scalar && !@analysis.empty && !@analysis.multiline) || + check_empty_sequence || check_empty_mapping)) + end + + + # Anchor, Tag, and Scalar processors. + + def process_anchor(indicator) + if @event.anchor.nil? + @prepared_anchor = nil + return nil + end + @prepared_anchor = prepare_anchor(@event.anchor) if @prepared_anchor.nil? + write_indicator(indicator+@prepared_anchor, true) if @prepared_anchor && !@prepared_anchor.empty? + @prepared_anchor = nil + end + + def process_tag + tag = @event.tag + if ScalarEvent === @event + @style = choose_scalar_style if @style.nil? + if ((!@canonical || tag.nil?) && ((@style == "" && @event.implicit[0]) || (@style != "" && @event.implicit[1]))) + @prepared_tag = nil + return + end + if @event.implicit[0] && tag.nil? + tag = "!" + @prepared_tag = nil + end + else + if (!@canonical || tag.nil?) && @event.implicit + @prepared_tag = nil + return + end + end + raise EmitterError.new("tag is not specified") if tag.nil? + @prepared_tag = prepare_tag(tag) if @prepared_tag.nil? + write_indicator(@prepared_tag, true) if @prepared_tag && !@prepared_tag.empty? + @prepared_tag = nil + end + + def choose_scalar_style + @analysis = analyze_scalar(@event.value) if @analysis.nil? + return '"' if @event.style == '"' || @canonical + if !@event.style && @event.implicit[0] + if !(@simple_key_context && (@analysis.empty || @analysis.multiline)) && ((@flow_level!=0 && @analysis.allow_flow_plain) || (@flow_level == 0 && @analysis.allow_block_plain)) + return "" + end + end + + if !@event.style && @event.implicit && (!(@simple_key_context && (@analysis.empty || @analysis.multiline)) && + (@flow_level!=0 && @analysis.allow_flow_plain || (@flow_level==0 && @analysis.allow_block_plain))) + return "" + end + return @event.style if @event.style && /^[|>]$/ =~ @event.style && @flow_level==0 && @analysis.allow_block + return "'" if (!@event.style || @event.style == "'") && (@analysis.allow_single_quoted && !(@simple_key_context && @analysis.multiline)) + return '"' + end + + def process_scalar + @analysis = analyze_scalar(@event.value) if @analysis.nil? + @style = choose_scalar_style if @style.nil? + split = !@simple_key_context + if @style == '"' + write_double_quoted(@analysis.scalar, split) + elsif @style == "'" + write_single_quoted(@analysis.scalar, split) + elsif @style == ">" + write_folded(@analysis.scalar) + elsif @style == "|" + write_literal(@analysis.scalar) + else + write_plain(@analysis.scalar, split) + end + @analysis = nil + @style = nil + end + + # Analyzers. + + def prepare_version(version) + major, minor = version + raise EmitterError.new("unsupported YAML version: #{major}.#{minor}") if major != 1 + "#{major}.#{minor}" + end + + def prepare_tag_handle(handle) + raise EmitterError.new("tag handle must not be empty") if handle.nil? || handle.empty? + raise EmitterError("tag handle must start and end with '!': #{handle}") if handle[0] != ?! || handle[-1] != ?! + raise EmitterError.new("invalid character #{$&} in the tag handle: #{handle}") if /[^-\w]/ =~ handle[1...-1] + handle + end + + def prepare_tag_prefix(prefix) + raise EmitterError.new("tag prefix must not be empty") if prefix.nil? || prefix.empty? + chunks = [] + start = ending = 0 + ending = 1 if prefix[0] == ?! + ending += 1 while ending < prefix.length + chunks << prefix[start...ending] if start < ending + chunks.to_s + end + + def prepare_tag(tag) + raise EmitterError.new("tag must not be empty") if tag.nil? || tag.empty? + return tag if tag == "!" + handle = nil + suffix = tag + for prefix in @tag_prefixes.keys + if Regexp.new("^"+Regexp.escape(prefix)) =~ tag && (prefix == "!" || prefix.length < tag.length) + handle = @tag_prefixes[prefix] + suffix = tag[prefix.length..-1] + end + end + chunks = [] + start = ending = 0 + ending += 1 while ending < suffix.length + chunks << suffix[start...ending] if start < ending + suffix_text = chunks.to_s + if handle + "#{handle}#{suffix_text}" + else + "!<#{suffix_text}>" + end + end + + def prepare_anchor(anchor) + raise EmitterError.new("anchor must not be empty") if anchor.nil? || anchor.empty? + raise EmitterError.new("invalid character #{$&} in the anchor: #{anchor}") if /[^-\w]/ =~ anchor + anchor + end + + def analyze_scalar(scalar) + # Empty scalar is a special case. + return ScalarAnalysis.new(scalar,true,false,false,true,true,true,false) if scalar.nil? || scalar.empty? + # Indicators and special characters. + block_indicators = false + flow_indicators = false + line_breaks = false + special_characters = false + + # Whitespaces. + inline_spaces = false # non-space space+ non-space + inline_breaks = false # non-space break+ non-space + leading_spaces = false # ^ space+ (non-space | $) + leading_breaks = false # ^ break+ (non-space | $) + trailing_spaces = false # (^ | non-space) space+ $ + trailing_breaks = false # (^ | non-space) break+ $ + inline_breaks_spaces = false # non-space break+ space+ non-space + mixed_breaks_spaces = false # anything else + + # Check document indicators. + if /^(---|\.\.\.)/ =~ scalar + block_indicators = true + flow_indicators = true + end + + # First character or preceded by a whitespace. + preceeded_by_space = true + + # Last character or followed by a whitespace. + followed_by_space = scalar.length == 1 || "\0 \t\r\n\x85".include?(scalar[1]) + + # The current series of whitespaces contain plain spaces. + spaces = false + + # The current series of whitespaces contain line breaks. + breaks = false + + # The current series of whitespaces contain a space followed by a + # break. + mixed = false + + # The current series of whitespaces start at the beginning of the + # scalar. + leading = false + + index = 0 + while index < scalar.length + ch = scalar[index] + + # Check for indicators. + + if index == 0 + # Leading indicators are special characters. + if "#,[]{}#&*!|>'\"%@`".include?(ch) + flow_indicators = true + block_indicators = true + end + if "?:".include?(ch) + flow_indicators = true + if followed_by_space + block_indicators = true + end + end + if ch == ?- && followed_by_space + flow_indicators = true + block_indicators = true + end + else + # Some indicators cannot appear within a scalar as well. + flow_indicators = true if ",?[]{}".include?(ch) + if ch == ?: + flow_indicators = true + block_indicators = true if followed_by_space + end + if ch == ?# && preceeded_by_space + flow_indicators = true + block_indicators = true + end + end + # Check for line breaks, special, and unicode characters. + line_breaks = true if "\n\x85".include?(ch) + if !(ch == ?\n || (?\x20 <= ch && ch <= ?\x7E)) + special_characters = true + end + # Spaces, line breaks, and how they are mixed. State machine. + + # Start or continue series of whitespaces. + if " \n\x85".include?(ch) + if spaces && breaks + mixed = true if ch != 32 # break+ (space+ break+) => mixed + elsif spaces + if ch != 32 # (space+ break+) => mixed + breaks = true + mixed = true + end + elsif breaks + spaces = true if ch == 32 # break+ space+ + else + leading = (index == 0) + if ch == 32 # space+ + spaces = true + else # break+ + breaks = true + end + end + # Series of whitespaces ended with a non-space. + elsif spaces || breaks + if leading + if spaces && breaks + mixed_breaks_spaces = true + elsif spaces + leading_spaces = true + elsif breaks + leading_breaks = true + end + else + if mixed + mixed_breaks_spaces = true + elsif spaces && breaks + inline_breaks_spaces = true + elsif spaces + inline_spaces = true + elsif breaks + inline_breaks = true + end + end + spaces = breaks = mixed = leading = false + end + + # Series of whitespaces reach the end. + if (spaces || breaks) && (index == scalar.length-1) + if spaces && breaks + mixed_breaks_spaces = true + elsif spaces + trailing_spaces = true + leading_spaces = true if leading + elsif breaks + trailing_breaks = true + leading_breaks = true if leading + end + spaces = breaks = mixed = leading = false + end + # Prepare for the next character. + index += 1 + preceeded_by_space = "\0 \t\r\n\x85".include?(ch) + followed_by_space = index+1 >= scalar.length || "\0 \t\r\n\x85".include?(scalar[index+1]) + end + # Let's decide what styles are allowed. + allow_flow_plain = true + allow_block_plain = true + allow_single_quoted = true + allow_double_quoted = true + allow_block = true + # Leading and trailing whitespace are bad for plain scalars. We also + # do not want to mess with leading whitespaces for block scalars. + allow_flow_plain = allow_block_plain = allow_block = false if leading_spaces || leading_breaks || trailing_spaces + + # Trailing breaks are fine for block scalars, but unacceptable for + # plain scalars. + allow_flow_plain = allow_block_plain = false if trailing_breaks + + # The combination of (space+ break+) is only acceptable for block + # scalars. + allow_flow_plain = allow_block_plain = allow_single_quoted = false if inline_breaks_spaces + + # Mixed spaces and breaks, as well as special character are only + # allowed for double quoted scalars. + allow_flow_plain = allow_block_plain = allow_single_quoted = allow_block = false if mixed_breaks_spaces || special_characters + + # We don't emit multiline plain scalars. + allow_flow_plain = allow_block_plain = false if line_breaks + + # Flow indicators are forbidden for flow plain scalars. + allow_flow_plain = false if flow_indicators + + # Block indicators are forbidden for block plain scalars. + allow_block_plain = false if block_indicators + + ScalarAnalysis.new(scalar,false,line_breaks,allow_flow_plain,allow_block_plain,allow_single_quoted,allow_double_quoted,allow_block) + end + + # Writers. + + def flush_stream + @stream.flush if @stream.respond_to?(:flush) + end + + def write_stream_start + end + + def write_stream_end + flush_stream + end + + def write_indicator(indicator, need_whitespace,whitespace=false,indention=false) + if @whitespace || !need_whitespace + data = indicator + else + data = " "+indicator + end + + @whitespace = whitespace + @indention = @indention && indention + @column += data.length + @stream.write(data) + end + + def write_indent + indent = @indent || 0 + write_line_break if !@indention || @column > indent || (@column == indent && !@whitespace) + if @column < indent + @whitespace = true + data = " "*(indent-@column) + @column = indent + @stream.write(data) + end + end + + def write_line_break(data=nil) + data = @best_line_break if data.nil? + @whitespace = true + @indention = true + @line += 1 + @column = 0 + @stream.write(data) + end + + + def write_version_directive(version_text) + data = "%YAML #{version_text}" + @stream.write(data) + write_line_break + end + + def write_tag_directive(handle_text, prefix_text) + data = "%TAG #{handle_text} #{prefix_text}" + @stream.write(data) + write_line_break + end + + # Scalar streams. + + def write_single_quoted(text, split=true) + write_indicator("'",true) + spaces = false + breaks = false + start = ending = 0 + while ending <= text.length + ch = nil + ch = text[ending] if ending < text.length + if spaces + if ch.nil? || ch != 32 + if start+1 == ending && @column > @best_width && split && start != 0 && ending != text.length + write_indent + else + data = text[start...ending] + @column += data.length + @stream.write(data) + end + start = ending + end + elsif breaks + if ch.nil? or !"\n\x85".include?(ch) + write_line_break if text[start] == ?\n + (text[start...ending]).each_byte { |br| + if br == ?\n + write_line_break + else + write_line_break(br) + end + } + write_indent + start = ending + end + else + if ch.nil? || "' \n\x85".include?(ch) + if start < ending + data = text[start...ending] + @column += data.length + @stream.write(data) + start = ending + end + if ch == ?' + data = "''" + @column += 2 + @stream.write(data) + start = ending + 1 + end + end + end + + if !ch.nil? + spaces = ch == 32 + breaks = "\n\x85".include?(ch) + end + + ending += 1 + end + write_indicator("'", false) + end + + ESCAPE_REPLACEMENTS = { + ?\0 => "0", + ?\x07 => "a", + ?\x08 => "b", + ?\x09 => "t", + ?\x0A => "n", + ?\x0B => "v", + ?\x0C => "f", + ?\x0D => "r", + ?\x1B => "e", + ?" => "\"", + ?\\ => "\\", + ?\x85 => "N", + ?\xA0 => "_" + } + + def write_double_quoted(text, split=true) + write_indicator('"', true) + start = ending = 0 + while ending <= text.length + ch = nil + ch = text[ending] if ending < text.length + if ch.nil? || "\"\\\x85".include?(ch) || !(?\x20 <= ch && ch <= ?\x7E) + if start < ending + data = text[start...ending] + @column += data.length + @stream.write(data) + start = ending + end + if !ch.nil? + if ESCAPE_REPLACEMENTS.include?(ch) + data = "\\"+ESCAPE_REPLACEMENTS[ch] + elsif ch <= ?\xFF + data = "\\x%02X" % ch + end + @column += data.length + @stream.write(data) + start = ending+1 + end + end + if (0 < ending && ending < text.length-1) && (ch == 32 || start >= ending) && @column+(ending-start) > @best_width && split + data = text[start...ending]+"\\" + start = ending if start < ending + @column += data.length + @stream.write(data) + write_indent + @whitespace = false + @indention = false + if text[start] == 32 + data = "\\" + @column += data.length + @stream.write(data) + end + end + ending += 1 + end + write_indicator('"', false) + end + + def determine_chomp(text) + tail = text[-2..-1] + tail = " "+tail while tail.length < 2 + "\n\x85".include?(tail[-1])? ("\n\x85".include?(tail[-2])? "+" : "" ) : "-" + end + + def write_folded(text) + chomp = determine_chomp(text) + write_indicator(">"+chomp, true) + write_indent + leading_space = false + spaces = false + breaks = false + start = ending = 0 + while ending <= text.length + ch = nil + ch = text[ending] if ending < text.length + if breaks + if ch.nil? || !"\n\x85".include?(ch) + write_line_break if !leading_space && !ch.nil? && ch != 32 && text[start] == ?\n + leading_space = ch == 32 + (text[start...ending]).each_byte { |br| + if br == ?\n + write_line_break + else + write_line_break(br) + end + } + write_indent if !ch.nil? + start = ending + end + elsif spaces + if ch != 32 + if start+1 == ending && @column > @best_width + write_indent + else + data = text[start...ending] + @column += data.length + @stream.write(data) + end + start = ending + end + else + if ch.nil? || " \n\x85".include?(ch) + data = text[start...ending] + @stream.write(data) + write_line_break if ch.nil? + start = ending + end + end + if !ch.nil? + breaks = "\n\x85".include?(ch) + spaces = ch == 32 + end + ending += 1 + end + end + + def write_literal(text) + chomp = determine_chomp(text) + write_indicator("|"+chomp, true) + write_indent + breaks = false + start = ending = 0 + while ending <= text.length + ch = nil + ch = text[ending] if ending < text.length + if breaks + if ch.nil? || !"\n\x85".include?(ch) + (text[start...ending]).each_byte { |br| + if br == ?\n + write_line_break + else + write_line_break(br) + end + } + write_indent if !ch.nil? + start = ending + end + else + if ch.nil? || "\n\x85".include?(ch) + data = text[start...ending] + @stream.write(data) + write_line_break if ch.nil? + start = ending + end + end + breaks = "\n\x85".include?(ch) if !ch.nil? + ending += 1 + end + end + + def write_plain(text, split=true) + return nil if text.nil? || text.empty? + if !@whitespace + data = " " + @column += data.length + @stream.write(data) + end + @writespace = false + @indention = false + spaces = false + breaks = false + start = ending = 0 + while ending <= text.length + ch = nil + ch = text[ending] if ending < text.length + if spaces + if ch != 32 + if start+1 == ending && @column > @best_width && split + write_indent + @writespace = false + @indention = false + else + data = text[start...ending] + @column += data.length + @stream.write(data) + end + start = ending + end + elsif breaks + if !"\n\x85".include?(ch) + write_line_break if text[start] == ?\n + (text[start...ending]).each_byte { |br| + if br == ?\n + write_line_break + else + write_line_break(br) + end + } + write_indent + @whitespace = false + @indention = false + start = ending + end + else + if ch.nil? || " \n\x85".include?(ch) + data = text[start...ending] + @column += data.length + @stream.write(data) + start = ending + end + end + if !ch.nil? + spaces = ch == 32 + breaks = "\n\x85".include?(ch) + end + ending += 1 + end + end + end +end diff --git a/lib/ruby/site_ruby/1.8/rbyaml/error.rb b/lib/ruby/site_ruby/1.8/rbyaml/error.rb new file mode 100644 index 00000000000..2f9467ce313 --- /dev/null +++ b/lib/ruby/site_ruby/1.8/rbyaml/error.rb @@ -0,0 +1,75 @@ + +module RbYAML + Mark = Struct.new(:name,:column,:buffer,:pointer) + class Mark + def get_snippet(indent=4, max_length=75) + return nil if buffer.nil? + head = "" + start = pointer + while start > 0 && !"\0\r\n\x85".include?(buffer[start-1]) + start -= 1 + if pointer-start > max_length/2-1 + head = " ... " + start += 5 + break + end + end + tail = "" + tend = pointer + while tend < buffer.length && !"\0\r\n\x85".include?(buffer[tend]) + tend += 1 + if tend-pointer > max_length/2-1 + tail = " ... " + tend -= 5 + break + end + end + snippet = buffer[start..tend] + ' ' * indent + "#{head}#{snippet}#{tail}\n" + ' '*(indent+pointer-start+head.length) + ' ' + end + + def to_s + snippet = get_snippet() + where = " in \"#{name}\", line ?, column #{column+1}" + if snippet + where << ":\n" << snippet + end + end + + def hash + object_id + end + end + + class YAMLError < StandardError + end + + class TypeError < YAMLError + end + + class MarkedYAMLError < YAMLError + def initialize(context=nil, context_mark=nil, problem=nil, problem_mark=nil, note=nil) + super() + @context = context + @context_mark = context_mark + @problem = problem + @problem_mark = problem_mark + @note = note + end + + def to_s + lines = [] + + lines << @context if @context + if @context_mark && (@problem.nil? || @problem_mark.nil? || + @context_mark.name != @problem_mark.name || + @context_mark.column != @problem_mark.column) + lines << @context_mark.to_s + end + lines << @problem if @problem + lines << @problem_mark.to_s if @problem_mark + lines << @note if @note + lines.join("\n") + end + end +end diff --git a/lib/ruby/site_ruby/1.8/rbyaml/events.rb b/lib/ruby/site_ruby/1.8/rbyaml/events.rb new file mode 100644 index 00000000000..de1a81fe02f --- /dev/null +++ b/lib/ruby/site_ruby/1.8/rbyaml/events.rb @@ -0,0 +1,117 @@ + +module RbYAML + Event = Struct.new(:start_mark,:end_mark) + class Event + def hash + object_id + end + def to_s + attributes = ["@anchor","@tag","@implicit","@value"] & self.instance_variables + args = attributes.collect {|val| "#{val[1..-1]}=" + eval("#{val}").to_s}.join(", ") + "#{self.class.name}(#{args})" + end + def __is_node; false; end + def __is_collection_start; false; end + def __is_collection_end; false; end + def __is_stream_start; false; end + def __is_stream_end; false; end + def __is_document_start; false; end + def __is_document_end; false; end + def __is_alias; false; end + def __is_scalar; false; end + def __is_sequence_start; false; end + def __is_sequence_end; false; end + def __is_mapping_start; false; end + def __is_mapping_end; false; end + end + + class NodeEvent < Event + attr_reader :anchor + def initialize(anchor, start_mark=nil, end_mark=nil) + super(start_mark,end_mark) + @anchor = anchor + end + def __is_node; true; end + end + + class CollectionStartEvent < NodeEvent + attr_reader :tag, :implicit, :flow_style + def initialize(anchor,tag,implicit,start_mark=nil, end_mark=nil,flow_style=nil) + super(anchor,start_mark,end_mark) + @tag = tag + @implicit = implicit + @flow_style = flow_style + end + def __is_collection_start; true; end + end + + class CollectionEndEvent < Event + def __is_collection_end; true; end + end + + class StreamStartEvent < Event + attr_reader :encoding + def initialize(start_mark=nil,end_mark=nil,encoding=nil) + super(start_mark,end_mark) + @encoding = encoding + end + def __is_stream_start; true; end + end + + class StreamEndEvent < Event + def __is_stream_end; true; end + end + + class DocumentStartEvent < Event + attr_reader :explicit, :version, :tags + def initialize(start_mark=nil,end_mark=nil,explicit=nil,version=nil,tags=nil) + super(start_mark,end_mark) + @explicit = explicit + @version = version + @tags = tags + end + def __is_document_start; true; end + end + + class DocumentEndEvent < Event + attr_reader :explicit + def initialize(start_mark=nil,end_mark=nil,explicit=nil) + super(start_mark,end_mark) + @explicit = explicit + end + def __is_document_end; true; end + end + + class AliasEvent < NodeEvent + def __is_alias; true; end + end + + class ScalarEvent < NodeEvent + attr_reader :tag, :style, :value, :implicit + def initialize(anchor,tag,implicit,value,start_mark=nil, end_mark=nil,style=nil) + super(anchor,start_mark,end_mark) + @tag = tag + @style = style + @value = value + @implicit = implicit + end + def __is_scalar; true; end + end + + class SequenceStartEvent < CollectionStartEvent + def __is_sequence_start; true; end + end + + class SequenceEndEvent < CollectionEndEvent + def __is_sequence_end; true; end + end + + class MappingStartEvent < CollectionStartEvent + def __is_mapping_start; true; end + end + + class MappingEndEvent < CollectionEndEvent + def __is_mapping_end; true; end + end +end + diff --git a/lib/ruby/site_ruby/1.8/rbyaml/loader.rb b/lib/ruby/site_ruby/1.8/rbyaml/loader.rb new file mode 100644 index 00000000000..56b2d649dca --- /dev/null +++ b/lib/ruby/site_ruby/1.8/rbyaml/loader.rb @@ -0,0 +1,40 @@ +# This is a more or less straight translation of PyYAML3000 to Ruby + +require 'rbyaml/scanner' +require 'rbyaml/parser' +require 'rbyaml/composer' +require 'rbyaml/constructor' +require 'rbyaml/resolver' + +module RbYAML + class CommonLoader + attr_accessor :scanner, :parser, :composer, :constructor, :resolver + + def initialize(stream,scanner=Scanner,parser=Parser,composer=Composer,constructor=BaseConstructor,resolver=BaseResolver) + @scanner = scanner.new(stream) + @parser = parser.new(@scanner) + @resolver = resolver.new + @composer = composer.new(@parser,@resolver) + @constructor = constructor.new(@composer) + end + end + + class BaseLoader < CommonLoader + def initialize(stream) + super(stream,Scanner,Parser,Composer,BaseConstructor,BaseResolver) + end + end + + class SafeLoader < CommonLoader + def initialize(stream) + super(stream,Scanner,Parser,Composer,SafeConstructor,Resolver) + end + end + + class Loader < CommonLoader + def initialize(stream) + super(stream,Scanner,Parser,Composer,Constructor,Resolver) + end + end +end + diff --git a/lib/ruby/site_ruby/1.8/rbyaml/nodes.rb b/lib/ruby/site_ruby/1.8/rbyaml/nodes.rb new file mode 100644 index 00000000000..8fbcfef6848 --- /dev/null +++ b/lib/ruby/site_ruby/1.8/rbyaml/nodes.rb @@ -0,0 +1,56 @@ + +module RbYAML + Node = Struct.new(:tag, :value, :start_mark, :end_mark) + class Node + def hash + object_id + end + def to_s + "#{self.class.name}(tag=#{tag}, value=#{value})" + end + + def __is_scalar; false; end + def __is_collection; false; end + def __is_sequence; false; end + def __is_mapping; false; end + end + + class ScalarNode < Node + def tid + "scalar" + end + + attr_accessor :style + + def initialize(tag, value,start_mark=nil,end_mark=nil,style=nil) + super(tag,value,start_mark,end_mark) + @style = style + end + def __is_scalar; true; end + end + + class CollectionNode < Node + attr_accessor :flow_style + + def initialize(tag, value,start_mark=nil,end_mark=nil,flow_style=nil) + super(tag,value,start_mark,end_mark) + @flow_style = flow_style + end + def __is_collection; true; end + end + + class SequenceNode < CollectionNode + def tid + "sequence" + end + def __is_sequence; true; end + end + + class MappingNode < CollectionNode + def tid + "mapping" + end + def __is_mapping; true; end + end +end + diff --git a/lib/ruby/site_ruby/1.8/rbyaml/parser.rb b/lib/ruby/site_ruby/1.8/rbyaml/parser.rb new file mode 100644 index 00000000000..c8379ce2d5d --- /dev/null +++ b/lib/ruby/site_ruby/1.8/rbyaml/parser.rb @@ -0,0 +1,632 @@ + +# YAML can be parsed by an LL(1) parser! +# +# We use the following production rules: +# stream ::= STREAM-START implicit_document? explicit_document* STREAM-END +# explicit_document ::= DIRECTIVE* DOCUMENT-START block_node? DOCUMENT-END? +# implicit_document ::= block_node DOCUMENT-END? +# block_node ::= ALIAS | properties? block_content +# flow_node ::= ALIAS | properties? flow_content +# properties ::= TAG ANCHOR? | ANCHOR TAG? +# block_content ::= block_collection | flow_collection | SCALAR +# flow_content ::= flow_collection | SCALAR +# block_collection ::= block_sequence | block_mapping +# block_sequence ::= BLOCK-SEQUENCE-START (BLOCK-ENTRY block_node?)* BLOCK-END +# block_mapping ::= BLOCK-MAPPING_START ((KEY block_node_or_indentless_sequence?)? (VALUE block_node_or_indentless_sequence?)?)* BLOCK-END +# block_node_or_indentless_sequence ::= ALIAS | properties? (block_content | indentless_block_sequence) +# indentless_block_sequence ::= (BLOCK-ENTRY block_node?)+ +# flow_collection ::= flow_sequence | flow_mapping +# flow_sequence ::= FLOW-SEQUENCE-START (flow_sequence_entry FLOW-ENTRY)* flow_sequence_entry? FLOW-SEQUENCE-END +# flow_mapping ::= FLOW-MAPPING-START (flow_mapping_entry FLOW-ENTRY)* flow_mapping_entry? FLOW-MAPPING-END +# flow_sequence_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)? +# flow_mapping_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)? + +# TODO: support for BOM within a stream. +# stream ::= (BOM? implicit_document)? (BOM? explicit_document)* STREAM-END + +# FIRST sets: +# stream: { STREAM-START } +# explicit_document: { DIRECTIVE DOCUMENT-START } +# implicit_document: FIRST(block_node) +# block_node: { ALIAS TAG ANCHOR SCALAR BLOCK-SEQUENCE-START BLOCK-MAPPING-START FLOW-SEQUENCE-START FLOW-MAPPING-START } +# flow_node: { ALIAS ANCHOR TAG SCALAR FLOW-SEQUENCE-START FLOW-MAPPING-START } +# block_content: { BLOCK-SEQUENCE-START BLOCK-MAPPING-START FLOW-SEQUENCE-START FLOW-MAPPING-START SCALAR } +# flow_content: { FLOW-SEQUENCE-START FLOW-MAPPING-START SCALAR } +# block_collection: { BLOCK-SEQUENCE-START BLOCK-MAPPING-START } +# flow_collection: { FLOW-SEQUENCE-START FLOW-MAPPING-START } +# block_sequence: { BLOCK-SEQUENCE-START } +# block_mapping: { BLOCK-MAPPING-START } +# block_node_or_indentless_sequence: { ALIAS ANCHOR TAG SCALAR BLOCK-SEQUENCE-START BLOCK-MAPPING-START FLOW-SEQUENCE-START FLOW-MAPPING-START BLOCK-ENTRY } +# indentless_sequence: { ENTRY } +# flow_collection: { FLOW-SEQUENCE-START FLOW-MAPPING-START } +# flow_sequence: { FLOW-SEQUENCE-START } +# flow_mapping: { FLOW-MAPPING-START } +# flow_sequence_entry: { ALIAS ANCHOR TAG SCALAR FLOW-SEQUENCE-START FLOW-MAPPING-START KEY } +# flow_mapping_entry: { ALIAS ANCHOR TAG SCALAR FLOW-SEQUENCE-START FLOW-MAPPING-START KEY } + +require 'rbyaml/error' +require 'rbyaml/tokens' +require 'rbyaml/events' +require 'rbyaml/scanner' + +module RbYAML + class ParserError < MarkedYAMLError + end + + class Parser + DEFAULT_TAGS = { + '!' => '!', + '!!' => 'tag:yaml.org,2002:' + } + + def initialize(scanner) + @scanner = scanner + @current_event = nil + @yaml_version = nil + @events = nil + @working_events = nil + @tag_handles = { } + @parse_stack = nil + @start_mark = [] + @tks = [] + + end + + def check_event(*choices) + parse_stream + @current_event = parse_stream_next if @current_event.nil? + if @current_event + return true if choices.empty? + for choice in choices + return true if choice === @current_event + end + end + false + end + + def peek_event + parse_stream + @current_event = parse_stream_next unless @current_event + @current_event + end + + def get_event + parse_stream + @current_event = parse_stream_next unless @current_event + value = @current_event + @current_event = nil + value + end + + def each_event + parse_stream + while @current_event = parse_stream_next + yield @current_event + end + end + + def parse_stream + if !@parse_stack + @parse_stack = [:stream] + @tokens = nil + @tags = [] + @anchors = [] + @start_marks = [] + @end_marks = [] + end + end + + def parse_stream_next + if !@parse_stack.empty? + while true + meth = @parse_stack.pop +#puts "our method: :#{meth}" +#puts "--- with peeked: :#{@scanner.peek_token.class} #{if @scanner.peek_token.respond_to?(:value): @scanner.peek_token.value.inspect; end}" + val = send(meth) + if !val.nil? +#puts "returning: #{val}" + return val + end + end + else + @tokens = nil + @tags = [] + @anchors = [] + @start_marks = [] + @end_marks = [] + return nil + end + end + +#TERMINALS, definitions + + def stream_start + token = @scanner.get_token + StreamStartEvent.new(token.start_mark, token.end_mark,token.encoding) + end + + def stream_end + token = @scanner.get_token + StreamEndEvent.new(token.start_mark, token.end_mark) + end + + def document_start_implicit + token = @scanner.peek_token + version, tags = process_directives + DocumentStartEvent.new(token.start_mark,token.start_mark,false) + end + + def document_start + token = @scanner.peek_token + start_mark = token.start_mark + version, tags = process_directives + raise ParserError.new(nil, nil,"expected '', but found #{token.tid}",token.start_mark) unless @scanner.peek_token.__is_document_start + @token = token = @scanner.get_token + end_mark = token.end_mark + DocumentStartEvent.new(start_mark, end_mark,true,version,tags) + end + + def document_end + token = @scanner.peek_token + start_mark = end_mark = token.start_mark + explicit = false + while @scanner.peek_token.__is_document_end + @tokens = token = @scanner.get_token + end_mark = token.end_mark + explicit = true + end + DocumentEndEvent.new(start_mark, end_mark, explicit) + end + + def _alias + token = @scanner.get_token + AliasEvent.new(token.value, token.start_mark, token.end_mark) + end + + def block_sequence_start + end_mark = @scanner.peek_token.start_mark + implicit = @tags.last.nil? || @tags.last == ?! + @tokens = token = @scanner.get_token + SequenceStartEvent.new(@anchors.last, @tags.last, implicit, @start_marks.last, end_mark,false) + end + + def block_indentless_sequence_start + end_mark = @scanner.peek_token.end_mark + implicit = @tags.last.nil? || @tags.last == ?! + SequenceStartEvent.new(@anchors.last, @tags.last, implicit, @start_marks.last, end_mark,false) + end + + def block_sequence_end + if !@scanner.peek_token.__is_block_end + token = @scanner.peek_token + raise ParserError.new("while scanning a block collection", @start_marks.last,"expected , but found #{token.tid}: #{token.inspect}", token.start_mark) + end + token = @scanner.get_token + SequenceEndEvent.new(token.start_mark, token.end_mark) + end + + def block_indentless_sequence_end + @tokens = token = @scanner.peek_token + SequenceEndEvent.new(token.start_mark, token.end_mark) + end + + def block_mapping_start + end_mark = @scanner.peek_token.start_mark + implicit = @tags.last.nil? || @tags.last == ?! + @tokens = token = @scanner.get_token + MappingStartEvent.new(@anchors.last, @tags.last, implicit, @start_marks.last, end_mark,false) + end + + def block_mapping_end + if !@scanner.peek_token.__is_block_end + token = @scanner.peek_token + raise ParserError.new("while scanning a block mapping", @start_marks.last,"expected , but found #{token.tid}", token.start_mark) + end + @tokens = token = @scanner.get_token + MappingEndEvent.new(token.start_mark, token.end_mark) + end + + def flow_sequence_start + end_mark = @scanner.peek_token.end_mark + implicit = @tags.last.nil? || @tags.last == ?! + @tokens = token = @scanner.get_token + SequenceStartEvent.new(@anchors.last, @tags.last, implicit, @start_marks.last, end_mark,true) + end + + def flow_sequence_end + @tokens = token = @scanner.get_token + SequenceEndEvent.new(token.start_mark, token.end_mark) + end + + def flow_internal_mapping_start + @tokens = token = @scanner.get_token + MappingStartEvent.new(nil,nil,true,token.start_mark, token.end_mark,true) + end + + def flow_internal_mapping_end + token = peek_token + MappingEndEvent.new(token.start_mark, token.start_mark) + end + + def flow_mapping_start + end_mark = @scanner.peek_token.end_mark + implicit = @tags.last.nil? || @tags.last == ?! + @tokens = token = @scanner.get_token + MappingStartEvent.new(@anchors.last, @tags.last, implicit, @start_marks.last, end_mark,true) + end + + def flow_mapping_end + @tokens = token = @scanner.get_token + MappingEndEvent.new(token.start_mark, token.end_mark) + end + + def scalar + token = @scanner.get_token + end_mark = token.end_mark + if (token.plain && @tags.last.nil?) || @tags.last == ?! + implicit = [true, false] + elsif @tags.last.nil? + implicit = [false, true] + else + implicit = [false, false] + end + ScalarEvent.new(@anchors.last, @tags.last, implicit, token.value, @start_marks.last, end_mark, token.style) + end + + def empty_scalar + process_empty_scalar(@tokens.end_mark) + end + + +# PRODUCTIONS + def stream + @parse_stack += [:stream_end, :explicit_document, :implicit_document] + stream_start + end + + def implicit_document + curr = @scanner.peek_token + unless curr.__is_directive || curr.__is_document_start || curr.__is_stream_end + @parse_stack += [:document_end, :block_node] + return document_start_implicit + end + nil + end + + def explicit_document + if !@scanner.peek_token.__is_stream_end + @parse_stack += [:explicit_document, :document_end, :block_node] + return document_start + end + nil + end + + def block_node + curr = @scanner.peek_token + if curr.__is_directive || curr.__is_document_start || curr.__is_document_end || curr.__is_stream_end + return empty_scalar + else + if curr.__is_alias + return _alias + else + @parse_stack << :un_properties + properties + return block_content + end + end + end + + def flow_node + if @scanner.peek_token.__is_alias + return _alias + else + @parse_stack << :un_properties + properties + return flow_content + end + end + + def properties + anchor = nil + tag = nil + start_mark = end_mark = tag_mark = nil + if @scanner.peek_token.__is_anchor + token = @scanner.get_token + start_mark = token.start_mark + end_mark = token.end_mark + anchor = token.value + if @scanner.peek_token.__is_tag + token = @scanner.get_token + tag_mark = token.start_mark + end_mark = token.end_mark + tag = token.value + end + elsif @scanner.peek_token.__is_tag + token = @scanner.get_token + start_mark = tag_mark = token.start_mark + end_mark = token.end_mark + tag = token.value + if @scanner.peek_token.__is_anchor + token = @scanner.get_token + end_mark = token.end_mark + anchor = token.value + end + end + + if !tag.nil? and tag != "!" + handle, suffix = tag + if !handle.nil? + raise ParserError.new("while parsing a node", start_mark,"found undefined tag handle #{handle}",tag_mark) if !@tag_handles.include?(handle) + tag = @tag_handles[handle]+suffix + else + tag = suffix + end + end + if start_mark.nil? + start_mark = end_mark = @scanner.peek_token.start_mark + end + @anchors << anchor + @tags << tag + @start_marks << start_mark + @end_marks << end_mark + nil + end + + def un_properties + @anchors.pop + @tags.pop + @start_marks.pop + @end_marks.pop + nil + end + + def block_content + token = @scanner.peek_token + if token.__is_block_sequence_start + return block_sequence + elsif token.__is_block_mapping_start + return block_mapping + elsif token.__is_flow_sequence_start + return flow_sequence + elsif token.__is_flow_mapping_start + return flow_mapping + elsif token.__is_scalar + return scalar + else + raise ParserError.new("while scanning a node", @start_marks.last,"expected the node content, but found #{token.tid}",token.start_mark) + end + end + + def flow_content + token = @scanner.peek_token + if token.__is_flow_sequence_start + return flow_sequence + elsif token.__is_flow_mapping_start + return flow_mapping + elsif token.__is_scalar + return scalar + else + raise ParserError.new("while scanning a flow node", @start_marks.last,"expected the node content, but found #{token.tid}",token.start_mark) + end + end + + def block_sequence_entry + if @scanner.peek_token.__is_block_entry + @tokens = token = @scanner.get_token + if !(@scanner.peek_token.__is_block_entry || @scanner.peek_token.__is_block_end) + @parse_stack += [:block_sequence_entry] + return block_node + else + @parse_steck += [:block_sequence_entry] + return empty_scalar + end + end + nil + end + + def block_mapping_entry + # ((KEY block_node_or_indentless_sequence?)? (VALUE block_node_or_indentless_sequence?)?)* + if @scanner.peek_token.__is_key || @scanner.peek_token.__is_value + if @scanner.check_token(KeyToken) + @tokens = token = @scanner.get_token + curr = @scanner.peek_token + if !(curr.__is_key || curr.__is_value || curr.__is_block_end) + @parse_stack += [:block_mapping_entry,:block_mapping_entry_value] + return block_node_or_indentless_sequence + else + @parse_stack += [:block_mapping_entry,:block_mapping_entry_value] + return empty_scalar + end + else + @parse_stack += [:block_mapping_entry,:block_mapping_entry_value] + return empty_scalar + end + end + nil + end + + def block_mapping_entry_value + if @scanner.peek_token.__is_key || @scanner.peek_token.__is_value + if @scanner.peek_token.__is_value + @tokens = token = @scanner.get_token + curr = @scanner.peek_token + if !(curr.__is_key || curr.__is_value || curr.__is_block_end) + return block_node_or_indentless_sequence + else + return empty_scalar + end + else + @tokens = token = @scanner.peek_token + return empty_scalar + end + end + nil + end + + def block_sequence + @parse_stack += [:block_sequence_end,:block_sequence_entry] + block_sequence_start + end + + def block_mapping + @parse_stack += [:block_mapping_end,:block_mapping_entry] + block_mapping_start + end + + def block_node_or_indentless_sequence + if @scanner.peek_token.__is_alias + return _alias + else + if @scanner.peek_token.__is_block_entry + properties + return indentless_block_sequence + else + properties + return block_content + end + end + end + + def indentless_block_sequence + @parse_stack += [:block_indentless_sequence_end,:indentless_block_sequence_entry] + block_indentless_sequence_start + end + + def indentless_block_sequence_entry + if @scanner.peek_token.__is_block_entry + @tokens = @scanner.get_token + curr = @scanner.peek_token + if !(curr.__is_block_entry || curr.__is_key || curr.__is_value || curr.__is_block_end) + @parse_stack << :indentless_block_sequence_entry + return block_node + else + @parse_stack << :indentless_block_sequence_entry + return empty_scalar + end + end + nil + end + + def flow_sequence + @parse_stack += [:flow_sequence_end,:flow_sequence_entry] + flow_sequence_start + end + + def flow_mapping + @parse_stack += [:flow_mapping_end,:flow_mapping_entry] + flow_mapping_start + end + + def flow_sequence_entry + if !@scanner.peek_token.__is_flow_sequence_end + if @scanner.peek_token.__is_key + @parse_stack += [:flow_sequence_entry,:flow_entry_marker,:flow_internal_mapping_end,:flow_internal_value,:flow_internal_content] + return flow_internal_mapping_start + else + @parse_stack += [:flow_sequence_entry,:flow_node] + return flow_entry_marker + end + end + nil + end + + def flow_internal_content + token = @scanner.peek_token + if !(token.__is_value || token.__is_flow_entry || token.__is_flow_sequence_end) + flow_node + else + empty_scalar + end + end + + def flow_internal_value + if @scanner.peek_token.__is_value + @tokens = token = @scanner.get_token + if !(@scanner.peek_token.__is_flow_entry || @scanner.peek_token.__is_flow_sequence_end) + flow_node + else + empty_scalar + end + else + @tokens = token = @scanner.peek_token + empty_scalar + end + end + + def flow_entry_marker + if @scanner.peek_token.__is_flow_entry + @scanner.get_token + end + nil + end + + def flow_mapping_entry + if !@scanner.peek_token.__is_flow_mapping_end + if @scanner.peek_token.__is_key + @parse_stack += [:flow_mapping_entry,:flow_entry_marker,:flow_mapping_internal_value] + return flow_mapping_internal_content + else + @parse_stack += [:flow_mapping_entry,:flow_node] + return flow_entry_marker + end + end + nil + end + + def flow_mapping_internal_content + curr = @scanner.peek_token + if !(curr.__is_value || curr.__is_flow_entry || curr.__is_flow_mapping_end) + @tokens = token = @scanner.get_token + flow_node + else + empty_scalar + end + end + + def flow_mapping_internal_value + if @scanner.peek_token.__is_value + @tokens = token = @scanner.get_token + if !(@scanner.peek_token.__is_flow_entry || @scanner.peek_token.__is_flow_mapping_end) + flow_node + else + empty_scalar + end + else + @tokens = token = @scanner.peek_token + empty_scalar + end + end + + + def process_directives + # DIRECTIVE* + while @scanner.peek_token.__is_directive + token = @scanner.get_token + if token.name == "YAML" + raise ParserError.new(nil, nil,"found duplicate YAML directive", token.start_mark) if !@yaml_version.nil? + major, minor = token.value[0].to_i, token.value[1].to_i + raise ParserError.new(nil,nil,"found incompatible YAML document (version 1.* is required)",token.start_mark) if major != 1 + @yaml_version = [major,minor] + elsif token.name == "TAG" + handle, prefix = token.value + raise ParserError.new(nil,nil,"duplicate tag handle #{handle}",token.start_mark) if @tag_handles.member?(handle) + @tag_handles[handle] = prefix + end + end + if !@tag_handles.empty? + value = @yaml_version, @tag_handles.dup + else + value = @yaml_version, nil + end + for key in DEFAULT_TAGS.keys + @tag_handles[key] = DEFAULT_TAGS[key] if !@tag_handles.include?(key) + end + value + end + + def process_empty_scalar(mark) + ScalarEvent.new(nil, nil, [true, false], "", mark, mark) + end + end +end + diff --git a/lib/ruby/site_ruby/1.8/rbyaml/reader.rb b/lib/ruby/site_ruby/1.8/rbyaml/reader.rb new file mode 100644 index 00000000000..93984141505 --- /dev/null +++ b/lib/ruby/site_ruby/1.8/rbyaml/reader.rb @@ -0,0 +1,127 @@ +# This is a more or less straight translation of PyYAML3000 to Ruby + +# the big difference in this implementation is that unicode support is not here... + +require 'rbyaml/error' + +module RbYAML + + # Reader: + # - checks if characters are in allowed range, + # - adds '\0' to the end. + # Reader accepts + # - a String object + # - a duck-typed IO object + module Reader + def initialize_reader(stream) + @stream = nil + @stream_pointer = 0 + @eof = true + @buffer = "" + @pointer = 0 + @index = 0 + @line = 0 + @column = 0 + if String === stream + @name = "" + @raw_buffer = stream + else + @stream = stream + @name = stream.respond_to?(:path) ? stream.path : stream.inspect + @eof = false + @raw_buffer = "" + end + end + + def peek(index=0) + update(index+1) if @pointer+index+1 >= @buffer.length + @buffer[@pointer+index] + end + + def prefix(length=1) + update(length) if @pointer+length >= @buffer.length + @buffer[@pointer...@pointer+length] + end + + def forward(length=1) + update(length+1) if @pointer+length+1 >= @buffer.length + length.times { |k| + ch = @buffer[@pointer] + @pointer += 1 + @index += 1 + if "\n\x85".include?(ch) || (ch == ?\r && @buffer[@pointer+1] != ?\n) + @line += 1 + @column = 0 + else + @column += 1 + end + } + end + + def get_mark + if @stream.nil? + Mark.new(@name,@index,@line,@column,@buffer,@pointer) + else + Mark.new(@name,@index,@line,@column,nil,nil) + end + end + + NON_PRINTABLE = /[^\x09\x0A\x0D\x20-\x7E\x85\xA0-\xFF]/ + def check_printable(data) + if NON_PRINTABLE =~ data + position = @index+@buffer.length-@pointer+($~.offset(0)[0]) + raise ReaderError.new(@name, position, $&,"unicode","special characters are not allowed"),"special characters are not allowed" + end + end + + def update(length) + return if @raw_buffer.nil? + @buffer = @buffer[@pointer..-1] + @pointer = 0 + while @buffer.length < length + unless @eof + update_raw + end + data = @raw_buffer + converted = data.length + check_printable(data) + @buffer << data + @raw_buffer = @raw_buffer[converted..-1] + if @eof + @buffer << ?\0 + @raw_buffer = nil + break + end + end + end + + def update_raw(size=1024) + data = @stream.read(size) + if data && !data.empty? + @raw_buffer << data + @stream_pointer += data.length + else + @eof = true + end + end + end + + class ReaderError < YAMLError + def initialize(name, position, character, encoding, reason) + @name = name + @position = position + @character = character + @encoding = encoding + @reason = reason + end + + def to_s + if String === @character + "'#{@encoding}' codec can't decode byte #x%02x: #{@reason}\n in \"#{@name}\", position #{@position}" % @character.to_i + else + "unacceptable character #x%04x: #{@reason}\n in \"#{@name}\", position #{@position}" % @character.to_i + end + end + end +end + diff --git a/lib/ruby/site_ruby/1.8/rbyaml/representer.rb b/lib/ruby/site_ruby/1.8/rbyaml/representer.rb new file mode 100644 index 00000000000..7336f539470 --- /dev/null +++ b/lib/ruby/site_ruby/1.8/rbyaml/representer.rb @@ -0,0 +1,248 @@ + +require 'set' + +require 'rbyaml/error' +require 'rbyaml/nodes' + +module RbYAML + class RepresenterError < YAMLError + end + + class BaseRepresenter + @@yaml_representers = {} + @@yaml_multi_representers = {} + + def initialize(serializer, default_style=nil, default_flow_style=nil) + @serializer = serializer + @default_style = default_style + @default_flow_style = default_flow_style + @represented_objects = {} + end + + def represent(data) + node = represent_data(data) + @serializer.serialize(node) + represented_objects = {} + end + + CLASSOBJ_TYPE = Class + INSTANCE_TYPE = Object + FUNCTION_TYPE = Method + BUILTIN_FUNCTION_TYPE = Method + MODULE_TYPE = Module + + def get_classobj_bases(cls) + [cls] + cls.ancestors + end + + def represent_data(data) + if ignore_aliases(data) + alias_key = nil + else + alias_key = data.object_id + end + + if !alias_key.nil? + if @represented_objects.include?(alias_key) + node = @represented_objects[alias_key] + raise RepresenterError.new("recursive objects are not allowed: #{data}") if node.nil? + return node + end + @represented_objects[alias_key] = nil + end + + data_types = data.class.ancestors + data_types = get_classobj_bases(data.class) + data_types if INSTANCE_TYPE === data + + if @@yaml_representers.include?(data_types[0]) + node = send(@@yaml_representers[data_types[0]],data) + else + rerun = true + for data_type in data_types + if @@yaml_multi_representers.include?(data_type) + node = send(@@yaml_multi_representers[data_type],data) + rerun = false + break + elsif @@yaml_representers.include?(data_type) + node = send(@@yaml_representers[data_type],data) + rerun = false + break + end + end + if rerun + if @@yaml_multi_representers.include?(nil) + node = send(@@yaml_multi_representers[nil],data) + elsif @@yaml_representers.include?(nil) + node = send(@@yaml_representers[nil],data) + else + node = ScalarNode.new(nil, data) + end + end + end + + @represented_objects[alias_key] = node if !alias_key.nil? + node + end + + def self.add_representer(data_type, representer) + @@yaml_representers[data_type] = representer + end + + def self.add_multi_representer(data_type, representer) + @@yaml_multi_representers[data_type] = representer + end + + def represent_scalar(tag, value, style=nil) + style ||= @default_style + ScalarNode.new(tag, value, style) + end + + def represent_sequence(tag, sequence, flow_style=nil) + best_style = true + value = sequence.map {|item| + node_item = represent_data(item) + best_style = false if !node_item.__is_scalar && !node_item.flow_style + node_item + } + flow_style ||= (@flow_default_style || best_style) + SequenceNode.new(tag, value, flow_style) + end + + def represent_mapping(tag, mapping, flow_style=nil) + best_style = true + if mapping.respond_to?(:keys) + value = {} + for item_key,item_value in mapping + node_key = represent_data(item_key) + node_value = represent_data(item_value) + best_style = false if !node_key.__is_scalar && !node_key.flow_style + best_style = false if !node_value.__is_scalar && !node_value.flow_style + value[node_key] = node_value + end + else + value = [] + for item_key, item_value in mapping + node_key = represent_data(item_key) + node_value = represent_data(item_value) + best_style = false if !node_key.__is_scalar && !node_key.flow_style + best_style = false if !node_value.__is_scalar && !node_value.flow_style + value << [node_key, node_value] + end + end + flow_style ||= (@flow_default_style || best_style) + MappingNode.new(tag, value, flow_style) + end + + def ignore_aliases(data) + false + end + end + + class SafeRepresenter < BaseRepresenter + + def ignore_aliases(data) + data.nil? || data.__is_str || TrueClass === data || FalseClass === data || data.__is_int || Float === data + end + + def represent_none(data) + represent_scalar(data.taguri,"null") + end + + def represent_str(data) + represent_scalar(data.taguri,data,nil) + end + + def represent_symbol(data) + represent_scalar(data.taguri,data.to_s) + end + + def represent_bool(data) + value = data ? "true" : "false" + represent_scalar('tag:yaml.org,2002:bool',value) + end + + def represent_int(data) + represent_scalar(data.taguri,data.to_s) + end + + def represent_float(data) + if data.infinite? == 1 + value = ".inf" + elsif data.infinite? == -1 + value = "-.inf" + elsif data.nan? || data != data + value = ".nan" + else + value = data.to_s + end + represent_scalar(data.taguri, value) + end + + def represent_list(data) +#no support for pairs right now. should probably be there, though... + represent_sequence(data.taguri, data) + end + + def represent_dict(data) + represent_mapping(data.taguri, data) + end + + def represent_set(data) + value = {} + for key in data + value[key] = nil + end + represent_mapping(data.taguri, value) + end + + def represent_datetime(data) + value = "%04d-%02d-%02d %02d:%02d:%02d" % [data.year, data.month, data.day, data.hour, data.min, data.sec] + if data.usec != 0 + value += "." + (data.usec/1000000.0).to_s.split(/\./)[1] + end + if data.utc_offset != 0 + value += data.utc_offset.to_s + end + represent_scalar(data.taguri, value) + end + + def represent_ruby(data) + state = data.to_yaml_properties + map = {} + state.each do |m| + map[m[1..-1]] = data.instance_variable_get(m) + end + represent_mapping("!ruby/object:#{data.class.yaml_tag_class_name}", map,nil) + end + + def represent_yaml_object(tag, data, flow_style=nil) + state = data.to_yaml_properties + map = {} + state.each do |m| + map[m[1..-1]] = data.instance_variable_get(m) + end + represent_mapping(tag, map, flow_style) + end + + def represent_undefined(data) + raise RepresenterError.new("cannot represent an object: #{data}") + end + end + + BaseRepresenter.add_representer(NilClass,:represent_none) + BaseRepresenter.add_representer(String,:represent_str) + BaseRepresenter.add_representer(Symbol,:represent_symbol) + BaseRepresenter.add_representer(TrueClass,:represent_bool) + BaseRepresenter.add_representer(FalseClass,:represent_bool) + BaseRepresenter.add_representer(Integer,:represent_int) + BaseRepresenter.add_representer(Float,:represent_float) + BaseRepresenter.add_representer(Array,:represent_list) + BaseRepresenter.add_representer(Hash,:represent_dict) + BaseRepresenter.add_representer(Set,:represent_set) + BaseRepresenter.add_representer(Time,:represent_datetime) + BaseRepresenter.add_representer(Object,:represent_ruby) + BaseRepresenter.add_representer(nil,:represent_undefined) + + class Representer < SafeRepresenter + end +end diff --git a/lib/ruby/site_ruby/1.8/rbyaml/resolver.rb b/lib/ruby/site_ruby/1.8/rbyaml/resolver.rb new file mode 100644 index 00000000000..5224c7c0bc8 --- /dev/null +++ b/lib/ruby/site_ruby/1.8/rbyaml/resolver.rb @@ -0,0 +1,163 @@ +require 'rbyaml/util' +require 'rbyaml/nodes' +require 'rbyaml/error' + +module RbYAML + class ResolverError < MarkedYAMLError + end + + DEFAULT_SCALAR_TAG = 'tag:yaml.org,2002:str' + DEFAULT_SEQUENCE_TAG = 'tag:yaml.org,2002:seq' + DEFAULT_MAPPING_TAG = 'tag:yaml.org,2002:map' + + class BaseResolver + @@yaml_implicit_resolvers = {} + @@yaml_path_resolvers = {} + + def initialize + @resolver_exact_paths = [] + @resolver_prefix_paths = [] + end + + def self.add_implicit_resolver(tag, regexp, first) + if first.nil? + first = "" + end + first.each_byte { |ch| + @@yaml_implicit_resolvers[ch] ||= [] + @@yaml_implicit_resolvers[ch] << [tag,regexp] + } + end + + def self.add_path_resolver(tag, path, kind=nil) + new_path = [] + for element in path + if element.__is_a + if element.length == 2 + node_check, index_check = element + elsif element.length == 1 + node_check = element[0] + index_check = true + else + raise ResolverError.new("Invalid path element: #{element}") + end + else + node_check = nil + index_check = element + end + if String == node_check + node_check = ScalarNode + elsif Array == node_check + node_check = SequenceNode + elsif Hash == node_check + node_check = MappingNode + elsif ![ScalarNode, SequenceNode, MappingNode].include?(node_check) && !node_check.__is_sym && !node_check.nil? + raise ResolverError.new("Invalid node checker: #{node_check}") + end + if !(index_check.__is_str || index_check.__is_int) && !index_check.nil? + raise ResolverError.new("Invalid index checker: #{index_check}") + end + new_path << [node_check, index_check] + end + if String == kind + kind = ScalarNode + elsif Array == kind + kind = SequenceNode + elsif Hash == kind + kind = MappingNode + elsif ![ScalarNode, SequenceNode, MappingNode].include?(kind) && !kind.nil? + raise ResolverError.new("Invalid node kind: #{kind}") + end + @@yaml_path_resolvers[[[new_path], kind]] = tag + end + + def descend_resolver(current_node, current_index) + exact_paths = {} + prefix_paths = [] + if current_node + depth = @resolver_prefix_paths.length + for path, kind in @resolver_prefix_paths[-1] + if check_resolver_prefix(depth, path, kind,current_node, current_index) + if path.length > depth + prefix_paths << [path, kind] + else + exact_paths[kind] = @@yaml_path_resolvers[[path, kind]] + end + end + end + else + for path, kind in @@yaml_path_resolvers + if !path + exact_paths[kind] = @@yaml_path_resolvers[[path, kind]] + else + prefix_paths << [path, kind] + end + end + end + @resolver_exact_paths << exact_paths + @resolver_prefix_paths << prefix_paths + end + + def ascend_resolver + @resolver_exact_paths.pop + @resolver_prefix_paths.pop + end + + def check_resolver_prefix(depth, path, kind, current_node, current_index) + node_check, index_check = path[depth-1] + if node_check.__is_str + return false if current_node.tag != node_check + elsif !node_check.nil? + return false if !node_check === current_node + end + return false if index_check==true && !current_index.nil? + return false if !index_check && current_index.nil? + if index_check.__is_str + return false if !(current_index.__is_scalar && index_check == current_index.value) + elsif index_check.__is_int + return false if index_check != current_index + end + true + end + + def resolve(kind, value, implicit) + if ScalarNode == kind && implicit[0] + if value == "" + resolvers = @@yaml_implicit_resolvers.fetch("",[]) + else + resolvers = @@yaml_implicit_resolvers.fetch(value[0],[]) + end + resolvers += @@yaml_implicit_resolvers.fetch(nil,[]) + for tag, regexp in resolvers + return tag if regexp =~ value + end + implicit = implicit[1] + end + exact_paths = @resolver_exact_paths[-1] + return exact_paths[kind] if exact_paths.include?(kind) + return exact_paths[nil] if exact_paths.include?(nil) + if ScalarNode == kind + return RbYAML::DEFAULT_SCALAR_TAG + elsif SequenceNode == kind + return RbYAML::DEFAULT_SEQUENCE_TAG + elsif MappingNode == kind + return RbYAML::DEFAULT_MAPPING_TAG + end + end + end + + class Resolver < BaseResolver + end + + BaseResolver.add_implicit_resolver('tag:yaml.org,2002:bool',/^(?:y|Y|yes|Yes|YES|n|N|no|No|NO|true|True|TRUE|false|False|FALSE|on|On|ON|off|Off|OFF)$/,'yYnNtTfFoO') + BaseResolver.add_implicit_resolver('tag:yaml.org,2002:float',/^(?:[-+]?(?:[0-9][0-9_]*)?\.[0-9_]*(?:[eE][-+][0-9]+)?|[-+]?[0-9][0-9_]*(?::[0-5]?[0-9])+\.[0-9_]*|[-+]?\.(?:inf|Inf|INF)|\.(?:nan|NaN|NAN))$/,'-+0123456789.') + BaseResolver.add_implicit_resolver('tag:yaml.org,2002:int',/^(?:[-+]?0b[0-1_]+|[-+]?0[0-7_]+|[-+]?(?:0|[1-9][0-9_]*)|[-+]?0x[0-9a-fA-F_]+|[-+]?[1-9][0-9_]*(?::[0-5]?[0-9])+)$/,'-+0123456789') + BaseResolver.add_implicit_resolver('tag:yaml.org,2002:merge',/^(?:<<)$/,'<') + BaseResolver.add_implicit_resolver('tag:yaml.org,2002:null',/^(?: ~|null|Null|NULL| )$/,'~nN' + ?\0.chr) + BaseResolver.add_implicit_resolver('tag:yaml.org,2002:timestamp',/^(?:[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]|[0-9][0-9][0-9][0-9]-[0-9][0-9]?-[0-9][0-9]?(?:[Tt]|[ \t]+)[0-9][0-9]?:[0-9][0-9]:[0-9][0-9](?:\.[0-9]*)?(?:[ \t]*(?:Z|[-+][0-9][0-9]?(?::[0-9][0-9])?))?)$/,'0123456789') + BaseResolver.add_implicit_resolver('tag:yaml.org,2002:value',/^(?:=)$/,'=') + # The following implicit resolver is only for documentation purposes. It cannot work + # because plain scalars cannot start with '!', '&', or '*'. + BaseResolver.add_implicit_resolver('tag:yaml.org,2002:yaml',/^(?:!|&|\*)$/,'!&*') +end + diff --git a/lib/ruby/site_ruby/1.8/rbyaml/scanner.rb b/lib/ruby/site_ruby/1.8/rbyaml/scanner.rb new file mode 100644 index 00000000000..34cc1a377fa --- /dev/null +++ b/lib/ruby/site_ruby/1.8/rbyaml/scanner.rb @@ -0,0 +1,1350 @@ +# Scanner produces tokens of the following types: +# STREAM-START +# STREAM-END +# DIRECTIVE(name, value) +# DOCUMENT-START +# DOCUMENT-END +# BLOCK-SEQUENCE-START +# BLOCK-MAPPING-START +# BLOCK-END +# FLOW-SEQUENCE-START +# FLOW-MAPPING-START +# FLOW-SEQUENCE-END +# FLOW-MAPPING-END +# BLOCK-ENTRY +# FLOW-ENTRY +# KEY +# VALUE +# ALIAS(value) +# ANCHOR(value) +# TAG(value) +# SCALAR(value, plain) +# +# Read comments in the Scanner code for more details. +# + +require 'rbyaml/util' +require 'rbyaml/error' +require 'rbyaml/tokens' + +module RbYAML + class ScannerError < MarkedYAMLError + end + class ReaderError < YAMLError + def initialize(name, position, character, encoding, reason) + @name = name + @position = position + @character = character + @encoding = encoding + @reason = reason + end + + def to_s + if @character.__is_str + "'#{@encoding}' codec can't decode byte #x%02x: #{@reason}\n in \"#{@name}\", position #{@position}" % @character.to_i + else + "unacceptable character #x%04x: #{@reason}\n in \"#{@name}\", position #{@position}" % @character.to_i + end + end + end + + SimpleKey = Struct.new(:token_number, :required, :index, :line, :column, :mark) + + class Scanner + attr_reader :column, :stream, :stream_pointer, :eof, :buffer, :pointer, :index, :line + def initialize(stream) + # Had we reached the end of the stream? + @done = false + + # The number of unclosed '{' and '['. `flow_level == 0` means block + # context. + @flow_level = 0 + @flow_zero = true + + # List of processed tokens that are not yet emitted. + @tokens = [] + + # Add the STREAM-START token. + fetch_stream_start + + # Number of tokens that were emitted through the `get_token` method. + @tokens_taken = 0 + + # The current indentation level. + @indent = -1 + + # Past indentation levels. + @indents = [] + + # Variables related to simple keys treatment. + + # A simple key is a key that is not denoted by the '?' indicator. + # Example of simple keys: + # --- + # block simple key: value + # ? not a simple key: + # : { flow simple key: value } + # We emit the KEY token before all keys, so when we find a potential + # simple key, we try to locate the corresponding ':' indicator. + # Simple keys should be limited to a single line and 1024 characters. + + # Can a simple key start at the current position? A simple key may + # start: + # - at the beginning of the line, not counting indentation spaces + # (in block context), + # - after '{', '[', ',' (in the flow context), + # - after '?', ':', '-' (in the block context). + # In the block context, this flag also signifies if a block collection + # may start at the current position. + @allow_simple_key = true + + # Keep track of possible simple keys. This is a dictionary. The key + # is `flow_level`; there can be no more that one possible simple key + # for each level. The value is a SimpleKey record: + # (token_number, required, index, line, column, mark) + # A simple key may start with ALIAS, ANCHOR, TAG, SCALAR(flow), + # '[', or '{' tokens. + @possible_simple_keys = {} + + @stream = nil + @stream_pointer = 0 + @eof = true + @buffer = "" + @buffer_length = 0 + @pointer = 0 + @pointer1 = 1 + @column = 0 + if stream.__is_str + @name = "" + @raw_buffer = stream + else + @stream = stream + @name = stream.respond_to?(:path) ? stream.path : stream.inspect + @eof = false + @raw_buffer = "" + end + end + + def peek(index=0) + peekn(index) + end + + def peek0 + update(1) unless @pointer1 < @buffer_length + @buffer[@pointer] + end + + def peek1 + update(2) unless @pointer1+1 < @buffer_length + @buffer[@pointer1] + end + + def peek2 + update(3) unless @pointer1+2 < @buffer_length + @buffer[@pointer1+1] + end + + def peek3 + update(4) unless @pointer1+3 < @buffer_length + @buffer[@pointer1+2] + end + + def peekn(index=0) + pix = @pointer1+index + unless pix < @buffer_length + update(index+1) + pix = @pointer1+index + end + @buffer[pix-1] + end + + def prefix(length=1) + update(length) unless @pointer+length < @buffer_length + @buffer[@pointer...@pointer+length] + end + + def prefix2() + update(2) unless @pointer1+1 < @buffer_length + @buffer[@pointer..@pointer1] + end + + def forward(length=1) + case length + when 0: forward0 + when 1: forward1 + when 2: forward2 + when 3: forward3 + when 4: forward4 + when 5: forward5 + when 6: forward6 + else forwardn(length) + end + end + + def forward0 + update(1) unless @pointer1 < @buffer_length + end + + LINE_BR = "\n\x85" + + def forward1 + update(2) unless @pointer1+1 < @buffer_length + buff = @buffer[@pointer...@pointer1+1] + index = buff.rindex(LINE_BR_REG) + @column = index ? -index : column+1 + @pointer += 1 + @pointer1 += 1 + end + + def forward2 + update(3) unless @pointer1+2 < @buffer_length + buff = @buffer[@pointer...@pointer1+2] + index = buff.rindex(LINE_BR_REG) + @column = index ? 1-index : column+2 + @pointer += 2 + @pointer1 += 2 + end + + def forward3 + update(4) unless @pointer1+3 < @buffer_length + buff = @buffer[@pointer...@pointer1+3] + index = buff.rindex(LINE_BR_REG) + @column = index ? 2-index : column+3 + @pointer += 3 + @pointer1 += 3 + end + + def forward4 + update(5) unless @pointer1+4 < @buffer_length + buff = @buffer[@pointer...@pointer1+4] + index = buff.rindex(LINE_BR_REG) + @column = index ? 3-index : column+4 + @pointer += 4 + @pointer1 += 4 + end + + def forward5 + update(6) unless @pointer1+5 < @buffer_length + buff = @buffer[@pointer...@pointer1+5] + index = buff.rindex(LINE_BR_REG) + @column = index ? 4-index : column+5 + @pointer += 5 + @pointer1 += 5 + end + + def forward6 + update(7) unless @pointer1+6 < @buffer_length + buff = @buffer[@pointer...@pointer1+6] + index = buff.rindex(LINE_BR_REG) + @column = index ? 5-index : column+6 + @pointer += 6 + @pointer1 += 6 + end + + LINE_BR_REG = /[\n\x85]|(?:\r[^\n])/ + def forwardn(length) + update(length + 1) unless @pointer1+length < @buffer_length + buff = @buffer[@pointer...@pointer+length] + index = buff.rindex(LINE_BR_REG) + @column = index ? (length-index)-1 : column+length + @pointer += length + @pointer1 += length + end + + def get_mark + if @stream.nil? + Mark.new(@name,@column,@buffer,@pointer) + else + Mark.new(@name,@column,nil,nil) + end + end + + NON_PRINTABLE = /[^\x09\x0A\x0D\x20-\x7E\x85\xA0-\xFF]/ + def check_printable(data) + if NON_PRINTABLE =~ data + position = @buffer.length-@pointer+($~.offset(0)[0]) + raise ReaderError.new(@name, position, $&,"unicode","special characters are not allowed"),"special characters are not allowed" + end + end + + + def update(length) + return if @raw_buffer.nil? + @buffer = @buffer[@pointer..-1] + @pointer = 0 + while @buffer.length < length + unless @eof + data = @stream.read(1024) + if data && !data.empty? + @buffer << data + @stream_pointer += data.length + @raw_buffer = "" + else + @eof = true + @buffer << ?\0 + @raw_buffer = nil + break + end + else + @buffer << @raw_buffer << ?\0 + @raw_buffer = nil + break + end + end + @buffer_length = @buffer.length + if @eof + check_printable(@buffer[(-length)..-2]) + else + check_printable(@buffer[(-length)..-1]) + end + @pointer1 = @pointer+1 + end + + def check_token(*choices) + # Check if the next token is one of the given types. + fetch_more_tokens while need_more_tokens + unless @tokens.empty? + return true if choices.empty? + for choice in choices + return true if choice === @tokens[0] + end + end + return false + end + + def peek_token + # Return the next token, but do not delete if from the queue. + fetch_more_tokens while need_more_tokens + return @tokens[0] unless @tokens.empty? + end + + def get_token + # Return the next token. + fetch_more_tokens while need_more_tokens + unless @tokens.empty? + @tokens_taken += 1 + @tokens.shift + end + end + + def each_token + fetch_more_tokens while need_more_tokens + while !@tokens.empty? + @tokens_taken += 1 + yield @tokens.shift + fetch_more_tokens while need_more_tokens + end + end + + def need_more_tokens + return false if @done + @tokens.empty? || next_possible_simple_key == @tokens_taken + end + + ENDING = /^---[\0 \t\r\n\x85]$/ + START = /^\.\.\.[\0 \t\r\n\x85]$/ + NULL_OR_OTHER = "\0 \t\r\n\x85" + BEG = /^([^\0 \t\r\n\x85\-?:,\[\]{}#&*!|>'"%@`]|([\-?:][^\0 \t\r\n\x85]))/ + def fetch_more_tokens + # Eat whitespaces and comments until we reach the next token. + scan_to_next_token + + # Remove obsolete possible simple keys. +# stale_possible_simple_keys + + # Compare the current indentation and column. It may add some tokens + # and decrease the current indentation level. + unwind_indent(@column) + + # Peek the next character. + ch = peek0 + colz = @column == 0 + + case ch + when ?\0: return fetch_stream_end + when ?': return fetch_single + when ?": return fetch_double + when ??: if !@flow_zero || NULL_OR_OTHER.include?(peek1): return fetch_key end + when ?:: if !@flow_zero || NULL_OR_OTHER.include?(peek1): return fetch_value end + when ?%: if colz: return fetch_stream_end end + when ?-: if colz && ENDING =~ prefix(4): return fetch_document_start; elsif NULL_OR_OTHER.include?(peek1): return fetch_block_entry end + when ?.: if colz && START =~ prefix(4): return fetch_document_end end + when ?[: return fetch_flow_sequence_start + when ?{: return fetch_flow_mapping_start + when ?]: return fetch_flow_sequence_end + when ?}: return fetch_flow_mapping_end + when ?,: return fetch_flow_entry + when ?*: return fetch_alias + when ?&: return fetch_anchor + when ?!: return fetch_tag + when ?|: if @flow_zero: return fetch_literal end + when ?>: if @flow_zero: return fetch_folded end + end + return fetch_plain if BEG =~ prefix(2) + raise ScannerError.new("while scanning for the next token", nil,"found character #{ch.chr}(#{ch}) that cannot start any token",get_mark) + end + + # Simple keys treatment. + + def next_possible_simple_key + # Return the number of the nearest possible simple key. Actually we + # don't need to loop through the whole dictionary. + @possible_simple_keys.each_value {|key| return key.token_number if key.token_number} + nil + end + + def save_possible_simple_key + # The next token may start a simple key. We check if it's possible + # and save its position. This function is called for + # ALIAS, ANCHOR, TAG, SCALAR(flow), '[', and '{'. + # The next token might be a simple key. Let's save it's number and + # position. + @possible_simple_keys[@flow_level] = SimpleKey.new(@tokens_taken+@tokens.length, @flow_zero && @indent == @column,index,line,column,get_mark) if @allow_simple_key + end + + # Indentation functions. + + def unwind_indent(col) + ## In flow context, tokens should respect indentation. + ## Actually the condition should be `@indent >= column` according to + ## the spec. But this condition will prohibit intuitively correct + ## constructions such as + ## key : { + ## } + #if @flow_level and @indent > column + # raise ScannerError(nil, nil, + # "invalid intendation or unclosed '[' or '{'", + # get_mark) + + # In the flow context, indentation is ignored. We make the scanner less + # restrictive then specification requires. + return nil if !@flow_zero + # In block context, we may need to issue the BLOCK-END tokens. + while @indent > col + mark = get_mark + @indent = @indents.pop + @tokens << BlockEndToken.new(mark, mark) + end + end + + def add_indent(column) + # Check if we need to increase indentation. + if @indent < column + @indents << @indent + @indent = column + return true + end + return false + end + + # Fetchers. + + def fetch_stream_start + # We always add STREAM-START as the first token and STREAM-END as the + # last token. + # Read the token. + mark = get_mark + # Add STREAM-START. + @tokens << StreamStartToken.new(mark, mark, @encoding) + end + + + def fetch_stream_end + # Set the current intendation to -1. + unwind_indent(-1) + # Reset everything (not really needed). + @allow_simple_key = false + @possible_simple_keys = {} + # Read the token. + mark = get_mark + # Add STREAM-END. + @tokens << StreamEndToken.new(mark, mark) + # The stream is finished. + @done = true + end + + def fetch_directive + # Set the current intendation to -1. + unwind_indent(-1) + # Reset simple keys. + @allow_simple_key = false + # Scan and add DIRECTIVE. + @tokens << scan_directive + end + + def fetch_document_start + fetch_document_indicator(DocumentStartToken) + end + + def fetch_document_end + fetch_document_indicator(DocumentEndToken) + end + + def fetch_document_indicator(token) + # Set the current intendation to -1. + unwind_indent(-1) + # Reset simple keys. Note that there could not be a block collection + # after '---'. + @allow_simple_key = false + # Add DOCUMENT-START or DOCUMENT-END. + start_mark = get_mark + forward3 + end_mark = get_mark + @tokens << token.new(start_mark, end_mark) + end + + def fetch_flow_sequence_start + fetch_flow_collection_start(FlowSequenceStartToken) + end + + def fetch_flow_mapping_start + fetch_flow_collection_start(FlowMappingStartToken) + end + + def fetch_flow_collection_start(token) + # '[' and '{' may start a simple key. + save_possible_simple_key + # Increase the flow level. + @flow_level += 1 + @flow_zero = false + # Simple keys are allowed after '[' and '{'. + @allow_simple_key = true + # Add FLOW-SEQUENCE-START or FLOW-MAPPING-START. + start_mark = get_mark + forward1 + end_mark = get_mark + @tokens << token.new(start_mark, end_mark) + end + + def fetch_flow_sequence_end + fetch_flow_collection_end(FlowSequenceEndToken) + end + + def fetch_flow_mapping_end + fetch_flow_collection_end(FlowMappingEndToken) + end + + def fetch_flow_collection_end(token) + # Decrease the flow level. + @flow_level -= 1 + if @flow_level == 0 + @flow_zero = true + end + # No simple keys after ']' or '}'. + @allow_simple_key = false + # Add FLOW-SEQUENCE-END or FLOW-MAPPING-END. + start_mark = get_mark + forward1 + end_mark = get_mark + @tokens << token.new(start_mark, end_mark) + end + + def fetch_flow_entry + # Simple keys are allowed after ','. + @allow_simple_key = true + # Add FLOW-ENTRY. + start_mark = get_mark + forward1 + end_mark = get_mark + @tokens << FlowEntryToken.new(start_mark, end_mark) + end + + def fetch_block_entry + # Block context needs additional checks. + if @flow_zero + raise ScannerError.new(nil,nil,"sequence entries are not allowed here",get_mark) if !@allow_simple_key + # We may need to add BLOCK-SEQUENCE-START. + if add_indent(column) + mark = get_mark + @tokens << BlockSequenceStartToken.new(mark, mark) + end + # It's an error for the block entry to occur in the flow context, + # but we let the parser detect this. + end + # Simple keys are allowed after '-'. + @allow_simple_key = true + # Add BLOCK-ENTRY. + start_mark = get_mark + forward1 + end_mark = get_mark + @tokens << BlockEntryToken.new(start_mark, end_mark) + end + + def fetch_key + # Block context needs additional checks. + if @flow_zero + # Are we allowed to start a key (not nessesary a simple)? + raise ScannerError.new(nil,nil,"mapping keys are not allowed here",get_mark) if !@allow_simple_key + # We may need to add BLOCK-MAPPING-START. + if add_indent(column) + mark = get_mark + @tokens << BlockMappingStartToken.new(mark, mark) + end + end + # Simple keys are allowed after '?' in the block context. + @allow_simple_key = @flow_zero + # Add KEY. + start_mark = get_mark + forward1 + end_mark = get_mark + @tokens << KeyToken.new(start_mark, end_mark) + end + + def fetch_value + key = @possible_simple_keys[@flow_level] + # Do we determine a simple key? + if key.nil? + # Block context needs additional checks. + # (Do we really need them? They will be catched by the parser + # anyway.) + if @flow_zero + # We are allowed to start a complex value if and only if + # we can start a simple key. + raise ScannerError.new(nil,nil,"mapping values are not allowed here",get_mark) if !@allow_simple_key + # Simple keys are allowed after ':' in the block context. + @allow_simple_key = true + end + else + # Add KEY. + @possible_simple_keys.delete(@flow_level) + + # If this key starts a new block mapping, we need to add + # BLOCK-MAPPING-START. + se = (@flow_zero && add_indent(key.column)) ? [BlockMappingStartToken.new(key.mark, key.mark)] : [] + se << KeyToken.new(key.mark, key.mark) + @tokens.insert(key.token_number-@tokens_taken,*se) + # There cannot be two simple keys one after another. + @allow_simple_key = false + # It must be a part of a complex key. + end + # Add VALUE. + start_mark = get_mark + forward1 + end_mark = get_mark + @tokens << ValueToken.new(start_mark, end_mark) + end + + def fetch_alias + # ALIAS could be a simple key. + save_possible_simple_key + # No simple keys after ALIAS. + @allow_simple_key = false + # Scan and add ALIAS. + @tokens << scan_anchor(AliasToken) + end + + def fetch_anchor + # ANCHOR could start a simple key. + save_possible_simple_key + # No simple keys after ANCHOR. + @allow_simple_key = false + # Scan and add ANCHOR. + @tokens << scan_anchor(AnchorToken) + end + + def fetch_tag + # TAG could start a simple key. + save_possible_simple_key + # No simple keys after TAG. + @allow_simple_key = false + # Scan and add TAG. + @tokens << scan_tag + end + + def fetch_literal + fetch_block_scalar(?|) + end + + def fetch_folded + fetch_block_scalar(?>) + end + + def fetch_block_scalar(style) + # A simple key may follow a block scalar. + @allow_simple_key = true + # Scan and add SCALAR. + @tokens << scan_block_scalar(style) + end + + def fetch_single + fetch_flow_scalar(?') + end + + def fetch_double + fetch_flow_scalar(?") + end + + def fetch_flow_scalar(style) + # A flow scalar could be a simple key. + save_possible_simple_key + # No simple keys after flow scalars. + @allow_simple_key = false + # Scan and add SCALAR. + @tokens << scan_flow_scalar(style) + end + + def fetch_plain + # A plain scalar could be a simple key. + save_possible_simple_key + # No simple keys after plain scalars. But note that `scan_plain` will + # change this flag if the scan is finished at the beginning of the + # line. + @allow_simple_key = false + # Scan and add SCALAR. May change `allow_simple_key`. + @tokens << scan_plain + end + + + # Scanners. + NULL_OR_LINEBR = "\0\r\n\x85" + def scan_to_next_token + # We ignore spaces, line breaks and comments. + # If we find a line break in the block context, we set the flag + # `allow_simple_key` on. + # + # TODO: We need to make tab handling rules more sane. A good rule is + # Tabs cannot precede tokens + # BLOCK-SEQUENCE-START, BLOCK-MAPPING-START, BLOCK-END, + # KEY(block), VALUE(block), BLOCK-ENTRY + # So the checking code is + # if : + # @allow_simple_keys = false + # We also need to add the check for `allow_simple_keys == true` to + # `unwind_indent` before issuing BLOCK-END. + # Scanners for block, flow, and plain scalars need to be modified. + while true + peek_0 = peek0 + while peek_0 == 32 + forward1 + peek_0 = peek0 + end + if peek_0 == ?# + while !NULL_OR_LINEBR.include?(peek0) + forward1 + peek_0 = peek0 + end + end + + if !scan_line_break.empty? + @allow_simple_key = true if @flow_zero + else + break + end + end + end + + def scan_directive + # See the specification for details. + start_mark = get_mark + forward1 + name = scan_directive_name(start_mark) + value = nil + if name == "YAML" + value = scan_yaml_directive_value(start_mark) + end_mark = get_mark + elsif name == "TAG" + value = scan_tag_directive_value(start_mark) + end_mark = get_mark + else + end_mark = get_mark + forward1 while !NULL_OR_LINEBR.include?(peek0) + end + scan_directive_ignored_line(start_mark) + DirectiveToken.new(name, value, start_mark, end_mark) + end + + ALPHA_REG = /[-0-9A-Za-z_]/ + NULL_BL_LINEBR = "\0 \r\n\x85" + NULL_BL_T_LINEBR = "\0 \t\r\n\x85" + def scan_directive_name(start_mark) + # See the specification for details. + length = 0 + ch = peek(length) + zlen = false + while ALPHA_REG =~ ch.chr + zlen = true + length += 1 + ch = peek(length) + end + raise ScannerError.new("while scanning a directive", start_mark,"expected alphabetic or numeric character, but found #{ch.to_s}",get_mark) if zlen + value = prefix(length) + forward(length) + ch = peek0 + raise ScannerError.new("while scanning a directive", start_mark,"expected alphabetic or numeric character, but found #{ch.to_s}",get_mark) if !NULL_BL_LINEBR.include?(ch) + value + end + + def scan_yaml_directive_value(start_mark) + # See the specification for details. + forward1 while peek0 == 32 + major = scan_yaml_directive_number(start_mark) + raise ScannerError.new("while scanning a directive", start_mark,"expected a digit or '.', but found #{peek.to_s}",get_mark) if peek0 != ?. + forward1 + minor = scan_yaml_directive_number(start_mark) + raise ScannerError.new("while scanning a directive", start_mark,"expected a digit or ' ', but found #{peek.to_s}",get_mark) if !NULL_BL_LINEBR.include?(peek0) + [major, minor] + end + + def scan_yaml_directive_number(start_mark) + # See the specification for details. + ch = peek0 + raise ScannerError.new("while scanning a directive", start_mark,"expected a digit, but found #{ch.to_s}",get_mark) if !(ch.__is_ascii_num) + length = 0 + length += 1 while (peek(length).__is_ascii_num) + value = prefix(length) + forward(length) + value + end + + def scan_tag_directive_value(start_mark) + # See the specification for details. + forward1 while peek0 == 32 + handle = scan_tag_directive_handle(start_mark) + forward1 while peek0 == 32 + prefix = scan_tag_directive_prefix(start_mark) + [handle, prefix] + end + + def scan_tag_directive_handle(start_mark) + # See the specification for details. + value = scan_tag_handle("directive", start_mark) + raise ScannerError.new("while scanning a directive", start_mark,"expected ' ', but found #{peek0}",get_mark()) if peek0 != 32 + value + end + + def scan_tag_directive_prefix(start_mark) + # See the specification for details. + value = scan_tag_uri("directive", start_mark) + raise ScannerError.new("while scanning a directive", start_mark,"expected ' ', but found #{peek0}",get_mark()) if !NULL_BL_LINEBR.include?(peek0) + value + end + + def scan_directive_ignored_line(start_mark) + # See the specification for details. + forward1 while peek0 == 32 + if peek0 == ?# + forward1 while !NULL_OR_LINEBR.include?(peek0) + end + ch = peek0 + raise ScannerError.new("while scanning a directive", start_mark,"expected a comment or a line break, but found #{peek0.to_s}",get_mark()) if !NULL_OR_LINEBR.include?(peek0) + scan_line_break + end + + NON_ALPHA = /[^-0-9A-Za-z_]/ + NON_ALPHA_OR_NUM = "\0 \t\r\n\x85?:,]}%@`" + def scan_anchor(token) + # The specification does not restrict characters for anchors and + # aliases. This may lead to problems, for instance, the document: + # [ *alias, value ] + # can be interpteted in two ways, as + # [ "value" ] + # and + # [ *alias , "value" ] + # Therefore we restrict aliases to numbers and ASCII letters. + start_mark = get_mark + indicator = peek0 + name = (indicator == ?*) ? "alias":"anchor" + forward1 + length = 0 + chunk_size = 16 + while true + chunk = prefix(chunk_size) + if length = (NON_ALPHA =~ chunk) + break + end + chunk_size += 16 + end + raise ScannerError.new("while scanning an #{name}", start_mark,"expected alphabetic or numeric character, but found something else...",get_mark) if length==0 + value = prefix(length) + forward(length) + if !NON_ALPHA_OR_NUM.include?(peek0) + raise ScannerError.new("while scanning an #{name}", start_mark,"expected alphabetic or numeric character, but found #{peek0}",get_mark) + end + end_mark = get_mark + token.new(value, start_mark, end_mark) + end + + NULL_T_BL_LINEBR = "\0 \t\r\n\x85" + def scan_tag + # See the specification for details. + start_mark = get_mark + ch = peek1 + if ch == ?< + handle = nil + forward2 + suffix = scan_tag_uri("tag", start_mark) + raise ScannerError.new("while parsing a tag", start_mark,"expected '>', but found #{peek.to_s}",get_mark) if peek0 != ?> + forward1 + elsif NULL_T_BL_LINEBR.include?(ch) + handle = nil + suffix = "!" + forward1 + else + length = 1 + use_handle = false + while !NULL_T_BL_LINEBR.include?(ch) + if ch == ?! + use_handle = true + break + end + length += 1 + ch = peek(length) + end + handle = "!" + if use_handle + handle = scan_tag_handle("tag", start_mark) + else + handle = "!" + forward1 + end + suffix = scan_tag_uri("tag", start_mark) + end + raise ScannerError.new("while scanning a tag",start_mark,"expected ' ', but found #{peek0}",get_mark) if !NULL_BL_LINEBR.include?(peek0) + value = [handle, suffix] + end_mark = get_mark + TagToken.new(value, start_mark, end_mark) + end + + BLANK_T = " \t" + def scan_block_scalar(style) + # See the specification for details. + folded = style== ?> + chunks = [] + start_mark = get_mark + # Scan the header. + forward1 + chomping, increment = scan_block_scalar_indicators(start_mark) + scan_block_scalar_ignored_line(start_mark) + # Determine the indentation level and go to the first non-empty line. + min_indent = @indent+1 + min_indent = 1 if min_indent < 1 + if increment.nil? + breaks, max_indent, end_mark = scan_block_scalar_indentation + indent = [min_indent, max_indent].max + else + indent = min_indent+increment-1 + breaks, end_mark = scan_block_scalar_breaks(indent) + end + line_break = '' + # Scan the inner part of the block scalar. + while column == indent and peek0 != ?\0 + chunks += breaks + leading_non_space = !BLANK_T.include?(peek0) + length = 0 + length += 1 while !NULL_OR_LINEBR.include?(peek(length)) + chunks << prefix(length) + forward(length) + line_break = scan_line_break + breaks, end_mark = scan_block_scalar_breaks(indent) + if column == indent && peek0 != 0 + # Unfortunately, folding rules are ambiguous. + # + # This is the folding according to the specification: + if folded && line_break == ?\n && leading_non_space && !BLANK_T.include?(peek0) + chunks << ' ' if breaks.empty? + else + chunks << line_break + end + # This is Clark Evans's interpretation (also in the spec + # examples): + # + #if folded and line_break == u'\n': + # if not breaks: + # if self.peek() not in ' \t': + # chunks.append(u' ') + # else: + # chunks.append(line_break) + #else: + # chunks.append(line_break) + else + break + end + end + + # Chomp the tail. + if chomping + chunks << line_break + chunks += breaks + end + + # We are done. + ScalarToken.new(chunks.to_s, false, start_mark, end_mark,style) + end + + PLUS_MIN = /[+-]/ + def scan_block_scalar_indicators(start_mark) + # See the specification for details. + chomping = nil + increment = nil + ch = peek0 + if PLUS_MIN =~ ch.chr + chomping = ch == ?+ + forward1 + ch = peek0 + if ch.__is_ascii_num + increment = ch.to_i + raise ScannerError.new("while scanning a block scalar", start_mark,"expected indentation indicator in the range 1-9, but found 0",get_mark) if increment == 0 + forward1 + end + elsif ch.__is_ascii_num + increment = ch + raise ScannerError.new("while scanning a block scalar", start_mark,"expected indentation indicator in the range 1-9, but found 0",get_mark) if increment == 0 + forward1 + ch = peek0 + if PLUS_MIN =~ ch.chr + chomping = ch == ?+ + forward1 + end + end + raise ScannerError.new("while scanning a block scalar", start_mark,"expected chomping or indentation indicators, but found #{peek0}",get_mark) if !NULL_BL_LINEBR.include?(peek0) + [chomping, increment] + end + + def scan_block_scalar_ignored_line(start_mark) + # See the specification for details. + forward1 while peek0 == 32 + if peek0 == ?# + forward1 while !NULL_OR_LINEBR.include?(peek0) + end + raise ScannerError.new("while scanning a block scalar", start_mark,"expected a comment or a line break, but found #{peek0}",get_mark) if !NULL_OR_LINEBR.include?(peek0) + scan_line_break + end + + BLANK_OR_LINEBR = " \r\n\x85" + def scan_block_scalar_indentation + # See the specification for details. + chunks = [] + max_indent = 0 + end_mark = get_mark + while BLANK_OR_LINEBR.include?(peek0) + if peek0 != 32 + chunks << scan_line_break + end_mark = get_mark + else + forward1 + max_indent = column if column > max_indent + end + end + [chunks, max_indent, end_mark] + end + + FULL_LINEBR = "\r\n\x85" + def scan_block_scalar_breaks(indent) + # See the specification for details. + chunks = [] + end_mark = get_mark + forward1 while @column < indent && peek0 == 32 + while FULL_LINEBR.include?(peek0) + chunks << scan_line_break + end_mark = get_mark + forward1 while @column < indent && peek0 == 32 + end + [chunks, end_mark] + end + + def scan_flow_scalar(style) + # See the specification for details. + # Note that we loose indentation rules for quoted scalars. Quoted + # scalars don't need to adhere indentation because " and ' clearly + # mark the beginning and the end of them. Therefore we are less + # restrictive then the specification requires. We only need to check + # that document separators are not included in scalars. + double = style == ?" + chunks = [] + start_mark = get_mark + quote = peek0 + forward1 + chunks += scan_flow_scalar_non_spaces(double, start_mark) + while peek0 != quote + chunks += scan_flow_scalar_spaces(double, start_mark) + chunks += scan_flow_scalar_non_spaces(double, start_mark) + end + forward1 + end_mark = get_mark + ScalarToken.new(chunks.to_s, false, start_mark, end_mark,style) + end + + ESCAPE_REPLACEMENTS = { + "0" => "\0", + "a" => "\x07", + "b" => "\x08", + "t" => "\x09", + "\t" => "\x09", + "n" => "\x0A", + "v" => "\x0B", + "f" => "\x0C", + "r" => "\x0D", + "e" => "\x1B", + " " => "\x20", + '"' => '"', + "\\" => "\\", + "N" => "\x85", + "_" => "\xA0" + } + + ESCAPE_CODES = { + 'x' => 2 + } + + SPACES_AND_STUFF = "'\"\\\0 \t\r\n\x85" + DOUBLE_ESC = "\"\\" + NOT_HEXA = /[^0-9A-Fa-f]/ + def scan_flow_scalar_non_spaces(double, start_mark) + # See the specification for details. + chunks = [] + while true + length = 0 + length += 1 while !SPACES_AND_STUFF.include?(peek(length)) + if length!=0 + chunks << prefix(length) + forward(length) + end + ch = peek0 + if !double && ch == ?' && peek1 == ?' + chunks << ?' + forward2 + elsif (double && ch == ?') || (!double && DOUBLE_ESC.include?(ch)) + chunks << ch + forward1 + elsif double && ch == ?\\ + forward1 + ch = peek0 + if ESCAPE_REPLACEMENTS.member?(ch.chr) + chunks << ESCAPE_REPLACEMENTS[ch.chr] + forward1 + elsif ESCAPE_CODES.member?(ch.chr) + length = ESCAPE_CODES[ch.chr] + forward1 + if NOT_HEXA =~ prefix(length) + raise ScannerError.new("while scanning a double-quoted scalar", start_mark, + "expected escape sequence of #{length} hexdecimal numbers, but found something else: #{prefix(length)}}",get_mark) + end + code = prefix(length).to_i.to_s(16) + chunks << code + forward(length) + elsif FULL_LINEBR.include?(ch) + scan_line_break + chunks += scan_flow_scalar_breaks(double, start_mark) + else + raise ScannerError.new("while scanning a double-quoted scalar", start_mark,"found unknown escape character #{ch}",get_mark) + end + else + return chunks + end + end + end + + def scan_flow_scalar_spaces(double, start_mark) + # See the specification for details. + chunks = [] + length = 0 + length += 1 while BLANK_T.include?(peek(length)) + whitespaces = prefix(length) + forward(length) + ch = peek0 + if ch == ?\0 + raise ScannerError.new("while scanning a quoted scalar", start_mark,"found unexpected end of stream",get_mark) + elsif FULL_LINEBR.include?(ch) + line_break = scan_line_break + breaks = scan_flow_scalar_breaks(double, start_mark) + if line_break != ?\n + chunks << line_break + elsif breaks.empty? + chunks << ' ' + end + chunks += breaks + else + chunks << whitespaces + end + chunks + end + + def scan_flow_scalar_breaks(double, start_mark) + # See the specification for details. + chunks = [] + while true + # Instead of checking indentation, we check for document + # separators. + prefix = prefix(3) + if (prefix == "---" || prefix == "...") &&NULL_BL_T_LINEBR.include?(peek3) + raise ScannerError.new("while scanning a quoted scalar", start_mark,"found unexpected document separator", get_mark) + end + forward1 while BLANK_T.include?(peek0) + if FULL_LINEBR.include?(peek0) + chunks << scan_line_break + else + return chunks + end + end + end + + + R_flowzero = /[\0 \t\r\n\x85]|(:[\0 \t\r\n\x28])/ + R_flownonzero = /[\0 \t\r\n\x85\[\]{},:?]/ + S4 = "\0 \t\r\n\x28[]{}" + + def scan_plain + # See the specification for details. + # We add an additional restriction for the flow context: + # plain scalars in the flow context cannot contain ',', ':' and '?'. + # We also keep track of the `allow_simple_key` flag here. + # Indentation rules are loosed for the flow context. + chunks = [] + end_mark = start_mark = get_mark + indent = @indent+1 + # We allow zero indentation for scalars, but then we need to check for + # document separators at the beginning of the line. + #if indent == 0 + # indent = 1 + spaces = [] + if @flow_zero + f_nzero, r_check = false, R_flowzero + else + f_nzero, r_check = true, R_flownonzero + end + + while peek0 != ?# + length = 0 + chunk_size = 32 + chunk_size += 32 until length = (r_check =~ prefix(chunk_size)) + ch = peek(length) + if f_nzero && ch == ?: && !S4.include?(peek(length+1)) + forward(length) + raise ScannerError.new("while scanning a plain scalar",start_mark,"found unexpected ':'",get_mark,"Please check http://pyyaml.org/wiki/YAMLColonInFlowContext for details.") + end + break if length == 0 + @allow_simple_key = false + chunks += spaces + chunks << prefix(length) + forward(length) + end_mark = get_mark + spaces = scan_plain_spaces(indent, start_mark) + break if !spaces || (@flow_zero && @column < indent) + end + return ScalarToken.new(chunks.to_s, true, start_mark, end_mark) + end + + END_OR_START = /^(---|\.\.\.)[\0 \t\r\n\x85]$/ + def scan_plain_spaces(indent, start_mark) + # See the specification for details. + # The specification is really confusing about tabs in plain scalars. + # We just forbid them completely. Do not use tabs in YAML! + chunks = [] + length = 0 + length += 1 while peek(length) == 32 + whitespaces = prefix(length) + forward(length) + ch = peek0 + if FULL_LINEBR.include?(ch) + line_break = scan_line_break + @allow_simple_key = true + return if END_OR_START =~ prefix(4) + breaks = [] + while BLANK_OR_LINEBR.include?(peek0) + if peek0 == 32 + forward1 + else + breaks << scan_line_break + return if END_OR_START =~ prefix(4) + end + end + if line_break != '\n' + chunks << line_break + elsif breaks.empty? + chunks << ' ' + end + chunks += breaks + else + chunks << whitespaces + end + chunks + end + + + def scan_tag_handle(name, start_mark) + # See the specification for details. + # For some strange reasons, the specification does not allow '_' in + # tag handles. I have allowed it anyway. + ch = peek0 + raise ScannerError.new("while scanning a #{name}", start_mark,"expected '!', but found #{ch}",get_mark) if ch != ?! + length = 1 + ch = peek(length) + if ch != 32 + while ALPHA_REG =~ ch.chr + length += 1 + ch = peek(length) + end + if ch != ?! + forward(length) + raise ScannerError.new("while scanning a #{name}", start_mark,"expected '!', but found #{ch}",get_mark) + end + length += 1 + end + value = prefix(length) + forward(length) + value + end + + STRANGE_CHR = /[\]\[\-';\/?:@&=+$,.!~*()%\w]/ + def scan_tag_uri(name, start_mark) + # See the specification for details. + # Note: we do not check if URI is well-formed. + chunks = [] + length = 0 + ch = peek(length) + while STRANGE_CHR =~ ch.chr + if ch == ?% + chunks << prefix(length) + forward(length) + length = 0 + chunks << scan_uri_escapes(name, start_mark) + else + length += 1 + end + ch = peek(length) + end + if length!=0 + chunks << prefix(length) + forward(length) + length = 0 + end + + raise ScannerError.new("while parsing a #{name}", start_mark,"expected URI, but found #{ch}",get_mark) if chunks.empty? + chunks.to_s + end + + HEXA_REG = /[0-9A-Fa-f]/ + def scan_uri_escapes(name, start_mark) + # See the specification for details. + bytes = [] + mark = get_mark + while peek0 == ?% + forward1 + raise ScannerError.new("while scanning a #{name}", start_mark,"expected URI escape sequence of 2 hexdecimal numbers, but found #{peek1} and #{peek2}",get_mark) if HEXA_REG !~ peek1.chr || HEXA_REG !~ peek2.chr + bytes << prefix(2).to_i.to_s(16) + forward2 + end + bytes.to_s + end + + RN = "\r\n" + def scan_line_break + # Transforms: + # '\r\n' : '\n' + # '\r' : '\n' + # '\n' : '\n' + # '\x85' : '\n' + # default : '' + if FULL_LINEBR.include?(peek0) + if prefix2 == RN + forward2 + else + forward1 + end + return "\n" + end + "" + end + end +end + diff --git a/lib/ruby/site_ruby/1.8/rbyaml/serializer.rb b/lib/ruby/site_ruby/1.8/rbyaml/serializer.rb new file mode 100644 index 00000000000..55ea81199e4 --- /dev/null +++ b/lib/ruby/site_ruby/1.8/rbyaml/serializer.rb @@ -0,0 +1,117 @@ +require 'rbyaml/error' +require 'rbyaml/events' +require 'rbyaml/nodes' + +module RbYAML + class SerializerError < YAMLError + end + + class Serializer + ANCHOR_TEMPLATE = "id%03d" + + def initialize(emitter, resolver, explicit_start=nil, explicit_end=nil, version=nil, tags=nil) + @emitter = emitter + @resolver = resolver + @use_explicit_start = explicit_start + @use_explicit_end = explicit_end + @use_version = version + @use_tags = tags + @serialized_nodes = {} + @anchors = {} + @last_anchor_id = 0 + @closed = nil + end + + def open + if @closed.nil? + @emitter.emit(StreamStartEvent.new) + @closed = false + elsif @closed + raise SerializerError.new("serializer is closed") + else + raise SerializerError.new("serializer is already opened") + end + end + + def close + if @closed.nil? + raise SerializerError.new("serializer is not opened") + elsif !@closed + @emitter.emit(StreamEndEvent.new) + @closed = true + end + end + + def serialize(node) + if @closed.nil? + raise SerializerError.new("serializer is not opened") + elsif @closed + raise SerializerError.new("serializer is closed") + end + @emitter.emit(DocumentStartEvent.new(@use_explicit_start,@use_version,@use_tags)) + anchor_node(node) + serialize_node(node,nil,nil) + @emitter.emit(DocumentEndEvent.new(@use_explicit_end)) + @serialized_nodes = {} + @anchors = {} + @last_alias_id = 0 + end + + def anchor_node(node) + if @anchors.include?(node) + @anchors[node] ||= generate_anchor(node) + else + @anchors[node] = nil + if SequenceNode === node + for item in node.value + anchor_node(item) + end + elsif MappingNode === node + for key,val in node.value + anchor_node(key) + anchor_node(val) + end + end + end + end + + def generate_anchor(node) + @last_anchor_id += 1 + ANCHOR_TEMPLATE % @last_anchor_id + end + + def serialize_node(node,parent,index) + talias = @anchors[node] + if @serialized_nodes.include?(node) + @emitter.emit(AliasEvent.new(talias)) + else + @serialized_nodes[node] = true + @resolver.descend_resolver(parent, index) + if ScalarNode === node + detected_tag = @resolver.resolve(ScalarNode, node.value, [true,false]) + default_tag = @resolver.resolve(ScalarNode, node.value, [false,true]) + implicit = (node.tag == detected_tag), (node.tag == default_tag) + @emitter.emit(ScalarEvent.new(talias, node.tag, implicit, node.value,nil,nil,node.style)) + elsif SequenceNode === node + implicit = (node.tag == @resolver.resolve(SequenceNode, node.value, true)) + @emitter.emit(SequenceStartEvent.new(talias, node.tag, implicit,node.flow_style)) + index = 0 + for item in node.value + serialize_node(item,node,index) + index += 1 + end + @emitter.emit(SequenceEndEvent.new) + elsif MappingNode === node + implicit = (node.tag == @resolver.resolve(MappingNode, node.value, true)) + @emitter.emit(MappingStartEvent.new(talias, node.tag, implicit,node.flow_style)) + for key, value in node.value + serialize_node(key,node,nil) + serialize_node(value,node,key) + end + @emitter.emit(MappingEndEvent.new) + end + end + end + end +end + diff --git a/lib/ruby/site_ruby/1.8/rbyaml/test.rb b/lib/ruby/site_ruby/1.8/rbyaml/test.rb new file mode 100644 index 00000000000..612c72620fd --- /dev/null +++ b/lib/ruby/site_ruby/1.8/rbyaml/test.rb @@ -0,0 +1,56 @@ +require 'rbyaml/error' +require 'rbyaml/reader' +require 'rbyaml/scanner' +require 'rbyaml/parser' +require 'rbyaml/composer' +require 'rbyaml/constructor' +require 'rbyaml/detector' + +class RbYAMLTester + include RbYAML::Reader, RbYAML::Scanner, RbYAML::Parser + + def initialize(stream) + initialize_reader(stream) + initialize_scanner + initialize_parser + end +end + +class RbYAMLTester2 + include RbYAML::Reader, RbYAML::Scanner, RbYAML::Parser, RbYAML::Composer + + def initialize(stream) + initialize_reader(stream) + initialize_scanner + initialize_parser + initialize_composer + end +end + +class RbYAMLTester3 + include RbYAML::Reader, RbYAML::Scanner, RbYAML::Parser, RbYAML::Composer, RbYAML::Constructor, RbYAML::Detector + + def initialize(stream) + initialize_reader(stream) + initialize_scanner + initialize_parser + initialize_composer + initialize_constructor + end +end + +i=0 +begin + File.open(ARGV.shift) {|f| + tester = RbYAMLTester3.new(f) + tester.each_document {|doc| + puts "#{doc.inspect}" + # i += 1 + # if (i%10000) == 0 + # puts "token ##{i}" + # end + } + } +rescue RbYAML::MarkedYAMLError => err + puts "MarkedYAMLError: #{err}" +end diff --git a/lib/ruby/site_ruby/1.8/rbyaml/tokens.rb b/lib/ruby/site_ruby/1.8/rbyaml/tokens.rb new file mode 100644 index 00000000000..451a9a320a1 --- /dev/null +++ b/lib/ruby/site_ruby/1.8/rbyaml/tokens.rb @@ -0,0 +1,204 @@ + +module RbYAML + Token = Struct.new(:start_mark, :end_mark) + + class Token + def __is_document_start; false; end + def __is_document_end; false; end + def __is_stream_start; false; end + def __is_stream_end; false; end + def __is_directive; false; end + def __is_block_sequence_start; false; end + def __is_block_mapping_start; false; end + def __is_block_end; false; end + def __is_flow_sequence_start; false; end + def __is_flow_mapping_start; false; end + def __is_flow_sequence_end; false; end + def __is_flow_mapping_end; false; end + def __is_key; false; end + def __is_value; false; end + def __is_block_entry; false; end + def __is_flow_entry; false; end + def __is_alias; false; end + def __is_anchor; false; end + def __is_tag; false; end + def __is_scalar; false; end + def hash + object_id + end + end + + class DirectiveToken < Token + def __is_directive; true; end + def tid + "" + end + attr_reader :name, :value + def initialize(name, value, start_mark, end_mark) + super(start_mark,end_mark) + @name = name + @value = value + end + end + + class DocumentStartToken < Token + def __is_document_start; true; end + def tid + "" + end + end + + class DocumentEndToken < Token + def __is_document_end; true; end + def tid + "" + end + end + + class StreamStartToken < Token + def __is_stream_start; true; end + def tid + "" + end + attr_reader :encoding + def initialize(start_mark=nil, end_mark=nil, encoding=nil) + super(start_mark,end_mark) + @encoding = encoding + end + end + + class StreamEndToken < Token + def __is_stream_end; true; end + def tid + "" + end + end + + class BlockSequenceStartToken < Token + def __is_block_sequence_start; true; end + def tid + "" + end + end + + class BlockMappingStartToken < Token + def __is_block_mapping_start; true; end + def tid + "" + end + end + + class BlockEndToken < Token + def __is_block_end; true; end + def tid + "" + end + end + + class FlowSequenceStartToken < Token + def __is_flow_sequence_start; true; end + def tid + "[" + end + end + + class FlowMappingStartToken < Token + def __is_flow_mapping_start; true; end + def tid + "{" + end + end + + class FlowSequenceEndToken < Token + def __is_flow_sequence_end; true; end + def tid + "]" + end + end + + class FlowMappingEndToken < Token + def __is_flow_mapping_end; true; end + def tid + "}" + end + end + + class KeyToken < Token + def __is_key; true; end + def tid + "?" + end + end + + class ValueToken < Token + def __is_value; true; end + def tid + ":" + end + end + + class BlockEntryToken < Token + def __is_block_entry; true; end + def tid + "-" + end + end + + class FlowEntryToken < Token + def __is_flow_entry; true; end + def tid + "," + end + end + + class AliasToken < Token + def __is_alias; true; end + def tid + "" + end + attr_reader :value + def initialize(value, start_mark, end_mark) + super(start_mark,end_mark) + @value = value + end + end + + class AnchorToken < Token + def __is_anchor; true; end + def tid + "" + end + attr_reader :value + def initialize(value, start_mark, end_mark) + super(start_mark,end_mark) + @value = value + end + end + + class TagToken < Token + def __is_tag; true; end + def tid + "" + end + attr_reader :value + def initialize(value, start_mark, end_mark) + super(start_mark,end_mark) + @value = value + end + end + + class ScalarToken < Token + def __is_scalar; true; end + def tid + "" + end + attr_reader :value, :plain, :style + alias :implicit :plain #Until all references to ScalarToken.implicit has been removed + def initialize(value, plain, start_mark, end_mark, style=nil) + super(start_mark, end_mark) + @value = value + @plain = plain + @style = style + end + end +end diff --git a/lib/ruby/site_ruby/1.8/rbyaml/util.rb b/lib/ruby/site_ruby/1.8/rbyaml/util.rb new file mode 100644 index 00000000000..8af4f7acc22 --- /dev/null +++ b/lib/ruby/site_ruby/1.8/rbyaml/util.rb @@ -0,0 +1,28 @@ +class Object + def __is_str; false end + def __is_sym; false end + def __is_a; false end + def __is_int; false end +end + +class String + def __is_str; true end +end + +class Symbol + def __is_sym; true end +end + +class Array + def __is_a; true end +end + +class Integer + def __is_int; true end +end + +class Fixnum + def __is_ascii_num + self <= ?9 && self >= ?0 + end +end diff --git a/lib/ruby/site_ruby/1.8/rbyaml/yaml.rb b/lib/ruby/site_ruby/1.8/rbyaml/yaml.rb new file mode 100644 index 00000000000..c4051640839 --- /dev/null +++ b/lib/ruby/site_ruby/1.8/rbyaml/yaml.rb @@ -0,0 +1,136 @@ +require 'stringio' + +require 'rbyaml/error' + +require 'rbyaml/tokens' +require 'rbyaml/events' +require 'rbyaml/nodes' + +require 'rbyaml/loader' +require 'rbyaml/dumper' + +module RbYAML + def self._scan(stream, loader=Loader) + l = loader.new(stream) + yield l.scanner.get_token while l.scanner.check_token + end + + def self._parse(stream, loader=Loader) + l = loader.new(stream) + yield l.parser.get_event while l.parser.check_event + end + + def self._compose(stream, loader=Loader) + l = loader.new(stream) + l.composer.get_node if l.composer.check_node + end + + def self._compose_all(stream, loader=Loader) + l = loader.new(stream) + yield l.composer.get_node while l.composer.check_node + end + + def self._load_all(stream, loader=Loader) + l = loader.new(stream) + yield l.constructor.get_data while l.constructor.check_data + end + + def self._load(stream, loader=Loader) + l = loader.new(stream) + l.constructor.get_data if l.constructor.check_data + end + + def self._safe_load_all(stream) + _load_all(stream, SafeLoader) + end + + def self._safe_load(stream) + _load(stream, SafeLoader) + end + + def self._emit(events, stream=nil, dumper=Dumper,default_style=nil,default_flow_style=nil,canonical=nil, indent=nil, width=nil,line_break=nil) + if stream.nil? + require 'stringio' + stream = StringIO.new + end + dumper = dumper.new(stream,default_style,default_flow_style,canonical,indent,width,line_break) + for event in events + dumper.emit(event) + end + stream.string if StringIO === stream + end + + def self._serialize_all(nodes,stream=nil,dumper=Dumper,default_style=nil,default_flow_style=nil,canonical=nil,indent=nil,width=nil,line_break=nil,explicit_start=nil,explicit_end=nil,version=nil,tags=nil) + if stream.nil? + require 'stringio' + stream = StringIO.new + end + dumper = dumper.new(stream,default_style,default_flow_style,canonical,indent,width,line_break,version,tags,explicit_start,explicit_end) + dumper.serializer.open + for node in nodes + dumper.serializer.serialize(node) + end + dumper.serializer.close + stream.string if StringIO === stream + end + + def self._serialize(node, stream=nil, dumper=Dumper, *kwds) + _serialize_all([node], stream, dumper, *kwds) + end + + def self._dump_all(documents,stream=nil,dumper=Dumper,default_style=nil,default_flow_style=nil,canonical=nil,indent=nil,width=nil,line_break=nil,explicit_start=nil,explicit_end=nil,version=nil,tags=nil) + if stream.nil? + require 'stringio' + stream = StringIO.new + end + dumper = dumper.new(stream,default_style,default_flow_style,canonical,indent,width,line_break,version,tags,explicit_start,explicit_end) + dumper.serializer.open + for data in documents + dumper.representer.represent(data) + end + dumper.serializer.close + stream.string if StringIO === stream + end + + def self._dump(data, stream=nil, dumper=Dumper, *kwds) + _dump_all([data], stream, dumper, *kwds) + end + + def self._safe_dump_all(documents, stream=nil, *kwds) + _dump_all(documents, stream, SafeDumper, *kwds) + end + + def self._safe_dump(data, stream=nil, *kwds) + _dump_all([data], stream, SafeDumper, *kwds) + end + + def self._add_implicit_resolver(tag, regexp, first=nil, loader=Loader, dumper=Dumper) + loader.add_implicit_resolver(tag, regexp, first) + dumper.add_implicit_resolver(tag, regexp, first) + end + + def self._add_path_resolver(tag, path, kind=nil, loader=Loader, dumper=Dumper) + loader.add_path_resolver(tag, path, kind) + lumper.add_path_resolver(tag, path, kind) + end + + def self._add_constructor(tag, constructor, loader=Loader) + loader.add_constructor(tag, constructor) + end + + def self._add_multi_constructor(tag_prefix, multi_constructor, loader=Loader) + loader.add_multi_constructor(tag_prefix, multi_constructor) + end + + def self._add_representer(data_type, representer, dumper=Dumper) + dumper.add_representer(data_type, representer) + end + + def self._add_multi_representer(data_type, multi_representer, dumper=Dumper) + dumper.add_multi_representer(data_type, multi_representer) + end + + def self._dump_ruby_object(data, dumper=Dumper) + _dump(data,nil,dumper) + end +end