aws-eventstream-1.1.0/0000755000004100000410000000000013704150541014666 5ustar www-datawww-dataaws-eventstream-1.1.0/aws-eventstream.gemspec0000644000004100000410000000261413704150541021363 0ustar www-datawww-data######################################################### # This file has been automatically generated by gem2tgz # ######################################################### # -*- encoding: utf-8 -*- # stub: aws-eventstream 1.1.0 ruby lib Gem::Specification.new do |s| s.name = "aws-eventstream".freeze s.version = "1.1.0" s.required_rubygems_version = Gem::Requirement.new(">= 0".freeze) if s.respond_to? :required_rubygems_version= s.metadata = { "changelog_uri" => "https://github.com/aws/aws-sdk-ruby/tree/master/gems/aws-eventstream/CHANGELOG.md", "source_code_uri" => "https://github.com/aws/aws-sdk-ruby/tree/master/gems/aws-eventstream" } if s.respond_to? :metadata= s.require_paths = ["lib".freeze] s.authors = ["Amazon Web Services".freeze] s.date = "2020-04-08" s.description = "Amazon Web Services event stream library. Decodes and encodes binary stream under `vnd.amazon.event-stream` content-type".freeze s.files = ["lib/aws-eventstream.rb".freeze, "lib/aws-eventstream/decoder.rb".freeze, "lib/aws-eventstream/encoder.rb".freeze, "lib/aws-eventstream/errors.rb".freeze, "lib/aws-eventstream/header_value.rb".freeze, "lib/aws-eventstream/message.rb".freeze, "lib/aws-eventstream/types.rb".freeze] s.homepage = "https://github.com/aws/aws-sdk-ruby".freeze s.licenses = ["Apache-2.0".freeze] s.rubygems_version = "2.5.2.1".freeze s.summary = "AWS Event Stream Library".freeze end aws-eventstream-1.1.0/lib/0000755000004100000410000000000013704150541015434 5ustar www-datawww-dataaws-eventstream-1.1.0/lib/aws-eventstream/0000755000004100000410000000000013704150541020561 5ustar www-datawww-dataaws-eventstream-1.1.0/lib/aws-eventstream/header_value.rb0000644000004100000410000000211413704150541023530 0ustar www-datawww-datamodule Aws module EventStream class HeaderValue def initialize(options) @type = options.fetch(:type) @value = options[:format] ? format_value(options.fetch(:value)) : options.fetch(:value) end attr_reader :value # @return [String] type of the header value # complete type list see Aws::EventStream::Types attr_reader :type private def format_value(value) case @type when 'timestamp' then format_timestamp(value) when 'uuid' then format_uuid(value) else value end end def format_uuid(value) bytes = value.bytes # For user-friendly uuid representation, # format binary bytes into uuid string format uuid_pattern = [ [ 3, 2, 1, 0 ], [ 5, 4 ], [ 7, 6 ], [ 8, 9 ], 10..15 ] uuid_pattern.map {|p| p.map {|n| "%02x" % bytes.to_a[n] }.join }.join('-') end def format_timestamp(value) # millis_since_epoch to sec_since_epoch Time.at(value / 1000.0) end end end end aws-eventstream-1.1.0/lib/aws-eventstream/encoder.rb0000644000004100000410000001063413704150541022531 0ustar www-datawww-datarequire 'zlib' module Aws module EventStream # This class provides #encode method for encoding # Aws::EventStream::Message into binary. # # * {#encode} - encode Aws::EventStream::Message into binary # when output IO-like object is provided, binary string # would be written to IO. If not, the encoded binary string # would be returned directly # # ## Examples # # message = Aws::EventStream::Message.new( # headers: { # "foo" => Aws::EventStream::HeaderValue.new( # value: "bar", type: "string" # ) # }, # payload: "payload" # ) # encoder = Aws::EventsStream::Encoder.new # file = Tempfile.new # # # encode into IO ouput # encoder.encode(message, file) # # # get encoded binary string # encoded_message = encoder.encode(message) # # file.read == encoded_message # # => true # class Encoder # bytes of total overhead in a message, including prelude # and 4 bytes total message crc checksum OVERHEAD_LENGTH = 16 # Maximum header length allowed (after encode) 128kb MAX_HEADERS_LENGTH = 131072 # Maximum payload length allowed (after encode) 16mb MAX_PAYLOAD_LENGTH = 16777216 # Encodes Aws::EventStream::Message to output IO when # provided, else return the encoded binary string # # @param [Aws::EventStream::Message] message # # @param [IO#write, nil] io An IO-like object that # responds to `#write`, encoded message will be # written to this IO when provided # # @return [nil, String] when output IO is provided, # encoded message will be written to that IO, nil # will be returned. Else, encoded binary string is # returned. def encode(message, io = nil) encoded = encode_message(message) if io io.write(encoded) io.close else encoded end end # Encodes an Aws::EventStream::Message # into String # # @param [Aws::EventStream::Message] message # # @return [String] def encode_message(message) # create context buffer with encode headers encoded_header = encode_headers(message) header_length = encoded_header.bytesize # encode payload if message.payload.length > MAX_PAYLOAD_LENGTH raise Aws::EventStream::Errors::EventPayloadLengthExceedError.new end encoded_payload = message.payload.read total_length = header_length + encoded_payload.bytesize + OVERHEAD_LENGTH # create message buffer with prelude section encoded_prelude = encode_prelude(total_length, header_length) # append message context (headers, payload) encoded_content = [ encoded_prelude, encoded_header, encoded_payload, ].pack('a*a*a*') # append message checksum message_checksum = Zlib.crc32(encoded_content) [encoded_content, message_checksum].pack('a*N') end # Encodes headers part of an Aws::EventStream::Message # into String # # @param [Aws::EventStream::Message] message # # @return [String] def encode_headers(message) header_entries = message.headers.map do |key, value| encoded_key = [key.bytesize, key].pack('Ca*') # header value pattern, value_length, type_index = Types.pattern[value.type] encoded_value = [type_index].pack('C') # boolean types doesn't need to specify value next [encoded_key, encoded_value].pack('a*a*') if !!pattern == pattern encoded_value = [encoded_value, value.value.bytesize].pack('a*S>') unless value_length [ encoded_key, encoded_value, pattern ? [value.value].pack(pattern) : value.value, ].pack('a*a*a*') end header_entries.join.tap do |encoded_header| break encoded_header if encoded_header.bytesize <= MAX_HEADERS_LENGTH raise Aws::EventStream::Errors::EventHeadersLengthExceedError.new end end private def encode_prelude(total_length, headers_length) prelude_body = [total_length, headers_length].pack('NN') checksum = Zlib.crc32(prelude_body) [prelude_body, checksum].pack('a*N') end end end end aws-eventstream-1.1.0/lib/aws-eventstream/message.rb0000644000004100000410000000072613704150541022537 0ustar www-datawww-datamodule Aws module EventStream class Message def initialize(options) @headers = options[:headers] || {} @payload = options[:payload] || StringIO.new end # @return [Hash] headers of a message attr_reader :headers # @return [IO] payload of a message, size not exceed 16MB. # StringIO is returned for <= 1MB payload # Tempfile is returned for > 1MB payload attr_reader :payload end end end aws-eventstream-1.1.0/lib/aws-eventstream/errors.rb0000644000004100000410000000237313704150541022427 0ustar www-datawww-datamodule Aws module EventStream module Errors # Raised when reading bytes exceed buffer total bytes class ReadBytesExceedLengthError < RuntimeError def initialize(target_byte, total_len) msg = "Attempting reading bytes to offset #{target_byte} exceeds"\ " buffer length of #{total_len}" super(msg) end end # Raise when insufficient bytes of a message is received class IncompleteMessageError < RuntimeError def initialize(*args) super('Not enough bytes for event message') end end class PreludeChecksumError < RuntimeError def initialize(*args) super('Prelude checksum mismatch') end end class MessageChecksumError < RuntimeError def initialize(*args) super('Message checksum mismatch') end end class EventPayloadLengthExceedError < RuntimeError def initialize(*args) super("Payload length of a message should be under 16mb.") end end class EventHeadersLengthExceedError < RuntimeError def initialize(*args) super("Encoded headers length of a message should be under 128kb.") end end end end end aws-eventstream-1.1.0/lib/aws-eventstream/types.rb0000644000004100000410000000146713704150541022262 0ustar www-datawww-datamodule Aws module EventStream # Message Header Value Types module Types def self.types [ 'bool_true', 'bool_false', 'byte', 'short', 'integer', 'long', 'bytes', 'string', 'timestamp', 'uuid' ] end # pack/unpack pattern, byte size, type idx def self.pattern { 'bool_true' => [true, 0, 0], 'bool_false' => [false, 0, 1], 'byte' => ["c", 1, 2], 'short' => ["s>", 2, 3], 'integer' => ["l>", 4, 4], 'long' => ["q>", 8, 5], 'bytes' => [nil, nil, 6], 'string' => [nil, nil, 7], 'timestamp' => ["q>", 8, 8], 'uuid' => [nil, 16, 9] } end end end end aws-eventstream-1.1.0/lib/aws-eventstream/decoder.rb0000644000004100000410000001550113704150541022515 0ustar www-datawww-datarequire 'stringio' require 'tempfile' require 'zlib' module Aws module EventStream # This class provides method for decoding binary inputs into # single or multiple messages (Aws::EventStream::Message). # # * {#decode} - decodes messages from an IO like object responds # to #read that containing binary data, returning decoded # Aws::EventStream::Message along the way or wrapped in an enumerator # # ## Examples # # decoder = Aws::EventStream::Decoder.new # # # decoding from IO # decoder.decode(io) do |message| # message.headers # # => { ... } # message.payload # # => StringIO / Tempfile # end # # # alternatively # message_pool = decoder.decode(io) # message_pool.next # # => Aws::EventStream::Message # # * {#decode_chunk} - decodes a single message from a chunk of data, # returning message object followed by boolean(indicating eof status # of data) in an array object # # ## Examples # # # chunk containing exactly one message data # message, chunk_eof = decoder.decode_chunk(chunk_str) # message # # => Aws::EventStream::Message # chunk_eof # # => true # # # chunk containing a partial message # message, chunk_eof = decoder.decode_chunk(chunk_str) # message # # => nil # chunk_eof # # => true # # chunk data is saved at decoder's message_buffer # # # chunk containing more that one data message # message, chunk_eof = decoder.decode_chunk(chunk_str) # message # # => Aws::EventStream::Message # chunk_eof # # => false # # extra chunk data is saved at message_buffer of the decoder # class Decoder include Enumerable ONE_MEGABYTE = 1024 * 1024 private_constant :ONE_MEGABYTE # bytes of prelude part, including 4 bytes of # total message length, headers length and crc checksum of prelude PRELUDE_LENGTH = 12 private_constant :PRELUDE_LENGTH # 4 bytes message crc checksum CRC32_LENGTH = 4 private_constant :CRC32_LENGTH # @param [Hash] options The initialization options. # @option options [Boolean] :format (true) When `false` it # disables user-friendly formatting for message header values # including timestamp and uuid etc. def initialize(options = {}) @format = options.fetch(:format, true) @message_buffer = '' end # Decodes messages from a binary stream # # @param [IO#read] io An IO-like object # that responds to `#read` # # @yieldparam [Message] message # @return [Enumerable, nil] Returns a new Enumerable # containing decoded messages if no block is given def decode(io, &block) raw_message = io.read decoded_message = decode_message(raw_message) return wrap_as_enumerator(decoded_message) unless block_given? # fetch message only raw_event, _eof = decoded_message block.call(raw_event) end # Decodes a single message from a chunk of string # # @param [String] chunk A chunk of string to be decoded, # chunk can contain partial event message to multiple event messages # When not provided, decode data from #message_buffer # # @return [Array] Returns single decoded message # and boolean pair, the boolean flag indicates whether this chunk # has been fully consumed, unused data is tracked at #message_buffer def decode_chunk(chunk = nil) @message_buffer = [@message_buffer, chunk].pack('a*a*') if chunk decode_message(@message_buffer) end private # exposed via object.send for testing attr_reader :message_buffer def wrap_as_enumerator(decoded_message) Enumerator.new do |yielder| yielder << decoded_message end end def decode_message(raw_message) # incomplete message prelude received return [nil, true] if raw_message.bytesize < PRELUDE_LENGTH prelude, content = raw_message.unpack("a#{PRELUDE_LENGTH}a*") # decode prelude total_length, header_length = decode_prelude(prelude) # incomplete message received, leave it in the buffer return [nil, true] if raw_message.bytesize < total_length content, checksum, remaining = content.unpack("a#{total_length - PRELUDE_LENGTH - CRC32_LENGTH}Na*") unless Zlib.crc32([prelude, content].pack('a*a*')) == checksum raise Errors::MessageChecksumError end # decode headers and payload headers, payload = decode_context(content, header_length) @message_buffer = remaining [Message.new(headers: headers, payload: payload), remaining.empty?] end def decode_prelude(prelude) # prelude contains length of message and headers, # followed with CRC checksum of itself content, checksum = prelude.unpack("a#{PRELUDE_LENGTH - CRC32_LENGTH}N") raise Errors::PreludeChecksumError unless Zlib.crc32(content) == checksum content.unpack('N*') end def decode_context(content, header_length) encoded_header, encoded_payload = content.unpack("a#{header_length}a*") [ extract_headers(encoded_header), extract_payload(encoded_payload) ] end def extract_headers(buffer) scanner = buffer headers = {} until scanner.bytesize == 0 # header key key_length, scanner = scanner.unpack('Ca*') key, scanner = scanner.unpack("a#{key_length}a*") # header value type_index, scanner = scanner.unpack('Ca*') value_type = Types.types[type_index] unpack_pattern, value_length = Types.pattern[value_type] value = if !!unpack_pattern == unpack_pattern # boolean types won't have value specified unpack_pattern else value_length, scanner = scanner.unpack('S>a*') unless value_length unpacked_value, scanner = scanner.unpack("#{unpack_pattern || "a#{value_length}"}a*") unpacked_value end headers[key] = HeaderValue.new( format: @format, value: value, type: value_type ) end headers end def extract_payload(encoded) encoded.bytesize <= ONE_MEGABYTE ? payload_stringio(encoded) : payload_tempfile(encoded) end def payload_stringio(encoded) StringIO.new(encoded) end def payload_tempfile(encoded) payload = Tempfile.new payload.binmode payload.write(encoded) payload.rewind payload end end end end aws-eventstream-1.1.0/lib/aws-eventstream.rb0000644000004100000410000000040513704150541021105 0ustar www-datawww-datarequire_relative 'aws-eventstream/decoder' require_relative 'aws-eventstream/encoder' require_relative 'aws-eventstream/message' require_relative 'aws-eventstream/header_value' require_relative 'aws-eventstream/types' require_relative 'aws-eventstream/errors'