beaneater-1.0.0/0000755000004100000410000000000012545466244013502 5ustar www-datawww-databeaneater-1.0.0/Rakefile0000644000004100000410000000147212545466244015153 0ustar www-datawww-datarequire "bundler/gem_tasks" require 'rake/testtask' require 'yard' require 'redcarpet' # rake test Rake::TestTask.new do |t| t.libs.push "lib" t.test_files = FileList[File.expand_path('../test/**/*_test.rb', __FILE__)] - FileList[File.expand_path('../test/**/beaneater_test.rb', __FILE__)] t.verbose = true end # rake test:integration Rake::TestTask.new("test:integration") do |t| t.libs.push "lib" t.test_files = FileList[File.expand_path('../test/**/beaneater_test.rb', __FILE__)] t.verbose = true end # rake test:full Rake::TestTask.new("test:full") do |t| t.libs.push "lib" t.test_files = FileList[File.expand_path('../test/**/*_test.rb', __FILE__)] t.verbose = true end YARD::Rake::YardocTask.new do |t| t.files = ['lib/beaneater/**/*.rb'] t.options = [] end task :default => 'test:full' beaneater-1.0.0/Gemfile0000644000004100000410000000037512545466244015002 0ustar www-datawww-datasource 'https://rubygems.org' # Specify your gem's dependencies in beaneater.gemspec gemspec group :development do gem 'redcarpet', '~> 1' gem 'github-markup' gem 'yard' end group :development, :test do gem 'coveralls', :require => false end beaneater-1.0.0/examples/0000755000004100000410000000000012545466244015320 5ustar www-datawww-databeaneater-1.0.0/examples/demo.rb0000644000004100000410000000365612545466244016603 0ustar www-datawww-datarequire 'term/ansicolor' class String; include Term::ANSIColor; end def step(msg); "\n[STEP] #{msg}...".yellow; end $:.unshift("../lib") require 'beaneater' # Establish a pool of beanstalks puts step("Connecting to Beanstalk") bc = Beaneater.new('localhost') puts bc # Print out key stats puts step("Print Stats") p bc.stats.keys p [bc.stats.total_connections, bc.stats[:total_connections], bc.stats['total_connections']] # find tube puts step("Find tube") tube = bc.tubes.find('tube2') puts tube # Put job onto tube puts step("Put job") response = tube.put "foo bar", :pri => 1000, :ttr => 10, :delay => 0 puts response # peek tube puts step("Peek tube") p tube.peek :ready # watch tube bc.tubes.watch!('tube2') # Check tube stats puts step("Get tube stats") p tube.stats.keys p tube.stats.name p tube.stats.current_jobs_ready # Reserve job from tube puts step("Reserve job") p job = bc.tubes.reserve jid = job.id # pause tube puts step("Pause tube") p tube.pause(1) # Register jobs puts step("Register jobs for tubes") bc.jobs.register('tube_test', :retry_on => [Timeout::Error]) do |job| p 'tube_test' p job raise Beaneater::AbortProcessingError end bc.jobs.register('tube_test2', :retry_on => [Timeout::Error]) do |job| p 'tube_test2' p job raise Beaneater::AbortProcessingError end p bc.jobs.processors response = bc.tubes.find('tube_test').put "foo register", :pri => 1000, :ttr => 10, :delay => 0 response = bc.tubes.find('tube_test2').put "foo baz", :pri => 1000, :ttr => 10, :delay => 0 # Process jobs puts step("Process jobs") 2.times { bc.jobs.process! } # Get job from id (peek job) puts step("Get job from id") p bc.jobs.find(jid) p bc.jobs.peek(jid) # Check job stats puts step("Get job stats") p job.stats.keys p job.stats.tube p job.stats.state # bury job puts step("Bury job") p job.bury # delete job puts step("Delete job") p job.delete # list tubes puts step("List tubes") p bc.tubes.watched p bc.tubes.used p bc.tubes.allbeaneater-1.0.0/LICENSE.txt0000644000004100000410000000205212545466244015324 0ustar www-datawww-dataCopyright (c) 2012 Nico Taing MIT License Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.beaneater-1.0.0/.travis.yml0000644000004100000410000000042512545466244015614 0ustar www-datawww-datalanguage: ruby rvm: - 1.9.3 - 2.0.0 before_install: - curl -L https://github.com/kr/beanstalkd/archive/v1.9.tar.gz | tar xz -C /tmp - cd /tmp/beanstalkd-1.9/ - make - ./beanstalkd & - cd $TRAVIS_BUILD_DIR script: - bundle install - bundle exec rake test:full beaneater-1.0.0/lib/0000755000004100000410000000000012545466244014250 5ustar www-datawww-databeaneater-1.0.0/lib/beaneater.rb0000644000004100000410000000361312545466244016526 0ustar www-datawww-datarequire 'thread' unless defined?(Mutex) %w(version configuration errors connection tube job stats).each do |f| require "beaneater/#{f}" end class Beaneater # @!attribute connection # @return returns the associated connection object attr_reader :connection # Initialize new instance of Beaneater # # @param [String] address in the form "host:port" # @example # Beaneater.new('127.0.0.1:11300') # # ENV['BEANSTALKD_URL'] = '127.0.0.1:11300' # @b = Beaneater.new # @b.connection.host # => '127.0.0.1' # @b.connection.port # => '11300' # def initialize(address) @connection = Connection.new(address) end # Returns Beaneater::Tubes object for accessing tube related functions. # # @return [Beaneater::Tubes] tubes object # @api public def tubes @tubes ||= Beaneater::Tubes.new(self) end # Returns Beaneater::Jobs object for accessing job related functions. # # @return [Beaneater::Jobs] jobs object # @api public def jobs @jobs ||= Beaneater::Jobs.new(self) end # Returns Beaneater::Stats object for accessing beanstalk stats. # # @return [Beaneater::Stats] stats object # @api public def stats @stats ||= Stats.new(self) end # Closes the related connection # # @example # @beaneater_instance.close # def close connection.close if connection end protected class << self # Yields a configuration block # # @example # Beaneater.configure do |config| # config.job_parser = lamda { |body| Yaml.load(body)} # end # def configure(&block) yield(configuration) if block_given? configuration end # Returns the configuration options set for Backburner # # @example # Beaneater.configuration.default_put_ttr => 120 # def configuration @_configuration ||= Configuration.new end end end # Beaneaterbeaneater-1.0.0/lib/beaneater/0000755000004100000410000000000012545466244016176 5ustar www-datawww-databeaneater-1.0.0/lib/beaneater/tube/0000755000004100000410000000000012545466244017135 5ustar www-datawww-databeaneater-1.0.0/lib/beaneater/tube/record.rb0000644000004100000410000001207512545466244020745 0ustar www-datawww-dataclass Beaneater # Beanstalk tube which contains jobs which can be inserted, reserved, et al. class Tube # @!attribute name # @return [String] name of the tube # @!attribute client # @return [Beaneater] returns the client instance attr_reader :name, :client # Fetches the specified tube. # # @param [Beaneater] client The beaneater client instance. # @param [String] name The name for this tube. # @example # Beaneater::Tube.new(@client, 'tube-name') # def initialize(client, name) @client = client @name = name.to_s @mutex = Mutex.new end # Delegates transmit to the connection object. # # @see Beaneater::Connection#transmit def transmit(command, options={}) client.connection.transmit(command, options) end # Inserts job with specified body onto tube. # # @param [String] body The data to store with this job. # @param [Hash{String => Integer}] options The settings associated with this job. # @option options [Integer] pri priority for this job # @option options [Integer] ttr time to respond for this job # @option options [Integer] delay delay for this job # @return [Hash{String => String, Number}] beanstalkd command response # @example # @tube.put "data", :pri => 1000, :ttr => 10, :delay => 5 # # @api public def put(body, options={}) safe_use do serialized_body = config.job_serializer.call(body) options = { :pri => config.default_put_pri, :delay => config.default_put_delay, :ttr => config.default_put_ttr }.merge(options) cmd_options = "#{options[:pri]} #{options[:delay]} #{options[:ttr]} #{serialized_body.bytesize}" transmit("put #{cmd_options}\r\n#{serialized_body}") end end # Peek at next job within this tube in given `state`. # # @param [String] state The job state to peek at (`ready`, `buried`, `delayed`) # @return [Beaneater::Job] The next job within this tube. # @example # @tube.peek(:ready) # => # # @api public def peek(state) safe_use do res = transmit("peek-#{state}") Job.new(client, res) end rescue Beaneater::NotFoundError # Return nil if not found nil end # Reserves the next job from tube. # # @param [Integer] timeout Number of seconds before timing out # @param [Proc] block Callback to perform on reserved job # @yield [job] Job that was reserved. # @return [Beaneater::Job] Job that was reserved. # @example # @tube.reserve # => # # @api public def reserve(timeout=nil, &block) client.tubes.watch!(self.name) client.tubes.reserve(timeout, &block) end # Kick specified number of jobs from buried to ready state. # # @param [Integer] bounds The number of jobs to kick. # @return [Hash{String => String, Number}] Beanstalkd command response # @example # @tube.kick(5) # # @api public def kick(bounds=1) safe_use { transmit("kick #{bounds}") } end # Returns related stats for this tube. # # @return [Beaneater::StatStruct] Struct of tube related values # @example # @tube.stats.current_jobs_delayed # => 24 # # @api public def stats res = transmit("stats-tube #{name}") StatStruct.from_hash(res[:body]) end # Pause the execution of this tube for specified `delay`. # # @param [Integer] delay Number of seconds to delay tube execution # @return [Array String, Number}>] Beanstalkd command response # @example # @tube.pause(10) # # @api public def pause(delay) transmit("pause-tube #{name} #{delay}") end # Clears all unreserved jobs in all states from the tube # # @example # @tube.clear # def clear client.tubes.watch!(self.name) %w(delayed buried ready).each do |state| while job = self.peek(state.to_sym) job.delete end end client.tubes.ignore(name) rescue Beaneater::UnexpectedResponse # swallow any issues end # String representation of tube. # # @return [String] Representation of tube including name. # @example # @tube.to_s # => "#" # def to_s "#" end alias :inspect :to_s protected # Transmits a beanstalk command that requires this tube to be set as used. # # @param [Proc] block Beanstalk command to transmit. # @return [Object] Result of block passed # @example # safe_use { transmit("kick 1") } # # => "Response to kick command" # def safe_use(&block) @mutex.lock client.tubes.use(self.name) yield ensure @mutex.unlock end # Returns configuration options for beaneater # # @return [Beaneater::Configuration] configuration object def config Beaneater.configuration end end # Tube end # Beaneater beaneater-1.0.0/lib/beaneater/tube/collection.rb0000644000004100000410000001154412545466244021622 0ustar www-datawww-dataclass Beaneater # Represents collection of tube related commands. class Tubes include Enumerable # @!attribute client # @return [Beaneater] returns the client instance attr_reader :client # Creates new tubes instance. # # @param [Beaneater] client The beaneater client instance. # @example # Beaneater::Tubes.new(@client) # def initialize(client) @client = client end def last_used client.connection.tube_used end def last_used=(tube_name) client.connection.tube_used = tube_name end # Delegates transmit to the connection object. # # @see Beaneater::Connection#transmit def transmit(command, options={}) client.connection.transmit(command, options) end # Finds the specified beanstalk tube. # # @param [String] tube_name Name of the beanstalkd tube # @return [Beaneater::Tube] specified tube # @example # @pool.tubes.find('tube2') # @pool.tubes['tube2'] # # => # # @api public def find(tube_name) Tube.new(client, tube_name) end alias_method :[], :find # Reserves a ready job looking at all watched tubes. # # @param [Integer] timeout Number of seconds before timing out. # @param [Proc] block Callback to perform on the reserved job. # @yield [job] Reserved beaneater job. # @return [Beaneater::Job] Reserved beaneater job. # @example # @client.tubes.reserve { |job| process(job) } # # => # # @api public def reserve(timeout=nil, &block) res = transmit( timeout ? "reserve-with-timeout #{timeout}" : 'reserve') job = Job.new(client, res) block.call(job) if block_given? job end # List of all known beanstalk tubes. # # @return [Array] List of all beanstalk tubes. # @example # @client.tubes.all # # => [, ] # # @api public def all transmit('list-tubes')[:body].map do |tube_name| Tube.new(client, tube_name) end end # Calls the given block once for each known beanstalk tube, passing that element as a parameter. # # @return An Enumerator is returned if no block is given. # @example # @pool.tubes.each {|t| puts t.name} # # @api public def each block_given? ? all.each(&Proc.new) : all.each end # List of watched beanstalk tubes. # # @return [Array] List of watched beanstalk tubes. # @example # @client.tubes.watched # # => [, ] # # @api public def watched last_watched = transmit('list-tubes-watched')[:body] client.connection.tubes_watched = last_watched.dup last_watched.map do |tube_name| Tube.new(client, tube_name) end end # Currently used beanstalk tube. # # @return [Beaneater::Tube] Currently used beanstalk tube. # @example # @client.tubes.used # # => # # @api public def used last_used = transmit('list-tube-used')[:id] Tube.new(client, last_used) end # Add specified beanstalkd tubes as watched. # # @param [*String] names Name of tubes to watch # @raise [Beaneater::InvalidTubeName] Tube to watch was invalid. # @example # @client.tubes.watch('foo', 'bar') # # @api public def watch(*names) names.each do |t| transmit "watch #{t}" client.connection.add_to_watched(t) end rescue BadFormatError => ex raise InvalidTubeName, "Tube in '#{ex.cmd}' is invalid!" end # Add specified beanstalkd tubes as watched and ignores all other tubes. # # @param [*String] names Name of tubes to watch # @raise [Beaneater::InvalidTubeName] Tube to watch was invalid. # @example # @client.tubes.watch!('foo', 'bar') # # @api public def watch!(*names) old_tubes = watched.map(&:name) - names.map(&:to_s) watch(*names) ignore(*old_tubes) end # Ignores specified beanstalkd tubes. # # @param [*String] names Name of tubes to ignore # @example # @client.tubes.ignore('foo', 'bar') # # @api public def ignore(*names) names.each do |w| transmit "ignore #{w}" client.connection.remove_from_watched(w) end end # Set specified tube as used. # # @param [String] tube Tube to be used. # @example # @conn.tubes.use("some-tube") # def use(tube) return tube if last_used == tube transmit("use #{tube}") last_used = tube rescue BadFormatError raise InvalidTubeName, "Tube cannot be named '#{tube}'" end end # Tubes end # Beaneater beaneater-1.0.0/lib/beaneater/configuration.rb0000644000004100000410000000141112545466244021367 0ustar www-datawww-dataclass Beaneater class Configuration attr_accessor :default_put_delay # default delay value to put a job attr_accessor :default_put_pri # default priority value to put a job attr_accessor :default_put_ttr # default ttr value to put a job attr_accessor :job_parser # default job_parser to parse job body attr_accessor :job_serializer # default serializer for job body attr_accessor :beanstalkd_url # default beanstalkd url def initialize @default_put_delay = 0 @default_put_pri = 65536 @default_put_ttr = 120 @job_parser = lambda { |body| body } @job_serializer = lambda { |body| body } @beanstalkd_url = nil end end # Configuration end # Beaneaterbeaneater-1.0.0/lib/beaneater/stats/0000755000004100000410000000000012545466244017334 5ustar www-datawww-databeaneater-1.0.0/lib/beaneater/stats/fast_struct.rb0000644000004100000410000000646512545466244022235 0ustar www-datawww-dataclass Beaneater # # Borrowed from: # https://github.com/dolzenko/faster_open_struct/blob/master/lib/faster_open_struct.rb # # Up to 40 (!) times more memory efficient version of OpenStruct # # Differences from Ruby MRI OpenStruct: # # 1. Doesn't `dup` passed initialization hash (NOTE: only reference to hash is stored) # # 2. Doesn't convert hash keys to symbols (by default string keys are used, # with fallback to symbol keys) # # 3. Creates methods on the fly on `OpenStruct` class, instead of singleton class. # Uses `module_eval` with string to avoid holding scope references for every method. # # 4. Refactored, crud clean, spec covered :) # # @private class FasterOpenStruct # Undefine particularly nasty interfering methods on Ruby 1.8 undef :type if method_defined?(:type) undef :id if method_defined?(:id) def initialize(hash = nil) @hash = hash || {} @initialized_empty = hash == nil end def method_missing(method_name_sym, *args) if method_name_sym.to_s[-1] == ?= if args.size != 1 raise ArgumentError, "wrong number of arguments (#{args.size} for 1)", caller(1) end if self.frozen? raise TypeError, "can't modify frozen #{self.class}", caller(1) end __new_ostruct_member__(method_name_sym.to_s.chomp("=")) send(method_name_sym, args[0]) elsif args.size == 0 __new_ostruct_member__(method_name_sym) send(method_name_sym) else raise NoMethodError, "undefined method `#{method_name_sym}' for #{self}", caller(1) end end def __new_ostruct_member__(method_name_sym) self.class.module_eval <<-END_EVAL, __FILE__, __LINE__ + 1 def #{ method_name_sym } @hash.fetch("#{ method_name_sym }", @hash[:#{ method_name_sym }]) # read by default from string key, then try symbol # if string key doesn't exist end END_EVAL unless method_name_sym.to_s[-1] == ?? # can't define writer for predicate method self.class.module_eval <<-END_EVAL, __FILE__, __LINE__ + 1 def #{ method_name_sym }=(val) if @hash.key?("#{ method_name_sym }") || @initialized_empty # write by default to string key (when it is present # in initialization hash or initialization hash # wasn't provided) @hash["#{ method_name_sym }"] = val # if it doesn't exist - write to symbol key else @hash[:#{ method_name_sym }] = val end end END_EVAL end end def empty? @hash.empty? end # # Compare this object and +other+ for equality. # def ==(other) return false unless other.is_a?(self.class) @hash == other.instance_variable_get(:@hash) end # # Returns a string containing a detailed summary of the keys and values. # def inspect str = "#<#{ self.class }" str << " #{ @hash.map { |k, v| "#{ k }=#{ v.inspect }" }.join(", ") }" unless @hash.empty? str << ">" end alias :to_s :inspect end # FasterOpenStruct end # Beaneaterbeaneater-1.0.0/lib/beaneater/stats/stat_struct.rb0000644000004100000410000000227512545466244022246 0ustar www-datawww-dataclass Beaneater # Represents a stats hash with proper underscored keys class StatStruct < FasterOpenStruct # Convert a stats hash into a struct. # # @param [Hash{String => String}] hash Hash Stats hash to convert to struct # @return [Beaneater::StatStruct, nil] Stats struct from hash # @example # s = StatStruct.from_hash(:foo => "bar") # s.foo # => 'bar' # def self.from_hash(hash) return unless hash.is_a?(Hash) underscore_hash = hash.inject({}) { |r, (k, v)| r[k.to_s.gsub(/-/, '_')] = v; r } self.new(underscore_hash) end # Access value for stat with specified key. # # @param [String] key Key to fetch from stats. # @return [String, Integer] Value for specified stat key. # @example # @stats['foo'] # => "bar" # def [](key) self.send(key.to_s) end # Returns set of keys within this struct # # @return [Array] Value for specified stat key. # @example # @stats.keys # => ['foo', 'bar', 'baz'] # def keys @hash.keys.map { |k| k.to_s } end # Returns the initialization hash # def to_h @hash end end # StatStruct end # Beaneater beaneater-1.0.0/lib/beaneater/connection.rb0000644000004100000410000001671312545466244020672 0ustar www-datawww-datarequire 'yaml' require 'socket' class Beaneater # Represents a connection to a beanstalkd instance. class Connection # Default number of retries to send a command to a connection MAX_RETRIES = 3 # Default retry interval DEFAULT_RETRY_INTERVAL = 1 # @!attribute address # @return [String] returns Beanstalkd server address # @example # @conn.address # => "localhost:11300" # @!attribute host # @return [String] returns Beanstalkd server host # @example # @conn.host # => "localhost" # @!attribute port # @return [Integer] returns Beanstalkd server port # @example # @conn.port # => "11300" # @!attribute connection # @return [Net::TCPSocket] returns connection object attr_reader :address, :host, :port, :connection # @!attribute tubes_watched # @returns [Array] returns currently watched tube names # @!attribute tube_used # @returns [String] returns currently used tube name attr_accessor :tubes_watched, :tube_used # Default port value for beanstalk connection DEFAULT_PORT = 11300 # Initializes new connection. # # @param [String] address beanstalkd instance address. # @example # Beaneater::Connection.new('127.0.0.1') # Beaneater::Connection.new('127.0.0.1:11300') # # ENV['BEANSTALKD_URL'] = '127.0.0.1:11300' # @b = Beaneater.new # @b.connection.host # => '127.0.0.1' # @b.connection.port # => '11300' # def initialize(address) @address = address || _host_from_env || Beaneater.configuration.beanstalkd_url @mutex = Mutex.new @tube_used = 'default' @tubes_watched = ['default'] establish_connection rescue _raise_not_connected! end # Send commands to beanstalkd server via connection. # # @param [Hash{String => String, Number}>] options Retained for compatibility # @param [String] command Beanstalkd command # @return [Array String, Number}>] Beanstalkd command response # @example # @conn = Beaneater::Connection.new # @conn.transmit('bury 123') # @conn.transmit('stats') # def transmit(command, options={}) _with_retry(options[:retry_interval], options[:init]) do @mutex.synchronize do _raise_not_connected! unless connection command = command.force_encoding('ASCII-8BIT') if command.respond_to?(:force_encoding) connection.write(command.to_s + "\r\n") res = connection.readline parse_response(command, res) end end end # Close connection with beanstalkd server. # # @example # @conn.close # def close if @connection @connection.close @connection = nil end end # Returns string representation of job. # # @example # @conn.inspect # def to_s "#" end alias :inspect :to_s def add_to_watched(tube_name) @tubes_watched << tube_name @tubes_watched.uniq end def remove_from_watched(tube_name) @tubes_watched.delete(tube_name) end protected # Establish a connection based on beanstalk address. # # @return [Net::TCPSocket] connection for specified address. # @raise [Beaneater::NotConnected] Could not connect to specified beanstalkd instance. # @example # establish_connection('localhost:3005') # def establish_connection @address = address.first if address.is_a?(Array) match = address.split(':') @host, @port = match[0], Integer(match[1] || DEFAULT_PORT) @connection = TCPSocket.new @host, @port end # Parses the response and returns the useful beanstalk response. # Will read the body if one is indicated by the status. # # @param [String] cmd Beanstalk command transmitted # @param [String] res Telnet command response # @return [Array String, Number}>] Beanstalk response with `status`, `id`, `body` # @raise [Beaneater::UnexpectedResponse] Response from beanstalk command was an error status # @example # parse_response("delete 56", "DELETED 56\nFOO") # # => { :body => "FOO", :status => "DELETED", :id => 56 } # def parse_response(cmd, res) status = res.chomp body_values = status.split(/\s/) status = body_values[0] raise UnexpectedResponse.from_status(status, cmd) if UnexpectedResponse::ERROR_STATES.include?(status) body = nil if ['OK','FOUND', 'RESERVED'].include?(status) bytes_size = body_values[-1].to_i raw_body = connection.read(bytes_size) body = status == 'OK' ? YAML.load(raw_body) : config.job_parser.call(raw_body) crlf = connection.read(2) # \r\n raise ExpectedCrlfError.new('EXPECTED_CRLF', cmd) if crlf != "\r\n" end id = body_values[1] response = { :status => status } response[:id] = id if id response[:body] = body if body response end # Returns configuration options for beaneater # # @return [Beaneater::Configuration] configuration object def config Beaneater.configuration end private def _initialize_tubes if @tubes_watched != ['default'] tubes_watched.each do |t| transmit("watch #{t}", init: false) end end transmit("use #{tube_used}", init: false) if @tube_used != 'default' end # Wrapper method for capturing certain failures and retry the payload block # # @param [Proc] block The command to execute. # @param [Integer] retry_interval The time to wait before the next retry # @param [Integer] tries The maximum number of tries in draining mode # @return [Object] Result of the block passed # def _with_retry(retry_interval, init=true, tries=MAX_RETRIES, &block) yield rescue EOFError, Errno::ECONNRESET, Errno::EPIPE, Errno::ECONNREFUSED => ex _reconnect(ex, retry_interval) _initialize_tubes if init retry rescue Beaneater::DrainingError tries -= 1 if tries.zero? close raise end sleep(retry_interval || DEFAULT_RETRY_INTERVAL) retry end # Tries to re-establish connection to the beanstalkd # # @param [Exception] original_exception The exception caused the retry # @param [Integer] retry_interval The time to wait before the next reconnect # @param [Integer] tries The maximum number of attempts to reconnect def _reconnect(original_exception, retry_interval, tries=MAX_RETRIES) close establish_connection rescue Errno::ECONNREFUSED tries -= 1 if tries.zero? _raise_not_connected! end sleep(retry_interval || DEFAULT_RETRY_INTERVAL) retry end # The host provided by BEANSTALKD_URL environment variable, if available. # # @return [String] A beanstalkd host address # @example # ENV['BEANSTALKD_URL'] = "localhost:1212" # # => 'localhost:1212' # def _host_from_env ENV['BEANSTALKD_URL'].respond_to?(:length) && ENV['BEANSTALKD_URL'].length > 0 && ENV['BEANSTALKD_URL'].strip end # Raises an error to be triggered when the connection has failed # @raise [Beaneater::NotConnected] Beanstalkd is no longer connected def _raise_not_connected! raise Beaneater::NotConnected, "Connection to beanstalk '#{@host}:#{@port}' is closed!" end end # Connection end # Beaneater beaneater-1.0.0/lib/beaneater/errors.rb0000644000004100000410000000624012545466244020041 0ustar www-datawww-dataclass Beaneater # Raises when a beanstalkd instance is no longer accessible. class NotConnected < RuntimeError; end # Raises when the tube name specified is invalid. class InvalidTubeName < RuntimeError; end # Raises when a job has not been reserved properly. class JobNotReserved < RuntimeError; end # Abstract class for errors that occur when a command does not complete successfully. class UnexpectedResponse < RuntimeError # Set of status states that are considered errors ERROR_STATES = %w(OUT_OF_MEMORY INTERNAL_ERROR BAD_FORMAT UNKNOWN_COMMAND JOB_TOO_BIG DRAINING TIMED_OUT DEADLINE_SOON NOT_FOUND NOT_IGNORED EXPECTED_CRLF) # @!attribute status # @return [String] returns beanstalkd response status # @example @ex.status # => "NOT_FOUND" # @!attribute cmd # @return [String] returns beanstalkd request command # @example @ex.cmd # => "stats-job 23" attr_reader :status, :cmd # Initialize unexpected response error # # @param [Beaneater::UnexpectedResponse] status Unexpected response object # @param [String] cmd Beanstalkd request command # # @example # Beaneater::UnexpectedResponse.new(NotFoundError, 'bury 123') # def initialize(status, cmd) @status, @cmd = status, cmd super("Response failed with: #{status}") end # Translate beanstalkd error status to ruby Exeception # # @param [String] status Beanstalkd error status # @param [String] cmd Beanstalkd request command # # @return [Beaneater::UnexpectedResponse] Exception for the status provided # @example # Beaneater::UnexpectedResponse.new('NOT_FOUND', 'bury 123') # def self.from_status(status, cmd) error_klazz_name = status.split('_').map { |w| w.capitalize }.join error_klazz_name << "Error" unless error_klazz_name =~ /Error$/ error_klazz = Beaneater.const_get(error_klazz_name) error_klazz.new(status, cmd) end end # Raises when the beanstalkd instance runs out of memory class OutOfMemoryError < UnexpectedResponse; end # Raises when the beanstalkd instance is draining and new jobs cannot be inserted class DrainingError < UnexpectedResponse; end # Raises when the job or tube cannot be found class NotFoundError < UnexpectedResponse; end # Raises when the job reserved is going to be released within a second. class DeadlineSoonError < UnexpectedResponse; end # Raises when a beanstalkd has an internal error. class InternalError < UnexpectedResponse; end # Raises when a command was not properly formatted. class BadFormatError < UnexpectedResponse; end # Raises when a command was sent that is unknown. class UnknownCommandError < UnexpectedResponse; end # Raises when command does not have proper CRLF suffix. class ExpectedCrlfError < UnexpectedResponse; end # Raises when the body of a job was too large. class JobTooBigError < UnexpectedResponse; end # Raises when a job was attempted to be reserved but the timeout occured. class TimedOutError < UnexpectedResponse; end # Raises when a tube could not be ignored because it is the last watched tube. class NotIgnoredError < UnexpectedResponse; end end beaneater-1.0.0/lib/beaneater/job.rb0000644000004100000410000000010112545466244017265 0ustar www-datawww-datarequire 'beaneater/job/record' require 'beaneater/job/collection'beaneater-1.0.0/lib/beaneater/tube.rb0000644000004100000410000000010312545466244017454 0ustar www-datawww-datarequire 'beaneater/tube/record' require 'beaneater/tube/collection'beaneater-1.0.0/lib/beaneater/version.rb0000644000004100000410000000010312545466244020202 0ustar www-datawww-dataclass Beaneater # Current version of gem. VERSION = "1.0.0" endbeaneater-1.0.0/lib/beaneater/stats.rb0000644000004100000410000000402112545466244017656 0ustar www-datawww-datarequire 'beaneater/stats/fast_struct' require 'beaneater/stats/stat_struct' class Beaneater # Represents stats related to the beanstalkd pool. class Stats # @!attribute client # @return [Beaneater] returns the client instance attr_reader :client # Creates new stats instance. # # @param [Beaneater] client The beaneater client instance. # @example # Beaneater::Stats.new(@client) # def initialize(client) @client = client end # Returns keys for stats data # # @return [Array] Set of keys for stats. # @example # @bp.stats.keys # => ["version", "total_connections"] # # @api public def keys data.keys end # Returns value for specified key. # # @param [String,Symbol] key Name of key to retrieve # @return [String,Integer] Value of specified key # @example # @bp.stats['total_connections'] # => 4 # def [](key) data[key] end # Delegates inspection to the real data structure # # @return [String] returns a string containing a detailed stats summary def inspect data.to_s end alias :to_s :inspect # Defines a cached method for looking up data for specified key # Protects against infinite loops by checking stacktrace # @api public def method_missing(name, *args, &block) if caller.first !~ /`(method_missing|data')/ && data.keys.include?(name.to_s) self.class.class_eval <<-CODE, __FILE__, __LINE__ def #{name}; data[#{name.inspect}]; end CODE data[name.to_s] else # no key matches or caught infinite loop super end end protected # Returns struct based on stats data from response. # # @return [Beaneater::StatStruct] the stats # @example # self.data # => { 'version' : 1.7, 'total_connections' : 23 } # self.data.total_connections # => 23 # def data StatStruct.from_hash(client.connection.transmit('stats')[:body]) end end # Stats end # Beaneaterbeaneater-1.0.0/lib/beaneater/job/0000755000004100000410000000000012545466244016750 5ustar www-datawww-databeaneater-1.0.0/lib/beaneater/job/record.rb0000644000004100000410000001365412545466244020564 0ustar www-datawww-dataclass Beaneater # Represents job related commands. class Job # @!attribute id # @return [Integer] id for the job. # @!attribute body # @return [String] the job's body. # @!attribute reserved # @return [Boolean] whether the job has been reserved. # @!attribute client # @return [Beaneater] returns the client instance attr_reader :id, :body, :reserved, :client # Initializes a new job object. # # @param [Hash{Symbol => String,Number}] res Result from beanstalkd response # def initialize(client, res) @client = client @id = res[:id] @body = res[:body] @reserved = res[:status] == 'RESERVED' end # Sends command to bury a reserved job. # # @param [Hash{Symbol => Integer}] options Settings to bury job # @option options [Integer] pri Assign new priority to job # @return [Hash{Symbol => String,Number}] Beanstalkd response for the command. # # @example # @job.bury({:pri => 100}) # # => {:status=>"BURIED", :body=>nil} # # @api public def bury(options={}) options = { :pri => stats.pri }.merge(options) with_reserved("bury #{id} #{options[:pri]}") do @reserved = false end end # Sends command to release a job back to ready state. # # @param [Hash{String => Integer}] options Settings to release job # @option options [Integer] pri Assign new priority to job # @option options [Integer] delay Assign new delay to job # @return [Hash{Symbol => String,Number}] Beanstalkd response for the command. # @example # @beaneater.jobs.find(123).release(:pri => 10, :delay => 5) # # => {:status=>"RELEASED", :body=>nil} # # @api public def release(options={}) options = { :pri => stats.pri, :delay => stats.delay }.merge(options) with_reserved("release #{id} #{options[:pri]} #{options[:delay]}") do @reserved = false end end # Sends command to touch job which extends the ttr. # # @return [Hash{Symbol => String,Number}] Beanstalkd response for the command. # @example # @beaneater.jobs.find(123).touch # # => {:status=>"TOUCHED", :body=>nil} # # @api public def touch with_reserved("touch #{id}") end # Sends command to delete a job. # # @return [Hash{Symbol => String,Number}] Beanstalkd response for the command. # @example # @beaneater.jobs.find(123).delete # # => {:status=>"DELETED", :body=>nil} # # @api public def delete transmit("delete #{id}") { @reserved = false } end # Sends command to kick a buried job. # # @return [Hash{Symbol => String,Number}] Beanstalkd response for the command. # @example # @beaneater.jobs.find(123).kick # # => {:status=>"KICKED", :body=>nil} # # @api public def kick transmit("kick-job #{id}") end # Sends command to get stats about job. # # @return [Beaneater::StatStruct] struct filled with relevant job stats # @example # @beaneater.jobs.find(123).stats # @job.stats.tube # => "some-tube" # # @api public def stats res = transmit("stats-job #{id}") StatStruct.from_hash(res[:body]) end # Check if job is currently in a reserved state. # # @return [Boolean] Returns true if the job is in a reserved state # @example # @beaneater.jobs.find(123).reserved? # # @api public def reserved? @reserved || self.stats.state == "reserved" end # Check if the job still exists. # # @return [Boolean] Returns true if the job still exists # @example # @beaneater.jobs.find(123).exists? # # @api public def exists? !self.stats.nil? rescue Beaneater::NotFoundError false end # Returns the name of the tube this job is in # # @return [String] The name of the tube for this job # @example # @beaneater.jobs.find(123).tube # # => "some-tube" # # @api public def tube @tube ||= self.stats.tube end # Returns the ttr of this job # # @return [Integer] The ttr of this job # @example # @beaneater.jobs.find(123).ttr # # => 123 # # @api public def ttr @ttr ||= self.stats.ttr end # Returns the pri of this job # # @return [Integer] The pri of this job # @example # @beaneater.jobs.find(123).pri # # => 1 # def pri self.stats.pri end # Returns the delay of this job # # @return [Integer] The delay of this job # @example # @beaneater.jobs.find(123).delay # # => 5 # def delay self.stats.delay end # Returns string representation of job # # @return [String] string representation # @example # @beaneater.jobs.find(123).to_s # @beaneater.jobs.find(123).inspect # def to_s "#" end alias :inspect :to_s protected # Transmit command to beanstalkd instance and fetch response. # # @param [String] cmd Beanstalkd command to send. # @return [Hash{Symbol => String,Number}] Beanstalkd response for the command. # @example # transmit('stats') # transmit('stats') { 'success' } # def transmit(cmd, &block) res = client.connection.transmit(cmd) yield if block_given? res end # Transmits a command which requires the job to be reserved. # # @param [String] cmd Beanstalkd command to send. # @return [Hash{Symbol => String,Number}] Beanstalkd response for the command. # @raise [Beaneater::JobNotReserved] Command cannot execute since job is not reserved. # @example # with_reserved("bury 26") { @reserved = false } # def with_reserved(cmd, &block) raise JobNotReserved unless reserved? transmit(cmd, &block) end end # Job end # Beaneater beaneater-1.0.0/lib/beaneater/job/collection.rb0000644000004100000410000001110212545466244021423 0ustar www-datawww-dataclass Beaneater # Exception to stop processing jobs during a `process!` loop. # Simply `raise AbortProcessingError` in any job process handler to stop the processing loop. class AbortProcessingError < RuntimeError; end # Represents collection of job-related commands. class Jobs # @!attribute processors # @return [Array] returns Collection of proc to handle beanstalkd jobs # @!attribute client # @return [Beaneater] returns the client instance # @!attribute current_job # @return [Beaneater] returns the currently processing job in the process loop attr_reader :processors, :client, :current_job # Number of retries to process a job. MAX_RETRIES = 3 # Delay in seconds before to make job ready again. RELEASE_DELAY = 1 # Number of seconds to wait for a job before checking a different server. RESERVE_TIMEOUT = nil # Creates new jobs instance. # # @param [Beaneater] client The beaneater client instance. # @example # Beaneater::Jobs.new(@client) # def initialize(client) @client = client end # Delegates transmit to the connection object. # # @see Beaneater::Connection#transmit def transmit(command, options={}) client.connection.transmit(command, options) end # Peek (or find) job by id from beanstalkd. # # @param [Integer] id Job id to find # @return [Beaneater::Job] Job matching given id # @example # @beaneater.jobs[123] # => # @beaneater.jobs.find(123) # => # @beaneater.jobs.peek(123) # => # # @api public def find(id) res = transmit("peek #{id}") Job.new(client, res) rescue Beaneater::NotFoundError nil end alias_method :peek, :find alias_method :[], :find # Register a processor to handle beanstalkd job on particular tube. # # @param [String] tube_name Tube name # @param [Hash{String=>RuntimeError}] options settings for processor # @param [Proc] block Process beanstalkd job # @option options [Integer] max_retries Number of retries to process a job # @option options [Array] retry_on Collection of errors to rescue and re-run processor # # @example # @beanstalk.jobs.register('some-tube', :retry_on => [SomeError]) do |job| # do_something(job) # end # # @beanstalk.jobs.register('other-tube') do |job| # do_something_else(job) # end # # @api public def register(tube_name, options={}, &block) @processors ||= {} max_retries = options[:max_retries] || MAX_RETRIES retry_on = Array(options[:retry_on]) @processors[tube_name.to_s] = { :block => block, :retry_on => retry_on, :max_retries => max_retries } end # Sets flag to indicate that process loop should stop after current job def stop! @stop = true end # Returns whether the process loop should stop # # @return [Boolean] if true the loop should stop after current processing def stop? !!@stop end # Watch, reserve, process and delete or bury or release jobs. # # @param [Hash{String => Integer}] options Settings for processing # @option options [Integer] release_delay Delay in seconds before to make job ready again # @option options [Integer] reserve_timeout Number of seconds to wait for a job before checking a different server # # @api public def process!(options={}) release_delay = options.delete(:release_delay) || RELEASE_DELAY reserve_timeout = options.delete(:reserve_timeout) || RESERVE_TIMEOUT client.tubes.watch!(*processors.keys) while !stop? do begin @current_job = client.tubes.reserve(reserve_timeout) processor = processors[@current_job.tube] begin processor[:block].call(@current_job) @current_job.delete rescue *processor[:retry_on] if @current_job.stats.releases < processor[:max_retries] @current_job.release(:delay => release_delay) end end rescue AbortProcessingError break rescue Beaneater::JobNotReserved, Beaneater::NotFoundError, Beaneater::TimedOutError retry rescue StandardError # handles unspecified errors @current_job.bury if @current_job ensure # bury if still reserved @current_job.bury if @current_job && @current_job.exists? && @current_job.reserved? @current_job = nil end end end # process! end # Jobs end # Beaneater beaneater-1.0.0/metadata.yml0000644000004100000410000000726312545466244016015 0ustar www-datawww-data--- !ruby/object:Gem::Specification name: beaneater version: !ruby/object:Gem::Version version: 1.0.0 platform: ruby authors: - Nico Taing autorequire: bindir: bin cert_chain: [] date: 2015-04-26 00:00:00.000000000 Z dependencies: - !ruby/object:Gem::Dependency name: minitest requirement: !ruby/object:Gem::Requirement requirements: - - ~> - !ruby/object:Gem::Version version: 4.1.0 type: :development prerelease: false version_requirements: !ruby/object:Gem::Requirement requirements: - - ~> - !ruby/object:Gem::Version version: 4.1.0 - !ruby/object:Gem::Dependency name: rake requirement: !ruby/object:Gem::Requirement requirements: - - '>=' - !ruby/object:Gem::Version version: '0' type: :development prerelease: false version_requirements: !ruby/object:Gem::Requirement requirements: - - '>=' - !ruby/object:Gem::Version version: '0' - !ruby/object:Gem::Dependency name: mocha requirement: !ruby/object:Gem::Requirement requirements: - - '>=' - !ruby/object:Gem::Version version: '0' type: :development prerelease: false version_requirements: !ruby/object:Gem::Requirement requirements: - - '>=' - !ruby/object:Gem::Version version: '0' - !ruby/object:Gem::Dependency name: term-ansicolor requirement: !ruby/object:Gem::Requirement requirements: - - '>=' - !ruby/object:Gem::Version version: '0' type: :development prerelease: false version_requirements: !ruby/object:Gem::Requirement requirements: - - '>=' - !ruby/object:Gem::Version version: '0' - !ruby/object:Gem::Dependency name: json requirement: !ruby/object:Gem::Requirement requirements: - - '>=' - !ruby/object:Gem::Version version: '0' type: :development prerelease: false version_requirements: !ruby/object:Gem::Requirement requirements: - - '>=' - !ruby/object:Gem::Version version: '0' description: Simple beanstalkd client for ruby email: - nico.taing@gmail.com executables: [] extensions: [] extra_rdoc_files: [] files: - .gitignore - .travis.yml - .yardopts - CHANGELOG.md - Gemfile - LICENSE.txt - README.md - Rakefile - TODO - beaneater.gemspec - examples/demo.rb - lib/beaneater.rb - lib/beaneater/configuration.rb - lib/beaneater/connection.rb - lib/beaneater/errors.rb - lib/beaneater/job.rb - lib/beaneater/job/collection.rb - lib/beaneater/job/record.rb - lib/beaneater/stats.rb - lib/beaneater/stats/fast_struct.rb - lib/beaneater/stats/stat_struct.rb - lib/beaneater/tube.rb - lib/beaneater/tube/collection.rb - lib/beaneater/tube/record.rb - lib/beaneater/version.rb - test/beaneater_test.rb - test/connection_test.rb - test/errors_test.rb - test/job_test.rb - test/jobs_test.rb - test/prompt_regexp_test.rb - test/stat_struct_test.rb - test/stats_test.rb - test/test_helper.rb - test/tube_test.rb - test/tubes_test.rb homepage: '' licenses: - MIT metadata: {} post_install_message: rdoc_options: [] require_paths: - lib required_ruby_version: !ruby/object:Gem::Requirement requirements: - - '>=' - !ruby/object:Gem::Version version: '0' required_rubygems_version: !ruby/object:Gem::Requirement requirements: - - '>=' - !ruby/object:Gem::Version version: '0' requirements: [] rubyforge_project: rubygems_version: 2.0.6 signing_key: specification_version: 4 summary: Simple beanstalkd client for ruby. test_files: - test/beaneater_test.rb - test/connection_test.rb - test/errors_test.rb - test/job_test.rb - test/jobs_test.rb - test/prompt_regexp_test.rb - test/stat_struct_test.rb - test/stats_test.rb - test/test_helper.rb - test/tube_test.rb - test/tubes_test.rb has_rdoc: beaneater-1.0.0/test/0000755000004100000410000000000012545466244014461 5ustar www-datawww-databeaneater-1.0.0/test/stat_struct_test.rb0000644000004100000410000000257512545466244020435 0ustar www-datawww-data# test/stat_struct_test.rb require File.expand_path('../test_helper', __FILE__) describe Beaneater::StatStruct do before do @hash = { :foo => "bar", :bar => "baz", :baz => "foo", :"under-score" => "demo" } @struct = Beaneater::StatStruct.from_hash(@hash) end describe "for #from_hash" do it "should have 4 keys" do assert_equal 'bar', @struct.foo assert_equal 'baz', @struct.bar assert_equal 'foo', @struct.baz assert_equal 'demo', @struct.under_score end end # from_hash describe "for [] access" do it "should have hash lookup" do assert_equal 'bar', @struct['foo'] assert_equal 'baz', @struct['bar'] end it "should convert keys to string" do assert_equal 'foo', @struct[:baz] assert_equal 'demo', @struct[:"under_score"] end end # [] describe "for #keys" do it "should return 4 keys" do assert_equal 4, @struct.keys.size end it "should return expected keys" do assert_equal ['foo', 'bar', 'baz', 'under_score'].sort, @struct.keys.sort end end # keys describe "for #to_h" do it "should return 4 keys / values" do assert_equal 4, @struct.to_h.size end it "should return expect hash" do assert_equal [['foo', 'bar'], ['bar', 'baz'], ['baz', 'foo'], ['under_score', 'demo']].sort, @struct.to_h.sort end end # to_h end # Beaneater::StatStruct beaneater-1.0.0/test/errors_test.rb0000644000004100000410000000245412545466244017366 0ustar www-datawww-data# test/errors_test.rb require File.expand_path('../test_helper', __FILE__) describe "Beaneater::Errors" do it 'should raise proper exception for invalid status NOT_FOUND' do @klazz = Beaneater::UnexpectedResponse.from_status("NOT_FOUND", "job-stats -1") assert_kind_of(Beaneater::NotFoundError, @klazz) assert_equal 'job-stats -1', @klazz.cmd assert_equal 'NOT_FOUND', @klazz.status end it 'should raise proper exception for invalid status BAD_FORMAT' do @klazz = Beaneater::UnexpectedResponse.from_status("BAD_FORMAT", "FAKE") assert_kind_of(Beaneater::BadFormatError, @klazz) assert_equal 'FAKE', @klazz.cmd assert_equal 'BAD_FORMAT', @klazz.status end it 'should raise proper exception for invalid status DEADLINE_SOON' do @klazz = Beaneater::UnexpectedResponse.from_status("DEADLINE_SOON", "reserve 0") assert_kind_of(Beaneater::DeadlineSoonError, @klazz) assert_equal 'reserve 0', @klazz.cmd assert_equal 'DEADLINE_SOON', @klazz.status end it 'should raise proper exception for invalid status EXPECTED_CRLF' do @klazz = Beaneater::UnexpectedResponse.from_status("EXPECTED_CRLF", "reserve 0") assert_kind_of(Beaneater::ExpectedCrlfError, @klazz) assert_equal 'reserve 0', @klazz.cmd assert_equal 'EXPECTED_CRLF', @klazz.status end end beaneater-1.0.0/test/job_test.rb0000644000004100000410000001352212545466244016622 0ustar www-datawww-data# test/job_test.rb require File.expand_path('../test_helper', __FILE__) describe Beaneater::Job do before do @beanstalk = Beaneater.new('localhost') @tube = @beanstalk.tubes.find 'tube' end describe "for #bury" do before do @time = Time.now.to_i @tube.put "foo bury #{@time}", :pri => 5 end it("should be buried with same pri") do job = @tube.reserve assert_equal "foo bury #{@time}", job.body assert_equal 'reserved', job.stats.state job.bury assert_equal 'buried', job.stats.state assert_equal 5, job.stats.pri assert_equal "foo bury #{@time}", @tube.peek(:buried).body end it("should be released with new pri") do job = @tube.reserve assert_equal "foo bury #{@time}", job.body assert_equal 'reserved', job.stats.state job.bury(:pri => 10) assert_equal 'buried', job.stats.state assert_equal 10, job.stats.pri assert_equal "foo bury #{@time}", @tube.peek(:buried).body end it "should not bury if not reserved" do job = @tube.peek(:ready) assert_raises(Beaneater::JobNotReserved) { job.bury } end it "should not bury if reserved and deleted" do job = @tube.reserve job.delete assert_equal false, job.reserved assert_raises(Beaneater::NotFoundError) { job.bury } end end # bury describe "for #release" do before do @time = Time.now.to_i @tube.put "foo release #{@time}", :pri => 5 end it("should be released with same pri") do job = @tube.reserve assert_equal "foo release #{@time}", job.body assert_equal 'reserved', job.stats.state job.release assert_equal 'ready', job.stats.state assert_equal 5, job.stats.pri assert_equal 0, job.stats.delay end it("should be released with new pri") do job = @tube.reserve assert_equal "foo release #{@time}", job.body assert_equal 'reserved', job.stats.state job.release :pri => 10, :delay => 2 assert_equal 'delayed', job.stats.state assert_equal 10, job.stats.pri assert_equal 2, job.stats.delay end it "should not released if not reserved" do job = @tube.peek(:ready) assert_raises(Beaneater::JobNotReserved) { job.release } end it "should not release if not reserved and buried" do job = @tube.reserve job.bury assert_raises(Beaneater::JobNotReserved) { job.release } end end # release describe "for #delete" do before do @tube.put 'foo' end it("should deletable") do job = @tube.peek(:ready) assert_equal 'foo', job.body job.delete assert_nil @tube.peek(:ready) end end # delete describe "for #touch" do before do @tube.put 'foo touch', :ttr => 1 end it("should be toucheable") do job = @tube.reserve assert_equal 'foo touch', job.body job.touch assert_equal 1, job.stats.reserves job.delete end it "should not touch if not reserved" do job = @tube.peek(:ready) assert_raises(Beaneater::JobNotReserved) { job.touch } end it "should not touch if not reserved and released" do job = @tube.reserve job.release assert_raises(Beaneater::JobNotReserved) { job.touch } end it "should not touch if reserved and deleted" do job = @tube.reserve job.delete assert_raises(Beaneater::NotFoundError) { job.touch } end end # touch describe "for #kick" do before do @tube.put 'foo touch', :ttr => 1 end it("should be toucheable") do job = @tube.reserve assert_equal 'foo touch', job.body job.bury assert_equal 1, @tube.stats.current_jobs_buried if @beanstalk.stats.version.to_f > 1.7 job.kick assert_equal 0, @tube.stats.current_jobs_buried assert_equal 1, @tube.stats.current_jobs_ready end end end # kick describe "for #stats" do before do @tube.put 'foo' @job = @tube.peek(:ready) end it("should have stats") do assert_equal 'tube', @job.stats['tube'] assert_equal 'ready', @job.stats.state end it "should return nil for deleted job with no stats" do @job.delete assert_raises(Beaneater::NotFoundError) { @job.stats } end end # stats describe "for #reserved?" do before do @tube.put 'foo' @job = @tube.peek(:ready) end it("should have stats") do assert_equal false, @job.reserved? job = @tube.reserve assert_equal job.id, @job.id assert_equal true, @job.reserved? @job.delete assert_raises(Beaneater::NotFoundError) { @job.reserved? } end end # reserved? describe "for #exists?" do before do @tube.put 'foo' @job = @tube.peek(:ready) end it("should exists") { assert @job.exists? } it "should not exist" do @job.delete assert !@job.exists? end end # exists? describe "for #tube" do before do @tube.put 'bar' @job = @tube.peek(:ready) end it("should have stats") do job = @tube.reserve assert_equal @tube.name, job.tube job.release end end # tube describe "for #pri" do before do @tube.put 'bar', :pri => 1 @job = @tube.peek(:ready) end it("should return pri") do job = @tube.reserve assert_equal 1, job.pri job.release end end # pri describe "for #ttr" do before do @tube.put 'bar', :ttr => 5 @job = @tube.peek(:ready) end it("should return ttr") do job = @tube.reserve assert_equal 5, job.ttr job.release end end # ttr describe "for #delay" do before do @tube.put 'bar', :delay => 5 @job = @tube.peek(:delayed) end it("should return delay") do assert_equal 5, @job.delay end end # delay end # Beaneater::Job beaneater-1.0.0/test/test_helper.rb0000644000004100000410000000173012545466244017325 0ustar www-datawww-dataENV["TEST"] = 'true' require 'rubygems' require 'coveralls' Coveralls.wear! require 'minitest/autorun' $:.unshift File.expand_path("../../lib") require 'beaneater' require 'timeout' begin require 'mocha/setup' rescue LoadError require 'mocha' end require 'json' class MiniTest::Unit::TestCase # Cleans up all jobs from specific tubes # # @example # cleanup_tubes!(['foo'], @beanstalk) # def cleanup_tubes!(tubes, client=nil) client ||= @beanstalk tubes.each do |name| client.tubes.find(name).clear end end # Cleans up all jobs from all tubes known to the connection def flush_all(client=nil) client ||= @beanstalk # Do not continue if it is a mock or the connection has been closed return if !client.is_a?(Beaneater) || !client.connection.connection client.tubes.all.each do |tube| tube.clear end end # Run clean up after each test to ensure clean state in all tests def teardown flush_all end end beaneater-1.0.0/test/stats_test.rb0000644000004100000410000000224412545466244017205 0ustar www-datawww-data# test/stats_test.rb require File.expand_path('../test_helper', __FILE__) describe Beaneater::Stats do before do connection = stub(:transmit => { :body => { 'uptime' => 1, 'cmd-use' => 2 }, :status => "OK"}) @beanstalk = stub(:connection => connection) @stats = Beaneater::Stats.new(@beanstalk) end describe 'for #[]' do it "should return stats by key" do assert_equal 1, @stats[:uptime] end it "should return stats by underscore key" do assert_equal 2, @stats[:'cmd_use'] end end # [] describe 'for #keys' do it "should return list of keys" do assert_equal 2, @stats.keys.size assert @stats.keys.include?('uptime'), "Expected keys to include 'uptime'" assert @stats.keys.include?('cmd_use'), "Expected keys to include 'cmd-use'" end end # keys describe 'for #method_missing' do it "should return stats by key" do assert_equal 1, @stats.uptime end it "should return stats by underscore key" do assert_equal 2, @stats.cmd_use end it "should raise NoMethodError" do assert_raises(NoMethodError) { @stats.cmd } end end # method_missing end # Beaneater::Stats beaneater-1.0.0/test/beaneater_test.rb0000644000004100000410000000521612545466244017777 0ustar www-datawww-datarequire File.expand_path('../test_helper', __FILE__) describe "beanstalk-client" do before do @beanstalk = Beaneater.new('127.0.0.1:11300') @tubes = ['one', 'two', 'three'] # Put something on each tube so they exist tube_one = @beanstalk.tubes.find('one') tube_one.put('one') tube_two = @beanstalk.tubes.find('two') tube_two.put('two') end describe "test thread safe one" do before do # Create threads that will execute # A: use one # B: use one # B: put two # A: put one a = Thread.new do tube_one = @beanstalk.tubes.find('one') sleep 0.5 tube_one.put('one') end b = Thread.new do sleep 0.125 tube_two = @beanstalk.tubes.find('two') tube_two.put('two') end a.join b.join end it "should return correct current-jobs-ready for tube one" do one = @beanstalk.tubes.find('one').stats assert_equal 2, one.current_jobs_ready end it "should return correct current-jobs-ready for tube two" do two = @beanstalk.tubes.find('two').stats assert_equal 2, two.current_jobs_ready end end describe "test thread safe two" do before do a = Thread.new do tube_one = @beanstalk.tubes.find('one') sleep 0.5 tube_one.put('one') end b = Thread.new do tube_two = @beanstalk.tubes.find('two') sleep 0.125 tube_two.put('two') end a.join b.join end it "should return correct current-jobs-ready for tube one" do one = @beanstalk.tubes.find('one').stats assert_equal 2, one.current_jobs_ready end it "should return correct current-jobs-ready for tube two" do two = @beanstalk.tubes.find('two').stats assert_equal 2, two.current_jobs_ready end end describe "test delete job in reserved state" do before do @tube_three = @beanstalk.tubes.find('three') @tube_three.put('one') @job = @tube_three.reserve end it "should be deleted properly" do assert_equal 'one', @job.body assert_equal 'one', @beanstalk.jobs.find(@job.id).body @job.delete assert_nil @beanstalk.jobs.find(@job.id) end end describe "test delete job in buried state" do before do @tube_three = @beanstalk.tubes.find('three') @tube_three.put('two') @job = @tube_three.reserve end it "should delete job as expected in buried state" do assert_equal 'two', @job.body @job.bury assert_equal 'two', @tube_three.peek(:buried).body @job.delete assert_nil @beanstalk.jobs.find(@job.id) end end end beaneater-1.0.0/test/prompt_regexp_test.rb0000644000004100000410000000263512545466244020746 0ustar www-datawww-data# test/prompt_regexp_test.rb require File.expand_path('../test_helper', __FILE__) require 'socket' describe "Reading from socket client" do before do @fake_port = 11301 @tube_name = 'tube.to.test' @fake_server = Thread.start do server = TCPServer.new(@fake_port) loop do IO.select([server]) client = server.accept_nonblock while line = client.gets case line when /list-tubes-watched/i client.print "OK #{7+@tube_name.size}\r\n---\n- #{@tube_name}\n\r\n" when /watch #{@tube_name}/i client.print "WATCHING 1\r\n" when /reserve/i client.print "RESERVED 17 25\r\n" client.print "[first part]" # Emulate network delay sleep 0.5 client.print "[second part]\r\n" else client.print "ERROR\r\n" end end end end slept = 0 while @beanstalk.nil? begin @beanstalk = Beaneater.new("localhost:#{@fake_port}") rescue Beaneater::NotConnected raise 'Could not connect to fake beanstalkd server' if slept > 1 sleep 0.1 slept += 0.1 end end end it 'should reserve job with full body' do job = @beanstalk.tubes[@tube_name].reserve assert_equal '[first part][second part]', job.body end after do @beanstalk.close @fake_server.kill end end beaneater-1.0.0/test/tubes_test.rb0000644000004100000410000001107712545466244017175 0ustar www-datawww-data# test/tubes_test.rb require File.expand_path('../test_helper', __FILE__) describe Beaneater::Tubes do describe "for #find" do before do @beanstalk = stub @tubes = Beaneater::Tubes.new(@beanstalk) end it("should return Tube obj") { assert_kind_of Beaneater::Tube, @tubes.find(:foo) } it("should return Tube name") { assert_equal "foo", @tubes.find(:foo).name } it("should support hash syntax") { assert_equal "bar", @tubes["bar"].name } end # find describe "for #use" do before do @beanstalk = Beaneater.new('localhost') end it "should switch to used tube for valid name" do tube = Beaneater::Tube.new(@beanstalk, 'some_name') @beanstalk.tubes.use('some_name') assert_equal 'some_name', @beanstalk.tubes.used.name end it "should raise for invalid tube name" do assert_raises(Beaneater::InvalidTubeName) { @beanstalk.tubes.use('; ') } end end # use describe "for #watch & #watched" do before do @beanstalk = Beaneater.new('localhost') end it 'should watch specified tubes' do @beanstalk.tubes.watch('foo') @beanstalk.tubes.watch('bar') assert_equal ['default', 'foo', 'bar'].sort, @beanstalk.tubes.watched.map(&:name).sort end it 'should raise invalid name for bad tube' do assert_raises(Beaneater::InvalidTubeName) { @beanstalk.tubes.watch('; ') } end end # watch! & watched describe "for #all" do before do @beanstalk = Beaneater.new('localhost') @beanstalk.tubes.find('foo').put 'bar' @beanstalk.tubes.find('bar').put 'foo' end it 'should retrieve all tubes' do ['default', 'foo', 'bar'].each do |t| assert @beanstalk.tubes.all.map(&:name).include?(t) end end end # all describe "for Enumerable" do before do @beanstalk = Beaneater.new('localhost') @beanstalk.tubes.find('foo').put 'bar' @beanstalk.tubes.find('bar').put 'foo' end it 'should map tubes' do ['default', 'foo', 'bar'].each do |t| assert @beanstalk.tubes.map(&:name).include?(t) end end end describe "for #used" do before do @beanstalk = Beaneater.new('localhost') @beanstalk.tubes.find('foo').put 'bar' @beanstalk.tubes.find('bar').put 'foo' end it 'should retrieve used tube' do assert_equal 'bar', @beanstalk.tubes.used.name end it 'should support dashed tubes' do @beanstalk.tubes.find('der-bam').put 'foo' assert_equal 'der-bam', @beanstalk.tubes.used.name end end # used describe "for #watch!" do before do @beanstalk = Beaneater.new('localhost') end it 'should watch specified tubes' do @beanstalk.tubes.watch!(:foo) @beanstalk.tubes.watch!('bar') assert_equal ['bar'].sort, @beanstalk.tubes.watched.map(&:name).sort end end # watch! describe "for #ignore" do before do @beanstalk = Beaneater.new('localhost') end it 'should ignore specified tubes' do @beanstalk.tubes.watch('foo') @beanstalk.tubes.watch('bar') @beanstalk.tubes.ignore('foo') assert_equal ['default', 'bar'].sort, @beanstalk.tubes.watched.map(&:name).sort end end # ignore describe "for #reserve" do before do @beanstalk = Beaneater.new('localhost') @tube = @beanstalk.tubes.find 'tube' @time = Time.now.to_i @tube.put "foo reserve #{@time}" end it("should reserve job") do @beanstalk.tubes.watch 'tube' job = @beanstalk.tubes.reserve assert_equal "foo reserve #{@time}", job.body job.delete end it("should reserve job with block") do @beanstalk.tubes.watch 'tube' job = nil @beanstalk.tubes.reserve { |j| job = j; job.delete } assert_equal "foo reserve #{@time}", job.body end it("should reserve job with block and timeout") do @beanstalk.tubes.watch 'tube' job = nil res = @beanstalk.tubes.reserve(0) { |j| job = j; job.delete } assert_equal "foo reserve #{@time}", job.body end it "should raise TimedOutError with timeout" do @beanstalk.tubes.watch 'tube' @beanstalk.tubes.reserve(0) { |j| job = j; job.delete } assert_raises(Beaneater::TimedOutError) { @beanstalk.tubes.reserve(0) } end it "should raise DeadlineSoonError with ttr 1" do @tube.reserve.delete @tube.put "foo reserve #{@time}", :ttr => 1 @beanstalk.tubes.watch 'tube' @beanstalk.tubes.reserve assert_raises(Beaneater::DeadlineSoonError) { @beanstalk.tubes.reserve(0) } end end # reserve end # Beaneater::Tubes beaneater-1.0.0/test/jobs_test.rb0000644000004100000410000000574112545466244017011 0ustar www-datawww-data# test/jobs_test.rb require File.expand_path('../test_helper', __FILE__) describe Beaneater::Jobs do before do @beanstalk = Beaneater.new('localhost') @jobs = Beaneater::Jobs.new(@beanstalk) @tube = @beanstalk.tubes.find('baz') end describe "for #find" do before do @time = Time.now.to_i @tube.put("foo find #{@time}") @job = @tube.peek(:ready) end it "should return job from id" do assert_equal "foo find #{@time}", @jobs.find(@job.id).body end it "should return job using peek" do assert_equal "foo find #{@time}", @jobs.find(@job.id).body end it "should return job using hash syntax" do assert_equal "foo find #{@time}", @jobs.find(@job.id).body end it "should return nil for invalid id" do assert_nil @jobs.find(-1) end end # find describe "for #register!" do before do $foo = 0 @jobs.register('tube', :retry_on => [Timeout::Error]) do |job| $foo += 1 end end it "should store processor" do assert_equal 'tube', @jobs.processors.keys.first assert_equal [Timeout::Error], @jobs.processors.values.first[:retry_on] end it "should store block for 'tube'" do @jobs.processors['tube'][:block].call nil assert_equal 1, $foo end end # register! describe "for process!" do before do $foo = [] @jobs.register('tube_success', :retry_on => [Timeout::Error]) do |job| # p job.body $foo << job.body raise Beaneater::AbortProcessingError if job.body =~ /abort/ end @jobs.register('tube_release', :retry_on => [Timeout::Error], :max_retries => 2) do |job| $foo << job.body raise Timeout::Error end @jobs.register('tube_buried') do |job| $foo << job.body raise RuntimeError end cleanup_tubes!(['tube_success', 'tube_release', 'tube_buried']) @beanstalk.tubes.find('tube_success').put("success abort", :pri => 2**31 + 1) @beanstalk.tubes.find('tube_success').put("success 2", :pri => 1) @beanstalk.tubes.find('tube_release').put("released") @beanstalk.tubes.find('tube_buried').put("buried") @jobs.process!(:release_delay => 0) end it "should process all jobs" do assert_equal ['success 2', 'released', 'released', 'released', 'buried', 'success abort'], $foo end it "should clear successful_jobs" do assert_equal 0, @beanstalk.tubes.find('tube_success').stats.current_jobs_ready assert_equal 1, @beanstalk.tubes.find('tube_success').stats.current_jobs_buried assert_equal 0, @beanstalk.tubes.find('tube_success').stats.current_jobs_reserved end it "should retry release jobs 2 times" do assert_equal 2, @beanstalk.tubes.find('tube_release').peek(:buried).stats.releases end it "should bury unexpected exception" do assert_equal 1, @beanstalk.tubes.find('tube_buried').stats.current_jobs_buried end end # for_process! end # Beaneater::Jobs beaneater-1.0.0/test/connection_test.rb0000644000004100000410000000667512545466244020222 0ustar www-datawww-data# test/connection_test.rb require File.expand_path('../test_helper', __FILE__) describe Beaneater::Connection do describe 'for #new' do before do @host = 'localhost' @bc = Beaneater::Connection.new(@host) end it "should store address, host and port" do assert_equal 'localhost', @bc.address assert_equal 'localhost', @bc.host assert_equal 11300, @bc.port end it "should init connection" do assert_kind_of TCPSocket, @bc.connection if @bc.connection.peeraddr[0] == 'AF_INET' assert_equal '127.0.0.1', @bc.connection.peeraddr[3] else assert_equal 'AF_INET6', @bc.connection.peeraddr[0] assert_equal '::1', @bc.connection.peeraddr[3] end assert_equal 11300, @bc.connection.peeraddr[1] end it "should raise on invalid connection" do assert_raises(Beaneater::NotConnected) { Beaneater::Connection.new("localhost:8544") } end it "should support array connection to single connection" do @bc2 = Beaneater::Connection.new([@host]) assert_equal 'localhost', @bc.address assert_equal 'localhost', @bc.host assert_equal 11300, @bc.port end end # new describe 'for #transmit' do before do @host = 'localhost' @bc = Beaneater::Connection.new(@host) end it "should return yaml loaded response" do res = @bc.transmit 'stats' refute_nil res[:body]['current-connections'] assert_equal 'OK', res[:status] end it "should return id" do res = @bc.transmit "put 0 0 100 1\r\nX" assert res[:id] assert_equal 'INSERTED', res[:status] end it "should support dashes in response" do res = @bc.transmit "use foo-bar\r\n" assert_equal 'USING', res[:status] assert_equal 'foo-bar', res[:id] end it "should pass crlf through without changing its length" do res = @bc.transmit "put 0 0 100 2\r\n\r\n" assert_equal 'INSERTED', res[:status] end it "should handle *any* byte value without changing length" do res = @bc.transmit "put 0 0 100 256\r\n"+(0..255).to_a.pack("c*") assert_equal 'INSERTED', res[:status] end it "should retry command with success after one connection failure" do TCPSocket.any_instance.expects(:readline).times(2). raises(EOFError.new).then. returns("DELETED 56\nFOO") res = @bc.transmit "delete 56\r\n" assert_equal 'DELETED', res[:status] end it "should fail after exceeding retries with DrainingError" do TCPSocket.any_instance.expects(:readline).times(3). raises(Beaneater::UnexpectedResponse.from_status("DRAINING", "delete 56")) assert_raises(Beaneater::DrainingError) { @bc.transmit "delete 56\r\n" } end it "should fail after exceeding reconnect max retries" do # next connection attempts should fail TCPSocket.stubs(:new).times(3).raises(Errno::ECONNREFUSED.new) TCPSocket.any_instance.stubs(:readline).times(1).raises(EOFError.new) assert_raises(Beaneater::NotConnected) { @bc.transmit "delete 56\r\n" } end end # transmit describe 'for #close' do before do @host = 'localhost' @bc = Beaneater::Connection.new(@host) end it "should clear connection" do assert_kind_of TCPSocket, @bc.connection @bc.close assert_nil @bc.connection assert_raises(Beaneater::NotConnected) { @bc.transmit 'stats' } end end # close end # Beaneater::Connection beaneater-1.0.0/test/tube_test.rb0000644000004100000410000001247312545466244017013 0ustar www-datawww-data# test/tube_test.rb require File.expand_path('../test_helper', __FILE__) describe Beaneater::Tube do before do @beanstalk = Beaneater.new('localhost') @tube = Beaneater::Tube.new(@beanstalk, 'baz') end describe "for #put" do before do @time = Time.now.to_i end it "should insert a job" do @tube.put "bar put #{@time}" assert_equal "bar put #{@time}", @tube.peek(:ready).body assert_equal 120, @tube.peek(:ready).ttr assert_equal 65536, @tube.peek(:ready).pri assert_equal 0, @tube.peek(:ready).delay end it "should insert a delayed job" do @tube.put "delayed put #{@time}", :delay => 1 assert_equal "delayed put #{@time}", @tube.peek(:delayed).body end it "should support custom serializer" do Beaneater.configure.job_serializer = lambda { |b| JSON.dump(b) } @tube.put({ foo: "bar"}) assert_equal '{"foo":"bar"}', @tube.peek(:ready).body end after do Beaneater::Connection.any_instance.unstub(:transmit) Beaneater.configure.job_serializer = lambda { |b| b } end end # put describe "for #peek" do before do @time = Time.now.to_i end it "should peek delayed" do @tube.put "foo delay #{@time}", :delay => 1 assert_equal "foo delay #{@time}", @tube.peek(:delayed).body end it "should peek ready" do @tube.put "foo ready #{@time}", :delay => 0 assert_equal "foo ready #{@time}", @tube.peek(:ready).body end it "should peek buried" do @tube.put "foo buried #{@time}" @tube.reserve.bury assert_equal "foo buried #{@time}", @tube.peek(:buried).body end it "should return nil for empty peek" do assert_nil @tube.peek(:buried) end it "supports valid JSON" do json = '{ "foo" : "bar" }' @tube.put(json) assert_equal 'bar', JSON.parse(@tube.peek(:ready).body)['foo'] end it "supports non valid JSON" do json = '{"message":"hi"}' @tube.put(json) assert_equal 'hi', JSON.parse(@tube.peek(:ready).body)['message'] end it "supports passing crlf through" do @tube.put("\r\n") assert_equal "\r\n", @tube.peek(:ready).body end it "supports passing any byte value through" do bytes = (0..255).to_a.pack("c*") @tube.put(bytes) assert_equal bytes, @tube.peek(:ready).body end it "should support custom parser" do Beaneater.configure.job_parser = lambda { |b| JSON.parse(b) } json = '{"message":"hi"}' @tube.put(json) assert_equal 'hi', @tube.peek(:ready).body['message'] end after do Beaneater.configure.job_parser = lambda { |b| b } end end # peek describe "for #reserve" do before do @time = Time.now.to_i @json = %Q&{ "foo" : "#{@time} bar" }& @tube.put @json end it "should reserve job" do @job = @tube.reserve assert_equal "#{@time} bar", JSON.parse(@job.body)['foo'] @job.delete end it "should reserve job with block" do job = nil @tube.reserve { |j| job = j; job.delete } assert_equal "#{@time} bar", JSON.parse(job.body)['foo'] end it "should support custom parser" do Beaneater.configure.job_parser = lambda { |b| JSON.parse(b) } @job = @tube.reserve assert_equal "#{@time} bar", @job.body['foo'] @job.delete end after do Beaneater.configure.job_parser = lambda { |b| b } end end # reserve describe "for #pause" do before do @time = Time.now.to_i @tube = Beaneater::Tube.new(@beanstalk, 'bam') @tube.put "foo pause #{@time}" end it "should allow tube pause" do assert_equal 0, @tube.stats.pause @tube.pause(1) assert_equal 1, @tube.stats.pause end end # pause describe "for #stats" do before do @time = Time.now.to_i @tube.put "foo stats #{@time}" @stats = @tube.stats end it "should return total number of jobs in tube" do assert_equal 1, @stats['current_jobs_ready'] assert_equal 0, @stats.current_jobs_delayed end it "should raise error for empty tube" do assert_raises(Beaneater::NotFoundError) { @beanstalk.tubes.find('fake_tube').stats } end end # stats describe "for #kick" do before do @time, @time2 = 2.times.map { Time.now.to_i } @tube.put "kick #{@time}" @tube.put "kick #{@time2}" 2.times.map { @tube.reserve.bury } end it "should kick 2 buried jobs" do assert_equal 2, @tube.stats.current_jobs_buried @tube.kick(2) assert_equal 0, @tube.stats.current_jobs_buried assert_equal 2, @tube.stats.current_jobs_ready end end # kick describe "for #clear" do @time = Time.now.to_i before do 2.times { |i| @tube.put "to clear success #{i} #{@time}" } 2.times { |i| @tube.put "to clear delayed #{i} #{@time}", :delay => 5 } 2.times { |i| @tube.put "to clear bury #{i} #{@time}", :pri => 1 } @tube.reserve.bury while @tube.peek(:ready).stats['pri'] == 1 end it "should clear all jobs in tube" do tube_counts = lambda { %w(ready buried delayed).map { |s| @tube.stats["current_jobs_#{s}"] } } assert_equal [2, 2, 2], tube_counts.call @tube.clear stats = @tube.stats assert_equal [0, 0, 0], tube_counts.call end end # clear end # Beaneater::Tube beaneater-1.0.0/.gitignore0000644000004100000410000000023212545466244015467 0ustar www-datawww-data*.gem *.rbc .bundle .config .yardoc Gemfile.lock InstalledFiles _yardoc coverage doc/ lib/bundler/man pkg rdoc spec/reports test/tmp test/version_tmp tmp beaneater-1.0.0/beaneater.gemspec0000644000004100000410000000171112545466244016775 0ustar www-datawww-data# -*- encoding: utf-8 -*- lib = File.expand_path('../lib', __FILE__) $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) require 'beaneater/version' Gem::Specification.new do |gem| gem.name = "beaneater" gem.version = Beaneater::VERSION gem.authors = ["Nico Taing"] gem.email = ["nico.taing@gmail.com"] gem.description = %q{Simple beanstalkd client for ruby} gem.summary = %q{Simple beanstalkd client for ruby.} gem.homepage = "" gem.license = 'MIT' gem.files = `git ls-files`.split($/) gem.executables = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) } gem.test_files = gem.files.grep(%r{^(test|spec|features)/}) gem.require_paths = ["lib"] gem.add_development_dependency 'minitest', "~> 4.1.0" gem.add_development_dependency 'rake' gem.add_development_dependency 'mocha' gem.add_development_dependency 'term-ansicolor' gem.add_development_dependency 'json' end beaneater-1.0.0/.yardopts0000644000004100000410000000022412545466244015346 0ustar www-datawww-data--output-dir doc/ --readme README.md --title Beaneater --markup-provider=redcarpet --markup=markdown --protected --no-private lib/beaneater/**/*.rb beaneater-1.0.0/TODO0000644000004100000410000000015412545466244014172 0ustar www-datawww-data- Remove connection from pool if it's not responding and be able to add more connections and reattempt laterbeaneater-1.0.0/CHANGELOG.md0000644000004100000410000000316512545466244015320 0ustar www-datawww-data# CHANGELOG for Beaneater ## 1.0.0 (April 26th 2015) * Beginning from version 1.0.0 the support for `Beaneater::Pool` has been dropped (@alup) * `Jobs#find_all` method has been removed, since it is no longer necessary after removing pool (@alup) * `Tubes` is now an enumerable allowing `tubes` to be handled as a collection (@Aethelflaed) ## 0.3.3 (August 16th 2014) * Fix failure when job is not defined and fix exception handling for jobs (@nicholasorenrawlings) * Add reserve_timeout option to job processing (@nicholasorenrawlings) * Add travis-ci badge (@tdg5) * Fix tests to run more reliably (@tdg5) ## 0.3.2 (Sept 15 2013) * Fix #29 ExpectedCrlfError name and invocation (@tdg5) ## 0.3.1 (Jun 28 2013) * Fixes issue with "chomp" nil exception when losing connection (Thanks @simao) * Better handling of unknown or invalid commands during transmit * Raise proper CRLF exception (Thanks @carlosmoutinho) ## 0.3.0 (Jan 23 2013) * Replace Telnet with tcpsocket thanks @vidarh ## 0.2.2 (Dec 2 2012) * Fixes status and ID parsing in a response (Thanks @justincase) ## 0.2.1 (Dec 1 2012) * Convert command to ASCII_8Bit to avoid gsub issues ## 0.2.0 (Nov 12 2012) * Fix 1.8.7 compatibility issues * Add configuration block to beaneater and better job parsing * BREAKING: json jobs now return as string by default not hashes ## 0.1.2 (Nov 7 2012) * Add timeout: false to transmit to allow longer reserve ## 0.1.1 (Nov 4 2012) * Add `Jobs#find_all` to fix #10 * Fixed issue with `tubes-list` by merging results * Add `Job#ttr`, `Job#pri`, `Job#delay` * Improved yardocs coverage and accuracy ## 0.1.0 (Nov 1 2012) * Initial release! beaneater-1.0.0/README.md0000644000004100000410000004302512545466244014765 0ustar www-datawww-data# Beaneater [![Build Status](https://secure.travis-ci.org/beanstalkd/beaneater.png)](http://travis-ci.org/beanstalkd/beaneater) [![Coverage Status](https://coveralls.io/repos/beanstalkd/beaneater/badge.png?branch=master)](https://coveralls.io/r/beanstalkd/beaneater) Beaneater is the best way to interact with beanstalkd from within Ruby. [Beanstalkd](http://kr.github.com/beanstalkd/) is a simple, fast work queue. Its interface is generic, but was originally designed for reducing the latency of page views in high-volume web applications by running time-consuming tasks asynchronously. Read the [yardocs](http://rdoc.info/github/beanstalkd/beaneater) and/or the [beanstalk protocol](https://github.com/kr/beanstalkd/blob/master/doc/protocol.md) for more details. **Important Note**: This README is **for branch 1.0.x which is under development**. Please switch to latest `0.x` branch for stable version. ## Why Beanstalk? Illya has an excellent blog post [Scalable Work Queues with Beanstalk](http://www.igvita.com/2010/05/20/scalable-work-queues-with-beanstalk/) and Adam Wiggins posted [an excellent comparison](http://adam.heroku.com/past/2010/4/24/beanstalk_a_simple_and_fast_queueing_backend/). You will find that **beanstalkd** is an underrated but incredibly powerful project that is extremely well-suited as a job or messaging queue. Significantly better suited for this task than Redis or a traditional RDBMS. Beanstalk is a simple, and fast work queue service rolled into a single binary - it is the memcached of work queues. Originally built to power the backend for the 'Causes' Facebook app, it is a mature and production ready open source project. [PostRank](http://www.postrank.com) has used beanstalk to reliably process millions of jobs a day. A single instance of Beanstalk is perfectly capable of handling thousands of jobs a second (or more, depending on your job size) because it is an in-memory, event-driven system. Powered by libevent under the hood, it requires zero setup (launch and forget, à la memcached), optional log based persistence, an easily parsed ASCII protocol, and a rich set of tools for job management that go well beyond a simple FIFO work queue. Beanstalkd supports the following features out of the box: | Feature | Description | | ------- | ------------------------------- | | **Easy Setup** | Quick to install, no files to edit, no settings to tweak. | | **Speed** | Process thousands of jobs per second without breaking a sweat. | | **Client Support** | Client libraries exist for over 21 languages including Python, Ruby, and Go. | | **Tubes** | Supports multiple work queues created automatically on demand. | | **Reliable** | Beanstalk’s reserve, work, delete cycle ensures reliable processing. | | **Scheduling** | Delay enqueuing jobs by a specified interval to be processed later. | | **Priorities** | Important jobs go to the head of the queue and get processed sooner. | | **Persistence** | Jobs are stored in memory for speed, but logged to disk for safe keeping. | | **Scalability** | Client-side federation provides effortless horizontal scalability. | | **Error Handling** | Bury any job which causes an error for later debugging and inspection. | | **Simple Debugging** | Talk directly to the beanstalkd server over telnet to get a handle on your app. | | **Efficiency** | Each beanstalkd process can handle tens of thousands of open connections. | | **Memory Usage** | Use the built-in `ulimit` OS feature to cap beanstalkd's memory consumption. | Keep in mind that these features are supported out of the box with beanstalk and requires no special ruby specific logic. In the end, **beanstalk is the ideal job queue** and has the added benefit of being easy to setup and configure. ## Installation Install beanstalkd: Mac OS ``` brew install beanstalkd beanstalkd -p 11300 ``` Ubuntu ``` apt-get install beanstalkd beanstalkd -p 11300 ``` Install beaneater as a gem: ``` gem install beaneater ``` or add this to your Gemfile: ```ruby # Gemfile gem 'beaneater' ``` and run `bundle install` to install the dependency. ## Breaking Changes since 1.0! Starting in 1.0, we removed the concept of the `Beaneater::Pool` which introduced considerable complexity into this gem. * Beginning from version 1.0.0 the support for `Beaneater::Pool` has been dropped. The specific feature may be supported again in the next versions as separate module or through a separate gem. If you want to use the pool feature you should switch to 0.x stable branches instead. * `Jobs#find_all` method has been removed, since it is no longer necessary. To manage a pool of beanstalkd instances, we'd prefer to leave the handling to the developer or other higher-level libraries. ## Quick Overview: The concise summary of how to use beaneater: ```ruby # Connect to pool @beanstalk = Beaneater.new('localhost:11300') # Enqueue jobs to tube @tube = @beanstalk.tubes["my-tube"] @tube.put '{ "key" : "foo" }', :pri => 5 @tube.put '{ "key" : "bar" }', :delay => 3 # Process jobs from tube while @tube.peek(:ready) job = @tube.reserve puts "job value is #{JSON.parse(job.body)["key"]}!" job.delete end # Disconnect the pool @beanstalk.close ``` For a more detailed rundown, check out the __Usage__ section below. ## Usage ### Configuration To setup advanced options for beaneater, you can pass configuration options using: ```ruby Beaneater.configure do |config| # config.default_put_delay = 0 # config.default_put_pri = 65536 # config.default_put_ttr = 120 # config.job_parser = lambda { |body| body } # config.job_serializer = lambda { |body| body } # config.beanstalkd_url = 'localhost:11300' end ``` The above options are all defaults, so only include a configuration block if you need to make changes. ### Connection To interact with a beanstalk queue, first establish a connection by providing an address: ```ruby @beanstalk = Beaneater.new('10.0.1.5:11300') # Or if ENV['BEANSTALKD_URL'] == '127.0.0.1:11300' @beanstalk = Beaneater.new @beanstalk.connectiont # => localhost:11300 ``` You can conversely close and dispose of a connection at any time with: ```ruby @beanstalk.close ``` ### Tubes Beanstalkd has one or more tubes which can contain any number of jobs. Jobs can be inserted (put) into the used tube and pulled out (reserved) from watched tubes. Each tube consists of a _ready_, _delayed_, and _buried_ queue for jobs. When a client connects, its watch list is initially just the tube named `default`. Tube names are at most 200 bytes. It specifies the tube to use. If the tube does not exist, it will be automatically created. To interact with a tube, first `find` the tube: ```ruby @tube = @beanstalk.tubes.find "some-tube-here" # => ``` To reserve jobs from beanstalk, you will need to 'watch' certain tubes: ```ruby # Watch only the tubes listed below (!) @beanstalk.tubes.watch!('some-tube') # Append tubes to existing set of watched tubes @beanstalk.tubes.watch('another-tube') # You can also ignore tubes that have been watched previously @beanstalk.tubes.ignore('some-tube') ``` You can easily get a list of all, used or watched tubes: ```ruby # The list-tubes command returns a list of all existing tubes @beanstalk.tubes.all # => [, ] # Returns the tube currently being used by the client (for insertion) @beanstalk.tubes.used # => # Returns a list tubes currently being watched by the client (for consumption) @beanstalk.tubes.watched # => [] ``` You can also temporarily 'pause' the execution of a tube by specifying the time: ```ruby tube = @beanstalk.tubes["some-tube-here"] tube.pause(3) # pauses tube for 3 seconds ``` or even clear the tube of all jobs: ```ruby tube = @beanstalk.tubes["some-tube-here"] tube.clear # tube will now be empty ``` In summary, each beanstalk client manages two separate concerns: which tube newly created jobs are put into, and which tube(s) jobs are reserved from. Accordingly, there are two separate sets of functions for these concerns: * **use** and **using** affect where 'put' places jobs * **watch** and **watching** control where reserve takes jobs from Note that these concerns are fully orthogonal: for example, when you 'use' a tube, it is not automatically 'watched'. Neither does 'watching' a tube affect the tube you are 'using'. ### Jobs A job in beanstalk gets inserted by a client and includes the 'body' and job metadata. Each job is enqueued into a tube and later reserved and processed. Here is a picture of the typical job lifecycle: ``` put reserve delete -----> [READY] ---------> [RESERVED] --------> *poof* ``` A job at any given time is in one of three states: **ready**, **delayed**, or **buried**: | State | Description | | ------- | ------------------------------- | | ready | waiting to be `reserved` and processed after being `put` onto a tube. | | delayed | waiting to become `ready` after the specified delay. | | buried | waiting to be kicked, usually after job fails to process | In addition, there are several actions that can be performed on a given job, you can: * **reserve** which locks a job from the ready queue for processing. * **touch** which extends the time before a job is autoreleased back to ready. * **release** which places a reserved job back onto the ready queue. * **delete** which removes a job from beanstalk. * **bury** which places a reserved job into the buried state. * **kick** which places a buried job from the buried queue back to ready. You can insert a job onto a beanstalk tube using the `put` command: ```ruby @tube.put "job-data-here" ``` Beanstalkd can only stores strings as job bodies, but you can easily encode your data into a string: ```ruby @tube.put({:foo => 'bar'}.to_json) ``` Moreover, you can provide a default job serializer by setting the corresponding configuration option (`job_serializer`), in order to apply the encoding on each job body which is going to be send using the `put` command. For example, to encode a ruby object to JSON format: ```ruby Beaneater.configure do |config| config.job_serializer = lambda { |body| JSON.dump(body) } end ``` Each job has various metadata associated such as `priority`, `delay`, and `ttr` which can be specified as part of the `put` command: ```ruby # defaults are priority 0, delay of 0 and ttr of 120 seconds @tube.put "job-data-here", :pri => 1000, :delay => 50, :ttr => 200 ``` The `priority` argument is an integer < 2**32. Jobs with a smaller priority take precedence over jobs with larger priorities. The `delay` argument is an integer number of seconds to wait before putting the job in the ready queue. The `ttr` argument is the time to run -- is an integer number of seconds to allow a worker to run this job. ### Processing Jobs (Manually) In order to process jobs, the client should first specify the intended tubes to be watched. If not specified, this will default to watching just the `default` tube. ```ruby @beanstalk = Beaneater.new('10.0.1.5:11300') @beanstalk.tubes.watch!('tube-name', 'other-tube') ``` Next you can use the `reserve` command which will return the first available job within the watched tubes: ```ruby job = @beanstalk.tubes.reserve # => puts job.body # prints 'job-data-here' print job.stats.state # => 'reserved' ``` By default, reserve will wait indefinitely for the next job. If you want to specify a timeout, simply pass that in seconds into the command: ```ruby job = @beanstalk.tubes.reserve(5) # wait 5 secs for a job, then return # => ``` You can 'release' a reserved job back onto the ready queue to retry later: ```ruby job = @beanstalk.tubes.reserve # ...job has ephemeral fail... job.release :delay => 5 print job.stats.state # => 'delayed' ``` You can also 'delete' jobs that are finished: ```ruby job = @beanstalk.tubes.reserve job.touch # extends ttr for job # ...process job... job.delete ``` Beanstalk jobs can also be buried if they fail, rather than being deleted: ```ruby job = @beanstalk.tubes.reserve # ...job fails... job.bury print job.stats.state # => 'buried' ``` Burying a job means that the job is pulled out of the queue into a special 'holding' area for later inspection or reuse. To reanimate this job later, you can 'kick' buried jobs back into being ready: ```ruby @beanstalk.tubes['some-tube'].kick(3) ``` This kicks 3 buried jobs for 'some-tube' back into the 'ready' state. Jobs can also be inspected using the 'peek' commands. To find and peek at a particular job based on the id: ```ruby @beanstalk.jobs.find(123) # => ``` or you can peek at jobs within a tube: ```ruby @tube = @beanstalk.tubes.find('foo') @tube.peek(:ready) # => @tube.peek(:buried) # => @tube.peek(:delayed) # => ``` When dealing with jobs there are a few other useful commands available: ```ruby job = @beanstalk.tubes.reserve print job.tube # => "some-tube-name" print job.reserved? # => true print job.exists? # => true job.delete print job.exists? # => false ``` ### Processing Jobs (Automatically) Instead of using `watch` and `reserve`, you can also use the higher level `register` and `process` methods to process jobs. First you can 'register' how to handle jobs from various tubes: ```ruby @beanstalk.jobs.register('some-tube', :retry_on => [SomeError]) do |job| do_something(job) end @beanstalk.jobs.register('other-tube') do |job| do_something_else(job) end ``` Once you have registered the handlers for known tubes, calling `process!` will begin a loop processing jobs as defined by the registered processor blocks: ```ruby @beanstalk.jobs.process! ``` Processing runs the following steps: 1. Watch all registered tubes 1. Reserve the next job 1. Once job is reserved, invoke the registered handler based on the tube name 1. If no exceptions occur, delete the job (success) 1. If 'retry_on' exceptions occur, call 'release' (retry) 1. If other exception occurs, call 'bury' (error) 1. Repeat steps 2-5 The `process!` command is ideally suited for a beanstalk job processing daemon. Even though `process!` is intended to be a long-running process, you can stop the loop at any time by raising `AbortProcessingError` while processing is running. ### Handling Errors While using Beaneater, certain errors may be encountered. Errors are encountered when a command is sent to beanstalk and something unexpected happens. The most common errors are listed below: | Errors | Description | | -------------------- | ------------- | | Beaneater::NotConnected | Client connection to beanstalk cannot be established. | | Beaneater::InvalidTubeName | Specified tube name for use or watch is not valid. | | Beaneater::NotFoundError | Specified job or tube could not be found. | | Beaneater::TimedOutError | Job could not be reserved within time specified. | | Beaneater::JobNotReserved | Job has not been reserved and action cannot be taken. | There are other exceptions that are less common such as `OutOfMemoryError`, `DrainingError`, `DeadlineSoonError`, `InternalError`, `BadFormatError`, `UnknownCommandError`, `ExpectedCRLFError`, `JobTooBigError`, `NotIgnoredError`. Be sure to check the [beanstalk protocol](https://github.com/kr/beanstalkd/blob/master/doc/protocol.md) for more information. ### Stats Beanstalk has plenty of commands for introspecting the state of the queues and jobs. To get stats for beanstalk overall: ```ruby # Get overall stats about the job processing that has occurred print @beanstalk.stats # => # 1 ``` For stats on a particular tube: ```ruby # Get statistical information about the specified tube if it exists print @beanstalk.tubes['some_tube_name'].stats # => { 'current_jobs_ready': 0, 'current_jobs_reserved': 0, ... } ``` For stats on an individual job: ```ruby # Get statistical information about the specified job if it exists print @beanstalk.jobs[some_job_id].stats # => {'age': 0, 'id': 2, 'state': 'reserved', 'tube': 'default', ... } ``` Be sure to check the [beanstalk protocol](https://github.com/kr/beanstalkd/blob/master/doc/protocol.md) for more details about the stats commands. ## Resources There are other resources helpful when learning about beanstalk: * [Beaneater Yardocs](http://rdoc.info/github/beanstalkd/beaneater) * [Beaneater on Rubygems](https://rubygems.org/gems/beaneater) * [Beanstalkd homepage](http://kr.github.com/beanstalkd/) * [beanstalk on github](https://github.com/kr/beanstalkd) * [beanstalk protocol](https://github.com/kr/beanstalkd/blob/master/doc/protocol.md) * [Backburner](https://github.com/nesquena/backburner) - Ruby job queue for Rails/Sinatra * [BeanCounter](https://github.com/gemeraldbeanstalk/bean_counter) - TestUnit/MiniTest assertions and RSpec matchers for testing code that relies on Beaneater ## Contributors - [Nico Taing](https://github.com/Nico-Taing) - Creator and co-maintainer - [Nathan Esquenazi](https://github.com/nesquena) - Contributor and co-maintainer - [Keith Rarick](https://github.com/kr) - Much code inspired and adapted from beanstalk-client - [Vidar Hokstad](https://github.com/vidarh) - Replaced telnet with correct TCP socket handling - [Andreas Loupasakis](https://github.com/alup) - Improve test coverage, improve job configuration