ruby-stud-0.0.20/000755 000765 000024 00000000000 12543132344 013661 5ustar00tpotstaff000000 000000 ruby-stud-0.0.20/CHANGELIST000644 000765 000024 00000000000 12543132341 015210 0ustar00tpotstaff000000 000000 ruby-stud-0.0.20/Gemfile000644 000765 000024 00000000046 12543132341 015151 0ustar00tpotstaff000000 000000 source "https://rubygems.org" gemspec ruby-stud-0.0.20/Gemfile.lock000644 000765 000024 00000001027 12543132341 016100 0ustar00tpotstaff000000 000000 PATH remote: . specs: stud (0.0.18) GEM remote: https://rubygems.org/ specs: diff-lcs (1.2.5) insist (1.0.0) rspec (3.1.0) rspec-core (~> 3.1.0) rspec-expectations (~> 3.1.0) rspec-mocks (~> 3.1.0) rspec-core (3.1.7) rspec-support (~> 3.1.0) rspec-expectations (3.1.2) diff-lcs (>= 1.2.0, < 2.0) rspec-support (~> 3.1.0) rspec-mocks (3.1.3) rspec-support (~> 3.1.0) rspec-support (3.1.2) PLATFORMS java ruby DEPENDENCIES insist rspec stud! ruby-stud-0.0.20/lib/000755 000765 000024 00000000000 12543132341 014424 5ustar00tpotstaff000000 000000 ruby-stud-0.0.20/LICENSE000644 000765 000024 00000001076 12543132341 014667 0ustar00tpotstaff000000 000000 Copyright 2012-2013 Jordan Sissel and contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ruby-stud-0.0.20/Makefile000644 000765 000024 00000001100 12543132341 015306 0ustar00tpotstaff000000 000000 GEMSPEC=$(shell ls *.gemspec | head -1) VERSION=$(shell ruby -rubygems -e 'puts Gem::Specification.load("$(GEMSPEC)").version') PROJECT=$(shell ruby -rubygems -e 'puts Gem::Specification.load("$(GEMSPEC)").name') GEM=$(PROJECT)-$(VERSION).gem .PHONY: test test: bundle exec rspec .PHONY: package package: $(GEM) # Always build the gem .PHONY: $(GEM) $(GEM): gem build $(PROJECT).gemspec showdocs: yard server --plugin yard-tomdoc -r clean: -rm -r .yardoc/ doc/ *.gem .PHONY: install install: $(GEM) gem install $< .PHONY: publish publish: $(GEM) gem push $(GEM) ruby-stud-0.0.20/README.md000644 000765 000024 00000002076 12543132341 015142 0ustar00tpotstaff000000 000000 # Stud. Ruby's stdlib is missing many things I use to solve most of my software problems. Things like like retrying on a failure, supervising workers, resource pools, etc. In general, I started exploring solutions to these things in code over in my [software-patterns](https://github.com/jordansissel/software-patterns) repo. This library (stud) aims to be a well-tested, production-quality implementation of the patterns described in that repo. For now, these all exist in a single repo because, so far, implementations of each 'pattern' are quite small by code size. ## Features * {Stud::Try} (and {Stud.try}) - retry on failure, with back-off, where failure is any exception. * {Stud::Pool} - generic resource pools * {Stud::Task} - tasks (threads that can return values, exceptions, etc) * {Stud.interval} - interval execution (do X every N seconds) * {Stud::Buffer} - batch & flush behavior. ## TODO: * Make sure all things are documented. rubydoc.info should be able to clearly show folks how to use features of this library. * Add tests to cover all supported features. ruby-stud-0.0.20/spec/000755 000765 000024 00000000000 12543132341 014610 5ustar00tpotstaff000000 000000 ruby-stud-0.0.20/stud.gemspec000644 000765 000024 00000001315 12543132341 016202 0ustar00tpotstaff000000 000000 Gem::Specification.new do |spec| spec.name = "stud" spec.version = "0.0.20" spec.summary = "stud - common code techniques" spec.description = "small reusable bits of code I'm tired of writing over " \ "and over. A library form of my software-patterns github repo." files = [] dirs = %w{lib} dirs.each do |dir| files += Dir["#{dir}/**/*"] end files << "LICENSE" files << "CHANGELIST" files << "README.md" spec.files = files spec.require_paths << "lib" spec.author = "Jordan Sissel" spec.email = "jls@semicomplete.com" spec.homepage = "https://github.com/jordansissel/ruby-stud" spec.add_development_dependency("rspec") spec.add_development_dependency("insist") end ruby-stud-0.0.20/spec/spec_env.rb000644 000765 000024 00000000127 12543132341 016737 0ustar00tpotstaff000000 000000 $:.unshift(File.expand_path("#{__FILE__}/../../lib")) require "rspec" require "insist" ruby-stud-0.0.20/spec/stud/000755 000765 000024 00000000000 12543132341 015567 5ustar00tpotstaff000000 000000 ruby-stud-0.0.20/spec/stud/buffer_spec.rb000644 000765 000024 00000021331 12543132341 020377 0ustar00tpotstaff000000 000000 require "stud/buffer" require "stud/try" require "spec_env" # from the top level spec/ directory Thread.abort_on_exception = true class BufferSubject include Stud::Buffer attr_accessor :buffer_state def initialize(options=nil) buffer_initialize(options) if options end def flush(items, group=nil); end def on_flush_error(exception); end end describe Stud::Buffer do it "should raise an error if included in a class which does not define flush" do class DoesntDefineFlush include Stud::Buffer end insist { DoesntDefineFlush.new.buffer_initialize }.raises ArgumentError end describe "buffer_full?" do it "should be false when when we have less than max_items" do subject = BufferSubject.new(:max_items => 2) subject.buffer_receive('one') insist { subject.buffer_full? } == false end it "should be true when we have more than max_items" do subject = BufferSubject.new(:max_items => 1) # take lock to prevent buffer_receive from flushing immediately subject.buffer_state[:flush_mutex].lock # so we'll accept this item, and not block, but won't flush it. subject.buffer_receive('one') insist { subject.buffer_full? } == true end end describe "buffer_receive" do it "should initialize buffer if necessary" do subject = BufferSubject.new insist { subject.buffer_state }.nil? subject.buffer_receive('item') insist { subject.buffer_state }.is_a?(Hash) end it "should block if max_items has been reached" do subject = BufferSubject.new(:max_interval => 2, :max_items => 5) # set up internal state so we're full. subject.buffer_state[:pending_count] = 5 subject.buffer_state[:pending_items][nil] = [1,2,3,4,5] subject.should_receive(:flush).with([1,2,3,4,5], nil) start = Time.now subject.buffer_receive(6) # we were hung for max_interval, when the timer kicked in and cleared out some events insist { Time.now-start } > 2 end it "should block while pending plus outgoing exceeds max_items" do subject = BufferSubject.new(:max_interval => 10, :max_items => 5) # flushes are slow this time. subject.stub(:flush) { sleep 4 } subject.buffer_receive(1) subject.buffer_receive(2) subject.buffer_receive(3) thread_started = false Thread.new do thread_started = true # this will take 4 seconds to complete subject.buffer_flush(:force => true) end # best effort at ensuring batch_flush is underway # we want the inital 3 items to move from pending to outgoing before # we proceed while (!thread_started) do end sleep 0.1 # now we accept 2 more events into pending subject.buffer_receive(4) subject.buffer_receive(5) # now we're full insist { subject.buffer_full? } == true # more buffer_receive calls should block until the slow # flush completes and decrements the number of outgoing items start = Time.now subject.buffer_receive(6) # if we waited this long, it had to be while we were blocked on the slow flush # this proves that outgoing items are counted when determining whether to block or not insist { Time.now - start } > 3.8 end it "should call on_full_buffer_receive if defined when buffer is full" do class DefinesOnFullBufferReceive < BufferSubject attr_reader :full_buffer_notices def initialize(options=nil) super @full_buffer_notices = [] end # we'll get a lot of calls to this method def on_full_buffer_receive(*args) @full_buffer_notices << args end end subject = DefinesOnFullBufferReceive.new(:max_items => 1) # start out with a full buffer subject.buffer_state[:pending_items][nil] = "waiting to flush" subject.buffer_state[:pending_count] = 1 Thread.new do sleep 0.5 subject.buffer_flush end # will block until the other thread calls buffer_flush subject.buffer_receive "will be blocked" insist { subject.full_buffer_notices.size } > 0 subject.full_buffer_notices.each do |notice| insist { notice } == [{:pending => 1, :outgoing => 0}] end end end # these test both buffer_recieve and buffer_flush describe "buffer and flush behaviors" do it "should accept new items to pending list" do subject = BufferSubject.new(:max_items => 2) subject.should_receive(:flush).with(['something', 'something else'], nil) subject.buffer_receive('something') subject.buffer_receive('something else') end it "should accept optional grouping key" do subject = BufferSubject.new(:max_items => 2) # we get 2 flush calls, one for each key subject.should_receive(:flush).with(['something'], 'key1', nil) subject.should_receive(:flush).with(['something else'], 'key2', nil) subject.buffer_receive('something', 'key1') subject.buffer_receive('something else', 'key2') end it "should accept non-string grouping keys" do subject = BufferSubject.new(:max_items => 2) subject.should_receive(:flush).with(['something'], {:key => 1, :foo => :yes}, nil) subject.should_receive(:flush).with(['something else'], {:key => 2, :foo => :no}, nil) subject.buffer_receive('something', :key => 1, :foo => :yes) subject.buffer_receive('something else', :key => 2, :foo => :no) end end describe "buffer_flush" do it "should call on_flush_error and retry if an exception occurs" do subject = BufferSubject.new(:max_items => 1) error = RuntimeError.new("blah!") # first flush will raise an exception subject.should_receive(:flush).and_raise(error) # which will be passed to on_flush_error subject.should_receive(:on_flush_error).with(error) # then we'll retry and succeed. (w/o this we retry forever) subject.should_receive(:flush) subject.buffer_receive('item') end it "should retry if no on_flush_error is defined" do class DoesntDefineOnFlushError include Stud::Buffer def flush; end; end subject = DoesntDefineOnFlushError.new subject.buffer_initialize(:max_items => 1) # first flush will raise an exception subject.should_receive(:flush).and_raise("boom!") # then we'll retry and succeed. (w/o this we retry forever) subject.should_receive(:flush) subject.buffer_receive('item') end it "should return if it cannot get a lock" do subject = BufferSubject.new(:max_items => 1, :max_interval => 100) subject.buffer_state[:pending_items][nil] << 'message' subject.buffer_state[:pending_count] = 1 subject.buffer_state[:flush_mutex].lock # we should have flushed a message (since :max_items has been reached), # but can't due to the lock. insist { subject.buffer_flush } == 0 subject.buffer_state[:flush_mutex].unlock # and now we flush successfully insist { subject.buffer_flush } == 1 end it "flushes when pending count is less than max items if forced" do subject = BufferSubject.new(:max_items => 5) subject.buffer_receive('one') subject.buffer_receive('two') subject.buffer_receive('three') insist { subject.buffer_flush(:force => true) } == 3 end it "should block until lock can be acquired if final option is used" do subject = BufferSubject.new(:max_items => 2, :max_interval => 100) subject.buffer_receive 'message' lock_acquired = false Thread.new do subject.buffer_state[:flush_mutex].lock lock_acquired = true sleep 0.5 subject.buffer_state[:flush_mutex].unlock end while (!lock_acquired) do end # we'll block for 0.5 seconds and then succeed in flushing our message insist { subject.buffer_flush(:final => true) } == 1 end it "does not flush if no items are pending" do subject = BufferSubject.new(:max_items => 5) insist { subject.buffer_flush } == 0 end it "does not flush if pending count is less than max items" do subject = BufferSubject.new(:max_items => 5) subject.buffer_receive('hi!') insist { subject.buffer_flush } == 0 end it "flushes when time since last flush exceeds max_interval" do class AccumulatingBufferSubject < BufferSubject attr_reader :flushed def flush(items, final=false) @flushed = items end end subject = AccumulatingBufferSubject.new(:max_items => 5, :max_interval => 1) subject.buffer_receive('item!') Stud.try(10.times) do insist { subject.flushed } == ['item!'] end end end end ruby-stud-0.0.20/spec/stud/interval_spec.rb000644 000765 000024 00000001614 12543132341 020754 0ustar00tpotstaff000000 000000 require "stud/interval" require "spec_env" # from the top level spec/ directory describe Stud do describe "#interval" do let(:interval) { 1 } it "allows the interval to sleep before running" do start_time = Time.now Stud.interval(interval, :sleep_then_run => true) do end_time = Time.now expect(end_time - start_time).to be >= interval break end end it "defaults to run than sleep" do start_time = Time.now Stud.interval(interval) do end_time = Time.now expect(end_time - start_time).to be < interval break end end it 'should be able to interrupt an interval defined as a task' do counter = 0 task = Stud::Task.new do Stud.interval(0.5) do counter += 1 task.stop! end end sleep(1) expect(counter).to eq(1) end end end ruby-stud-0.0.20/spec/stud/secret_spec.rb000644 000765 000024 00000000642 12543132341 020415 0ustar00tpotstaff000000 000000 require "stud/secret" require "spec_env" # from the top level spec/ directory describe Stud::Secret do subject { Stud::Secret.new("hello") } it "should hide the secret value from inspection" do insist { subject.inspect } == "" insist { subject.to_s } == "" end context "#value" do it "should expose the secret value" do insist { subject.value } == "hello" end end end ruby-stud-0.0.20/spec/stud/task_spec.rb000644 000765 000024 00000000667 12543132341 020101 0ustar00tpotstaff000000 000000 require "stud/task" require "spec_env" # from the top level spec/ directory describe Stud::Task do context "#wait" do it "should return the return value of the Task block" do task = Stud::Task.new { "Hello" } insist { task.wait } == "Hello" end it "should raise exception if the Task block raises such" do task = Stud::Task.new { raise Insist::Failure } insist { task.wait }.fails end end end ruby-stud-0.0.20/spec/stud/temporary_spec.rb000644 000765 000024 00000002216 12543132341 021151 0ustar00tpotstaff000000 000000 require "stud/temporary" require "spec_env" # from the top level spec/ directory describe Stud::Temporary do include Stud::Temporary # make these methods available in scope describe "#pathname" do it "should return a string" do insist { pathname }.is_a?(String) end it "should respect TMP" do old = ENV["TMP"] ENV["TMP"] = "/pants" # Make sure the leading part of the pathname is /pants/ insist { pathname } =~ Regexp.new("^#{Regexp.quote(ENV["TMP"])}/") ENV["TMP"] = old end end describe "#file" do context "without a block" do subject { file } after(:each) do subject.close File.delete(subject) end it "should return a File" do insist { subject }.is_a?(File) end end # without a block context "with a block" do it "should pass a File to the block" do path = "" file { |fd| insist { fd }.is_a?(File) } end it "should clean up after the block closes" do path = "" file { |fd| path = fd.path } reject { File }.exists?(path) end end # with a block end # #file end ruby-stud-0.0.20/spec/stud/trap_spec.rb000644 000765 000024 00000002455 12543132341 020102 0ustar00tpotstaff000000 000000 require "stud/trap" require "stud/try" require "spec_env" # from the top level spec/ directory require "timeout" def windows? ["mingw32", "mswin32"].include?(RbConfig::CONFIG["host_os"]) end describe "Stud#trap", :if => !windows? do it "should call multiple traps for a single signal" do queue = Queue.new Stud.trap("USR2") { queue << 1 } Stud.trap("USR2") { queue << 2 } Stud.trap("USR2") { queue << 3 } Process.kill("USR2", Process.pid) Stud.try(10.times) do insist { queue.size } == 3 end insist { queue.pop } == 1 insist { queue.pop } == 2 insist { queue.pop } == 3 end it "should keep any original traps set with Kernel#trap" do hupped = false studded = false queue = Queue.new # Set a standard signal using the ruby stdlib method Kernel.trap("HUP") { queue << :kernel } # This should still keep the previous trap. Stud.trap("HUP") { queue << :stud } # Send SIGHUP Process.kill("HUP", Process.pid) # Wait for both signal handlers to get called. Stud.try(10.times) do insist { queue.size } == 2 end # Kernel handler should get called first since it was # there first. insist { queue.pop } == :kernel # Our stud.trap should get called second insist { queue.pop } == :stud end end ruby-stud-0.0.20/spec/stud/try_spec.rb000644 000765 000024 00000005664 12543132341 017757 0ustar00tpotstaff000000 000000 require "stud/try" require "spec_env" # from the top level spec/ directory class FakeFailure < StandardError; end class DummyException < Exception; end class OtherException < Exception; end class RetryableException < Exception; end describe Stud::Try do class FastTry < Stud::Try def failure(*args) # do nothing end end # class FastTry subject { FastTry.new } it "should give up after N tries when given 'N.times'" do count = 0 total = 5 # This 'try' should always fail. insist do subject.try(total.times) do count += 1 raise Insist::Failure, "intentional" end end.fails # But it should try 'total' times. insist { count } == total end it "should pass the current iteration value to the block" do count = 0 total = 5 # This 'try' should always fail. values = total.times.to_a insist do subject.try(total.times) do |value| count += 1 insist { value } == values.shift raise FakeFailure, "intentional" end end.raises(FakeFailure) # But it should try 'total' times. insist { count } == total end it "should appear to try forever by default" do # This is really the 'halting problem' but if we # try enough times, consider that it is likely to continue forever. count = 0 value = subject.try do count += 1 raise FakeFailure if count < 1000 end insist { count } == 1000 end it "should return the block return value on success" do insist { subject.try(1.times) { 42 } } == 42 end it "should raise the last exception on final failure" do insist do subject.try(1.times) { raise FakeFailure } end.raises(FakeFailure) end it "should expose a default try as Stud.try" do # Replace the log method with a noop. class << Stud::TRY ; def log_failure(*args) ; end ; end insist do Stud.try(3.times) { raise FakeFailure } end.raises(FakeFailure) end context "when specifying exceptions" do let(:total) { 5 } it "allows to specify retryable exceptions" do count = 0 insist do Stud.try(total.times, DummyException) do count += 1 raise DummyException end end.raises(DummyException) insist { count } == total end it "You can specify a list" do count = 0 insist do Stud.try(total.times, [DummyException, RetryableException]) do count += 1 if count < 2 raise DummyException else raise RetryableException end end end.raises(RetryableException) insist { count } == total end it "doesnt retry if the exception is not in the list" do count = 0 insist do Stud.try(total.times, DummyException) do count += 1 raise RetryableException end end.raises(RetryableException) insist { count } == 1 end end end ruby-stud-0.0.20/spec/stud/with_spec.rb000644 000765 000024 00000000562 12543132341 020104 0ustar00tpotstaff000000 000000 require "stud/with" require "spec_env" # from the top level spec/ directory describe Stud::With do include Stud::With # make these methods available in scope it "should work" do # ☺ count = 0 with("hello world") do |v| count += 1 insist { v } == "hello world" end # Make sure the block is called. insist { count } == 1 end end ruby-stud-0.0.20/lib/stud/000755 000765 000024 00000000000 12543132341 015403 5ustar00tpotstaff000000 000000 ruby-stud-0.0.20/lib/stud/buffer.rb000644 000765 000024 00000022255 12543132341 017207 0ustar00tpotstaff000000 000000 module Stud # @author {Alex Dean}[http://github.com/alexdean] # # Implements a generic framework for accepting events which are later flushed # in batches. Flushing occurs whenever +:max_items+ or +:max_interval+ (seconds) # has been reached. # # Including class must implement +flush+, which will be called with all # accumulated items either when the output buffer fills (+:max_items+) or # when a fixed amount of time (+:max_interval+) passes. # # == batch_receive and flush # General receive/flush can be implemented in one of two ways. # # === batch_receive(event) / flush(events) # +flush+ will receive an array of events which were passed to +buffer_receive+. # # batch_receive('one') # batch_receive('two') # # will cause a flush invocation like # # flush(['one', 'two']) # # === batch_receive(event, group) / flush(events, group) # flush() will receive an array of events, plus a grouping key. # # batch_receive('one', :server => 'a') # batch_receive('two', :server => 'b') # batch_receive('three', :server => 'a') # batch_receive('four', :server => 'b') # # will result in the following flush calls # # flush(['one', 'three'], {:server => 'a'}) # flush(['two', 'four'], {:server => 'b'}) # # Grouping keys can be anything which are valid Hash keys. (They don't have to # be hashes themselves.) Strings or Fixnums work fine. Use anything which you'd # like to receive in your +flush+ method to help enable different handling for # various groups of events. # # == on_flush_error # Including class may implement +on_flush_error+, which will be called with an # Exception instance whenever buffer_flush encounters an error. # # * +buffer_flush+ will automatically re-try failed flushes, so +on_flush_error+ # should not try to implement retry behavior. # * Exceptions occurring within +on_flush_error+ are not handled by # +buffer_flush+. # # == on_full_buffer_receive # Including class may implement +on_full_buffer_receive+, which will be called # whenever +buffer_receive+ is called while the buffer is full. # # +on_full_buffer_receive+ will receive a Hash like {:pending => 30, # :outgoing => 20} which describes the internal state of the module at # the moment. # # == final flush # Including class should call buffer_flush(:final => true) # during a teardown/shutdown routine (after the last call to buffer_receive) # to ensure that all accumulated messages are flushed. module Buffer public # Initialize the buffer. # # Call directly from your constructor if you wish to set some non-default # options. Otherwise buffer_initialize will be called automatically during the # first buffer_receive call. # # Options: # * :max_items, Max number of items to buffer before flushing. Default 50. # * :max_interval, Max number of seconds to wait between flushes. Default 5. # * :logger, A logger to write log messages to. No default. Optional. # # @param [Hash] options def buffer_initialize(options={}) if ! self.class.method_defined?(:flush) raise ArgumentError, "Any class including Stud::Buffer must define a flush() method." end @buffer_config = { :max_items => options[:max_items] || 50, :max_interval => options[:max_interval] || 5, :logger => options[:logger] || nil, :has_on_flush_error => self.class.method_defined?(:on_flush_error), :has_on_full_buffer_receive => self.class.method_defined?(:on_full_buffer_receive) } @buffer_state = { # items accepted from including class :pending_items => {}, :pending_count => 0, # guard access to pending_items & pending_count :pending_mutex => Mutex.new, # items which are currently being flushed :outgoing_items => {}, :outgoing_count => 0, # ensure only 1 flush is operating at once :flush_mutex => Mutex.new, # data for timed flushes :last_flush => Time.now, :timer => Thread.new do loop do sleep(@buffer_config[:max_interval]) buffer_flush(:force => true) end end } # events we've accumulated buffer_clear_pending end # Determine if +:max_items+ has been reached. # # buffer_receive calls will block while buffer_full? == true. # # @return [bool] Is the buffer full? def buffer_full? @buffer_state[:pending_count] + @buffer_state[:outgoing_count] >= @buffer_config[:max_items] end # Save an event for later delivery # # Events are grouped by the (optional) group parameter you provide. # Groups of events, plus the group name, are later passed to +flush+. # # This call will block if +:max_items+ has been reached. # # @see Stud::Buffer The overview has more information on grouping and flushing. # # @param event An item to buffer for flushing later. # @param group Optional grouping key. All events with the same key will be # passed to +flush+ together, along with the grouping key itself. def buffer_receive(event, group=nil) buffer_initialize if ! @buffer_state # block if we've accumulated too many events while buffer_full? do on_full_buffer_receive( :pending => @buffer_state[:pending_count], :outgoing => @buffer_state[:outgoing_count] ) if @buffer_config[:has_on_full_buffer_receive] sleep 0.1 end @buffer_state[:pending_mutex].synchronize do @buffer_state[:pending_items][group] << event @buffer_state[:pending_count] += 1 end buffer_flush end # Try to flush events. # # Returns immediately if flushing is not necessary/possible at the moment: # * :max_items have not been accumulated # * :max_interval seconds have not elapased since the last flush # * another flush is in progress # # buffer_flush(:force => true) will cause a flush to occur even # if +:max_items+ or +:max_interval+ have not been reached. A forced flush # will still return immediately (without flushing) if another flush is # currently in progress. # # buffer_flush(:final => true) is identical to buffer_flush(:force => true), # except that if another flush is already in progress, buffer_flush(:final => true) # will block/wait for the other flush to finish before proceeding. # # @param [Hash] options Optional. May be {:force => true} or {:final => true}. # @return [Fixnum] The number of items successfully passed to +flush+. def buffer_flush(options={}) force = options[:force] || options[:final] final = options[:final] # final flush will wait for lock, so we are sure to flush out all buffered events if options[:final] @buffer_state[:flush_mutex].lock elsif ! @buffer_state[:flush_mutex].try_lock # failed to get lock, another flush already in progress return 0 end items_flushed = 0 begin time_since_last_flush = (Time.now - @buffer_state[:last_flush]) return 0 if @buffer_state[:pending_count] == 0 return 0 if (!force) && (@buffer_state[:pending_count] < @buffer_config[:max_items]) && (time_since_last_flush < @buffer_config[:max_interval]) @buffer_state[:pending_mutex].synchronize do @buffer_state[:outgoing_items] = @buffer_state[:pending_items] @buffer_state[:outgoing_count] = @buffer_state[:pending_count] buffer_clear_pending end @buffer_config[:logger].debug("Flushing output", :outgoing_count => @buffer_state[:outgoing_count], :time_since_last_flush => time_since_last_flush, :outgoing_events => @buffer_state[:outgoing_items], :batch_timeout => @buffer_config[:max_interval], :force => force, :final => final ) if @buffer_config[:logger] @buffer_state[:outgoing_items].each do |group, events| begin if group.nil? flush(events,final) else flush(events, group, final) end @buffer_state[:outgoing_items].delete(group) events_size = events.size @buffer_state[:outgoing_count] -= events_size items_flushed += events_size rescue => e @buffer_config[:logger].warn("Failed to flush outgoing items", :outgoing_count => @buffer_state[:outgoing_count], :exception => e, :backtrace => e.backtrace ) if @buffer_config[:logger] if @buffer_config[:has_on_flush_error] on_flush_error e end sleep 1 retry end @buffer_state[:last_flush] = Time.now end ensure @buffer_state[:flush_mutex].unlock end return items_flushed end private def buffer_clear_pending @buffer_state[:pending_items] = Hash.new { |h, k| h[k] = [] } @buffer_state[:pending_count] = 0 end end end ruby-stud-0.0.20/lib/stud/interval.rb000644 000765 000024 00000002154 12543132341 017556 0ustar00tpotstaff000000 000000 require "stud/task" module Stud # This implementation tries to keep clock more accurately. # Prior implementations still permitted skew, where as this one # will attempt to correct for skew. # # The execution patterns of this method should be that # the start time of 'block.call' should always be at time T*interval def self.interval(time, opts = {}, &block) start = Time.now while true break if Task.interrupted? if opts[:sleep_then_run] start = sleep_for_interval(time, start) block.call else block.call start = sleep_for_interval(time, start) end end # loop forever end # def interval def self.sleep_for_interval(time, start) duration = Time.now - start # Sleep only if the duration was less than the time interval if duration < time sleep(time - duration) start += time else # Duration exceeded interval time, reset the clock and do not sleep. start = Time.now end end def interval(time, opts = {}, &block) return Stud.interval(time, opts, &block) end # def interval end # module Stud ruby-stud-0.0.20/lib/stud/pool.rb000644 000765 000024 00000015242 12543132341 016705 0ustar00tpotstaff000000 000000 require "thread" module Stud # Public: A thread-safe, generic resource pool. # # This class is agnostic as to the resources in the pool. You can put # database connections, sockets, threads, etc. It's up to you! # # Examples: # # pool = Pool.new # pool.add(Sequel.connect("postgres://pg-readonly-1/prod")) # pool.add(Sequel.connect("postgres://pg-readonly-2/prod")) # pool.add(Sequel.connect("postgres://pg-readonly-3/prod")) # # pool.fetch # => Returns one of the Sequel::Database values from the pool class Pool class Error < StandardError; end # An error indicating a given resource is busy. class ResourceBusy < Error; end # An error indicating a given resource is not found. class NotFound < Error; end # You performed an invalid action. class InvalidAction < Error; end # Default all methods to private. See the bottom of the class definition # for public method declarations. private # Public: initialize a new pool. # # max_size - if specified, limits the number of resources allowed in the pool. def initialize(max_size=nil) # Available resources @available = Hash.new # Busy resources @busy = Hash.new # The pool lock @lock = Mutex.new # Locks for blocking {#fetch} calls if the pool is full. @full_lock = Mutex.new @full_cv = ConditionVariable.new # Maximum size of this pool. @max_size = max_size end # def initialize # Private: Is this pool size-limited? # # Returns true if this pool was created with a max_size. False, otherwise. def sized? return !@max_size.nil? end # def sized? # Private: Is this pool full? # # Returns true if the pool is sized and the count of resources is at maximum. def full? return sized? && (count == @max_size) end # def full? # Public: the count of resources in the pool # # Returns the count of resources in the pool. def count return (@busy.size + @available.size) end # def count # Public: Add a new resource to this pool. # # The resource, once added, is assumed to be available for use. # That means once you add it, you must not use it unless you receive it from # {Pool#fetch} # # resource - the object resource to add to the pool. # # Returns nothing def add(resource) @lock.synchronize do @available[resource.object_id] = resource end return nil end # def add # Public: Fetch an available resource. # # If no resource is available, and the pool is not full, the # default_value_block will be called and the return value of it used as the # resource. # # If no resource is availabe, and the pool is full, this call will block # until a resource is available. # # Returns a resource ready to be used. def fetch(&default_value_block) @lock.synchronize do object_id, resource = @available.shift if !resource.nil? @busy[resource.object_id] = resource return resource end end @full_lock.synchronize do if full? # This should really use a logger. puts "=> Pool is full and nothing available. Waiting for a release..." @full_cv.wait(@full_lock) return fetch(&default_value_block) end end # TODO(sissel): If no block is given, we should block until a resource is # available. # If we get here, no resource is available and the pool is not full. resource = default_value_block.call # Only add the resource if the default_value_block returned one. if !resource.nil? add(resource) return fetch end end # def fetch # Public: Remove a resource from the pool. # # This is useful if the resource is no longer useful. For example, if it is # a database connection and that connection has failed. # # This resource *MUST* be available and not busy. # # Raises Pool::NotFound if no such resource is found. # Raises Pool::ResourceBusy if the resource is found but in use. def remove(resource) # Find the object by object_id #p [:internal, :busy => @busy, :available => @available] @lock.synchronize do if available?(resource) raise InvalidAction, "This resource must be busy for you to remove " \ "it (ie; it must be fetched from the pool)" end @busy.delete(resource.object_id) end end # def remove # Private: Verify this resource is in the pool. # # You *MUST* call this method only when you are holding @lock. # # Returns :available if it is available, :busy if busy, false if not in the pool. def include?(resource) if @available.include?(resource.object_id) return :available elsif @busy.include?(resource.object_id) return :busy else return false end end # def include? # Private: Is this resource available? # You *MUST* call this method only when you are holding @lock. # # Returns true if this resource is available in the pool. # Raises NotFound if the resource given is not in the pool at all. def available?(resource) case include?(resource) when :available; return true when :busy; return false else; raise NotFound, "No resource, #{resource.inspect}, found in pool" end end # def avilable? # Private: Is this resource busy? # # You *MUST* call this method only when you are holding @lock. # # Returns true if this resource is busy. # Raises NotFound if the resource given is not in the pool at all. def busy?(resource) return !available?(resource) end # def busy? # Public: Release this resource back to the pool. # # After you finish using a resource you received with {#fetch}, you must # release it back to the pool using this method. # # Alternately, you can {#remove} it if you want to remove it from the pool # instead of releasing it. def release(resource) @lock.synchronize do if !include?(resource) raise NotFound, "No resource, #{resource.inspect}, found in pool" end # Release is a no-op if this resource is already available. #return if available?(resource) @busy.delete(resource.object_id) @available[resource.object_id] = resource # Notify any threads waiting on a resource from the pool. @full_lock.synchronize { @full_cv.signal } end end # def release public(:add, :remove, :fetch, :release, :sized?, :count, :initialize) end # class Pool end # module Stud ruby-stud-0.0.20/lib/stud/secret.rb000644 000765 000024 00000002263 12543132341 017220 0ustar00tpotstaff000000 000000 # A class for holding a secret. The main goal is to prevent the common mistake # of accidentally logging or printing passwords or other secrets. # # See # # for a discussion of why this implementation is useful. module Stud class Secret # Initialize a new secret with a given value. # # value - anything you want to keep secret from loggers, etc. def initialize(secret_value) # Redefine the 'value' method on this instance. This exposes no instance # variables to be accidentally leaked by things like awesome_print, etc. # This makes any #value call return the secret value. (class << self; self; end).class_eval do define_method(:value) { secret_value } end end # def initialize # Emit simply "" when printed or logged. def to_s return "" end # def to_s alias_method :inspect, :to_s # Get the secret value. def value # Nothing, this will be filled in by Secret.new # But we'll still document this so rdoc/yard know the method exists. end # def value end # class Secret end # class Stud ruby-stud-0.0.20/lib/stud/task.rb000644 000765 000024 00000001613 12543132341 016673 0ustar00tpotstaff000000 000000 require "thread" module Stud class Task def initialize(*args, &block) # A queue to receive the result of the block # TODO(sissel): Don't use a queue, just store it in an instance variable. @queue = Queue.new @thread = Thread.new(@queue, *args) do |queue, *args| begin result = block.call(*args) queue << [:return, result] rescue => e queue << [:exception, e] end end # thread end # def initialize def wait @thread.join reason, result = @queue.pop if reason == :exception #raise StandardError.new(result) raise result else return result end end # def wait def stop! Thread.current[:stud_task_interrupted] = true end def self.interrupted? Thread.current[:stud_task_interrupted] end end # class Task end # module Stud ruby-stud-0.0.20/lib/stud/temporary.rb000644 000765 000024 00000003151 12543132341 017752 0ustar00tpotstaff000000 000000 require "securerandom" # for uuid generation require "fileutils" module Stud module Temporary DEFAULT_PREFIX = "studtmp" # Returns a string for a randomly-generated temporary path. # # This does not create any files. def pathname(prefix=DEFAULT_PREFIX) root = ENV["TMP"] || ENV["TMPDIR"] || ENV["TEMP"] || "/tmp" return File.join(root, "#{prefix}-#{SecureRandom.hex(30)}") end # Return a File handle to a randomly-generated path. # # Any arguments beyond the first (prefix) argument will be # given to File.new. # # If no file args are given, the default file mode is "w+" def file(prefix=DEFAULT_PREFIX, *args, &block) args << "w+" if args.empty? file = File.new(pathname(prefix), *args) if block_given? begin block.call(file) ensure file.close unless file.closed? File.unlink(file.path) end else return file end end # Make a temporary directory. # # If given a block, the directory path is given to the block. WHen the # block finishes, the directory and all its contents will be deleted. # # If no block given, it will return the path to a newly created directory. # You are responsible for then cleaning up. def directory(prefix=DEFAULT_PREFIX, &block) path = pathname(prefix) Dir.mkdir(path) if block_given? begin block.call(path) ensure FileUtils.rm_r(path) end else return path end end extend self end # module Temporary end # module Stud ruby-stud-0.0.20/lib/stud/trap.rb000644 000765 000024 00000004716 12543132341 016706 0ustar00tpotstaff000000 000000 module Stud # Bind a block to be called when a certain signal is received. # # Same arguments to Signal::trap. # # The behavior of this method is different than Signal::trap because # multiple handlers can request notification for the same signal. # # For example, this is valid: # # Stud.trap("INT") { puts "Hello" } # Stud.trap("INT") { puts "World" } # # When SIGINT is received, both callbacks will be invoked, in order. # # This helps avoid the situation where a library traps a signal outside of # your control. # # If something has already used Signal::trap, that callback will be saved # and scheduled the same way as any other Stud::trap. def self.trap(signal, &block) @traps ||= Hash.new { |h,k| h[k] = [] } if !@traps.include?(signal) # First trap call for this signal, tell ruby to invoke us. previous_trap = Signal::trap(signal) { simulate_signal(signal) } # If there was a previous trap (via Kernel#trap) set, make sure we remember it. if previous_trap.is_a?(Proc) # MRI's default traps are "DEFAULT" string # JRuby's default traps are Procs with a source_location of "(internal") if RUBY_ENGINE != "jruby" || previous_trap.source_location.first != "(internal)" @traps[signal] << previous_trap end end end @traps[signal] << block return block.object_id end # def self.trap # Simulate a signal. This lets you force an interrupt without # sending a signal to yourself. def self.simulate_signal(signal) #puts "Simulate: #{signal} w/ #{@traps[signal].count} callbacks" @traps[signal].each(&:call) end # def self.simulate_signal # Remove a previously set signal trap. # # 'signal' is the name of the signal ("INT", etc) # 'id' is the value returned by a previous Stud.trap() call def self.untrap(signal, id) @traps[signal].delete_if { |block| block.object_id == id } # Restore the default handler if there are no custom traps anymore. if @traps[signal].empty? @traps.delete(signal) Signal::trap(signal, "DEFAULT") end end # def self.untrap end # module Stud # Monkey-patch the main 'trap' stuff? This could be useful. #module Signal #def trap(signal, value=nil, &block) #if value.nil? #Stud.trap(signal, &block) #else ## do nothing? #end #end # def trap #end # #module Kernel #def trap(signal, value=nil, &block) #Signal.trap(signal, value, &block) #end #end ruby-stud-0.0.20/lib/stud/try.rb000644 000765 000024 00000010643 12543132341 016552 0ustar00tpotstaff000000 000000 module Stud # A class implementing 'retry-on-failure' # # Example: # # Try.new.try(5.times) { your_code } # # A failure is indicated by any exception being raised. # On success, the return value of the block is the return value of the try # call. # # On final failure (ran out of things to try), the last exception is raised. class Try # An infinite enumerator class Forever include Enumerable def each(&block) a = 0 yield a += 1 while true end end # class Forever FOREVER = Forever.new BACKOFF_SCHEDULE = [0.01, 0.02, 0.04, 0.08, 0.16, 0.32, 0.64, 1.28, 2.0] DEFAULT_CATCHABLE_EXCEPTIONS = [StandardError] # Log a failure. # # You should override this method if you want a better logger. def log_failure(exception, fail_count, message) puts "Failed (#{exception}). #{message}" end # def log_failure # This method is called when a try attempt fails. # # The default implementation will sleep with exponential backoff up to a # maximum of 2 seconds (see BACKOFF_SCHEDULE) # # exception - the exception causing the failure # fail_count - how many times we have failed. def failure(exception, fail_count) backoff = BACKOFF_SCHEDULE[fail_count] || BACKOFF_SCHEDULE.last log_failure(exception, fail_count, "Sleeping for #{backoff}") sleep(backoff) end # def failure # Public: try a block of code until either it succeeds or we give up. # # enumerable - an Enumerable or omitted/nil, #each is invoked and is tried # that number of times. If this value is omitted or nil, we will try until # success with no limit on the number of tries. # # exceptions - the type of exceptions to retry, we use `StandardError` by default. # # Returns the return value of the block once the block succeeds. # Raises the last seen exception if we run out of tries. # # Examples # # # Try 10 times to fetch http://google.com/ # response = try(10.times) { Net::HTTP.get_response("google.com", "/") } # # # Try many times, yielding the value of the enumeration to the block. # # This allows you to try different inputs. # response = try([0, 2, 4, 6]) { |val| 50 / val } # # Output: # Failed (divided by 0). Retrying in 0.01 seconds... # => 25 # # # Try forever # return_value = try { ... } def try(enumerable=FOREVER, exceptions=DEFAULT_CATCHABLE_EXCEPTIONS, &block) if block.arity == 0 # If the block takes no arguments, give none procedure = lambda { |val| return block.call } else # Otherwise, pass the current 'enumerable' value to the block. procedure = lambda { |val| return block.call(val) } end # Track the last exception so we can reraise it on failure. last_exception = nil # When 'enumerable' runs out of things, if we still haven't succeeded, # we'll reraise fail_count = 0 enumerable.each do |val| begin # If the 'procedure' (the block, really) succeeds, we'll break # and return the return value of the block. Win! return procedure.call(val) rescue NoMethodError, NameError # Abort immediately on exceptions that are unlikely to recover. raise rescue *exceptions => exception last_exception = exception fail_count += 1 # Note: Since we can't reliably detect the 'end' of an enumeration, we # will call 'failure' for the final iteration (if it failed) and sleep # even though there's no strong reason to backoff on the last error. failure(exception, fail_count) end end # enumerable.each # generally make the exception appear from the 'try' method itself, not from # any deeply nested enumeration/begin/etc # It is my hope that this makes the backtraces easier to read, not more # difficult. If you find this is not the case, please please please let me # know. last_exception.set_backtrace(StandardError.new.backtrace) raise last_exception end # def try end # class Stud::Try TRY = Try.new # A simple try method for the common case. def try(enumerable=Stud::Try::FOREVER, exceptions=Try::DEFAULT_CATCHABLE_EXCEPTIONS, &block) return TRY.try(enumerable, exceptions, &block) end # def try extend self end # module Stud ruby-stud-0.0.20/lib/stud/with.rb000644 000765 000024 00000001055 12543132341 016704 0ustar00tpotstaff000000 000000 module Stud module With # Run a block with arguments. This is sometimes useful in lieu of # explicitly assigning variables. # # I find mainly that using 'with' is a clue that I can factor out # a given segment of code into a method or function. # # Example usage: # # with(TCPSocket.new("google.com", 80)) do |s| # s.write("GET / HTTP/1.0\r\nHost: google.com\r\n\r\n") # puts s.read # s.close # end def with(*args, &block) block.call(*args) end extend self end end