pax_global_header00006660000000000000000000000064131355414320014513gustar00rootroot0000000000000052 comment=4fe838144d1fbf65bb3aa85c3474b538255817a9 parallel-1.12.0/000077500000000000000000000000001313554143200133705ustar00rootroot00000000000000parallel-1.12.0/.gitignore000066400000000000000000000000201313554143200153500ustar00rootroot00000000000000/rspec.failures parallel-1.12.0/.travis.yml000066400000000000000000000002051313554143200154760ustar00rootroot00000000000000language: ruby cache: bundler sudo: false branches: only: master rvm: - 1.9.3 - 2.0.0 - 2.1.6 - 2.2.4 - 2.3.1 - 2.4.1 parallel-1.12.0/Gemfile000066400000000000000000000003341313554143200146630ustar00rootroot00000000000000source "https://rubygems.org" gemspec gem 'bump' gem 'rake' gem 'rspec' gem 'activerecord', "~>4.2.8" gem 'ruby-progressbar' gem 'rspec-rerun' gem 'rspec-legacy_formatters' gem 'mysql2', :group => :mysql gem 'sqlite3' parallel-1.12.0/Gemfile.lock000066400000000000000000000025421313554143200156150ustar00rootroot00000000000000PATH remote: . specs: parallel (1.12.0) GEM remote: https://rubygems.org/ specs: activemodel (4.2.8) activesupport (= 4.2.8) builder (~> 3.1) activerecord (4.2.8) activemodel (= 4.2.8) activesupport (= 4.2.8) arel (~> 6.0) activesupport (4.2.8) i18n (~> 0.7) minitest (~> 5.1) thread_safe (~> 0.3, >= 0.3.4) tzinfo (~> 1.1) arel (6.0.4) builder (3.2.3) bump (0.5.3) diff-lcs (1.3) i18n (0.8.1) minitest (5.10.1) mysql2 (0.4.6) rake (12.0.0) rspec (3.6.0) rspec-core (~> 3.6.0) rspec-expectations (~> 3.6.0) rspec-mocks (~> 3.6.0) rspec-core (3.6.0) rspec-support (~> 3.6.0) rspec-expectations (3.6.0) diff-lcs (>= 1.2.0, < 2.0) rspec-support (~> 3.6.0) rspec-legacy_formatters (1.0.1) rspec (~> 3.0) rspec-mocks (3.6.0) diff-lcs (>= 1.2.0, < 2.0) rspec-support (~> 3.6.0) rspec-rerun (1.1.0) rspec (~> 3.0) rspec-support (3.6.0) ruby-progressbar (1.8.1) sqlite3 (1.3.13) thread_safe (0.3.6) thread_safe (0.3.6-java) tzinfo (1.2.3) thread_safe (~> 0.1) PLATFORMS java ruby DEPENDENCIES activerecord (~> 4.2.8) bump mysql2 parallel! rake rspec rspec-legacy_formatters rspec-rerun ruby-progressbar sqlite3 BUNDLED WITH 1.15.0 parallel-1.12.0/MIT-LICENSE.txt000066400000000000000000000020701313554143200156410ustar00rootroot00000000000000Copyright (C) 2013 Michael Grosser Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. parallel-1.12.0/Rakefile000066400000000000000000000002511313554143200150330ustar00rootroot00000000000000require 'bundler/setup' require 'bundler/gem_tasks' require 'bump/tasks' require "rspec/core/rake_task" require 'rspec-rerun/tasks' task :default => "rspec-rerun:spec" parallel-1.12.0/Readme.md000066400000000000000000000127211313554143200151120ustar00rootroot00000000000000Run any code in parallel Processes(> use all CPUs) or Threads(> speedup blocking operations).
Best suited for map-reduce or e.g. parallel downloads/uploads. Install ======= ```Bash gem install parallel ``` Usage ===== ```Ruby # 2 CPUs -> work in 2 processes (a,b + c) results = Parallel.map(['a','b','c']) do |one_letter| expensive_calculation(one_letter) end # 3 Processes -> finished after 1 run results = Parallel.map(['a','b','c'], in_processes: 3) { |one_letter| ... } # 3 Threads -> finished after 1 run results = Parallel.map(['a','b','c'], in_threads: 3) { |one_letter| ... } ``` Same can be done with `each` ```Ruby Parallel.each(['a','b','c']) { |one_letter| ... } ``` or `each_with_index` or `map_with_index` Produce one item at a time with `lambda` (anything that responds to `.call`) or `Queue`. ```Ruby items = [1,2,3] Parallel.each( -> { items.pop || Parallel::Stop }) { |number| ... } ``` You can also call `any?` or `all?`, which work the same way as `Array#any?` and `Array#all?`. ```Ruby Parallel.any?([1,2,3,4,5,6,7]) { |number| number == 4 } # => true Parallel.all?([1,2,nil,4,5]) { |number| number != nil } # => false ``` Processes/Threads are workers, they grab the next piece of work when they finish. ### Processes - Speedup through multiple CPUs - Speedup for blocking operations - Variables are protected from change - Extra memory used - Child processes are killed when your main process is killed through Ctrl+c or kill -2 ### Threads - Speedup for blocking operations - Variables can be shared/modified - No extra memory used ### ActiveRecord Try any of those to get working parallel AR ```Ruby # reproducibly fixes things (spec/cases/map_with_ar.rb) Parallel.each(User.all, in_processes: 8) do |user| user.update_attribute(:some_attribute, some_value) end User.connection.reconnect! # maybe helps: explicitly use connection pool Parallel.each(User.all, in_threads: 8) do |user| ActiveRecord::Base.connection_pool.with_connection do user.update_attribute(:some_attribute, some_value) end end # maybe helps: reconnect once inside every fork Parallel.each(User.all, in_processes: 8) do |user| @reconnected ||= User.connection.reconnect! || true user.update_attribute(:some_attribute, some_value) end ``` ### Break ```Ruby Parallel.map(User.all) do |user| raise Parallel::Break # -> stops after all current items are finished end ``` ### Kill Only use if whatever is executing in the sub-command is safe to kill at any point ```Ruby Parallel.map([1,2,3]) do |x| raise Parallel::Kill if x == 1# -> stop all sub-processes, killing them instantly sleep 100 # Do stuff end ``` ### Progress / ETA ```Ruby # gem install ruby-progressbar Parallel.map(1..50, progress: "Doing stuff") { sleep 1 } # Doing stuff | ETA: 00:00:02 | ==================== | Time: 00:00:10 ``` Use `:finish` or `:start` hook to get progress information. - `:start` has item and index - `:finish` has item, index, result They are called on the main process and protected with a mutex. ```Ruby Parallel.map(1..100, finish: -> (item, i, result) { ... do something ... }) { sleep 1 } ``` ### Worker number Use `Parallel.worker_number` to determine the worker slot in which your task is running. ```Ruby Parallel.each(1..5, :in_processes => 2) { |i| puts "Item: #{i}, Worker: #{Parallel.worker_number}" } Item: 1, Worker: 1 Item: 2, Worker: 0 Item: 3, Worker: 1 Item: 4, Worker: 0 Item: 5, Worker: 1 ``` Tips ==== Here are a few notable options. - [Benchmark/Test] Disable threading/forking with `in_threads: 0` or `in_processes: 0`, great to test performance or to debug parallel issues - [Isolation] Do not reuse previous worker processes: `isolation: true` - [Stop all processses with an alternate interrupt signal] `'INT'` (from `ctrl+c`) is caught by default. Catch `'TERM'` (from `kill`) with `interrupt_signal: 'TERM'` TODO ==== - Replace Signal trapping with simple `rescue Interrupt` handler Authors ======= ### [Contributors](https://github.com/grosser/parallel/graphs/contributors) - [Przemyslaw Wroblewski](https://github.com/lowang) - [TJ Holowaychuk](http://vision-media.ca/) - [Masatomo Nakano](https://twitter.com/masatomo2) - [Fred Wu](http://fredwu.me) - [mikezter](https://github.com/mikezter) - [Jeremy Durham](http://www.jeremydurham.com) - [Nick Gauthier](http://www.ngauthier.com) - [Andrew Bowerman](http://andrewbowerman.com) - [Byron Bowerman](http://blog.bm5k.com/) - [Mikko Kokkonen](https://github.com/mikian) - [brian p o'rourke](https://github.com/bpo) - [Norio Sato] - [Neal Stewart](https://github.com/n-time) - [Jurriaan Pruis](https://github.com/jurriaan) - [Rob Worley](https://github.com/robworley) - [Tasveer Singh](https://github.com/tazsingh) - [Joachim](https://github.com/jmozmoz) - [yaoguai](https://github.com/yaoguai) - [Bartosz Dziewoński](https://github.com/MatmaRex) - [yaoguai](https://github.com/yaoguai) - [Guillaume Hain](https://github.com/zedtux) - [Adam Wróbel](https://github.com/amw) - [Matthew Brennan](https://github.com/mattyb) - [Brendan Dougherty](https://github.com/brendar) - [Daniel Finnie](https://github.com/danfinnie) - [Philip M. White](https://github.com/philipmw) - [Arlan Jaska](https://github.com/ajaska) - [Sean Walbran](https://github.com/seanwalbran) - [Nathan Broadbent](https://github.com/ndbroadbent) [Michael Grosser](http://grosser.it)
michael@grosser.it
License: MIT
[![Build Status](https://travis-ci.org/grosser/parallel.png)](https://travis-ci.org/grosser/parallel) parallel-1.12.0/gem-public_cert.pem000066400000000000000000000023451313554143200171400ustar00rootroot00000000000000-----BEGIN CERTIFICATE----- MIIDcDCCAligAwIBAgIBATANBgkqhkiG9w0BAQUFADA/MRAwDgYDVQQDDAdtaWNo YWVsMRcwFQYKCZImiZPyLGQBGRYHZ3Jvc3NlcjESMBAGCgmSJomT8ixkARkWAml0 MB4XDTE0MDIwNDIwMjk0MVoXDTE1MDIwNDIwMjk0MVowPzEQMA4GA1UEAwwHbWlj aGFlbDEXMBUGCgmSJomT8ixkARkWB2dyb3NzZXIxEjAQBgoJkiaJk/IsZAEZFgJp dDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMorXo/hgbUq97+kII9H MsQcLdC/7wQ1ZP2OshVHPkeP0qH8MBHGg6eYisOX2ubNagF9YTCZWnhrdKrwpLOO cPLaZbjUjljJ3cQR3B8Yn1veV5IhG86QseTBjymzJWsLpqJ1UZGpfB9tXcsFtuxO 6vHvcIHdzvc/OUkICttLbH+1qb6rsHUceqh+JrH4GrsJ5H4hAfIdyS2XMK7YRKbh h+IBu6dFWJJByzFsYmV1PDXln3UBmgAt65cmCu4qPfThioCGDzbSJrGDGLmw/pFX FPpVCm1zgYSb1v6Qnf3cgXa2f2wYGm17+zAVyIDpwryFru9yF/jJxE38z/DRsd9R /88CAwEAAaN3MHUwCQYDVR0TBAIwADALBgNVHQ8EBAMCBLAwHQYDVR0OBBYEFLIj Z1x7SnjGGHK+MiVZkFjjS/iMMB0GA1UdEQQWMBSBEm1pY2hhZWxAZ3Jvc3Nlci5p dDAdBgNVHRIEFjAUgRJtaWNoYWVsQGdyb3NzZXIuaXQwDQYJKoZIhvcNAQEFBQAD ggEBAExBcUWfGuamYn+IddOA0Ws8jUKwB14RXoZRDrTiTAlMm3Bkg2OKyxS3uJXa 6Z+LwFiZwVYk62yHXqNzEJycQk4SEmY+xDWLj0p7X6qEeU4QZKwR1TwJ5z3PTrZ6 irJgM3q7NIBRvmTzRaAghWcQn+Eyr5YLOfMksjVBMUMnzh5/ZDgq53LphgJbGwvz ScJAgfNclLHnjk9q1mT1s0e1FPWbiAL3siBIR5HpH8qtSEiivTf2ntciebOqS93f F5etKHZg0j3eHO31/i2HnswY04lqGImUu6aM5EnijFTB7PPW2KwKKM4+kKDYFdlw /0WV1Ng2/Y6qsHwmqGg2VlYj2h4= -----END CERTIFICATE----- parallel-1.12.0/lib/000077500000000000000000000000001313554143200141365ustar00rootroot00000000000000parallel-1.12.0/lib/parallel.rb000066400000000000000000000330121313554143200162560ustar00rootroot00000000000000require 'rbconfig' require 'parallel/version' require 'parallel/processor_count' module Parallel extend Parallel::ProcessorCount class DeadWorker < StandardError end class Break < StandardError end class Kill < StandardError end class UndumpableException < StandardError attr_reader :backtrace def initialize(original) super "#{original.class}: #{original.message}" @backtrace = original.backtrace end end Stop = Object.new class ExceptionWrapper attr_reader :exception def initialize(exception) # Remove the bindings stack added by the better_errors gem, # because it cannot be marshalled if exception.instance_variable_defined? :@__better_errors_bindings_stack exception.send :remove_instance_variable, :@__better_errors_bindings_stack end @exception = begin Marshal.dump(exception) && exception rescue UndumpableException.new(exception) end end end class Worker attr_reader :pid, :read, :write attr_accessor :thread def initialize(read, write, pid) @read, @write, @pid = read, write, pid end def stop close_pipes wait # if it goes zombie, rather wait here to be able to debug end # might be passed to started_processes and simultaneously closed by another thread # when running in isolation mode, so we have to check if it is closed before closing def close_pipes read.close unless read.closed? write.close unless write.closed? end def work(data) begin Marshal.dump(data, write) rescue Errno::EPIPE raise DeadWorker end result = begin Marshal.load(read) rescue EOFError raise DeadWorker end raise result.exception if ExceptionWrapper === result result end private def wait Process.wait(pid) rescue Interrupt # process died end end class JobFactory def initialize(source, mutex) @lambda = (source.respond_to?(:call) && source) || queue_wrapper(source) @source = source.to_a unless @lambda # turn Range and other Enumerable-s into an Array @mutex = mutex @index = -1 @stopped = false end def next if producer? # - index and item stay in sync # - do not call lambda after it has returned Stop item, index = @mutex.synchronize do return if @stopped item = @lambda.call @stopped = (item == Parallel::Stop) return if @stopped [item, @index += 1] end else index = @mutex.synchronize { @index += 1 } return if index >= size item = @source[index] end [item, index] end def size if producer? Float::INFINITY else @source.size end end # generate item that is sent to workers # just index is faster + less likely to blow up with unserializable errors def pack(item, index) producer? ? [item, index] : index end # unpack item that is sent to workers def unpack(data) producer? ? data : [@source[data], data] end private def producer? @lambda end def queue_wrapper(array) array.respond_to?(:num_waiting) && array.respond_to?(:pop) && lambda { array.pop(false) } end end class UserInterruptHandler INTERRUPT_SIGNAL = :SIGINT class << self # kill all these pids or threads if user presses Ctrl+c def kill_on_ctrl_c(pids, options) @to_be_killed ||= [] old_interrupt = nil signal = options.fetch(:interrupt_signal, INTERRUPT_SIGNAL) if @to_be_killed.empty? old_interrupt = trap_interrupt(signal) do $stderr.puts 'Parallel execution interrupted, exiting ...' @to_be_killed.flatten.each { |pid| kill(pid) } end end @to_be_killed << pids yield ensure @to_be_killed.pop # do not kill pids that could be used for new processes restore_interrupt(old_interrupt, signal) if @to_be_killed.empty? end def kill(thing) Process.kill(:KILL, thing) rescue Errno::ESRCH # some linux systems already automatically killed the children at this point # so we just ignore them not being there end private def trap_interrupt(signal) old = Signal.trap signal, 'IGNORE' Signal.trap signal do yield if old == "DEFAULT" raise Interrupt else old.call end end old end def restore_interrupt(old, signal) Signal.trap signal, old end end end class << self def in_threads(options={:count => 2}) count, _ = extract_count_from_options(options) Array.new(count) do |i| Thread.new { yield(i) } end.map!(&:value) end def in_processes(options = {}, &block) count, options = extract_count_from_options(options) count ||= processor_count map(0...count, options.merge(:in_processes => count), &block) end def each(array, options={}, &block) map(array, options.merge(:preserve_results => false), &block) end def any?(*args, &block) raise "You must provide a block when calling #any?" if block.nil? !each(*args) { |*a| raise Parallel::Kill if block.call(*a) } end def all?(*args, &block) raise "You must provide a block when calling #all?" if block.nil? !!each(*args) { |*a| raise Parallel::Kill unless block.call(*a) } end def each_with_index(array, options={}, &block) each(array, options.merge(:with_index => true), &block) end def map(source, options = {}, &block) options[:mutex] = Mutex.new if RUBY_PLATFORM =~ /java/ and not options[:in_processes] method = :in_threads size = options[method] || processor_count elsif options[:in_threads] method = :in_threads size = options[method] else method = :in_processes if Process.respond_to?(:fork) size = options[method] || processor_count else warn "Process.fork is not supported by this Ruby" size = 0 end end job_factory = JobFactory.new(source, options[:mutex]) size = [job_factory.size, size].min options[:return_results] = (options[:preserve_results] != false || !!options[:finish]) add_progress_bar!(job_factory, options) results = if size == 0 work_direct(job_factory, options, &block) elsif method == :in_threads work_in_threads(job_factory, options.merge(:count => size), &block) else work_in_processes(job_factory, options.merge(:count => size), &block) end if results options[:return_results] ? results : source end end def map_with_index(array, options={}, &block) map(array, options.merge(:with_index => true), &block) end def worker_number Thread.current[:parallel_worker_number] end def worker_number=(worker_num) Thread.current[:parallel_worker_number] = worker_num end private def add_progress_bar!(job_factory, options) if progress_options = options[:progress] raise "Progressbar can only be used with array like items" if job_factory.size == Float::INFINITY require 'ruby-progressbar' if progress_options == true progress_options = { title: "Progress" } elsif progress_options.respond_to? :to_str progress_options = { title: progress_options.to_str } end progress_options = { total: job_factory.size, format: '%t |%E | %B | %a' }.merge(progress_options) progress = ProgressBar.create(progress_options) old_finish = options[:finish] options[:finish] = lambda do |item, i, result| old_finish.call(item, i, result) if old_finish progress.increment end end end def work_direct(job_factory, options, &block) self.worker_number = 0 results = [] exception = nil begin while set = job_factory.next item, index = set results << with_instrumentation(item, index, options) do call_with_index(item, index, options, &block) end end rescue exception = $! end handle_exception(exception, results) ensure self.worker_number = nil end def work_in_threads(job_factory, options, &block) raise "interrupt_signal is no longer supported for threads" if options[:interrupt_signal] results = [] results_mutex = Mutex.new # arrays are not thread-safe on jRuby exception = nil in_threads(options) do |worker_num| self.worker_number = worker_num # as long as there are more jobs, work on one of them while !exception && set = job_factory.next begin item, index = set result = with_instrumentation item, index, options do call_with_index(item, index, options, &block) end results_mutex.synchronize { results[index] = result } rescue exception = $! end end end handle_exception(exception, results) end def work_in_processes(job_factory, options, &blk) workers = if options[:isolation] [] # we create workers per job and not beforehand else create_workers(job_factory, options, &blk) end results = [] results_mutex = Mutex.new # arrays are not thread-safe exception = nil UserInterruptHandler.kill_on_ctrl_c(workers.map(&:pid), options) do in_threads(options) do |i| worker = workers[i] begin loop do break if exception item, index = job_factory.next break unless index if options[:isolation] worker = replace_worker(job_factory, workers, i, options, blk) end worker.thread = Thread.current begin result = with_instrumentation item, index, options do worker.work(job_factory.pack(item, index)) end results_mutex.synchronize { results[index] = result } # arrays are not threads safe on jRuby rescue exception = $! if Parallel::Kill === exception (workers - [worker]).each do |w| w.thread.kill unless w.thread.nil? UserInterruptHandler.kill(w.pid) end end end end ensure worker.stop if worker end end end handle_exception(exception, results) end def replace_worker(job_factory, workers, i, options, blk) options[:mutex].synchronize do # old worker is no longer used ... stop it worker = workers[i] worker.stop if worker # create a new replacement worker running = workers - [worker] workers[i] = worker(job_factory, options.merge(started_workers: running, worker_number: i), &blk) end end def create_workers(job_factory, options, &block) workers = [] Array.new(options[:count]).each_with_index do |_, i| workers << worker(job_factory, options.merge(started_workers: workers, worker_number: i), &block) end workers end def worker(job_factory, options, &block) child_read, parent_write = IO.pipe parent_read, child_write = IO.pipe pid = Process.fork do self.worker_number = options[:worker_number] begin options.delete(:started_workers).each(&:close_pipes) parent_write.close parent_read.close process_incoming_jobs(child_read, child_write, job_factory, options, &block) ensure child_read.close child_write.close end end child_read.close child_write.close Worker.new(parent_read, parent_write, pid) end def process_incoming_jobs(read, write, job_factory, options, &block) until read.eof? data = Marshal.load(read) item, index = job_factory.unpack(data) result = begin call_with_index(item, index, options, &block) rescue ExceptionWrapper.new($!) end Marshal.dump(result, write) end end def handle_exception(exception, results) return nil if [Parallel::Break, Parallel::Kill].include? exception.class raise exception if exception results end # options is either a Integer or a Hash with :count def extract_count_from_options(options) if options.is_a?(Hash) count = options[:count] else count = options options = {} end [count, options] end def call_with_index(item, index, options, &block) args = [item] args << index if options[:with_index] if options[:return_results] block.call(*args) else block.call(*args) nil # avoid GC overhead of passing large results around end end def with_instrumentation(item, index, options) on_start = options[:start] on_finish = options[:finish] options[:mutex].synchronize { on_start.call(item, index) } if on_start result = yield options[:mutex].synchronize { on_finish.call(item, index, result) } if on_finish result unless options[:preserve_results] == false end end end parallel-1.12.0/lib/parallel/000077500000000000000000000000001313554143200157325ustar00rootroot00000000000000parallel-1.12.0/lib/parallel/processor_count.rb000066400000000000000000000067401313554143200215150ustar00rootroot00000000000000if RUBY_VERSION.to_f >= 2.2 require 'etc' end module Parallel module ProcessorCount # Number of processors seen by the OS and used for process scheduling. # # * AIX: /usr/sbin/pmcycles (AIX 5+), /usr/sbin/lsdev # * BSD: /sbin/sysctl # * Cygwin: /proc/cpuinfo # * Darwin: /usr/bin/hwprefs, /usr/sbin/sysctl # * HP-UX: /usr/sbin/ioscan # * IRIX: /usr/sbin/sysconf # * Linux: /proc/cpuinfo # * Minix 3+: /proc/cpuinfo # * Solaris: /usr/sbin/psrinfo # * Tru64 UNIX: /usr/sbin/psrinfo # * UnixWare: /usr/sbin/psrinfo # def processor_count @processor_count ||= begin if defined?(Etc) && Etc.respond_to?(:nprocessors) Etc.nprocessors else os_name = RbConfig::CONFIG["target_os"] if os_name =~ /mingw|mswin/ require 'win32ole' result = WIN32OLE.connect("winmgmts://").ExecQuery( "select NumberOfLogicalProcessors from Win32_Processor") result.to_enum.collect(&:NumberOfLogicalProcessors).reduce(:+) elsif File.readable?("/proc/cpuinfo") IO.read("/proc/cpuinfo").scan(/^processor/).size elsif File.executable?("/usr/bin/hwprefs") IO.popen("/usr/bin/hwprefs thread_count").read.to_i elsif File.executable?("/usr/sbin/psrinfo") IO.popen("/usr/sbin/psrinfo").read.scan(/^.*on-*line/).size elsif File.executable?("/usr/sbin/ioscan") IO.popen("/usr/sbin/ioscan -kC processor") do |out| out.read.scan(/^.*processor/).size end elsif File.executable?("/usr/sbin/pmcycles") IO.popen("/usr/sbin/pmcycles -m").read.count("\n") elsif File.executable?("/usr/sbin/lsdev") IO.popen("/usr/sbin/lsdev -Cc processor -S 1").read.count("\n") elsif File.executable?("/usr/sbin/sysconf") and os_name =~ /irix/i IO.popen("/usr/sbin/sysconf NPROC_ONLN").read.to_i elsif File.executable?("/usr/sbin/sysctl") IO.popen("/usr/sbin/sysctl -n hw.ncpu").read.to_i elsif File.executable?("/sbin/sysctl") IO.popen("/sbin/sysctl -n hw.ncpu").read.to_i else $stderr.puts "Unknown platform: " + RbConfig::CONFIG["target_os"] $stderr.puts "Assuming 1 processor." 1 end end end end # Number of physical processor cores on the current system. # def physical_processor_count @physical_processor_count ||= begin ppc = case RbConfig::CONFIG["target_os"] when /darwin1/ IO.popen("/usr/sbin/sysctl -n hw.physicalcpu").read.to_i when /linux/ cores = {} # unique physical ID / core ID combinations phy = 0 IO.read("/proc/cpuinfo").scan(/^physical id.*|^core id.*/) do |ln| if ln.start_with?("physical") phy = ln[/\d+/] elsif ln.start_with?("core") cid = phy + ":" + ln[/\d+/] cores[cid] = true if not cores[cid] end end cores.count when /mswin|mingw/ require 'win32ole' result_set = WIN32OLE.connect("winmgmts://").ExecQuery( "select NumberOfCores from Win32_Processor") result_set.to_enum.collect(&:NumberOfCores).reduce(:+) else processor_count end # fall back to logical count if physical info is invalid ppc > 0 ? ppc : processor_count end end end end parallel-1.12.0/lib/parallel/version.rb000066400000000000000000000000631313554143200177430ustar00rootroot00000000000000module Parallel VERSION = Version = '1.12.0' end parallel-1.12.0/parallel.gemspec000066400000000000000000000007021313554143200165300ustar00rootroot00000000000000name = "parallel" $LOAD_PATH << File.expand_path('../lib', __FILE__) require "#{name}/version" Gem::Specification.new name, Parallel::VERSION do |s| s.summary = "Run any kind of code in parallel processes" s.authors = ["Michael Grosser"] s.email = "michael@grosser.it" s.homepage = "https://github.com/grosser/#{name}" s.files = `git ls-files lib MIT-LICENSE.txt`.split("\n") s.license = "MIT" s.required_ruby_version = '>= 1.9.3' end parallel-1.12.0/spec/000077500000000000000000000000001313554143200143225ustar00rootroot00000000000000parallel-1.12.0/spec/cases/000077500000000000000000000000001313554143200154205ustar00rootroot00000000000000parallel-1.12.0/spec/cases/after_interrupt.rb000066400000000000000000000001631313554143200211620ustar00rootroot00000000000000require './spec/cases/helper' Parallel.map([1, 2], :in_processes => 2) { } puts Signal.trap(:SIGINT, "IGNORE") parallel-1.12.0/spec/cases/all_false.rb000066400000000000000000000006241313554143200176710ustar00rootroot00000000000000require './spec/cases/helper' STDOUT.sync = true # otherwise results can go weird... results = [] [{in_processes: 2}, {in_threads: 2}, {in_threads: 0}].each do |options| x = [nil,nil,nil,nil,nil,nil,nil,nil] results << Parallel.all?(x, options) do |x| x end x = [42,42,42,42,42,42,42,5,42,42,42] results << Parallel.all?(x, options) do |x| x == 42 end end print results.join(',') parallel-1.12.0/spec/cases/all_true.rb000066400000000000000000000010011313554143200175440ustar00rootroot00000000000000require './spec/cases/helper' STDOUT.sync = true # otherwise results can go weird... results = [] [{in_processes: 2}, {in_threads: 2}, {in_threads: 0}].each do |options| x = [nil,nil,nil,nil,nil,nil,nil,nil] results << Parallel.all?(x, options) do |x| x.nil? end x = [42,42,42,42,42,42,42,42,42,42,42] results << Parallel.all?(x, options) do |x| x == 42 end # Empty array should return true x = [] results << Parallel.all?(x, options) do |x| x end end print results.join(',') parallel-1.12.0/spec/cases/any_false.rb000066400000000000000000000007471313554143200177160ustar00rootroot00000000000000require './spec/cases/helper' STDOUT.sync = true # otherwise results can go weird... results = [] [{in_processes: 2}, {in_threads: 2}, {in_threads: 0}].each do |options| x = [nil,nil,nil,nil,nil,nil,nil,nil] results << Parallel.any?(x, options) do |x| x end x = 10.times results << Parallel.any?(x, options) do |x| false end # Empty array should return false x = [] results << Parallel.any?(x, options) do |x| x == 42 end end print results.join(',') parallel-1.12.0/spec/cases/any_true.rb000066400000000000000000000006211313554143200175720ustar00rootroot00000000000000require './spec/cases/helper' STDOUT.sync = true # otherwise results can go weird... results = [] [{in_processes: 2}, {in_threads: 2}, {in_threads: 0}].each do |options| x = [nil,nil,nil,nil,42,nil,nil,nil] results << Parallel.any?(x, options) do |x| x end x = [true,true,true,false,true,true,true] results << Parallel.any?(x, options) do |x| x end end print results.join(',') parallel-1.12.0/spec/cases/closes_processes_at_runtime.rb000066400000000000000000000001611313554143200235500ustar00rootroot00000000000000require './spec/cases/helper' process_diff do Parallel.each((0..10).to_a, :in_processes => 5) { |a| a*2 } end parallel-1.12.0/spec/cases/count_open_pipes.rb000066400000000000000000000002241313554143200213140ustar00rootroot00000000000000require './spec/cases/helper' results = Parallel.map(Array.new(20), :in_processes => 20) do `lsof | grep pipe | wc -l`.to_i end puts results.max parallel-1.12.0/spec/cases/double_interrupt.rb000066400000000000000000000003041313554143200213300ustar00rootroot00000000000000require './spec/cases/helper' Signal.trap :SIGINT do sleep 0.5 puts "YES" exit 0 end Parallel.map(Array.new(20), :in_processes => 2) do sleep 10 puts "I should be killed earlier" end parallel-1.12.0/spec/cases/each.rb000066400000000000000000000002741313554143200166500ustar00rootroot00000000000000require './spec/cases/helper' STDOUT.sync = true # otherwise results can go weird... x = ['a','b','c','d'] result = Parallel.each(x) do |x| sleep 0.1 if x == 'a' end print result * ' ' parallel-1.12.0/spec/cases/each_in_place.rb000066400000000000000000000002421313554143200204750ustar00rootroot00000000000000require './spec/cases/helper' STDOUT.sync = true # otherwise results can go weird... x = ['a'] Parallel.each(x, :in_threads => 1) { |x| x << 'b' } print x.first parallel-1.12.0/spec/cases/each_with_ar_sqlite.rb000066400000000000000000000014371313554143200217500ustar00rootroot00000000000000require './spec/cases/helper' require "active_record" require "sqlite3" STDOUT.sync = true in_worker_type = "in_#{ENV.fetch('WORKER_TYPE')}".to_sym Tempfile.open("db") do |temp| ActiveRecord::Schema.verbose = false ActiveRecord::Base.establish_connection( :adapter => "sqlite3", :database => temp.path ) class User < ActiveRecord::Base end # create tables unless User.table_exists? ActiveRecord::Schema.define(:version => 1) do create_table :users do |t| t.string :name end end end User.delete_all 3.times { User.create!(:name => "X") } puts "Parent: #{User.first.name}" Parallel.each([1], in_worker_type => 1) do puts "Parallel (#{in_worker_type}): #{User.all.map(&:name).join}" end puts "Parent: #{User.first.name}" end parallel-1.12.0/spec/cases/each_with_index.rb000066400000000000000000000001661313554143200210720ustar00rootroot00000000000000require './spec/cases/helper' Parallel.each_with_index(['a','b'], :in_threads => 2) do |x, i| print "#{x}#{i}" end parallel-1.12.0/spec/cases/eof_in_process.rb000066400000000000000000000002071313554143200207410ustar00rootroot00000000000000require './spec/cases/helper' begin Parallel.map([1]){ raise EOFError } rescue EOFError puts 'Yep, EOF' else puts 'WHOOOPS' end parallel-1.12.0/spec/cases/exit_in_process.rb000066400000000000000000000002121313554143200211350ustar00rootroot00000000000000require './spec/cases/helper' begin Parallel.map([1]){ exit } rescue Parallel::DeadWorker puts "Yep, DEAD" else puts "WHOOOPS" end parallel-1.12.0/spec/cases/fatal_queue.rb000066400000000000000000000002301313554143200202330ustar00rootroot00000000000000require './spec/cases/helper' queue = Queue.new queue.push 1 queue.push 2 queue.push 3 Parallel.map(queue, :in_threads => 2) { |(i, id)| "ITEM-#{i}" } parallel-1.12.0/spec/cases/helper.rb000066400000000000000000000005151313554143200172250ustar00rootroot00000000000000require 'bundler/setup' require 'parallel' def process_diff cmd = "ps uaxw|grep ruby|wc -l" processes_before = `#{cmd}`.to_i yield sleep 1 processes_after = `#{cmd}`.to_i if processes_before == processes_after print 'OK' else print "FAIL: before:#{processes_before} -- after:#{processes_after}" end end parallel-1.12.0/spec/cases/map_isolation.rb000066400000000000000000000002401313554143200205770ustar00rootroot00000000000000require './spec/cases/helper' process_diff do result = Parallel.map([1,2,3,4], in_processes: 2, isolation: true) do |i| $i ||= i end puts result end parallel-1.12.0/spec/cases/map_with_ar.rb000066400000000000000000000016031313554143200202370ustar00rootroot00000000000000require './spec/cases/helper' require "active_record" Tempfile.open("xxx") do |f| database = "parallel_with_ar_test" `mysql #{database} -e '' || mysql -e 'create database #{database}'` ActiveRecord::Schema.verbose = false ActiveRecord::Base.establish_connection( :adapter => "mysql2", :database => database ) class User < ActiveRecord::Base end # create tables unless User.table_exists? ActiveRecord::Schema.define(:version => 1) do create_table :users do |t| t.string :name end end end User.delete_all User.create!(:name => "X") Parallel.map(1..8) do |i| User.create!(:name => i) end puts "User.count: #{User.count}" puts User.connection.reconnect!.inspect Parallel.map(1..8, :in_threads => 4) do |i| User.create!(:name => i) end User.create!(:name => "X") puts User.all.map(&:name).sort.join("-") end parallel-1.12.0/spec/cases/map_with_index.rb000066400000000000000000000001701313554143200207420ustar00rootroot00000000000000require './spec/cases/helper' result = Parallel.map_with_index(['a','b']) do |x, i| "#{x}#{i}" end print result * '' parallel-1.12.0/spec/cases/map_with_index_empty.rb000066400000000000000000000001611313554143200221600ustar00rootroot00000000000000require './spec/cases/helper' result = Parallel.map_with_index([]) do |x, i| "#{x}#{i}" end print result * '' parallel-1.12.0/spec/cases/map_with_killed_worker_before_read.rb000066400000000000000000000002441313554143200250070ustar00rootroot00000000000000require './spec/cases/helper' begin Parallel.map([1,2,3]) do |x, i| Process.kill("SIGKILL", Process.pid) end rescue Parallel::DeadWorker puts "DEAD" end parallel-1.12.0/spec/cases/map_with_killed_worker_before_write.rb000066400000000000000000000005251313554143200252300ustar00rootroot00000000000000require './spec/cases/helper' Parallel::Worker.class_eval do alias_method :work_without_kill, :work def work(*args) Process.kill("SIGKILL", pid) sleep 0.5 work_without_kill(*args) end end begin Parallel.map([1,2,3]) do |x, i| Process.kill("SIGKILL", Process.pid) end rescue Parallel::DeadWorker puts "DEAD" end parallel-1.12.0/spec/cases/map_with_nested_arrays_and_nil.rb000066400000000000000000000001641313554143200241650ustar00rootroot00000000000000require './spec/cases/helper' result = Parallel.map([1,2,[3]]) do |x| [x, x] if x != 1 end print result.inspect parallel-1.12.0/spec/cases/map_worker_number_isolation.rb000066400000000000000000000003021313554143200235370ustar00rootroot00000000000000require './spec/cases/helper' process_diff do result = Parallel.map([1,2,3,4], in_processes: 2, isolation: true) do |i| Parallel.worker_number end puts result.uniq.sort.join(',') end parallel-1.12.0/spec/cases/no_dump_with_each.rb000066400000000000000000000004271313554143200214240ustar00rootroot00000000000000require './spec/cases/helper' class NotDumpable def marshal_dump raise "NOOOO" end def to_s 'not dumpable' end end Parallel.each([1]) do print 'no dump for result' NotDumpable.new end Parallel.each([NotDumpable.new]) do print 'no dump for each' 1 end parallel-1.12.0/spec/cases/no_gc_with_each.rb000066400000000000000000000001501313554143200210410ustar00rootroot00000000000000require './spec/cases/helper' Parallel.each(1..1000, :in_threads => 2) do |i| "xxxx" * 1_000_000 end parallel-1.12.0/spec/cases/parallel_break_better_errors.rb000066400000000000000000000007511313554143200236510ustar00rootroot00000000000000require './spec/cases/helper' require 'stringio' class MyException < StandardError def initialize(object) @object = object end end begin Parallel.in_processes(2) do ex = Parallel::Break.new # better_errors sets an instance variable that contains an array of bindings. ex.instance_variable_set :@__better_errors_bindings_stack, [ex.send(:binding)] raise ex end puts "NOTHING WAS RAISED" rescue puts $!.message puts "BACKTRACE: #{$!.backtrace.first}" end parallel-1.12.0/spec/cases/parallel_fast_exit.rb000066400000000000000000000001611313554143200216050ustar00rootroot00000000000000require './spec/cases/helper' Parallel.map([1,2,3], :in_processes => 2) do puts "I finished..." end sleep 10 parallel-1.12.0/spec/cases/parallel_high_fork_rate.rb000066400000000000000000000001601313554143200225710ustar00rootroot00000000000000require './spec/cases/helper' Parallel.each((0..200).to_a, :in_processes=>200) do |x| sleep 1 end print 'OK' parallel-1.12.0/spec/cases/parallel_influence_outside_data.rb000066400000000000000000000001351313554143200243150ustar00rootroot00000000000000require './spec/cases/helper' x = 'yes' Parallel.in_processes(2) do x = 'no' end print x parallel-1.12.0/spec/cases/parallel_kill.rb000066400000000000000000000004211313554143200205510ustar00rootroot00000000000000require './spec/cases/helper' results = Parallel.map([1,2,3]) do |x| if x == 1 # -> stop all sub-processes, killing them instantly sleep 0.1 puts "DEAD" raise Parallel::Kill elsif x == 3 sleep 10 else x end end puts "Works #{results.inspect}" parallel-1.12.0/spec/cases/parallel_map.rb000066400000000000000000000001611313554143200203740ustar00rootroot00000000000000require './spec/cases/helper' result = Parallel.map(['a','b','c','d']) do |x| "-#{x}-" end print result * ' ' parallel-1.12.0/spec/cases/parallel_map_complex_objects.rb000066400000000000000000000003441313554143200236370ustar00rootroot00000000000000require './spec/cases/helper' object = ["\nasd#{File.read('Gemfile')}--#{File.read('Rakefile')}"*100, 12345, {:b=>:a}] result = Parallel.map([1,2]) do |x| object end print 'YES' if result.inspect == [object, object].inspect parallel-1.12.0/spec/cases/parallel_map_range.rb000066400000000000000000000001371313554143200215530ustar00rootroot00000000000000require './spec/cases/helper' result = Parallel.map(1..5) do |x| x end print result.inspect parallel-1.12.0/spec/cases/parallel_map_sleeping.rb000066400000000000000000000001241313554143200222610ustar00rootroot00000000000000require './spec/cases/helper' Parallel.map(['a','b','c','d']) do |x| sleep 1 end parallel-1.12.0/spec/cases/parallel_map_uneven.rb000066400000000000000000000001261313554143200217550ustar00rootroot00000000000000require './spec/cases/helper' Parallel.map([1,2,1,2]) do |x| sleep 2 if x == 1 end parallel-1.12.0/spec/cases/parallel_raise.rb000066400000000000000000000002221313554143200207200ustar00rootroot00000000000000require './spec/cases/helper' begin Parallel.in_processes(2) do raise "TEST" end puts "FAIL" rescue RuntimeError puts $!.message end parallel-1.12.0/spec/cases/parallel_raise_undumpable.rb000066400000000000000000000005021313554143200231350ustar00rootroot00000000000000require './spec/cases/helper' require 'stringio' class MyException < StandardError def initialize(object) @object = object end end begin Parallel.in_processes(2) do raise MyException.new(StringIO.new) end puts "NOTHING WAS RAISED" rescue puts $!.message puts "BACKTRACE: #{$!.backtrace.first}" end parallel-1.12.0/spec/cases/parallel_sleeping_2.rb000066400000000000000000000001111313554143200216410ustar00rootroot00000000000000require './spec/cases/helper' Parallel.in_processes(5) do sleep 2 end parallel-1.12.0/spec/cases/parallel_start_and_kill.rb000066400000000000000000000006471313554143200226220ustar00rootroot00000000000000require './spec/cases/helper' method = case ARGV[0] when "PROCESS" then :in_processes when "THREAD" then :in_threads else raise "Learn to use this!" end options = {} options[:count] = 2 if ARGV.length > 1 options[:interrupt_signal] = ARGV[1].to_s trap('SIGINT') { puts 'Wrapper caught SIGINT' } if ARGV[1] != 'SIGINT' end Parallel.send(method, options) do sleep 5 puts "I should have been killed earlier..." end parallel-1.12.0/spec/cases/parallel_with_detected_cpus.rb000066400000000000000000000001211313554143200234610ustar00rootroot00000000000000require './spec/cases/helper' x = Parallel.in_processes do "HELLO" end puts x parallel-1.12.0/spec/cases/parallel_with_nil_uses_detected_cpus.rb000066400000000000000000000001261313554143200253670ustar00rootroot00000000000000require './spec/cases/helper' x = Parallel.in_processes(nil) do "HELLO" end puts x parallel-1.12.0/spec/cases/parallel_with_set_processes.rb000066400000000000000000000001241313554143200235320ustar00rootroot00000000000000require './spec/cases/helper' x = Parallel.in_processes(5) do "HELLO" end puts x parallel-1.12.0/spec/cases/profile_memory.rb000066400000000000000000000011111313554143200207670ustar00rootroot00000000000000def count_objects old = Hash.new(0) cur = Hash.new(0) GC.start ObjectSpace.each_object { |o| old[o.class] += 1 } yield GC.start GC.start ObjectSpace.each_object { |o| cur[o.class] += 1 } Hash[cur.map{|k,v| [k, v - old[k]] }].reject{|k,v|v==0} end require './spec/cases/helper' items = Array.new(1000) options = {"in_#{ARGV[0]}".to_sym => 2} # TODO not sure why this fails without 2.times in threading mode :( puts(count_objects { 2.times { Parallel.map(items, options) {} } }.inspect) puts(count_objects { 2.times { Parallel.map(items, options) {} } }.inspect) parallel-1.12.0/spec/cases/progress.rb000066400000000000000000000002601313554143200176070ustar00rootroot00000000000000require './spec/cases/helper' title = (ENV["TITLE"] == "true" ? true : "Doing stuff") Parallel.map(1..50, :progress => title) do sleep 1 if $stdout.tty? # for debugging end parallel-1.12.0/spec/cases/progress_with_finish.rb000066400000000000000000000003421313554143200222030ustar00rootroot00000000000000require './spec/cases/helper' sum = 0 finish = lambda { |item, index, result| sum += result } Parallel.map(1..50, :progress => "Doing stuff", :finish => finish) do sleep 1 if $stdout.tty? # for debugging 2 end puts sum parallel-1.12.0/spec/cases/progress_with_options.rb000066400000000000000000000011701313554143200224160ustar00rootroot00000000000000require './spec/cases/helper' # ruby-progressbar ignores the format string you give it # unless the output is a TTY. When running in the test, # the output is not a TTY, so we cannot test that the format # string you pass overrides parallel's default. So, we pretend # that stdout is a TTY to test that the options are merged # in the correct way. tty_stdout = $stdout class << tty_stdout def tty? true end end parallel_options = { :progress => { :title => "Reticulating Splines", :progress_mark => ';', :format => "%t %w", :output => tty_stdout } } Parallel.map(1..50, parallel_options) do 2 end parallel-1.12.0/spec/cases/synchronizes_start_and_finish.rb000066400000000000000000000004741313554143200241070ustar00rootroot00000000000000require './spec/cases/helper' start = lambda {|item,index| print item * 5 sleep rand * 0.2 puts item * 5 } finish = lambda {|item,index,result| print result * 5 sleep rand * 0.2 puts result * 5 } Parallel.map(['a', 'b', 'c'], :start => start, :finish => finish) do |i| sleep rand * 0.2 i.upcase end parallel-1.12.0/spec/cases/with_break.rb000066400000000000000000000010201313554143200200550ustar00rootroot00000000000000require './spec/cases/helper' STDOUT.sync = true # otherwise results can go weird... method = ENV.fetch('METHOD') in_worker_type = "in_#{ENV.fetch('WORKER_TYPE')}".to_sym worker_size = (ENV['WORKER_SIZE'] || 4).to_i result = Parallel.public_send(method, 1..100, in_worker_type => worker_size) do |x| sleep 0.1 # so all workers get started print x raise Parallel::Break if x == 1 sleep 0.2 # so now no work gets queued before Parallel::Break is raised x end print " Parallel::Break raised - result #{result.inspect}" parallel-1.12.0/spec/cases/with_break_before_finish.rb000066400000000000000000000006531313554143200227520ustar00rootroot00000000000000require './spec/cases/helper' method = ENV.fetch('METHOD') in_worker_type = "in_#{ENV.fetch('WORKER_TYPE')}".to_sym finish = lambda do |_item, _index, _result| sleep 0.1 print "finish hook called" end result = Parallel.public_send(method, 1..100, in_worker_type => 4, finish: finish) do |x| sleep 0.1 # let workers start raise Parallel::Break if x == 1 sleep 0.2 print x x end print " Parallel::Break raised" parallel-1.12.0/spec/cases/with_exception.rb000066400000000000000000000007551313554143200210050ustar00rootroot00000000000000require './spec/cases/helper' STDOUT.sync = true # otherwise results can go weird... method = ENV.fetch('METHOD') in_worker_type = "in_#{ENV.fetch('WORKER_TYPE')}".to_sym worker_size = (ENV['WORKER_SIZE'] || 4).to_i begin Parallel.public_send(method, 1..100, in_worker_type => worker_size) do |x| sleep 0.1 # so all workers get started print x raise 'foo' if x == 1 sleep 0.2 # so now no work gets queued before exception is raised x end rescue print ' raised' end parallel-1.12.0/spec/cases/with_exception_before_finish.rb000066400000000000000000000006751313554143200236700ustar00rootroot00000000000000require './spec/cases/helper' method = ENV.fetch('METHOD') in_worker_type = "in_#{ENV.fetch('WORKER_TYPE')}".to_sym class ParallelTestError < StandardError end begin finish = lambda do |_item, _index, _result| print " called" end Parallel.public_send(method, 1..10, in_worker_type => 4, finish: finish) do |x| if x != 3 sleep 0.2 raise ParallelTestError end print x x end rescue ParallelTestError end parallel-1.12.0/spec/cases/with_exception_in_finish.rb000066400000000000000000000007161313554143200230300ustar00rootroot00000000000000require './spec/cases/helper' method = ENV.fetch('METHOD') in_worker_type = "in_#{ENV.fetch('WORKER_TYPE')}".to_sym begin finish = lambda do |x, _index, _result| raise 'foo' if x == 1 end Parallel.public_send(method, 1..100, in_worker_type => 4, finish: finish) do |x| sleep 0.1 # so all workers get started print x sleep 0.2 unless x == 1 # so now no work gets queued before exception is raised x end rescue print ' raised' end parallel-1.12.0/spec/cases/with_exception_in_start.rb000066400000000000000000000006771313554143200227130ustar00rootroot00000000000000require './spec/cases/helper' method = ENV.fetch('METHOD') in_worker_type = "in_#{ENV.fetch('WORKER_TYPE')}".to_sym begin start = lambda do |_item, _index| @started = @started ? @started + 1 : 1 raise 'foo' if @started == 4 end Parallel.public_send(method, 1..100, in_worker_type => 4, start: start) do |x| print x sleep 0.2 # so now no work gets queued before exception is raised x end rescue print ' raised' end parallel-1.12.0/spec/cases/with_exception_in_start_before_finish.rb000066400000000000000000000007701313554143200255670ustar00rootroot00000000000000require './spec/cases/helper' method = ENV.fetch('METHOD') in_worker_type = "in_#{ENV.fetch('WORKER_TYPE')}".to_sym class ParallelTestError < StandardError end begin start = lambda do |item, _index| if item != 3 sleep 0.2 raise ParallelTestError end end finish = lambda do |_item, _index, _result| print " called" end Parallel.public_send(method, 1..10, in_worker_type => 4, start: start, finish: finish) do |x| print x x end rescue ParallelTestError end parallel-1.12.0/spec/cases/with_lambda.rb000066400000000000000000000004411313554143200202170ustar00rootroot00000000000000require './spec/cases/helper' type = case ARGV[0] when "PROCESSES" then :in_processes when "THREADS" then :in_threads else raise "Use PROCESSES or THREADS" end all = [3,2,1] produce = lambda { all.pop || Parallel::Stop } puts Parallel.map(produce, type => 2) { |(i, id)| "ITEM-#{i}" } parallel-1.12.0/spec/cases/with_queue.rb000066400000000000000000000005331313554143200201250ustar00rootroot00000000000000require './spec/cases/helper' type = case ARGV[0] when "PROCESSES" then :in_processes when "THREADS" then :in_threads else raise "Use PROCESSES or THREADS" end queue = Queue.new Thread.new do sleep 0.2 queue.push 1 queue.push 2 queue.push 3 queue.push Parallel::Stop end puts Parallel.map(queue, type => 2) { |(i, id)| "ITEM-#{i}" } parallel-1.12.0/spec/cases/with_worker_number.rb000066400000000000000000000004001313554143200216530ustar00rootroot00000000000000require './spec/cases/helper' method = ENV.fetch('METHOD') in_worker_type = "in_#{ENV.fetch('WORKER_TYPE')}".to_sym Parallel.public_send(method, 1..100, in_worker_type => 4) do sleep 0.1 # so all workers get started print Parallel.worker_number end parallel-1.12.0/spec/parallel_spec.rb000066400000000000000000000541261313554143200174650ustar00rootroot00000000000000require 'spec_helper' describe Parallel do worker_types = (Process.respond_to?(:fork) ? ["processes", "threads"] : ["threads"]) def time_taken t = Time.now.to_f yield RUBY_ENGINE == "jruby" ? 0: Time.now.to_f - t # jruby is super slow ... don't blow up all the tests ... end def kill_process_with_name(file, signal='INT') running_processes = `ps -f`.split("\n").map{ |line| line.split(/\s+/) } pid_index = running_processes.detect { |p| p.include?("UID") }.index("UID") + 1 parent_pid = running_processes.detect { |p| p.include?(file) and not p.include?("sh") }[pid_index] `kill -s #{signal} #{parent_pid}` end def execute_start_and_kill(command, amount, signal='INT') t = nil lambda { t = Thread.new { `ruby spec/cases/parallel_start_and_kill.rb #{command} 2>&1 && echo "FINISHED"` } sleep 1.5 kill_process_with_name('spec/cases/parallel_start_and_kill.rb', signal) sleep 1 }.should change { `ps`.split("\n").size }.by amount t.value end describe ".processor_count" do before do Parallel.instance_variable_set(:@processor_count, nil) end it "returns a number" do (1..999).should include(Parallel.processor_count) end if RUBY_PLATFORM =~ /darwin10/ it 'works if hwprefs in not available' do Parallel.should_receive(:hwprefs_available?).and_return false (1..999).should include(Parallel.processor_count) end end if RUBY_VERSION.to_f >= 2.2 it 'uses Etc.nprocessors in Ruby 2.2+' do defined?(Etc).should == "constant" Etc.respond_to?(:nprocessors).should == true end else it 'doesnt use Etc.nprocessors in Ruby 2.1 and below' do defined?(Etc).should == "constant" end end end describe ".physical_processor_count" do before do Parallel.instance_variable_set(:@physical_processor_count, nil) end it "returns a number" do (1..999).should include(Parallel.physical_processor_count) end it "is even factor of logical cpus" do (Parallel.processor_count % Parallel.physical_processor_count).should == 0 end end describe ".in_processes" do def cpus Parallel.processor_count end it "executes with detected cpus" do `ruby spec/cases/parallel_with_detected_cpus.rb`.should == "HELLO\n" * cpus end it "executes with detected cpus when nil was given" do `ruby spec/cases/parallel_with_nil_uses_detected_cpus.rb`.should == "HELLO\n" * cpus end it "set amount of parallel processes" do `ruby spec/cases/parallel_with_set_processes.rb`.should == "HELLO\n" * 5 end it "does not influence outside data" do `ruby spec/cases/parallel_influence_outside_data.rb`.should == "yes" end it "kills the processes when the main process gets killed through ctrl+c" do time_taken { result = execute_start_and_kill "PROCESS", 0 result.should_not include "FINISHED" }.should be <= 3 end it "kills the processes when the main process gets killed through a custom interrupt" do time_taken { execute_start_and_kill "PROCESS SIGTERM", 0, "TERM" }.should be <= 3 end it "kills the threads when the main process gets killed through ctrl+c" do time_taken { result = execute_start_and_kill "THREAD", 0 result.should_not include "FINISHED" }.should be <= 3 end it "does not kill processes when the main process gets sent an interrupt besides the custom interrupt" do time_taken { result = execute_start_and_kill "PROCESS SIGTERM", 4 result.should include 'FINISHED' result.should include 'Wrapper caught SIGINT' result.should include 'I should have been killed earlier' }.should be <= 7 end it "does not kill threads when the main process gets sent an interrupt besides the custom interrupt" do time_taken { result = execute_start_and_kill "THREAD SIGTERM", 2 result.should include 'FINISHED' result.should include 'Wrapper caught SIGINT' result.should include 'I should have been killed earlier' }.should be <= 7 end it "does not kill anything on ctrl+c when everything has finished" do time_taken do t = Thread.new { `ruby spec/cases/parallel_fast_exit.rb 2>&1` } sleep 2 kill_process_with_name("spec/cases/parallel_fast_exit.rb") #simulates Ctrl+c sleep 1 result = t.value result.scan(/I finished/).size.should == 3 result.should_not include("Parallel execution interrupted") end.should <= 4 end it "preserves original intrrupts" do t = Thread.new { `ruby spec/cases/double_interrupt.rb 2>&1 && echo FIN` } sleep 2 kill_process_with_name("spec/cases/double_interrupt.rb") #simulates Ctrl+c sleep 1 result = t.value result.should include("YES") result.should include("FIN") end it "restores original intrrupts" do `ruby spec/cases/after_interrupt.rb 2>&1`.should == "DEFAULT\n" end it "saves time" do time_taken{ `ruby spec/cases/parallel_sleeping_2.rb` }.should < 3.5 end it "raises when one of the processes raises" do `ruby spec/cases/parallel_raise.rb`.strip.should == 'TEST' end it "can raise an undumpable exception" do out = `ruby spec/cases/parallel_raise_undumpable.rb`.strip out.sub!(Dir.pwd, '.') # relative paths out.gsub!(/(\d+)\:.*/, "\\1") # no diff in ruby version xyz.rb:123:in `block in
' out.should == "MyException: MyException\nBACKTRACE: spec/cases/parallel_raise_undumpable.rb:12" end it "can handle Break exceptions when the better_errors gem is installed" do out = `ruby spec/cases/parallel_break_better_errors.rb`.strip out.should == "NOTHING WAS RAISED" end it 'can handle to high fork rate' do unless RbConfig::CONFIG["target_os"] =~ /darwin1/ `ruby spec/cases/parallel_high_fork_rate.rb`.should == 'OK' end end it 'does not leave processes behind while running' do skip if ENV['TRAVIS'] # this randomly fails on travis all the time :( `ruby spec/cases/closes_processes_at_runtime.rb`.should == 'OK' end it "does not open unnecessary pipes" do open_pipes = `lsof | grep pipe | wc -l`.to_i max_pipes = `ruby spec/cases/count_open_pipes.rb`.to_i (max_pipes - open_pipes).should < 400 end end describe ".in_threads" do it "saves time" do time_taken{ Parallel.in_threads(3){ sleep 2 } }.should < 3 end it "does not create new processes" do lambda{ Thread.new{ Parallel.in_threads(2){sleep 1} } }.should_not change{`ps`.split("\n").size} end it "returns results as array" do Parallel.in_threads(4){|i| "XXX#{i}"}.should == ["XXX0",'XXX1','XXX2','XXX3'] end it "raises when a thread raises" do lambda{ Parallel.in_threads(2){|i| raise "TEST"} }.should raise_error("TEST") end end describe ".map" do it "saves time" do time_taken{ `ruby spec/cases/parallel_map_sleeping.rb` }.should <= 3.5 end it "executes with given parameters" do `ruby spec/cases/parallel_map.rb`.should == "-a- -b- -c- -d-" end it "can dump/load complex objects" do `ruby spec/cases/parallel_map_complex_objects.rb`.should == "YES" end it "starts new process imediatly when old exists" do time_taken{ `ruby spec/cases/parallel_map_uneven.rb` }.should <= 3.5 end it "does not flatten results" do Parallel.map([1,2,3], :in_threads=>2){|x| [x,x]}.should == [[1,1],[2,2],[3,3]] end it "can run in threads" do Parallel.map([1,2,3,4,5,6,7,8,9], :in_threads=>4){|x| x+2 }.should == [3,4,5,6,7,8,9,10,11] end it 'supports all Enumerable-s' do `ruby spec/cases/parallel_map_range.rb`.should == '[1, 2, 3, 4, 5]' end it 'handles nested arrays and nil correctly' do `ruby spec/cases/map_with_nested_arrays_and_nil.rb`.should == '[nil, [2, 2], [[3], [3]]]' end worker_types.each do |type| it "stops all workers when one fails in #{type}" do `METHOD=map WORKER_TYPE=#{type} ruby spec/cases/with_exception.rb 2>&1`.should =~ /^\d{4} raised$/ end it "stops all workers when one raises Break in #{type}" do `METHOD=map WORKER_TYPE=#{type} ruby spec/cases/with_break.rb 2>&1`.should =~ /^\d{4} Parallel::Break raised - result nil$/ end it "stops all workers when a start hook fails with #{type}" do `METHOD=map WORKER_TYPE=#{type} ruby spec/cases/with_exception_in_start.rb 2>&1`.should =~ /^\d{3} raised$/ end it "stops all workers when a finish hook fails with #{type}" do `METHOD=map WORKER_TYPE=#{type} ruby spec/cases/with_exception_in_finish.rb 2>&1`.should =~ /^\d{4} raised$/ end it "does not call the finish hook when a worker fails with #{type}" do `METHOD=map WORKER_TYPE=#{type} ruby spec/cases/with_exception_before_finish.rb 2>&1`.should == '3 called' end it "does not call the finish hook when a worker raises Break in #{type}" do `METHOD=map WORKER_TYPE=#{type} ruby spec/cases/with_break_before_finish.rb 2>&1`.should =~ /^\d{3}(finish hook called){3} Parallel::Break raised$/ end it "does not call the finish hook when a start hook fails with #{type}" do `METHOD=map WORKER_TYPE=#{type} ruby spec/cases/with_exception_in_start_before_finish.rb 2>&1`.should == '3 called' end it "sets Parallel.worker_number with 4 #{type}" do out = `METHOD=map WORKER_TYPE=#{type} ruby spec/cases/with_worker_number.rb 2>&1` out.should =~ /\A[0123]+\z/ %w(0 1 2 3).each { |number| out.should include number } end it "sets Parallel.worker_number with 0 #{type}" do type_key = "in_#{type}".to_sym Parallel.map([1,2,3,4,5,6,7,8,9], type_key => 0) { |x| Parallel.worker_number }.uniq.should == [0] Parallel.worker_number.should be_nil end end it "can run with 0 threads" do Thread.should_not_receive(:exclusive) Parallel.map([1,2,3,4,5,6,7,8,9], :in_threads => 0){|x| x+2 }.should == [3,4,5,6,7,8,9,10,11] end it "can run with 0 processes" do Process.should_not_receive(:fork) Parallel.map([1,2,3,4,5,6,7,8,9], :in_processes => 0){|x| x+2 }.should == [3,4,5,6,7,8,9,10,11] end it "notifies when an item of work is dispatched to a worker process" do monitor = double('monitor', :call => nil) monitor.should_receive(:call).once.with(:first, 0) monitor.should_receive(:call).once.with(:second, 1) monitor.should_receive(:call).once.with(:third, 2) Parallel.map([:first, :second, :third], :start => monitor, :in_processes => 3) {} end it "notifies when an item of work is dispatched with 0 processes" do monitor = double('monitor', :call => nil) monitor.should_receive(:call).once.with(:first, 0) monitor.should_receive(:call).once.with(:second, 1) monitor.should_receive(:call).once.with(:third, 2) Parallel.map([:first, :second, :third], :start => monitor, :in_processes => 0) {} end it "notifies when an item of work is completed by a worker process" do monitor = double('monitor', :call => nil) monitor.should_receive(:call).once.with(:first, 0, 123) monitor.should_receive(:call).once.with(:second, 1, 123) monitor.should_receive(:call).once.with(:third, 2, 123) Parallel.map([:first, :second, :third], :finish => monitor, :in_processes => 3) { 123 } end it "notifies when an item of work is completed with 0 processes" do monitor = double('monitor', :call => nil) monitor.should_receive(:call).once.with(:first, 0, 123) monitor.should_receive(:call).once.with(:second, 1, 123) monitor.should_receive(:call).once.with(:third, 2, 123) Parallel.map([:first, :second, :third], :finish => monitor, :in_processes => 0) { 123 } end it "notifies when an item of work is dispatched to a threaded worker" do monitor = double('monitor', :call => nil) monitor.should_receive(:call).once.with(:first, 0) monitor.should_receive(:call).once.with(:second, 1) monitor.should_receive(:call).once.with(:third, 2) Parallel.map([:first, :second, :third], :start => monitor, :in_threads => 3) {} end it "notifies when an item of work is dispatched with 0 threads" do monitor = double('monitor', :call => nil) monitor.should_receive(:call).once.with(:first, 0) monitor.should_receive(:call).once.with(:second, 1) monitor.should_receive(:call).once.with(:third, 2) Parallel.map([:first, :second, :third], :start => monitor, :in_threads => 0) {} end it "notifies when an item of work is completed by a threaded worker" do monitor = double('monitor', :call => nil) monitor.should_receive(:call).once.with(:first, 0, 123) monitor.should_receive(:call).once.with(:second, 1, 123) monitor.should_receive(:call).once.with(:third, 2, 123) Parallel.map([:first, :second, :third], :finish => monitor, :in_threads => 3) { 123 } end it "notifies when an item of work is completed with 0 threads" do monitor = double('monitor', :call => nil) monitor.should_receive(:call).once.with(:first, 0, 123) monitor.should_receive(:call).once.with(:second, 1, 123) monitor.should_receive(:call).once.with(:third, 2, 123) Parallel.map([:first, :second, :third], :finish => monitor, :in_threads => 0) { 123 } end it "spits out a useful error when a worker dies before read" do `ruby spec/cases/map_with_killed_worker_before_read.rb 2>&1`.should include "DEAD" end it "spits out a useful error when a worker dies before write" do `ruby spec/cases/map_with_killed_worker_before_write.rb 2>&1`.should include "DEAD" end it "raises DeadWorker when using exit so people learn to not kill workers and do not crash main process" do `ruby spec/cases/exit_in_process.rb 2>&1`.should include "Yep, DEAD" end it 'raises EOF (not DeadWorker) when a worker raises EOF in process' do `ruby spec/cases/eof_in_process.rb 2>&1`.should include 'Yep, EOF' end it "can be killed instantly" do result = `ruby spec/cases/parallel_kill.rb 2>&1` result.should == "DEAD\nWorks nil\n" end it "synchronizes :start and :finish" do out = `ruby spec/cases/synchronizes_start_and_finish.rb` %w{a b c}.each {|letter| out.sub! letter.downcase * 10, 'OK' out.sub! letter.upcase * 10, 'OK' } out.should == "OK\n" * 6 end it 'is equivalent to serial map' do l = Array.new(10_000){|i| i} Parallel.map(l, {in_threads: 4}){|x| x+1}.should == l.map{|x| x+1} end it 'can work in isolation' do skip if ENV['TRAVIS'] # this randomly hangs on travis out = `ruby spec/cases/map_isolation.rb` out.should == "1\n2\n3\n4\nOK" end it 'sets Parallel.worker_number when run with isolation' do skip if ENV['TRAVIS'] # this randomly hangs on travis out = `ruby spec/cases/map_worker_number_isolation.rb` out.should == "0,1\nOK" end end describe ".map_with_index" do it "yields object and index" do `ruby spec/cases/map_with_index.rb 2>&1`.should == 'a0b1' end it "does not crash with empty set" do `ruby spec/cases/map_with_index_empty.rb 2>&1`.should == '' end it "can run with 0 threads" do Thread.should_not_receive(:exclusive) Parallel.map_with_index([1,2,3,4,5,6,7,8,9], :in_threads => 0){|x,i| x+2 }.should == [3,4,5,6,7,8,9,10,11] end it "can run with 0 processes" do Process.should_not_receive(:fork) Parallel.map_with_index([1,2,3,4,5,6,7,8,9], :in_processes => 0){|x,i| x+2 }.should == [3,4,5,6,7,8,9,10,11] end end describe ".any?" do it "returns true if any result is truthy" do `ruby spec/cases/any_true.rb`.split(',').should == ['true'] * 3 * 2 end it "returns false if all results are falsy" do `ruby spec/cases/any_false.rb`.split(',').should == ['false'] * 3 * 3 end end describe ".all?" do it "returns true if all results are truthy" do `ruby spec/cases/all_true.rb`.split(',').should == ['true'] * 3 * 3 end it "returns false if any result is falsy" do `ruby spec/cases/all_false.rb`.split(',').should == ['false'] * 3 * 2 end end describe ".each" do it "returns original array, works like map" do `ruby spec/cases/each.rb`.should == 'a b c d' end it "passes result to :finish callback :in_processes`" do monitor = double('monitor', :call => nil) monitor.should_receive(:call).once.with(:first, 0, 123) monitor.should_receive(:call).once.with(:second, 1, 123) monitor.should_receive(:call).once.with(:third, 2, 123) Parallel.each([:first, :second, :third], :finish => monitor, :in_processes => 3) { 123 } end it "passes result to :finish callback :in_threads`" do monitor = double('monitor', :call => nil) monitor.should_receive(:call).once.with(:first, 0, 123) monitor.should_receive(:call).once.with(:second, 1, 123) monitor.should_receive(:call).once.with(:third, 2, 123) Parallel.each([:first, :second, :third], :finish => monitor, :in_threads => 3) { 123 } end it "does not use marshal_dump" do `ruby spec/cases/no_dump_with_each.rb 2>&1`.should == 'no dump for resultno dump for each' end it "does not slow down with lots of GC work in threads" do Benchmark.realtime { `ruby spec/cases/no_gc_with_each.rb 2>&1` }.should <= (ENV["TRAVIS"] ? 15 : 10) end it "can modify in-place" do `ruby spec/cases/each_in_place.rb`.should == 'ab' end worker_types.each do |type| it "works with SQLite in #{type}" do `WORKER_TYPE=#{type} ruby spec/cases/each_with_ar_sqlite.rb 2>&1`.should == "Parent: X\nParallel (in_#{type}): XXX\nParent: X\n" end it "stops all workers when one fails in #{type}" do `METHOD=each WORKER_TYPE=#{type} ruby spec/cases/with_exception.rb 2>&1`.should =~ /^\d{4} raised$/ end it "stops all workers when one raises Break in #{type}" do `METHOD=each WORKER_TYPE=#{type} ruby spec/cases/with_break.rb 2>&1`.should =~ /^\d{4} Parallel::Break raised - result nil$/ end it "stops all workers when a start hook fails with #{type}" do `METHOD=each WORKER_TYPE=#{type} ruby spec/cases/with_exception_in_start.rb 2>&1`.should =~ /^\d{3} raised$/ end it 'stops all workers when a finish hook fails with processes' do `METHOD=each WORKER_TYPE=#{type} ruby spec/cases/with_exception_in_finish.rb 2>&1`.should =~ /^\d{4} raised$/ end it "does not call the finish hook when a worker fails with #{type}" do `METHOD=each WORKER_TYPE=#{type} ruby spec/cases/with_exception_before_finish.rb 2>&1`.should == '3 called' end it "does not call the finish hook when a worker raises Break in #{type}" do `METHOD=each WORKER_TYPE=#{type} ruby spec/cases/with_break_before_finish.rb 2>&1`.should =~ /^\d{3}(finish hook called){3} Parallel::Break raised$/ end it "does not call the finish hook when a start hook fails with #{type}" do `METHOD=each WORKER_TYPE=#{type} ruby spec/cases/with_exception_in_start_before_finish.rb 2>&1`.should == '3 called' end it "sets Parallel.worker_number with #{type}" do out = `METHOD=each WORKER_TYPE=#{type} ruby spec/cases/with_worker_number.rb 2>&1` out.should =~ /\A[0123]+\z/ %w(0 1 2 3).each { |number| out.should include number } end end it "re-raises exceptions in work_direct" do `METHOD=each WORKER_TYPE=threads WORKER_SIZE=0 ruby spec/cases/with_exception.rb 2>&1` .should =~ /^1 raised$/ end it "handles Break in work_direct" do `METHOD=each WORKER_TYPE=threads WORKER_SIZE=0 ruby spec/cases/with_break.rb 2>&1` .should =~ /^1 Parallel::Break raised - result nil$/ end end describe ".each_with_index" do it "yields object and index" do ["a0b1", "b1a0"].should include `ruby spec/cases/each_with_index.rb 2>&1` end end describe "progress" do it "takes the title from :progress" do `ruby spec/cases/progress.rb 2>&1`.sub(/=+/, '==').strip.should == "Doing stuff: |==|" end it "takes true from :progress" do `TITLE=true ruby spec/cases/progress.rb 2>&1`.sub(/=+/, '==').strip.should == "Progress: |==|" end it "works with :finish" do `ruby spec/cases/progress_with_finish.rb 2>&1`.strip.sub(/=+/, '==').gsub(/\n+/,"\n").should == "Doing stuff: |==|\n100" end it "takes the title from :progress[:title] and passes options along" do `ruby spec/cases/progress_with_options.rb 2>&1`.should =~ /Reticulating Splines ;+ \d+ ;+/ end end ["lambda", "queue"].each do |thing| describe "lambdas" do let(:result) { "ITEM-1\nITEM-2\nITEM-3\n" } it "runs in threads" do `ruby spec/cases/with_#{thing}.rb THREADS 2>&1`.should == result end it "runs in processs" do `ruby spec/cases/with_#{thing}.rb PROCESSES 2>&1`.should == result end it "refuses to use progress" do lambda { Parallel.map(lambda{}, :progress => "xxx"){ raise "Ooops" } }.should raise_error("Progressbar can only be used with array like items") end end end it "fails when running with a prefilled queue without stop since there are no threads to fill it" do error = (RUBY_VERSION >= "2.0.0" ? "No live threads left. Deadlock?" : "deadlock detected (fatal)") `ruby spec/cases/fatal_queue.rb 2>&1`.should include error end describe "GC" do def normalize(result) result.sub(/\{(.*)\}/, "\\1").split(", ").reject { |x| x =~ /^(Hash|Array|String)=>(1|-1|-2)$/ } end worker_types.each do |type| it "does not leak memory in #{type}" do pending if RUBY_ENGINE == 'jruby' # lots of objects ... GC does not seem to work ... result = `ruby #{"-X+O" if RUBY_ENGINE == 'jruby'} spec/cases/profile_memory.rb #{type} 2>&1`.strip.split("\n").last normalize(result).should == [] end end end end parallel-1.12.0/spec/spec_helper.rb000066400000000000000000000004041313554143200171360ustar00rootroot00000000000000require 'parallel' require 'benchmark' require 'timeout' RSpec.configure do |config| config.expect_with(:rspec) { |c| c.syntax = :should } config.mock_with(:rspec) { |c| c.syntax = :should } config.around { |example| Timeout.timeout(30, &example) } end