pax_global_header00006660000000000000000000000064127554450140014521gustar00rootroot0000000000000052 comment=e86b9972c7e55554f9e2765367b7c66ca810282b benchmark-ips-2.7.2/000077500000000000000000000000001275544501400142545ustar00rootroot00000000000000benchmark-ips-2.7.2/.autotest000066400000000000000000000007401275544501400161260ustar00rootroot00000000000000# -*- ruby -*- require 'autotest/restart' # Autotest.add_hook :initialize do |at| # at.extra_files << "../some/external/dependency.rb" # # at.libs << ":../some/external" # # at.add_exception 'vendor' # # at.add_mapping(/dependency.rb/) do |f, _| # at.files_matching(/test_.*rb$/) # end # # %w(TestA TestB).each do |klass| # at.extra_class_map[klass] = "test/test_misc.rb" # end # end # Autotest.add_hook :run_command do |at| # system "rake build" # end benchmark-ips-2.7.2/.gitignore000066400000000000000000000000351275544501400162420ustar00rootroot00000000000000log tmp pkg doc Gemfile.lock benchmark-ips-2.7.2/.hoeignore000066400000000000000000000001041275544501400162270ustar00rootroot00000000000000.git/ .travis.yml .hoeignore .gitignore Gemfile *.gemspec examples/ benchmark-ips-2.7.2/.travis.yml000066400000000000000000000001521275544501400163630ustar00rootroot00000000000000language: ruby sudo: false rvm: - 1.9.2 - 1.9.3 - 2.0.0 - 2.1.0 - 2.1.2 - 2.2.2 - ruby-head benchmark-ips-2.7.2/Gemfile000066400000000000000000000002241275544501400155450ustar00rootroot00000000000000source 'https://rubygems.org' gem 'rake', '~> 10.5' gem 'hoe', '~> 3.14' gem 'minitest', :group => :test if RUBY_VERSION < "1.9" gem 'json' end benchmark-ips-2.7.2/History.txt000066400000000000000000000107171275544501400164640ustar00rootroot00000000000000=== 2.7.2 / 2016-08-18 * 1 bug fix: * Restore old accessors. Fixes #76 === 2.7.1 / 2016-08-08 Add missing files === 2.7.0 / 2016-08-05 * 1 minor features: * Add support for confidence intervals * 1 bug fixes: * Cleanup a few coding patterns * 2 doc fixes: * Add infos about benchark.fyi to Readme * Remove ancient releases * 3 merged PRs: * Merge pull request #65 from kbrock/fixup_inject * Merge pull request #67 from benoittgt/master * Merge pull request #69 from chrisseaton/kalibera-confidence-intervals === MISSING 2.6.0 and 2.6.1 === 2.5.0 / 2016-02-14 * 1 minor feature: * Add iterations option. * 1 bug fixes: * Don't tell people something is slower if it's within the error. * 2 merged PRs: * Merge pull request #58 from chrisseaton/iterations * Merge pull request #60 from chrisseaton/significance === 2.4.1 / 2016-02-12 * 1 bug fix: * Add missing files to gem === 2.4.0 / 2016-02-12 * 1 minor features * Add support for hold! and independent invocations. * 6 bug fixes * Separate messages for warming up and calculating. * Tighten timing loop. * Pass simple types into Job#create_report * More concise sorting * Fix runtime comparison * Use runtime if ips is not available * 5 doc fixes * Fix typo unsed --> used * Better document Report::Entry * Fix some typos in docs * Don't calculate mean 2 times * Add more tolerance to tests * 13 merged PRs * Merge pull request #44 from kbrock/job_extract * Merge pull request #45 from kbrock/runtime_only * Merge pull request #47 from kbrock/use_avg * Merge pull request #46 from kbrock/report_stdout * Merge pull request #48 from bquorning/fix-label-for-runtime-comparison * Merge pull request #50 from tjschuck/fix_typo * Merge pull request #51 from bquorning/all-reports-respond-to-ips * Merge pull request #52 from kbrock/document_reports * Merge pull request #53 from kbrock/interface_create_report * Merge pull request #54 from PragTob/patch-2 * Merge pull request #55 from chrisseaton/messages * Merge pull request #56 from chrisseaton/independence * Merge pull request #57 from chrisseaton/tighten-loop === 2.3.0 / 2015-07-20 * 2 minor features: * Support keyword arguments * Allow any datatype for labels (use #to_s conversion) * 1 doc/test changes: * Newer Travis for 1.8.7, ree, and 2.2.2 * 3 PRs merged: * Merge pull request #41 from kbrock/kwargs-support * Merge pull request #42 from kbrock/newer_travis * Merge pull request #43 from kbrock/non_to_s_labels === 2.2.0 / 2015-05-09 * 1 minor features: * Fix quiet mode * Allow passing a custom suite via config * Silent a job if a suite was passed and is quiet * Export report to json file. * Accept symbol as report's argument. * 2 doc fixes: * Squish duplicate `to` in README * Update copyright to 2015. [ci skip] * 9 PRs merged: * Merge pull request #37 from splattael/patch-1 * Merge pull request #36 from kirs/quiet-mode * Merge pull request #35 from JuanitoFatas/doc/suite * Merge pull request #34 from splattael/config-suite * Merge pull request #33 from splattael/suite-quiet * Merge pull request #32 from O-I/remove-gemfile-lock * Merge pull request #31 from JuanitoFatas/doc/bump-copyright-year * Merge pull request #29 from JuanitoFatas/feature/json-export * Merge pull request #26 from JuanitoFatas/feature/takes-symbol-as-report-parameter === 2.1.1 / 2015-01-12 * 1 minor fix: * Don't send label through printf so that % work directly * 1 documenation changes: * Use HEREDOC and wrap at 80 chars for example result description * 1 usage fix: * Add gemspec for use via bundler git * 1 PR merged: * Merge pull request #24 from zzak/simple-format-result-description === 2.1.0 / 2014-11-10 * Documentation changes: * Many documentation fixes by Juanito Fatas! * Minor readme fix by Will Leinweber * 2 minor features: * Displaying the total runtime for a job is suppressed unless interesting * Formatting of large values improved (human vs raw mode) * Contributed by Charles Oliver Nutter === 2.0.0 / 2014-06-18 * The 'Davy Stevenson' release! * Codename: Springtime Hummingbird Dance * Big API refactoring so the internal bits are easier to use * Bump to 2.0 because return types changed to make the API better * Contributors added: * Davy Stevenson * Juanito Fatas * Benoit Daloze * Matias * Tony Arcieri * Vipul A M * Zachary Scott * schneems (Richard Schneeman) === 1.0.0 / 2012-03-23 * 1 major enhancement * Birthday! benchmark-ips-2.7.2/Manifest.txt000066400000000000000000000005751275544501400165720ustar00rootroot00000000000000.autotest Gemfile.lock History.txt Manifest.txt README.md Rakefile lib/benchmark/compare.rb lib/benchmark/ips.rb lib/benchmark/ips/job.rb lib/benchmark/ips/job/entry.rb lib/benchmark/ips/job/stdout_report.rb lib/benchmark/ips/report.rb lib/benchmark/ips/share.rb lib/benchmark/ips/stats/bootstrap.rb lib/benchmark/ips/stats/sd.rb lib/benchmark/timing.rb test/test_benchmark_ips.rb benchmark-ips-2.7.2/README.md000066400000000000000000000200171275544501400155330ustar00rootroot00000000000000[![Gem Version](https://badge.fury.io/rb/benchmark-ips.svg)](http://badge.fury.io/rb/benchmark-ips) [![Build Status](https://secure.travis-ci.org/evanphx/benchmark-ips.svg)](http://travis-ci.org/evanphx/benchmark-ips) [![Inline docs](http://inch-ci.org/github/evanphx/benchmark-ips.svg)](http://inch-ci.org/github/evanphx/benchmark-ips) # benchmark-ips * https://github.com/evanphx/benchmark-ips * [documentation](http://rubydoc.info/gems/benchmark-ips) ## DESCRIPTION: An iterations per second enhancement to Benchmark. ## FEATURES/PROBLEMS: * benchmark/ips - benchmarks a blocks iterations/second. For short snippits of code, ips automatically figures out how many times to run the code to get interesting data. No more guessing at random iteration counts! ## SYNOPSIS: ```ruby require 'benchmark/ips' Benchmark.ips do |x| # Configure the number of seconds used during # the warmup phase (default 2) and calculation phase (default 5) x.config(:time => 5, :warmup => 2) # These parameters can also be configured this way x.time = 5 x.warmup = 2 # Typical mode, runs the block as many times as it can x.report("addition") { 1 + 2 } # To reduce overhead, the number of iterations is passed in # and the block must run the code the specific number of times. # Used for when the workload is very small and any overhead # introduces incorrectable errors. x.report("addition2") do |times| i = 0 while i < times 1 + 2 i += 1 end end # To reduce overhead even more, grafts the code given into # the loop that performs the iterations internally to reduce # overhead. Typically not needed, use the |times| form instead. x.report("addition3", "1 + 2") # Really long labels should be formatted correctly x.report("addition-test-long-label") { 1 + 2 } # Compare the iterations per second of the various reports! x.compare! end ``` This will generate the following report: ``` Calculating ------------------------------------- addition 71.254k i/100ms addition2 68.658k i/100ms addition3 83.079k i/100ms addition-test-long-label 70.129k i/100ms ------------------------------------------------- addition 4.955M (± 8.7%) i/s - 24.155M addition2 24.011M (± 9.5%) i/s - 114.246M addition3 23.958M (±10.1%) i/s - 115.064M addition-test-long-label 5.014M (± 9.1%) i/s - 24.545M Comparison: addition2: 24011974.8 i/s addition3: 23958619.8 i/s - 1.00x slower addition-test-long-label: 5014756.0 i/s - 4.79x slower addition: 4955278.9 i/s - 4.85x slower ``` Benchmark/ips will report the number of iterations per second for a given block of code. When analyzing the results, notice the percent of [standard deviation](http://en.wikipedia.org/wiki/Standard\_deviation) which tells us how spread out our measurements are from the average. A high standard deviation could indicate the results having too much variability. One benefit to using this method is benchmark-ips automatically determines the data points for testing our code, so we can focus on the results instead of guessing iteration counts as we do with the traditional Benchmark library. ### Custom Suite Pass a custom suite to disable garbage collection during benchmark: ```ruby require 'benchmark/ips' # Enable and start GC before each job run. Disable GC afterwards. # # Inspired by https://www.omniref.com/ruby/2.2.1/symbols/Benchmark/bm?#annotation=4095926&line=182 class GCSuite def warming(*) run_gc end def running(*) run_gc end def warmup_stats(*) end def add_report(*) end private def run_gc GC.enable GC.start GC.disable end end suite = GCSuite.new Benchmark.ips do |x| x.config(:suite => suite) x.report("job1") { ... } x.report("job2") { ... } end ``` ### Independent benchmarking If you are comparing multiple implementations of a piece of code you may want to benchmark them in separate invocations of Ruby so that the measurements are independent of each other. You can do this with the `hold!` command. ```ruby Benchmark.ips do |x| # Hold results between multiple invocations of Ruby x.hold! 'filename' end ``` This will run only one benchmarks each time you run the command, storing results in the specified file. The file is deleted when all results have been gathered and the report is shown. ### Multiple iterations In some cases you may want to run multiple iterations of the warmup and calculation stages and take only the last result for comparison. This is useful if you are benchmarking with an implementation of Ruby that optimizes using tracing or on-stack-replacement, because to those implementations the calculation phase may appear as new, unoptimized code. You can do this with the `iterations` option, which by default is `1`. The total time spent will then be `iterations * warmup + iterations * time` seconds. ```ruby Benchmark.ips do |x| x.config(:iterations => 3) # or x.iterations = 3 end ``` ### Online sharing If you want to share quickly your benchmark result with others. Run you benchmark with `SHARE=1` argument. I.e.: `SHARE=1 ruby my_benchmark.rb`. Result will be sent to [benchmark.fyi](https://benchmark.fyi/) and benchmark-ips will display the link to share the benchmark's result. ### Advanced Statistics By default, the margin of error shown is plus-minus one standard deviation. If a more advanced statistical test is wanted, a bootstrap confidence interval can be calculated instead. A bootstrap confidence interval has the advantages of arguably being more mathematically sound for this application than a standard deviation, it additionally produces an error for relative slowdowns, which the standard deviation does not, and it is arguably more intuitive and actionable. When a bootstrap confidence interval is used, a median of the interval is used rather than the mean of the samples, which is what you get with the default standard deviation. The bootstrap confidence interval used is the one described by Tomas Kalibera. Note that for this technique to be valid your benchmark should have reached a non-periodic steady state with statistically independent samples (it should have warmed up) by the time measurements start. Using a bootstrap confidence internal requires that the 'kalibera' gem is installed separately. This gem is not a formal dependency, as by default it is not needed. ``` gem install kalibera ``` ```ruby Benchmark.ips do |x| # The default is :stats => :sd, which doesn't have a configurable confidence x.config(:stats => :bootstrap, :confidence => 95) # or x.stats = :bootstrap x.confidence = 95 # confidence is 95% by default, so it can be omitted end ``` ## REQUIREMENTS: * None! ## INSTALL: $ gem install benchmark-ips ## DEVELOPERS: After checking out the source, run: $ rake newb This task will install any missing dependencies, run the tests/specs, and generate the RDoc. ## LICENSE: (The MIT License) Copyright (c) 2015 Evan Phoenix Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the 'Software'), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. benchmark-ips-2.7.2/Rakefile000066400000000000000000000010461275544501400157220ustar00rootroot00000000000000# -*- ruby -*- require 'rubygems' require 'hoe' Hoe.plugin :minitest Hoe.plugin :git Hoe.plugin :ignore hoe = Hoe.spec 'benchmark-ips' do developer('Evan Phoenix', 'evan@phx.io') self.readme_file = 'README.md' license "MIT" end file "#{hoe.spec.name}.gemspec" => ['Rakefile', "lib/benchmark/ips.rb"] do |t| puts "Generating #{t.name}" File.open(t.name, 'wb') { |f| f.write hoe.spec.to_ruby } end desc "Generate or update the standalone gemspec file for the project" task :gemspec => ["#{hoe.spec.name}.gemspec"] # vim: syntax=ruby benchmark-ips-2.7.2/benchmark-ips.gemspec000066400000000000000000000034301275544501400203440ustar00rootroot00000000000000# -*- encoding: utf-8 -*- # stub: benchmark-ips 2.1.0 ruby lib d = File.read(File.expand_path("../lib/benchmark/ips.rb", __FILE__)) if d =~ /VERSION = "(\d+\.\d+\.\d+)"/ version = $1 else version = "0.0.1" end Gem::Specification.new do |s| s.name = "benchmark-ips" s.version = version s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version= s.require_paths = ["lib"] s.authors = ["Evan Phoenix"] s.date = "2015-01-12" s.description = "A iterations per second enhancement to Benchmark." s.email = ["evan@phx.io"] s.extra_rdoc_files = ["History.txt", "Manifest.txt", "README.md"] s.files = [".autotest", ".gemtest", "History.txt", "Manifest.txt", "README.md", "Rakefile", "lib/benchmark/compare.rb", "lib/benchmark/ips.rb", "lib/benchmark/ips/job.rb", "lib/benchmark/ips/report.rb", "lib/benchmark/timing.rb", "test/test_benchmark_ips.rb"] s.homepage = "https://github.com/evanphx/benchmark-ips" s.licenses = ["MIT"] s.rdoc_options = ["--main", "README.md"] s.rubygems_version = "2.2.2" s.summary = "A iterations per second enhancement to Benchmark." s.test_files = ["test/test_benchmark_ips.rb"] if s.respond_to? :specification_version then s.specification_version = 4 if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then s.add_development_dependency(%q, ["~> 5.4"]) s.add_development_dependency(%q, ["~> 4.0"]) s.add_development_dependency(%q, ["~> 3.13"]) else s.add_dependency(%q, ["~> 5.4"]) s.add_dependency(%q, ["~> 4.0"]) s.add_dependency(%q, ["~> 3.13"]) end else s.add_dependency(%q, ["~> 5.4"]) s.add_dependency(%q, ["~> 4.0"]) s.add_dependency(%q, ["~> 3.13"]) end end benchmark-ips-2.7.2/examples/000077500000000000000000000000001275544501400160725ustar00rootroot00000000000000benchmark-ips-2.7.2/examples/advanced.rb000066400000000000000000000005271275544501400201700ustar00rootroot00000000000000#!/usr/bin/env ruby require 'benchmark/ips' Benchmark.ips do |x| # Use bootstrap confidence intervals x.stats = :bootstrap # Set confidence to 95% x.confidence = 95 # Run multiple iterations for better warmup x.iterations = 3 x.report("mul") { 2 * 2 * 2 * 2 * 2 * 2 * 2 * 2 } x.report("pow") { 2 ** 8 } x.compare! end benchmark-ips-2.7.2/examples/simple.rb000077500000000000000000000026351275544501400177210ustar00rootroot00000000000000#!/usr/bin/env ruby require 'benchmark/ips' Benchmark.ips do |x| # Configure the number of seconds used during # the warmup phase and calculation phase x.config(:time => 5, :warmup => 2) # These parameters can also be configured this way x.time = 5 x.warmup = 2 # Typical mode, runs the block as many times as it can x.report("addition") { 1 + 2 } # To reduce overhead, the number of iterations is passed in # and the block must run the code the specific number of times. # Used for when the workload is very small and any overhead # introduces incorrectable errors. x.report(:addition2) do |times| i = 0 while i < times 1 + 2 i += 1 end end # To reduce overhead even more, grafts the code given into # the loop that performs the iterations internally to reduce # overhead. Typically not needed, use the |times| form instead. x.report("addition3", "1 + 2") # Really long labels should be formatted correctly x.report("addition-test-long-label") { 1 + 2 } x.compare! end puts <<-EOD Typical results will show addition2 & addition3 to be the most performant, and they should perform reasonably similarly. You should see addition and addition-test-long-label to perform very similarly to each other (as they are running the same test, just with different labels), and they should both run in the neighborhood of 3.5 times slower than addition2 and addition3." EOD benchmark-ips-2.7.2/lib/000077500000000000000000000000001275544501400150225ustar00rootroot00000000000000benchmark-ips-2.7.2/lib/benchmark/000077500000000000000000000000001275544501400167545ustar00rootroot00000000000000benchmark-ips-2.7.2/lib/benchmark/compare.rb000066400000000000000000000043751275544501400207400ustar00rootroot00000000000000# encoding: utf-8 module Benchmark # Functionality of performaing comparison between reports. # # Usage: # # Add +x.compare!+ to perform comparison between reports. # # Example: # > Benchmark.ips do |x| # x.report('Reduce using tag') { [*1..10].reduce(:+) } # x.report('Reduce using to_proc') { [*1..10].reduce(&:+) } # x.compare! # end # # Calculating ------------------------------------- # Reduce using tag 19216 i/100ms # Reduce using to_proc 17437 i/100ms # ------------------------------------------------- # Reduce using tag 278950.0 (±8.5%) i/s - 1402768 in 5.065112s # Reduce using to_proc 247295.4 (±8.0%) i/s - 1238027 in 5.037299s # # Comparison: # Reduce using tag: 278950.0 i/s # Reduce using to_proc: 247295.4 i/s - 1.13x slower # # Besides regular Calculating report, this will also indicates which one is slower. module Compare # Compare between reports, prints out facts of each report: # runtime, comparative speed difference. # @param entries [Array] Reports to compare. def compare(*entries) return if entries.size < 2 sorted = entries.sort_by{ |e| e.stats.central_tendency }.reverse best = sorted.shift $stdout.puts "\nComparison:" $stdout.printf "%20s: %10.1f i/s\n", best.label, best.stats.central_tendency sorted.each do |report| name = report.label.to_s $stdout.printf "%20s: %10.1f i/s - ", name, report.stats.central_tendency best_low = best.stats.central_tendency - best.stats.error report_high = report.stats.central_tendency + report.stats.error overlaps = report_high > best_low if overlaps $stdout.print "same-ish: difference falls within error" else slowdown, error = report.stats.slowdown(best.stats) $stdout.printf "%.2fx ", slowdown if error $stdout.printf " (± %.2f)", error end $stdout.print " slower" end $stdout.puts end footer = best.stats.footer $stdout.puts footer.rjust(40) if footer $stdout.puts end end extend Benchmark::Compare end benchmark-ips-2.7.2/lib/benchmark/ips.rb000066400000000000000000000052441275544501400201010ustar00rootroot00000000000000# encoding: utf-8 require 'benchmark/timing' require 'benchmark/compare' require 'benchmark/ips/stats/sd' require 'benchmark/ips/stats/bootstrap' require 'benchmark/ips/report' require 'benchmark/ips/job/entry' require 'benchmark/ips/job/stdout_report' require 'benchmark/ips/job' # Performance benchmarking library module Benchmark # Benchmark in iterations per second, no more guessing! # @see https://github.com/evanphx/benchmark-ips module IPS # Benchmark-ips Gem version. VERSION = "2.7.2" # CODENAME of current version. CODENAME = "Cultivating Confidence" # Measure code in block, each code's benchmarked result will display in # iteration per second with standard deviation in given time. # @param time [Integer] Specify how long should benchmark your code in seconds. # @param warmup [Integer] Specify how long should Warmup time run in seconds. # @return [Report] def ips(*args) if args[0].is_a?(Hash) time, warmup, quiet = args[0].values_at(:time, :warmup, :quiet) else time, warmup, quiet = args end suite = nil sync, $stdout.sync = $stdout.sync, true if defined? Benchmark::Suite and Suite.current suite = Benchmark::Suite.current end quiet ||= (suite && suite.quiet?) job = Job.new({:suite => suite, :quiet => quiet }) job_opts = {} job_opts[:time] = time unless time.nil? job_opts[:warmup] = warmup unless warmup.nil? job.config job_opts yield job job.load_held_results if job.hold? && job.held_results? job.run $stdout.sync = sync job.run_comparison job.generate_json report = job.full_report if ENV['SHARE'] || ENV['SHARE_URL'] require 'benchmark/ips/share' share = Share.new report, job share.share end report end # Set options for running the benchmarks. # :format => [:human, :raw] # :human format narrows precision and scales results for readability # :raw format displays 6 places of precision and exact iteration counts def self.options @options ||= {:format => :human} end module Helpers def scale(value) scale = (Math.log10(value) / 3).to_i suffix = case scale when 1; 'k' when 2; 'M' when 3; 'B' when 4; 'T' when 5; 'Q' else # < 1000 or > 10^15, no scale or suffix scale = 0 ' ' end "%10.3f#{suffix}" % (value.to_f / (1000 ** scale)) end module_function :scale end end extend Benchmark::IPS # make ips available as module-level method end benchmark-ips-2.7.2/lib/benchmark/ips/000077500000000000000000000000001275544501400175475ustar00rootroot00000000000000benchmark-ips-2.7.2/lib/benchmark/ips/job.rb000066400000000000000000000243761275544501400206620ustar00rootroot00000000000000module Benchmark module IPS # Benchmark jobs. class Job # Microseconds per 100 millisecond. MICROSECONDS_PER_100MS = 100_000 # Microseconds per second. MICROSECONDS_PER_SECOND = Timing::MICROSECONDS_PER_SECOND # The percentage of the expected runtime to allow # before reporting a weird runtime MAX_TIME_SKEW = 0.05 # Two-element arrays, consisting of label and block pairs. # @return [Array] list of entries attr_reader :list # Determining whether to run comparison utility. # @return [Boolean] true if needs to run compare. attr_reader :compare # Determining whether to hold results between Ruby invocations # @return [Boolean] attr_accessor :hold # Report object containing information about the run. # @return [Report] the report object. attr_reader :full_report # Storing Iterations in time period. # @return [Hash] attr_reader :timing # Warmup time setter and getter (in seconds). # @return [Integer] attr_accessor :warmup # Calculation time setter and getter (in seconds). # @return [Integer] attr_accessor :time # Warmup and calculation iterations. # @return [Integer] attr_accessor :iterations # Statistics model. # @return [Object] attr_accessor :stats # Confidence. # @return [Integer] attr_accessor :confidence # Instantiate the Benchmark::IPS::Job. # @option opts [Benchmark::Suite] (nil) :suite Specify Benchmark::Suite. # @option opts [Boolean] (false) :quiet Suppress the printing of information. def initialize opts={} @suite = opts[:suite] || nil @stdout = opts[:quiet] ? nil : StdoutReport.new @list = [] @compare = false @json_path = false @held_path = nil @held_results = nil @timing = {} @full_report = Report.new # Default warmup and calculation time in seconds. @warmup = 2 @time = 5 @iterations = 1 # Default statistical model @stats = :sd @confidence = 95 end # Job configuration options, set +@warmup+ and +@time+. # @option opts [Integer] :warmup Warmup time. # @option opts [Integer] :time Calculation time. # @option iterations [Integer] :time Warmup and calculation iterations. def config opts @warmup = opts[:warmup] if opts[:warmup] @time = opts[:time] if opts[:time] @suite = opts[:suite] if opts[:suite] @iterations = opts[:iterations] if opts[:iterations] @stats = opts[:stats] if opts[:stats] @confidence = opts[:confidence] if opts[:confidence] end # Return true if job needs to be compared. # @return [Boolean] Need to compare? def compare? @compare end # Set @compare to true. def compare! @compare = true end # Return true if results are held while multiple Ruby invocations # @return [Boolean] Need to hold results between multiple Ruby invocations? def hold? !!@held_path end # Set @hold to true. def hold!(held_path) @held_path = held_path end # Return true if job needs to generate json. # @return [Boolean] Need to generate json? def json? !!@json_path end # Set @json_path to given path, defaults to "data.json". def json!(path="data.json") @json_path = path end # Registers the given label and block pair in the job list. # @param label [String] Label of benchmarked code. # @param str [String] Code to be benchmarked. # @param blk [Proc] Code to be benchmarked. # @raise [ArgumentError] Raises if str and blk are both present. # @raise [ArgumentError] Raises if str and blk are both absent. def item(label="", str=nil, &blk) # :yield: if blk and str raise ArgumentError, "specify a block and a str, but not both" end action = str || blk raise ArgumentError, "no block or string" unless action @list.push Entry.new(label, action) self end alias_method :report, :item # Calculate the cycles needed to run for approx 100ms, # given the number of iterations to run the given time. # @param [Float] time_msec Each iteration's time in ms. # @param [Integer] iters Iterations. # @return [Integer] Cycles per 100ms. def cycles_per_100ms time_msec, iters cycles = ((MICROSECONDS_PER_100MS / time_msec) * iters).to_i cycles <= 0 ? 1 : cycles end # Calculate the time difference of before and after in microseconds. # @param [Time] before time. # @param [Time] after time. # @return [Float] Time difference of before and after. def time_us before, after (after.to_f - before.to_f) * MICROSECONDS_PER_SECOND end # Calculate the interations per second given the number # of cycles run and the time in microseconds that elapsed. # @param [Integer] cycles Cycles. # @param [Integer] time_us Time in microsecond. # @return [Float] Iteration per second. def iterations_per_sec cycles, time_us MICROSECONDS_PER_SECOND * (cycles.to_f / time_us.to_f) end def held_results? File.exist?(@held_path) end def load_held_results require "json" @held_results = Hash[File.open(@held_path).map { |line| result = JSON.parse(line) [result['item'], result] }] end def run @stdout.start_warming if @stdout @iterations.times do run_warmup end @stdout.start_running if @stdout held = nil @iterations.times do |n| held = run_benchmark end @stdout.footer if @stdout if held puts puts 'Pausing here -- run Ruby again to measure the next benchmark...' end end # Run warmup. def run_warmup @list.each do |item| next if hold? && @held_results && @held_results.key?(item.label) @suite.warming item.label, @warmup if @suite @stdout.warming item.label, @warmup if @stdout Timing.clean_env before = Timing.now target = Timing.add_second before, @warmup warmup_iter = 0 while Timing.now < target item.call_times(1) warmup_iter += 1 end after = Timing.now warmup_time_us = Timing.time_us(before, after) @timing[item] = cycles_per_100ms warmup_time_us, warmup_iter @stdout.warmup_stats warmup_time_us, @timing[item] if @stdout @suite.warmup_stats warmup_time_us, @timing[item] if @suite break if hold? end end # Run calculation. def run_benchmark @list.each do |item| if hold? && @held_results && @held_results.key?(item.label) result = @held_results[item.label] create_report(item.label, result['measured_us'], result['iter'], create_stats(result['samples']), result['cycles']) next end @suite.running item.label, @time if @suite @stdout.running item.label, @time if @stdout Timing.clean_env iter = 0 measurements_us = [] # Running this number of cycles should take around 100ms. cycles = @timing[item] target = Timing.add_second Timing.now, @time while (before = Timing.now) < target item.call_times cycles after = Timing.now # If for some reason the timing said this took no time (O_o) # then ignore the iteration entirely and start another. iter_us = Timing.time_us before, after next if iter_us <= 0.0 iter += cycles measurements_us << iter_us end final_time = before measured_us = measurements_us.inject(:+) samples = measurements_us.map { |time_us| iterations_per_sec cycles, time_us } rep = create_report(item.label, measured_us, iter, create_stats(samples), cycles) if (final_time - target).abs >= (@time.to_f * MAX_TIME_SKEW) rep.show_total_time! end @stdout.add_report rep, caller(1).first if @stdout @suite.add_report rep, caller(1).first if @suite if hold? && item != @list.last File.open @held_path, "a" do |f| require "json" f.write JSON.generate({ :item => item.label, :measured_us => measured_us, :iter => iter, :samples => samples, :cycles => cycles }) f.write "\n" end return true end end if hold? && @full_report.entries.size == @list.size File.delete @held_path if File.exist?(@held_path) end false end def create_stats(samples) case @stats when :sd Stats::SD.new(samples) when :bootstrap Stats::Bootstrap.new(samples, @confidence) else raise "unknown stats #{@stats}" end end # Run comparison of entries in +@full_report+. def run_comparison @full_report.run_comparison if compare? end # Generate json from +@full_report+. def generate_json @full_report.generate_json @json_path if json? end # Create report by add entry to +@full_report+. # @param label [String] Report item label. # @param measured_us [Integer] Measured time in microsecond. # @param iter [Integer] Iterations. # @param samples [Array] Sampled iterations per second. # @param cycles [Integer] Number of Cycles. # @return [Report::Entry] Entry with data. def create_report(label, measured_us, iter, samples, cycles) @full_report.add_entry label, measured_us, iter, samples, cycles end end end end benchmark-ips-2.7.2/lib/benchmark/ips/job/000077500000000000000000000000001275544501400203215ustar00rootroot00000000000000benchmark-ips-2.7.2/lib/benchmark/ips/job/entry.rb000066400000000000000000000041341275544501400220110ustar00rootroot00000000000000module Benchmark module IPS # Benchmark jobs. class Job # Entries in Benchmark Jobs. class Entry # Instantiate the Benchmark::IPS::Job::Entry. # @param label [#to_s] Label of Benchmarked code. # @param action [String, Proc] Code to be benchmarked. # @raise [ArgumentError] Raises when action is not String or not responding to +call+. def initialize(label, action) @label = label if action.kind_of? String compile action @action = self @as_action = true else unless action.respond_to? :call raise ArgumentError, "invalid action, must respond to #call" end @action = action if action.respond_to? :arity and action.arity > 0 @call_loop = true else @call_loop = false end @as_action = false end end # The label of benchmarking action. # @return [#to_s] Label of action. attr_reader :label # The benchmarking action. # @return [String, Proc] Code to be called, could be String / Proc. attr_reader :action # Call action by given times, return if +@call_loop+ is present. # @param times [Integer] Times to call +@action+. # @return [Integer] Number of times the +@action+ has been called. def call_times(times) return @action.call(times) if @call_loop act = @action i = 0 while i < times act.call i += 1 end end # Compile code into +call_times+ method. # @param str [String] Code to be compiled. # @return [Symbol] :call_times. def compile(str) m = (class << self; self; end) code = <<-CODE def call_times(__total); __i = 0 while __i < __total #{str}; __i += 1 end end CODE m.class_eval code end end end end end benchmark-ips-2.7.2/lib/benchmark/ips/job/stdout_report.rb000066400000000000000000000026621275544501400235710ustar00rootroot00000000000000module Benchmark module IPS class Job class StdoutReport def start_warming $stdout.puts "Warming up --------------------------------------" end def start_running $stdout.puts "Calculating -------------------------------------" end def warming(label, _warmup) $stdout.print rjust(label) end def warmup_stats(_warmup_time_us, timing) case format when :human $stdout.printf "%s i/100ms\n", Helpers.scale(timing) else $stdout.printf "%10d i/100ms\n", timing end end alias_method :running, :warming def add_report(item, caller) $stdout.puts " #{item.body}" @last_item = item end def footer footer = @last_item.stats.footer $stdout.puts footer.rjust(40) if footer end private # @return [Symbol] format used for benchmarking def format Benchmark::IPS.options[:format] end # Add padding to label's right if label's length < 20, # Otherwise add a new line and 20 whitespaces. # @return [String] Right justified label. def rjust(label) label = label.to_s if label.size > 20 "#{label}\n#{' ' * 20}" else label.rjust(20) end end end end end end benchmark-ips-2.7.2/lib/benchmark/ips/report.rb000066400000000000000000000140171275544501400214120ustar00rootroot00000000000000# encoding: utf-8 module Benchmark module IPS # Report contains benchmarking entries. # Perform operations like add new entry, run comparison between entries. class Report # Represents benchmarking code data for Report. class Entry # Instantiate the Benchmark::IPS::Report::Entry. # @param [#to_s] label Label of entry. # @param [Integer] us Measured time in microsecond. # @param [Integer] iters Iterations. # @param [Object] stats Statistics. # @param [Integer] cycles Number of Cycles. def initialize(label, us, iters, stats, cycles) @label = label @microseconds = us @iterations = iters @stats = stats @measurement_cycle = cycles @show_total_time = false end # Label of entry. # @return [String] the label of entry. attr_reader :label # Measured time in microsecond. # @return [Integer] number of microseconds. attr_reader :microseconds # Number of Iterations. # @return [Integer] number of iterations. attr_reader :iterations # Statistical summary of samples. # @return [Object] statisical summary. attr_reader :stats # LEGACY: Iterations per second. # @return [Float] number of iterations per second. def ips @stats.central_tendency end # LEGACY: Standard deviation of iteration per second. # @return [Float] standard deviation of iteration per second. def ips_sd @stats.error end # Number of Cycles. # @return [Integer] number of cycles. attr_reader :measurement_cycle # Control if the total time the job took is reported. # Typically this value is not significant because it's very # close to the expected time, so it's supressed by default. def show_total_time! @show_total_time = true end # Return entry's microseconds in seconds. # @return [Float] +@microseconds+ in seconds. def seconds @microseconds.to_f / 1_000_000.0 end # Return entry's standard deviation of iteration per second in percentage. # @return [Float] +@ips_sd+ in percentage. def error_percentage 100.0 * (@stats.error.to_f / @stats.central_tendency) end alias_method :runtime, :seconds # Return Entry body text with left padding. # Body text contains information of iteration per second with # percentage of standard deviation, iterations in runtime. # @return [String] Left justified body. def body case Benchmark::IPS.options[:format] when :human left = "%s (±%4.1f%%) i/s" % [Helpers.scale(@stats.central_tendency), error_percentage] iters = Helpers.scale(@iterations) if @show_total_time left.ljust(20) + (" - %s in %10.6fs" % [iters, runtime]) else left.ljust(20) + (" - %s" % iters) end else left = "%10.1f (±%.1f%%) i/s" % [@stats.central_tendency, error_percentage] if @show_total_time left.ljust(20) + (" - %10d in %10.6fs" % [@iterations, runtime]) else left.ljust(20) + (" - %10d" % @iterations) end end end # Return header with padding if +@label+ is < length of 20. # @return [String] Right justified header (+@label+). def header @label.to_s.rjust(20) end # Return string repesentation of Entry object. # @return [String] Header and body. def to_s "#{header} #{body}" end # Print entry to current standard output ($stdout). def display $stdout.puts to_s end end # End of Entry # class Report # Entry to represent each benchmarked code in Report. # @return [Array] Entries in Report. attr_reader :entries # Instantiate the Report. def initialize @entries = [] @data = nil end # Add entry to report. # @param label [String] Entry label. # @param microseconds [Integer] Measured time in microsecond. # @param iters [Integer] Iterations. # @param stats [Object] Statistical results. # @param measurement_cycle [Integer] Number of cycles. # @return [Report::Entry] Last added entry. def add_entry label, microseconds, iters, stats, measurement_cycle entry = Entry.new(label, microseconds, iters, stats, measurement_cycle) @entries.delete_if { |e| e.label == label } @entries << entry entry end # Entries data in array for generate json. # Each entry is a hash, consists of: # name: Entry#label # ips: Entry#ips # stddev: Entry#ips_sd # microseconds: Entry#microseconds # iterations: Entry#iterations # cycles: Entry#measurement_cycles # @return [Array] Array of hashes def data @data ||= @entries.collect do |entry| { :name => entry.label, :central_tendency => entry.stats.central_tendency, :ips => entry.stats.central_tendency, # for backwards compatibility :error => entry.stats.error, :stddev => entry.stats.error, # for backwards compatibility :microseconds => entry.microseconds, :iterations => entry.iterations, :cycles => entry.measurement_cycle, } end end # Run comparison of entries. def run_comparison Benchmark.compare(*@entries) end # Generate json from Report#data to given path. # @param path [String] path to generate json. def generate_json(path) File.open path, "w" do |f| require "json" f.write JSON.pretty_generate(data) end end end end end benchmark-ips-2.7.2/lib/benchmark/ips/share.rb000066400000000000000000000017751275544501400212100ustar00rootroot00000000000000require 'net/http' require 'net/https' require 'json' module Benchmark module IPS class Share DEFAULT_URL = "https://benchmark.fyi" def initialize(report, job) @report = report @job = job end def share base = (ENV['SHARE_URL'] || DEFAULT_URL) url = URI(File.join(base, "reports")) req = Net::HTTP::Post.new(url) data = { "entries" => @report.data, "options" => { "compare" => @job.compare? } } req.body = JSON.generate(data) http = Net::HTTP.new(url.hostname, url.port) if url.scheme == "https" http.use_ssl = true http.ssl_version = :TLSv1_2 end res = http.start do |h| h.request req end if Net::HTTPOK === res data = JSON.parse res.body puts "Shared at: #{File.join(base, data["id"])}" else puts "Error sharing report" end end end end end benchmark-ips-2.7.2/lib/benchmark/ips/stats/000077500000000000000000000000001275544501400207055ustar00rootroot00000000000000benchmark-ips-2.7.2/lib/benchmark/ips/stats/bootstrap.rb000066400000000000000000000025531275544501400232540ustar00rootroot00000000000000module Benchmark module IPS module Stats class Bootstrap attr_reader :data def initialize(samples, confidence) dependencies @iterations = 10_000 @confidence = (confidence / 100.0).to_s @data = Kalibera::Data.new({[0] => samples}, [1, samples.size]) interval = @data.bootstrap_confidence_interval(@iterations, @confidence) @median = interval.median @error = interval.error end def central_tendency @median end def error @error end def slowdown(baseline) low, slowdown, high = baseline.data.bootstrap_quotient(@data, @iterations, @confidence) error = Timing.mean([slowdown - low, high - slowdown]) [slowdown, error] end def footer "with #{(@confidence.to_f * 100).round(1)}% confidence" end def dependencies require 'kalibera' rescue LoadError puts puts "Can't load the kalibera gem - this is required to use the :bootstrap stats options." puts "It's optional, so we don't formally depend on it and it isn't installed along with benchmark-ips." puts "You probably want to do something like 'gem install kalibera' to fix this." abort end end end end end benchmark-ips-2.7.2/lib/benchmark/ips/stats/sd.rb000066400000000000000000000010671275544501400216440ustar00rootroot00000000000000module Benchmark module IPS module Stats class SD def initialize(samples) @mean = Timing.mean(samples) @error = Timing.stddev(samples, @mean).round end def central_tendency @mean end def error @error end def slowdown(baseline) slowdown = baseline.central_tendency.to_f / central_tendency [slowdown, nil] end def footer nil end end end end end benchmark-ips-2.7.2/lib/benchmark/timing.rb000066400000000000000000000043301275544501400205700ustar00rootroot00000000000000module Benchmark # Perform caclulations on Timing results. module Timing # Microseconds per second. MICROSECONDS_PER_SECOND = 1_000_000 # Calculate (arithmetic) mean of given samples. # @param [Array] samples Samples to calculate mean. # @return [Float] Mean of given samples. def self.mean(samples) sum = samples.inject(:+) sum / samples.size end # Calculate variance of given samples. # @param [Float] m Optional mean (Expected value). # @return [Float] Variance of given samples. def self.variance(samples, m=nil) m ||= mean(samples) total = samples.inject(0) { |acc, i| acc + ((i - m) ** 2) } total / samples.size end # Calculate standard deviation of given samples. # @param [Array] samples Samples to calculate standard deviation. # @param [Float] m Optional mean (Expected value). # @return [Float] standard deviation of given samples. def self.stddev(samples, m=nil) Math.sqrt variance(samples, m) end # Recycle used objects by starting Garbage Collector. def self.clean_env # rbx if GC.respond_to? :run GC.run(true) else GC.start end end # Use a monotonic clock if available, otherwise use Time begin Process.clock_gettime Process::CLOCK_MONOTONIC, :float_microsecond # Get an object that represents now and can be converted to microseconds def self.now Process.clock_gettime Process::CLOCK_MONOTONIC, :float_microsecond end # Add one second to the time represenetation def self.add_second(t, s) t + (s * MICROSECONDS_PER_SECOND) end # Return the number of microseconds between the 2 moments def self.time_us(before, after) after - before end rescue NameError # Get an object that represents now and can be converted to microseconds def self.now Time.now end # Add one second to the time represenetation def self.add_second(t, s) t + s end # Return the number of microseconds between the 2 moments def self.time_us(before, after) (after.to_f - before.to_f) * MICROSECONDS_PER_SECOND end end end end benchmark-ips-2.7.2/test/000077500000000000000000000000001275544501400152335ustar00rootroot00000000000000benchmark-ips-2.7.2/test/test_benchmark_ips.rb000066400000000000000000000067301275544501400214320ustar00rootroot00000000000000require "minitest/autorun" require "benchmark/ips" require "stringio" class TestBenchmarkIPS < Minitest::Test def setup @old_stdout = $stdout $stdout = StringIO.new end def teardown $stdout = @old_stdout end def test_kwargs Benchmark.ips(:time => 1, :warmup => 1, :quiet => false) do |x| x.report("sleep 0.25") { sleep(0.25) } end assert $stdout.string.size > 0 end def test_output Benchmark.ips(1) do |x| x.report("operation") { 100 * 100 } end assert $stdout.string.size > 0 end def test_quiet Benchmark.ips(1, nil, true) do |x| x.report("operation") { 100 * 100 } end assert $stdout.string.size.zero? Benchmark.ips(:quiet => true) do |x| x.report("operation") { 100 * 100 } end assert $stdout.string.size.zero? end def test_ips report = Benchmark.ips do |x| x.config(:time => 1, :warmup => 1) x.report("sleep 0.25") { sleep(0.25) } x.report("sleep 0.05") { sleep(0.05) } x.compare! end rep1 = report.entries[0] rep2 = report.entries[1] assert_equal "sleep 0.25", rep1.label assert_equal 4, rep1.iterations assert_in_delta 4.0, rep1.ips, 0.2 assert_equal "sleep 0.05", rep2.label assert_in_delta 20.0, rep2.iterations.to_f, 1.0 assert_in_delta 20.0, rep2.ips, 2.0 end def test_ips_alternate_config report = Benchmark.ips do |x| x.time = 1 x.warmup = 1 x.report("sleep 0.25") { sleep(0.25) } end rep = report.entries.first assert_equal "sleep 0.25", rep.label assert_equal 4, rep.iterations assert_in_delta 4.0, rep.ips, 0.4 end def test_ips_old_config report = Benchmark.ips(1,1) do |x| x.report("sleep 0.25") { sleep(0.25) } end rep = report.entries.first assert_equal "sleep 0.25", rep.label assert_equal 4, rep.iterations assert_in_delta 4.0, rep.ips, 0.2 end def test_ips_config_suite suite = Struct.new(:calls) do def method_missing(method, *args) calls << method end end.new([]) Benchmark.ips(0.1, 0.1) do |x| x.config(:suite => suite) x.report("job") {} end assert_equal [:warming, :warmup_stats, :running, :add_report], suite.calls end def test_ips_defaults report = Benchmark.ips do |x| x.report("sleep 0.25") { sleep(0.25) } end rep = report.entries.first assert_equal "sleep 0.25", rep.label assert_equal 4*5, rep.iterations assert_in_delta 4.0, rep.ips, 0.2 end def test_ips_report_using_symbol report = Benchmark.ips do |x| x.report(:sleep_a_quarter_second) { sleep(0.25) } end rep = report.entries.first assert_equal :sleep_a_quarter_second, rep.label assert_equal 4*5, rep.iterations assert_in_delta 4.0, rep.ips, 0.2 end def test_ips_default_data report = Benchmark.ips do |x| x.report("sleep 0.25") { sleep(0.25) } end all_data = report.data assert all_data assert_equal "sleep 0.25", all_data[0][:name] assert all_data[0][:ips] assert all_data[0][:stddev] end def test_json_output json_file = Tempfile.new("data.json") Benchmark.ips do |x| x.report("sleep 0.25") { sleep(0.25) } x.json! json_file.path end json_data = json_file.read assert json_data data = JSON.parse json_data assert data assert_equal 1, data.size assert_equal "sleep 0.25", data[0]["name"] assert data[0]["ips"] assert data[0]["stddev"] end end