pax_global_header 0000666 0000000 0000000 00000000064 14522154247 0014520 g ustar 00root root 0000000 0000000 52 comment=a6df4c1749cd02c4820b3629d7d71b49bcd6680f
ruby-in-parallel-1.0.1/ 0000775 0000000 0000000 00000000000 14522154247 0014676 5 ustar 00root root 0000000 0000000 ruby-in-parallel-1.0.1/.github/ 0000775 0000000 0000000 00000000000 14522154247 0016236 5 ustar 00root root 0000000 0000000 ruby-in-parallel-1.0.1/.github/dependabot.yml 0000664 0000000 0000000 00000000202 14522154247 0021060 0 ustar 00root root 0000000 0000000 version: 2
updates:
- package-ecosystem: bundler
directory: "/"
schedule:
interval: weekly
open-pull-requests-limit: 10
ruby-in-parallel-1.0.1/.github/workflows/ 0000775 0000000 0000000 00000000000 14522154247 0020273 5 ustar 00root root 0000000 0000000 ruby-in-parallel-1.0.1/.github/workflows/release.yml 0000664 0000000 0000000 00000005425 14522154247 0022444 0 ustar 00root root 0000000 0000000 name: Release Gem
on: workflow_dispatch
jobs:
release:
runs-on: ubuntu-latest
if: github.repository == 'puppetlabs/in-parallel'
steps:
- uses: actions/checkout@v3
- name: Get Current Version
uses: actions/github-script@v6
id: cv
with:
script: |
const { data: response } = await github.rest.repos.getLatestRelease({
owner: context.repo.owner,
repo: context.repo.repo,
})
console.log(`The latest release is ${response.tag_name}`)
return response.tag_name
result-encoding: string
- name: Get Next Version
id: nv
run: |
version=$(grep VERSION lib/in-parallel/version.rb |rev |cut -d "'" -f2 |rev)
echo "version=$version" >> $GITHUB_OUTPUT
echo "Found version $version from lib/in-parallel/version.rb"
- name: Generate Changelog
uses: docker://githubchangeloggenerator/github-changelog-generator:1.16.2
with:
args: >-
--future-release ${{ steps.nv.outputs.version }}
env:
CHANGELOG_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: Validate Changelog
run : |
set -e
if [[ -n $(git status --porcelain) ]]; then
echo "Here is the current git status:"
git status
echo
echo "The following changes were detected:"
git --no-pager diff
echo "Uncommitted PRs found in the changelog. Please submit a release prep PR of changes after running `./update-changelog`"
exit 1
fi
- name: Generate Release Notes
uses: docker://githubchangeloggenerator/github-changelog-generator:1.16.2
with:
args: >-
--since-tag ${{ steps.cv.outputs.result }}
--future-release ${{ steps.nv.outputs.version }}
--base ''
--output release-notes.md
env:
CHANGELOG_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: Tag Release
uses: ncipollo/release-action@v1
with:
tag: ${{ steps.nv.outputs.version }}
token: ${{ secrets.GITHUB_TOKEN }}
bodyfile: release-notes.md
draft: false
prerelease: false
- name: Install Ruby 3.2
uses: ruby/setup-ruby@v1
with:
ruby-version: '3.2'
- name: Build gem
run: gem build *.gemspec
- name: Publish gem
run: |
mkdir -p $HOME/.gem
touch $HOME/.gem/credentials
chmod 0600 $HOME/.gem/credentials
printf -- "---\n:rubygems_api_key: ${GEM_HOST_API_KEY}\n" > $HOME/.gem/credentials
gem push *.gem
env:
GEM_HOST_API_KEY: '${{ secrets.RUBYGEMS_AUTH_TOKEN }}'
ruby-in-parallel-1.0.1/.github/workflows/security.yml 0000664 0000000 0000000 00000002234 14522154247 0022666 0 ustar 00root root 0000000 0000000 name: Security
on:
workflow_dispatch:
push:
branches:
- main
jobs:
scan:
name: Mend Scanning
runs-on: ubuntu-latest
steps:
- name: checkout repo content
uses: actions/checkout@v3
with:
fetch-depth: 1
- name: setup ruby
uses: ruby/setup-ruby@v1
with:
ruby-version: 2.7
# setup a package lock if one doesn't exist, otherwise do nothing
- name: check lock
run: '[ -f "Gemfile.lock" ] && echo "package lock file exists, skipping" || bundle lock'
# install java
- uses: actions/setup-java@v3
with:
distribution: 'temurin' # See 'Supported distributions' for available options
java-version: '17'
# download mend
- name: download_mend
run: curl -o wss-unified-agent.jar https://unified-agent.s3.amazonaws.com/wss-unified-agent.jar
- name: run mend
run: java -jar wss-unified-agent.jar
env:
WS_APIKEY: ${{ secrets.MEND_API_KEY }}
WS_WSS_URL: https://saas-eu.whitesourcesoftware.com/agent
WS_USERKEY: ${{ secrets.MEND_TOKEN }}
WS_PRODUCTNAME: RE
WS_PROJECTNAME: ${{ github.event.repository.name }}
ruby-in-parallel-1.0.1/.github/workflows/testing.yml 0000664 0000000 0000000 00000000751 14522154247 0022476 0 ustar 00root root 0000000 0000000 name: Testing
on:
pull_request:
branches:
- main
jobs:
spec_tests:
runs-on: ubuntu-latest
strategy:
matrix:
ruby-version:
- '3.2'
steps:
- uses: actions/checkout@v3
- name: Set up Ruby
uses: ruby/setup-ruby@v1
with:
ruby-version: ${{ matrix.ruby-version }}
bundler-cache: true # runs 'bundle install' and caches installed gems automatically
- name: Run spec tests
run: bundle exec rake test
ruby-in-parallel-1.0.1/.github_changelog_generator 0000664 0000000 0000000 00000000120 14522154247 0022227 0 ustar 00root root 0000000 0000000 project=in-parallel
user=puppetlabs
exclude_labels=maintenance
since-tag=0.1.13
ruby-in-parallel-1.0.1/CHANGELOG.md 0000664 0000000 0000000 00000013532 14522154247 0016513 0 ustar 00root root 0000000 0000000 # Changelog
## [1.0.1](https://github.com/puppetlabs/in-parallel/tree/1.0.1) (2023-08-24)
[Full Changelog](https://github.com/puppetlabs/in-parallel/compare/1.0.0...1.0.1)
**Merged pull requests:**
- Bump activesupport from 7.0.4.3 to 7.0.7.2 [\#23](https://github.com/puppetlabs/in-parallel/pull/23) ([dependabot[bot]](https://github.com/apps/dependabot))
## [1.0.0](https://github.com/puppetlabs/in-parallel/tree/1.0.0) (2023-03-15)
[Full Changelog](https://github.com/puppetlabs/in-parallel/compare/0.1.17...1.0.0)
**Merged pull requests:**
- \(RE-15226\) Update codeowners and add standard workflows [\#20](https://github.com/puppetlabs/in-parallel/pull/20) ([yachub](https://github.com/yachub))
- \(maint\) update exists? -\> exist? for ruby 3.2 compatibility [\#19](https://github.com/puppetlabs/in-parallel/pull/19) ([tvpartytonight](https://github.com/tvpartytonight))
## [0.1.17](https://github.com/puppetlabs/in-parallel/tree/0.1.17) (2017-02-07)
[Full Changelog](https://github.com/puppetlabs/in-parallel/compare/0.1.16...0.1.17)
**Merged pull requests:**
- \(maint\) Properly handle non-parallel enumerables [\#17](https://github.com/puppetlabs/in-parallel/pull/17) ([nicklewis](https://github.com/nicklewis))
## [0.1.16](https://github.com/puppetlabs/in-parallel/tree/0.1.16) (2017-02-06)
[Full Changelog](https://github.com/puppetlabs/in-parallel/compare/0.1.15...0.1.16)
## [0.1.15](https://github.com/puppetlabs/in-parallel/tree/0.1.15) (2017-02-03)
[Full Changelog](https://github.com/puppetlabs/in-parallel/compare/0.1.14...0.1.15)
**Merged pull requests:**
- \(maint\) Avoid deadlock with large results [\#16](https://github.com/puppetlabs/in-parallel/pull/16) ([nicklewis](https://github.com/nicklewis))
- \(maint\) Revert name change of in\_parallel.rb [\#15](https://github.com/puppetlabs/in-parallel/pull/15) ([samwoods1](https://github.com/samwoods1))
## [0.1.14](https://github.com/puppetlabs/in-parallel/tree/0.1.14) (2016-08-08)
[Full Changelog](https://github.com/puppetlabs/in-parallel/compare/0.1.13...0.1.14)
**Merged pull requests:**
- \(maint\) Consistent naming of in-parallel [\#14](https://github.com/puppetlabs/in-parallel/pull/14) ([samwoods1](https://github.com/samwoods1))
# experimental_in-parallel_bump_and_tag_master - History
## Tags
* [LATEST - 6 Feb, 2017 (8e97ff25)](#LATEST)
* [0.1.16 - 6 Feb, 2017 (0d5030c3)](#0.1.16)
* [0.1.15 - 3 Feb, 2017 (ff16929c)](#0.1.15)
* [0.1.14 - 8 Aug, 2016 (ce331dbd)](#0.1.14)
* [0.1.13 - 8 Aug, 2016 (26d19934)](#0.1.13)
## Details
### LATEST - 6 Feb, 2017 (8e97ff25)
* (GEM) update in-parallel version to 0.1.17 (8e97ff25)
* Merge pull request #17 from nicklewis/handle-non-parallel-enumerables (dca0b0a5)
```
Merge pull request #17 from nicklewis/handle-non-parallel-enumerables
(maint) Properly handle non-parallel enumerables
```
* (maint) Properly handle non-parallel enumerables (b041b864)
```
(maint) Properly handle non-parallel enumerables
For Enumerables containing 0 or 1 items, the #each_in_parallel method
was improperly calling the block without an argument, then calling it
again properly but not returning the result of the block.
The #each method returns the Enumerable that was it was called on,
rather than the value of the block. This needs to be #map instead, to
actually return an array of the one or zero values. The tests weren't
catching this because they were effectively passing `identity` as the
block, nullifying the distinction between #each and #map.
```
### 0.1.16 - 6 Feb, 2017 (0d5030c3)
* (HISTORY) update in-parallel history for gem release 0.1.16 (0d5030c3)
* (GEM) update in-parallel version to 0.1.16 (27b497ea)
### 0.1.15 - 3 Feb, 2017 (ff16929c)
* (HISTORY) update in-parallel history for gem release 0.1.15 (ff16929c)
* (GEM) update in-parallel version to 0.1.15 (206a62fe)
* Merge pull request #16 from nicklewis/support-large-results (4d644d88)
```
Merge pull request #16 from nicklewis/support-large-results
(maint) Avoid deadlock with large results
```
* (maint) Avoid deadlock with large results (a5a9c174)
```
(maint) Avoid deadlock with large results
Previously, if the result of a process was larger than the IO buffer
size (commonly 64k), execution would deadlock until the timeout.
In this scenario, the writer would fill the buffer and block until the
reader had cleared the buffer by reading. However, the reader will only
try to read once (to get the whole result) and will only attempt to read
after the child process has exited. This causes a deadlock, as the child
can't exit because it can't finish writing, but the the reader won't
read because the child hasn't exited.
This commit fixes the watcher loop to instead IO.select() from the
available result readers and read a partial result into a buffer whenever it's
available. This ensures that the writer will never remain blocked by a
full buffer. The reader now uses IO#eof? to determine whether the child
process has exited, at which point it will process the whole result as
before.
```
* Merge pull request #15 from samwoods1/add_jjb_pipelines (72d32635)
```
Merge pull request #15 from samwoods1/add_jjb_pipelines
(maint) Revert name change of in_parallel.rb
```
* (maint) Revert name change of in_parallel.rb (327c8fd5)
### 0.1.14 - 8 Aug, 2016 (ce331dbd)
* (HISTORY) update in-parallel history for gem release 0.1.14 (ce331dbd)
* (GEM) update in-parallel version to 0.1.14 (d026c624)
* Merge pull request #14 from samwoods1/add_jjb_pipelines (269bc350)
```
Merge pull request #14 from samwoods1/add_jjb_pipelines
(maint) Consistent naming of in-parallel
```
* (maint) Consistent naming of in-parallel (6fbb442d)
### 0.1.13 - 8 Aug, 2016 (26d19934)
* Initial release.
\* *This Changelog was automatically generated by [github_changelog_generator](https://github.com/github-changelog-generator/github-changelog-generator)*
ruby-in-parallel-1.0.1/CODEOWNERS 0000664 0000000 0000000 00000000622 14522154247 0016271 0 ustar 00root root 0000000 0000000 # This will cause RE to be assigned review of any opened PRs against
# the branches containing this file.
# See https://help.github.com/en/articles/about-code-owners for info on how to
# take ownership of parts of the code base that should be reviewed by another
# team.
# RE will be the default owners for everything in the repo.
* @puppetlabs/release-engineering
ruby-in-parallel-1.0.1/CONTRIBUTING.md 0000664 0000000 0000000 00000010525 14522154247 0017132 0 ustar 00root root 0000000 0000000 # How To Contribute To in-parallel
## Getting Started
* Make sure you have a [GitHub account](https://github.com/signup/free)
* Fork the [in-parallel repository on GitHub](https://github.com/puppetlabs/in-parallel)
## Making Changes
* Create a topic branch from where you want to base your work.
* This is the `master` branch in the case of in-parallel
* To quickly create a topic branch based on master use `git checkout -b my_contribution master`. Do not work directly on the `master` branch.
* Make commits of logical _working_ and _functional_ units.
* Check for unnecessary whitespace with `git diff --check` before committing.
* Make sure your commit messages are in the proper format.
(BKR-1234) Make the example in CONTRIBUTING imperative and concrete
Without this patch applied the example commit message in the CONTRIBUTING
document is not a concrete example. This is a problem because the
contributor is left to imagine what the commit message should look like
based on a description rather than an example. This patch fixes the
problem by making the example concrete and imperative.
The first line is a real life imperative statement with a ticket number
from our issue tracker. The body describes the behavior without the patch,
why this is a problem, and how the patch fixes the problem when applied.
* Make sure you have added [RSpec](http://rspec.info/) tests that exercise your new code. These test should be located in the appropriate `in-parallel/spec/` subdirectory. The addition of new methods/classes or the addition of code paths to existing methods/classes requires additional RSpec coverage.
* One should **NOT USE** the deprecated `should`/`stub` methods - **USE** `expect`/`allow`. Use of deprecated RSpec methods will result in your patch being rejected. See a nice blog post from 2013 on [RSpec's new message expectation syntax](http://teaisaweso.me/blog/2013/05/27/rspecs-new-message-expectation-syntax/).
* Run the spec unit tests to assure nothing else was accidentally broken, using `rake test`
* **Bonus**: if possible ensure that `rake test` runs without failures for additional Ruby versions (1.9, 2.0, etc). in-parallel supports Ruby 1.9+, and breakage of support for other rubies will cause a patch to be rejected.
* Make sure that if you have added new functionality of sufficiently high risk, and it can not be covered adequately via unit tests (mocking, requires disk, other classes, etc), you also include acceptance tests in your PR.
* Make sure that you have added documentation using [Yard](http://yardoc.org/), new methods/classes without apporpriate documentation will be rejected.
* Run the yardoc tool to ensure that your yard documentation is properly formatted and complete
* `[bundle exec] yard doc`
* Yard docs are great for other developers, but often are difficult to read for users. If your change impacts user-facing functionality, please include changes to the human-readable markdown docs starting at README.md
* During the time that you are working on your patch the master in-parallel branch may have changed - you'll want to [rebase](http://git-scm.com/book/en/Git-Branching-Rebasing) before you submit your PR with `git rebase master`. A successful rebase ensures that your patch will cleanly merge into in-parallel.
* Submitted patches will be smoke tested through a series of acceptance level tests that ensures basic in-parallel functionality - the results of these tests will be evaluated by a in-parallel team member. Failures associated with the submitted patch will result in the patch being rejected.
## Submitting Changes
* Sign the [Contributor License Agreement](http://links.puppet.com/cla).
* Push your changes to a topic branch in _your_ fork of the repository.
* Submit a pull request to [in-parallel](https://github.com/puppetlabs/in-parallel)
* PRs are reviewed as time permits.
# Additional Resources
* [More information on contributing](http://links.puppet.com/contribute-to-puppet)
* [Contributor License Agreement](http://links.puppet.com/cla)
* [General GitHub documentation](http://help.github.com/)
* [GitHub pull request documentation](http://help.github.com/send-pull-requests/)
* Questions? Comments? Contact the in-parallel team at qa-team@puppet.com
* The keyword `in-parallel` is monitored and we'll get back to you as quick as we can.
ruby-in-parallel-1.0.1/Gemfile 0000664 0000000 0000000 00000000546 14522154247 0016176 0 ustar 00root root 0000000 0000000 source 'https://rubygems.org'
# in the Rakefile, so we require it in all groups
gem 'rspec' ,'~> 3.1'
group :development do
gem 'simplecov'
gem 'rake', '>= 0.9.0'
#Documentation dependencies
gem 'yard' ,'~> 0'
gem 'markdown' ,'~> 0'
end
# Specify your gem's dependencies in in_parallel.gemspec
gemspec
ruby-in-parallel-1.0.1/Gemfile.lock 0000664 0000000 0000000 00000003122 14522154247 0017116 0 ustar 00root root 0000000 0000000 PATH
remote: .
specs:
in-parallel (1.0.1)
GEM
remote: https://rubygems.org/
specs:
activesupport (7.0.7.2)
concurrent-ruby (~> 1.0, >= 1.0.2)
i18n (>= 1.6, < 2)
minitest (>= 5.1)
tzinfo (~> 2.0)
concurrent-ruby (1.2.2)
diff-lcs (1.5.0)
docile (1.4.0)
i18n (1.14.1)
concurrent-ruby (~> 1.0)
iniparser (1.0.1)
kramdown (2.4.0)
rexml
logutils (0.6.1)
markdown (0.4.0)
kramdown (>= 0.13.7)
props (>= 0.2.0)
textutils (>= 0.2.0)
minitest (5.19.0)
props (1.2.0)
iniparser (>= 0.1.0)
rake (13.0.6)
rexml (3.2.5)
rspec (3.12.0)
rspec-core (~> 3.12.0)
rspec-expectations (~> 3.12.0)
rspec-mocks (~> 3.12.0)
rspec-core (3.12.1)
rspec-support (~> 3.12.0)
rspec-expectations (3.12.2)
diff-lcs (>= 1.2.0, < 2.0)
rspec-support (~> 3.12.0)
rspec-mocks (3.12.4)
diff-lcs (>= 1.2.0, < 2.0)
rspec-support (~> 3.12.0)
rspec-support (3.12.0)
rubyzip (2.3.2)
simplecov (0.22.0)
docile (~> 1.1)
simplecov-html (~> 0.11)
simplecov_json_formatter (~> 0.1)
simplecov-html (0.12.3)
simplecov_json_formatter (0.1.4)
textutils (1.4.0)
activesupport
logutils (>= 0.6.1)
props (>= 1.1.2)
rubyzip (>= 1.0.0)
tzinfo (2.0.6)
concurrent-ruby (~> 1.0)
webrick (1.7.0)
yard (0.9.28)
webrick (~> 1.7.0)
PLATFORMS
aarch64-linux
x86_64-linux
DEPENDENCIES
in-parallel!
markdown (~> 0)
rake (>= 0.9.0)
rspec (~> 3.1)
simplecov
yard (~> 0)
BUNDLED WITH
2.4.8
ruby-in-parallel-1.0.1/HISTORY.md 0000664 0000000 0000000 00000007002 14522154247 0016360 0 ustar 00root root 0000000 0000000 # experimental_in-parallel_bump_and_tag_master - History
## Tags
* [LATEST - 6 Feb, 2017 (8e97ff25)](#LATEST)
* [0.1.16 - 6 Feb, 2017 (0d5030c3)](#0.1.16)
* [0.1.15 - 3 Feb, 2017 (ff16929c)](#0.1.15)
* [0.1.14 - 8 Aug, 2016 (ce331dbd)](#0.1.14)
* [0.1.13 - 8 Aug, 2016 (26d19934)](#0.1.13)
## Details
### LATEST - 6 Feb, 2017 (8e97ff25)
* (GEM) update in-parallel version to 0.1.17 (8e97ff25)
* Merge pull request #17 from nicklewis/handle-non-parallel-enumerables (dca0b0a5)
```
Merge pull request #17 from nicklewis/handle-non-parallel-enumerables
(maint) Properly handle non-parallel enumerables
```
* (maint) Properly handle non-parallel enumerables (b041b864)
```
(maint) Properly handle non-parallel enumerables
For Enumerables containing 0 or 1 items, the #each_in_parallel method
was improperly calling the block without an argument, then calling it
again properly but not returning the result of the block.
The #each method returns the Enumerable that was it was called on,
rather than the value of the block. This needs to be #map instead, to
actually return an array of the one or zero values. The tests weren't
catching this because they were effectively passing `identity` as the
block, nullifying the distinction between #each and #map.
```
### 0.1.16 - 6 Feb, 2017 (0d5030c3)
* (HISTORY) update in-parallel history for gem release 0.1.16 (0d5030c3)
* (GEM) update in-parallel version to 0.1.16 (27b497ea)
### 0.1.15 - 3 Feb, 2017 (ff16929c)
* (HISTORY) update in-parallel history for gem release 0.1.15 (ff16929c)
* (GEM) update in-parallel version to 0.1.15 (206a62fe)
* Merge pull request #16 from nicklewis/support-large-results (4d644d88)
```
Merge pull request #16 from nicklewis/support-large-results
(maint) Avoid deadlock with large results
```
* (maint) Avoid deadlock with large results (a5a9c174)
```
(maint) Avoid deadlock with large results
Previously, if the result of a process was larger than the IO buffer
size (commonly 64k), execution would deadlock until the timeout.
In this scenario, the writer would fill the buffer and block until the
reader had cleared the buffer by reading. However, the reader will only
try to read once (to get the whole result) and will only attempt to read
after the child process has exited. This causes a deadlock, as the child
can't exit because it can't finish writing, but the the reader won't
read because the child hasn't exited.
This commit fixes the watcher loop to instead IO.select() from the
available result readers and read a partial result into a buffer whenever it's
available. This ensures that the writer will never remain blocked by a
full buffer. The reader now uses IO#eof? to determine whether the child
process has exited, at which point it will process the whole result as
before.
```
* Merge pull request #15 from samwoods1/add_jjb_pipelines (72d32635)
```
Merge pull request #15 from samwoods1/add_jjb_pipelines
(maint) Revert name change of in_parallel.rb
```
* (maint) Revert name change of in_parallel.rb (327c8fd5)
### 0.1.14 - 8 Aug, 2016 (ce331dbd)
* (HISTORY) update in-parallel history for gem release 0.1.14 (ce331dbd)
* (GEM) update in-parallel version to 0.1.14 (d026c624)
* Merge pull request #14 from samwoods1/add_jjb_pipelines (269bc350)
```
Merge pull request #14 from samwoods1/add_jjb_pipelines
(maint) Consistent naming of in-parallel
```
* (maint) Consistent naming of in-parallel (6fbb442d)
### 0.1.13 - 8 Aug, 2016 (26d19934)
* Initial release.
ruby-in-parallel-1.0.1/LICENSE 0000664 0000000 0000000 00000026135 14522154247 0015712 0 ustar 00root root 0000000 0000000 Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "{}"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright {yyyy} {name of copyright owner}
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
ruby-in-parallel-1.0.1/README.md 0000664 0000000 0000000 00000020106 14522154247 0016154 0 ustar 00root root 0000000 0000000 # in-parallel
- [in-parallel](#in-parallel)
- [Use Cases](#use-cases)
- [Install](#install)
- [Usage](#usage)
- [Methods](#methods)
- [run\_in\_parallel(timeout=nil, kill\_all\_on\_error = false, \&block)](#run_in_paralleltimeoutnil-kill_all_on_error--false-block)
- [Enumerable.each\_in\_parallel(identifier=nil, timeout=(InParallel::InParallelExecutor.timeout), kill\_all\_on\_error = false, \&block)](#enumerableeach_in_parallelidentifiernil-timeoutinparallelinparallelexecutortimeout-kill_all_on_error--false-block)
- [run\_in\_background(ignore\_results = true, \&block)](#run_in_backgroundignore_results--true-block)
- [wait\_for\_processes(timeout=nil, kill\_all\_on\_error = false)](#wait_for_processestimeoutnil-kill_all_on_error--false)
- [Global Options](#global-options)
- [Releasing](#releasing)
A lightweight Ruby library with very simple syntax, making use of Process.fork to execute code in parallel.
## Use Cases
Many other Ruby libraries that simplify parallel execution support one primary use case - crunching through a large queue of small, similar tasks as quickly and efficiently as possible. This library primarily supports the use case of executing a few larger and unrelated tasks in parallel, automatically managing the stdout and passing return values back to the main process. This library was created to be used by Puppet's Beaker test framework to enable parallel execution of some of the framework's tasks, and allow users to execute code in parallel within their tests.
If you are looking for something that excels at executing a large queue of tasks in parallel as efficiently as possible, you should take a look at the [parallel](https://github.com/grosser/parallel) project.
## Install
```gem install in-parallel```
## Usage
```include InParallel``` to use as a mix-in
The methods below allow you to fork processes to execute multiple methods or blocks within an enumerable in parallel. They all have this common behavior:
1. STDOUT is captured for each forked process and logged all at once when the process completes or is terminated.
1. By default execution of processes in parallel will wait until execution of all processes are complete before continuing (with the exception of run_in_background).
1. You can specify the parameter kill_all_on_error=true if you want to immediately exit all forked processes when an error executing any of the forked processes occurs.
1. When the forked process raises an exception or exits with a non zero exit code, an exception will be raised in the main process.
1. Terminating the main process with 'ctrl-c' or killing the process in some other way will immediately cause all forked processes to be killed and log their STDOUT up to that point.
1. If the result of the method or block can be marshalled, it will be returned as though it was executed within the same process. If the result cannot be marshalled a warning is produced and the return value will be nil.
1. NOTE: results of methods within run_in_parallel can be assigned to instance or class variables, but not local variables. See examples below.
1. Will timeout (stop execution and raise an exception) based on a global timeout value, or timeout parameter.
## Methods
### run_in_parallel(timeout=nil, kill_all_on_error = false, &block)
1. Each method in a block will be executed in parallel (unless the method is defined in Kernel or BaseObject).
1. Any methods further down the stack won't be affected, only the ones directly within the block.
1. Waits for each process in realtime and logs immediately upon completion of each process
```ruby
def method_with_param(name)
ret_val = "hello #{name} \n"
puts ret_val
ret_val
end
def method_without_param
# A result more complex than a string will be marshalled and unmarshalled and work
ret_val = {:foo => "bar"}
puts ret_val
return ret_val
end
# Example:
# will spawn 2 processes, (1 for each method) wait until they both complete, log chunked STDOUT/STDERR for
# each process and assign the method return values to instance variables:
run_in_parallel do
@result_1 = method_with_param('world')
@result_2 = method_without_param
end
puts "#{@result_1}, #{@result_2[:foo]}"
```
stdout:
```shell
Forked process for 'method_with_param' - PID = '49398'
Forked process for 'method_without_param' - PID = '49399'
------ Begin output for method_with_param - 49398
hello world
------ Completed output for method_with_param - 49398
------ Begin output for method_without_param - 49399
{:foo=>"bar"}
------ Completed output for method_without_param - 49399
hello world, bar
```
### Enumerable.each_in_parallel(identifier=nil, timeout=(InParallel::InParallelExecutor.timeout), kill_all_on_error = false, &block)
1. This is very similar to other solutions, except that it directly extends the Enumerable class with an each_in_parallel method, giving you the ability to pretty simply spawn a process for any item in an array or map.
1. Identifies the block location (or caller location if the block does not have a source_location) in the console log to make it clear which block is being executed
1. Identifier param is only for logging, otherwise it will use the block source location.
```ruby
["foo", "bar", "baz"].each_in_parallel { |item| puts item }
```
### run_in_background(ignore_results = true, &block)
1. This does basically the same thing as run_in_parallel, except it does not wait for execution of all processes to complete, it returns immediately.
1. You can optionally ignore results completely (default) or delay evaluating the results until later
1. You can run multiple blocks in the background and then at some later point evaluate all of the results
```ruby
TMP_FILE = '/tmp/test_file.txt'
def create_file_with_delay(file_path)
sleep 2
File.open(file_path, 'w') { |f| f.write('contents') }
return true
end
# Example 1 - ignore results
run_in_background { create_file_with_delay(TMP_FILE) }
# Should not exist immediately upon block completion
puts(File.exist?(TMP_FILE)) # false
sleep(3)
# Should exist once the delay from create_file_with_delay is done
puts(File.exist?(TMP_FILE)) # true
```
```ruby
# Example 2 - delay results
run_in_background(false) { @result = create_file_with_delay(TMP_FILE) }
# Do something else
run_in_background(false) { @result2 = create_file_with_delay('/tmp/someotherfile.txt') }
# @result has not been assigned yet
puts @result >> "unresolved_parallel_result_0"
# This assigns all instance variables within the block and writes STDOUT and STDERR from the process to console.
wait_for_processes
puts @result # true
puts @result2 # true
```
### wait_for_processes(timeout=nil, kill_all_on_error = false)
1. Used only after run_in_background with ignore_results=false
1. Optional args for timeout and kill_all_on_error
1. See run_in_background for examples
## Global Options
You can get or set the following values to set global defaults. These defaults can also be specified per execution by supplying the values as parameters to the parallel methods.
```ruby
# How many seconds to wait between logging a 'Waiting for child processes.' message. Defaults to 30 seconds
parallel_signal_interval
# How many seconds to wait before timing out a forked child process and raising an exception. Defaults to 30 minutes.
parallel_default_timeout
# The log level to log output.
# NOTE: The entire contents of STDOUT for forked processes will be printed to console regardless of
# the log level set here.
@logger.log_level
```
## Releasing
Follow these steps to publish a new GitHub release, and build and push the gem to .
1. Bump the "VERSION" in lib/in-parallel/version.rb appropriately based on changes in CHANGELOG.md since the last release.
2. Run `./release-prep` to update `Gemfile.lock` and `CHANGELOG.md`.
3. Commit and push changes to a new branch, then open a pull request against main and be sure to add the "maintenance" label.
4. After the pull request is approved and merged, then navigate to Actions --> Release Gem --> run workflow --> Branch: main --> Run workflow.
ruby-in-parallel-1.0.1/Rakefile 0000664 0000000 0000000 00000000311 14522154247 0016336 0 ustar 00root root 0000000 0000000 require "bundler/gem_tasks"
require 'rspec/core/rake_task'
task :default => :test
desc "Run spec tests"
RSpec::Core::RakeTask.new(:test) do |t|
t.rspec_opts = ['--color']
t.pattern = 'spec/'
end
ruby-in-parallel-1.0.1/in-parallel.gemspec 0000664 0000000 0000000 00000002412 14522154247 0020442 0 ustar 00root root 0000000 0000000 # coding: utf-8
lib = File.expand_path('../lib', __FILE__)
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
require 'in-parallel/version'
Gem::Specification.new do |spec|
spec.name = "in-parallel"
spec.version = InParallel::VERSION
spec.authors = ["samwoods1"]
spec.email = ["sam.woods@puppetlabs.com"]
spec.summary = "A lightweight library to execute a handful of tasks in parallel with simple syntax"
spec.description = "Many other Ruby libraries that simplify parallel execution support one primary use case - " +
"crunching through a large queue of small, similar tasks as quickly and efficiently as possible. This library " +
"primarily supports the use case of executing a few larger and unrelated tasks in parallel, automatically " +
"managing the stdout and passing return values back to the main process. This library was created to be used " +
"by Puppet's Beaker test framework to enable parallel execution of some of the framework's tasks, and allow " +
"users to execute code in parallel within their tests."
spec.homepage = "https://github.com/puppetlabs/in-parallel"
spec.license = "MIT"
spec.files = Dir['[A-Z]*[^~]'] + Dir['lib/**/*.rb'] + Dir['spec/*']
end
ruby-in-parallel-1.0.1/lib/ 0000775 0000000 0000000 00000000000 14522154247 0015444 5 ustar 00root root 0000000 0000000 ruby-in-parallel-1.0.1/lib/in-parallel/ 0000775 0000000 0000000 00000000000 14522154247 0017644 5 ustar 00root root 0000000 0000000 ruby-in-parallel-1.0.1/lib/in-parallel/version.rb 0000664 0000000 0000000 00000000052 14522154247 0021653 0 ustar 00root root 0000000 0000000 module InParallel
VERSION = '1.0.1'
end
ruby-in-parallel-1.0.1/lib/in_parallel.rb 0000664 0000000 0000000 00000042413 14522154247 0020257 0 ustar 00root root 0000000 0000000 require_relative 'parallel_logger'
require_relative 'parallel_enumerable'
require 'tempfile'
module InParallel
include ParallelLogger
class InParallelExecutor
# How many seconds between outputting to stdout that we are waiting for child processes.
# 0 or < 0 means no signaling.
@@parallel_signal_interval = 30
@@parallel_default_timeout = 1800
@@process_infos = []
def self.process_infos
@@process_infos
end
@@background_objs = []
@@result_id = 0
@@pids = []
@@main_pid = Process.pid
def self.main_pid
@@main_pid
end
def self.parallel_default_timeout
@@parallel_default_timeout
end
def self.parallel_default_timeout=(value)
@@parallel_default_timeout = value
end
def self.logger
@@logger
end
def self.logger=(value)
@@logger = value
end
# Runs all methods within the block in parallel and waits for them to complete
#
# Example - will spawn 2 processes, (1 for each method) wait until they both complete, and log STDOUT:
# InParallel.run_in_parallel do
# @result_1 = method1
# @result_2 = method2
# end
# NOTE: Only supports assigning instance variables within the block, not local variables
def self.run_in_parallel(timeout = @@parallel_default_timeout, kill_all_on_error = false, &block)
if fork_supported?
proxy = BlankBindingParallelProxy.new(block.binding)
proxy.instance_eval(&block)
return wait_for_processes(proxy, block.binding, timeout, kill_all_on_error)
end
# if fork is not supported
block.call
end
# Runs all methods within the block in parallel in the background
#
# Example - Will spawn a process in the background to run puppet agent on two agents and return immediately:
# Parallel.run_in_background do
# @result_1 = method1
# @result_2 = method2
# end
# # Do something else here before waiting for the process to complete
#
# # Optionally wait for the processes to complete before continuing.
# # Otherwise use run_in_background(true) to clean up the process status and output immediately.
# wait_for_processes(self)
#
# NOTE: must call get_background_results to allow instance variables in calling object to be set, otherwise @result_1 will evaluate to "unresolved_parallel_result_0"
def self.run_in_background(ignore_result = true, &block)
if fork_supported?
proxy = BlankBindingParallelProxy.new(block.binding)
proxy.instance_eval(&block)
if ignore_result
Process.detach(@@process_infos.last[:pid])
@@process_infos.pop
else
@@background_objs << { :proxy => proxy, :target => block.binding }
return process_infos.last[:tmp_result]
end
return
end
# if fork is not supported
result = block.call
return nil if ignore_result
result
end
# Waits for all processes to complete and logs STDOUT and STDERR in chunks from any processes that were triggered from this Parallel class
# @param [Object] proxy - The instance of the proxy class that the method was executed within (probably only useful when called by run_in_background)
# @param [Object] binding - The binding of the block to assign return values to instance variables (probably only useful when called by run_in_background)
# @param [Int] timeout Time in seconds to wait before giving up on a child process
# @param [Boolean] kill_all_on_error Whether to wait for all processes to complete, or fail immediately - killing all other forked processes - when one process errors.
def self.wait_for_processes(proxy = self, binding = nil, timeout = nil, kill_all_on_error = false)
raise_error = nil
timeout ||= @@parallel_default_timeout
send_int = false
trap(:INT) do
# Can't use logger inside of trap
puts "Warning, recieved interrupt. Processing child results and exiting."
send_int = true
kill_child_processes
end
return unless Process.respond_to?(:fork)
# Custom process to wait so that we can do things like time out, and kill child processes if
# one process returns with an error before the others complete.
results_map = Array.new(@@process_infos.count)
start_time = Time.now
timer = start_time
while !@@process_infos.empty? do
if @@parallel_signal_interval > 0 && Time.now > timer + @@parallel_signal_interval
@@logger.debug 'Waiting for child processes.'
timer = Time.now
end
if Time.now > start_time + timeout
kill_child_processes
raise_error = ::RuntimeError.new("Child process ran longer than timeout of #{timeout}")
end
if result = IO.select(@@process_infos.map {|p| p[:result]}, nil, nil, 0.5)
read_ios = result.first
read_ios.each do |reader|
process_info = @@process_infos.find {|p| p[:result] == reader}
process_info[:result_buffer] << reader.read
if reader.eof?
result = process_info[:result_buffer].string
# the process completed, get the result and rethrow on error.
begin
# Print the STDOUT and STDERR for each process with signals for start and end
@@logger.info "------ Begin output for #{process_info[:method_sym]} - #{process_info[:pid]}"
# Content from the other thread will already be pre-pended with log stuff (info, warn, date/time, etc)
# So don't use logger, just use puts.
puts " " + File.new(process_info[:std_out], 'r').readlines.join(" ")
@@logger.info "------ Completed output for #{process_info[:method_sym]} - #{process_info[:pid]}"
marshalled_result = (result.nil? || result.empty?) ? result : Marshal.load(result)
# Kill all other processes and let them log their stdout before re-raising
# if a child process raised an error.
if marshalled_result.is_a?(Exception)
raise_error = marshalled_result.dup
kill_child_processes if kill_all_on_error
marshalled_result = nil
end
results_map[process_info[:index]] = { process_info[:tmp_result] => marshalled_result }
ensure
File.delete(process_info[:std_out]) if File.exist?(process_info[:std_out])
# close the read end pipe
process_info[:result].close unless process_info[:result].closed?
@@process_infos.delete(process_info)
end
end
end
end
end
results = []
# pass in the 'self' from the block.binding which is the instance of the class
# that contains the initial binding call.
# This gives us access to the instance variables from that context.
results = result_lookup(proxy, binding, results_map) if binding
# If there are background_objs AND results, don't return the background obj results
# (which would mess up expected results from each_in_parallel),
# but do process their results in case they are assigned to instance variables
@@background_objs.each { |obj| result_lookup(obj[:proxy], obj[:target], results_map) }
@@background_objs.clear
Process.kill("INT", Process.pid) if send_int
raise raise_error unless raise_error.nil?
return results
end
# private method to execute a block of code in a separate process and store the STDOUT and return value for later retrieval
def self._execute_in_parallel(method_sym, obj = self, &block)
ret_val = nil
# Communicate the return value of the method or block
read_result, write_result = IO.pipe
Dir.mkdir('tmp') unless Dir.exist? 'tmp'
pid = fork do
stdout_file = File.new("tmp/pp_#{Process.pid}", 'w')
exit_status = 0
trap(:INT) do
# Can't use logger inside of trap
puts "Warning: Interrupt received in child process; exiting #{Process.pid}"
kill_child_processes
return
end
# IO buffer is 64kb, which isn't much... if debug logging is turned on,
# this can be exceeded before a process completes.
# Storing output in file rather than using IO.pipe
STDOUT.reopen(stdout_file)
STDERR.reopen(stdout_file)
begin
# close subprocess's copy of read_result since it only needs to write
read_result.close
ret_val = obj.instance_eval(&block)
ret_val = strip_singleton(ret_val)
# In case there are other types that can't be dumped
begin
# Write the result to the write_result IO stream.
Marshal.dump(ret_val, write_result) unless ret_val.nil?
rescue StandardError => err
@@logger.warn "Warning: return value from child process #{ret_val} " +
"could not be transferred to parent process: #{err.message}"
end
rescue Exception => err
@@logger.error "Error in process #{Process.pid}: #{err.message}"
# Return the error if an error is rescued so we can re-throw in the main process.
Marshal.dump(err, write_result)
exit_status = 1
ensure
write_result.close
exit exit_status
end
end
@@logger.info "Forked process for #{method_sym} - PID = '#{pid}'"
write_result.close
# Process.detach returns a thread that will be nil if the process is still running and thr if not.
# This allows us to check to see if processes have exited without having to call the blocking Process.wait functions.
wait_thread = Process.detach(pid)
# store the IO object with the STDOUT and waiting thread for each pid
process_info = { :wait_thread => wait_thread,
:pid => pid,
:method_sym => method_sym,
:std_out => "tmp/pp_#{pid}",
:result => read_result,
:tmp_result => "unresolved_parallel_result_#{@@result_id}",
:result_buffer => StringIO.new,
:index => @@process_infos.count }
@@process_infos.push(process_info)
@@result_id += 1
process_info
end
def self.fork_supported?
@@supported ||= Process.respond_to?(:fork)
@@logger.warn 'Warning: Fork is not supported on this OS, executing block normally' unless @@supported
@@supported
end
def self.kill_child_processes
@@process_infos.each do |process_info|
# Send INT to each child process so it returns and can print stdout and stderr to console before exiting.
begin
Process.kill("INT", process_info[:pid])
rescue Errno::ESRCH
# If one of the other processes has completed in the very short time before we try to kill it, handle the exception
end
end
end
private_class_method :kill_child_processes
def self.strip_singleton(obj)
unless (obj.nil? || obj.singleton_methods.empty?)
obj = obj.dup
end
begin
obj.singleton_class.class_eval do
instance_variables.each { |v| instance_eval("remove_instance_variable(:#{v})") }
end
rescue TypeError # if no singleton_class exists for the object it raises a TypeError
end
# Recursively check any objects assigned to instance variables for singleton methods, or variables
obj.instance_variables.each do |v|
obj.instance_variable_set(v, strip_singleton(obj.instance_variable_get(v)))
end
obj
end
private_class_method :strip_singleton
# Private method to lookup results from the results_map and replace the
# temp values with actual return values
def self.result_lookup(proxy_obj, target_obj, results_map)
target_obj = eval('self', target_obj)
proxy_obj ||= target_obj
vars = proxy_obj.instance_variables
results = []
results_map.each do |tmp_result|
results << tmp_result.values[0]
vars.each do |var|
if proxy_obj.instance_variable_get(var) == tmp_result.keys[0]
target_obj.instance_variable_set(var, tmp_result.values[0])
break
end
end
end
results
end
private_class_method :result_lookup
# Proxy class used to wrap each method execution in a block and run it in parallel
# A block from Parallel.run_in_parallel is executed with a binding of an instance of this class
class BlankBindingParallelProxy < BasicObject
# Don't worry about running methods like puts or other basic stuff in parallel
include ::Kernel
def initialize(obj)
@object = obj
@result_id = 0
end
# All methods within the block should show up as missing (unless defined in :Kernel)
def method_missing(method_sym, *args, &block)
if InParallelExecutor.main_pid == ::Process.pid
out = InParallelExecutor._execute_in_parallel("'#{method_sym.to_s}' #{caller[0].to_s}",
@object.eval('self')) { send(method_sym, *args, &block) }
out[:tmp_result]
end
end
end
end
InParallelExecutor.logger = @logger
# Gets how many seconds to wait between logging a 'Waiting for child processes.'
def parallel_signal_interval
InParallelExecutor.parallel_signal_interval
end
# Sets how many seconds to wait between logging a 'Waiting for child processes.'
# @param [Int] value Time in seconds to wait before logging 'Waiting for child processes.'
def parallel_signal_interval=(value)
InParallelExecutor.parallel_signal_interval = value
end
# Gets how many seconds to wait before timing out a forked child process and raising an exception
def parallel_default_timeout
InParallelExecutor.parallel_default_timeout
end
# Sets how many seconds to wait before timing out a forked child process and raising an exception
# @param [Int] value Time in seconds to wait before timing out and raising an exception
def parallel_default_timeout=(value)
InParallelExecutor.parallel_default_timeout = value
end
# Executes each method within a block in a different process.
#
# Example - Will spawn a process in the background to execute each method
# Parallel.run_in_parallel do
# @result_1 = method1
# @result_2 = method2
# end
# NOTE - Only instance variables can be assigned the return values of the methods within the block. Local variables will not be assigned any values.
# @param [Int] timeout Time in seconds to wait before giving up on a child process
# @param [Boolean] kill_all_on_error Whether to wait for all processes to complete, or fail immediately - killing all other forked processes - when one process errors.
# @param [Block] block This method will yield to a block of code passed by the caller
# @return [Array, Result] the return values of each method within the block
def run_in_parallel(timeout=nil, kill_all_on_error = false, &block)
timeout ||= InParallelExecutor.parallel_default_timeout
InParallelExecutor.run_in_parallel(timeout, kill_all_on_error, &block)
end
# Forks a process for each method within a block and returns immediately.
#
# Example 1 - Will fork a process in the background to execute each method and return immediately:
# Parallel.run_in_background do
# @result_1 = method1
# @result_2 = method2
# end
#
# Example 2 - Will fork a process in the background to execute each method, return immediately, then later
# wait for the process to complete, printing it's STDOUT and assigning return values to instance variables:
# Parallel.run_in_background(false) do
# @result_1 = method1
# @result_2 = method2
# end
# # Do something else here before waiting for the process to complete
#
# wait_for_processes
# NOTE: must call wait_for_processes to allow instance variables within the block to be set, otherwise results will evaluate to "unresolved_parallel_result_X"
# @param [Boolean] ignore_result True if you do not care about the STDOUT or return value of the methods executing in the background
# @param [Block] block This method will yield to a block of code passed by the caller
# @return [Array, Result] the return values of each method within the block
def run_in_background(ignore_result = true, &block)
InParallelExecutor.run_in_background(ignore_result, &block)
end
# Waits for all processes started by run_in_background to complete execution, then prints STDOUT and assigns return values to instance variables. See :run_in_background
# @param [Int] timeout Time in seconds to wait before giving up on a child process
# @param [Boolean] kill_all_on_error Whether to wait for all processes to complete, or fail immediately - killing all other forked processes - when one process errors.
# @return [Array, Result] the temporary return values of each method within the block
def wait_for_processes(timeout=nil, kill_all_on_error = false)
timeout ||= InParallelExecutor.parallel_default_timeout
InParallelExecutor.wait_for_processes(nil, nil, timeout, kill_all_on_error)
end
end
ruby-in-parallel-1.0.1/lib/parallel_enumerable.rb 0000664 0000000 0000000 00000002376 14522154247 0021774 0 ustar 00root root 0000000 0000000 # Extending Enumerable to make it easy to do any .each in parallel
module Enumerable
# Executes each iteration of the block in parallel
#
# Example - Will execute each iteration in a separate process, in parallel, log STDOUT per process, and return an array of results.
# my_array = [1,2,3]
# my_array.each_in_parallel { |int| my_method(int) }
# @param [String] identifier - Optional identifier for logging purposes only. Will use the block location by default.
# @param [Int] timeout - Seconds to wait for a forked process to complete before timing out
# @return [Array