librarian-puppet-3.0.0/0000755000004100000410000000000013236362373015030 5ustar www-datawww-datalibrarian-puppet-3.0.0/bin/0000755000004100000410000000000013236362373015600 5ustar www-datawww-datalibrarian-puppet-3.0.0/bin/librarian-puppet0000755000004100000410000000024713236362373021007 0ustar www-datawww-data#!/usr/bin/env ruby lib = File.expand_path('../../lib', __FILE__) $:.unshift(lib) unless $:.include?(lib) require 'librarian/puppet/cli' Librarian::Puppet::Cli.bin! librarian-puppet-3.0.0/lib/0000755000004100000410000000000013236362373015576 5ustar www-datawww-datalibrarian-puppet-3.0.0/lib/librarian/0000755000004100000410000000000013236362373017541 5ustar www-datawww-datalibrarian-puppet-3.0.0/lib/librarian/puppet/0000755000004100000410000000000013236362373021056 5ustar www-datawww-datalibrarian-puppet-3.0.0/lib/librarian/puppet/source.rb0000644000004100000410000000024513236362373022704 0ustar www-datawww-datarequire 'librarian/puppet/source/path' require 'librarian/puppet/source/git' require 'librarian/puppet/source/forge' require 'librarian/puppet/source/githubtarball' librarian-puppet-3.0.0/lib/librarian/puppet/action/0000755000004100000410000000000013236362373022333 5ustar www-datawww-datalibrarian-puppet-3.0.0/lib/librarian/puppet/action/install.rb0000644000004100000410000000106313236362373024326 0ustar www-datawww-datarequire 'librarian/action/install' module Librarian module Puppet module Action class Install < Librarian::Action::Install private def create_install_path install_path.rmtree if install_path.exist? && destructive? install_path.mkpath end def destructive? environment.config_db.local['destructive'] == 'true' end def check_specfile # don't fail if Puppetfile doesn't exist as we'll use the Modulefile or metadata.json end end end end end librarian-puppet-3.0.0/lib/librarian/puppet/action/resolve.rb0000644000004100000410000000111213236362373024332 0ustar www-datawww-datarequire 'librarian/action/resolve' module Librarian module Puppet module Action class Resolve < Librarian::Action::Resolve include Librarian::Puppet::Util def run super manifests = environment.lock.manifests.select{ |m| m.name } dupes = manifests.group_by{ |m| module_name(m.name) }.select { |k, v| v.size > 1 } dupes.each do |k,v| warn("Dependency on module '#{k}' is fullfilled by multiple modules and only one will be used: #{v.map{|m|m.name}}") end end end end end end librarian-puppet-3.0.0/lib/librarian/puppet/dependency.rb0000644000004100000410000000064713236362373023530 0ustar www-datawww-datamodule Librarian module Puppet class Dependency < Librarian::Dependency include Librarian::Puppet::Util def initialize(name, requirement, source) # Issue #235 fail if forge source is not defined raise Error, "forge entry is not defined in Puppetfile" if source.instance_of?(Array) && source.empty? super(normalize_name(name), requirement, source) end end end end librarian-puppet-3.0.0/lib/librarian/puppet/templates/0000755000004100000410000000000013236362373023054 5ustar www-datawww-datalibrarian-puppet-3.0.0/lib/librarian/puppet/templates/Puppetfile0000644000004100000410000000116413236362373025116 0ustar www-datawww-data#!/usr/bin/env ruby #^syntax detection forge "https://forgeapi.puppetlabs.com" # use dependencies defined in metadata.json metadata # use dependencies defined in Modulefile # modulefile # A module from the Puppet Forge # mod 'puppetlabs-stdlib' # A module from git # mod 'puppetlabs-ntp', # :git => 'git://github.com/puppetlabs/puppetlabs-ntp.git' # A module from a git branch/tag # mod 'puppetlabs-apt', # :git => 'https://github.com/puppetlabs/puppetlabs-apt.git', # :ref => '1.4.x' # A module from Github pre-packaged tarball # mod 'puppetlabs-apache', '0.6.0', :github_tarball => 'puppetlabs/puppetlabs-apache' librarian-puppet-3.0.0/lib/librarian/puppet/cli.rb0000644000004100000410000000653313236362373022161 0ustar www-datawww-datarequire 'librarian/helpers' require 'librarian/cli' require 'librarian/puppet' require 'librarian/puppet/action' module Librarian module Puppet class Cli < Librarian::Cli include Librarian::Puppet::Util module Particularity def root_module Puppet end end include Particularity extend Particularity source_root Pathname.new(__FILE__).dirname.join("templates") def init copy_file environment.specfile_name if File.exists? ".gitignore" gitignore = File.read('.gitignore').split("\n") else gitignore = [] end gitignore << ".tmp/" unless gitignore.include? ".tmp/" gitignore << "modules/" unless gitignore.include? "modules/" File.open(".gitignore", 'w') do |f| f.puts gitignore.join("\n") end end desc "install", "Resolves and installs all of the dependencies you specify." option "quiet", :type => :boolean, :default => false option "verbose", :type => :boolean, :default => false option "line-numbers", :type => :boolean, :default => false option "clean", :type => :boolean, :default => false option "strip-dot-git", :type => :boolean option "path", :type => :string option "destructive", :type => :boolean, :default => false option "local", :type => :boolean, :default => false option "use-v1-api", :type => :boolean, :default => true def install ensure! clean! if options["clean"] unless options["destructive"].nil? environment.config_db.local['destructive'] = options['destructive'].to_s end if options.include?("strip-dot-git") strip_dot_git_val = options["strip-dot-git"] ? "1" : nil environment.config_db.local["install.strip-dot-git"] = strip_dot_git_val end if options.include?("path") environment.config_db.local["path"] = options["path"] end environment.config_db.local['use-v1-api'] = options['use-v1-api'] ? '1' : nil environment.config_db.local['mode'] = options['local'] ? 'local' : nil resolve! debug { "Install: dependencies resolved"} install! end # only used to replace / to - in the module names def update(*names) warn("Usage of module/name is deprecated, use module-name") if names.any? {|n| n.include?("/")} super(*names.map{|n| normalize_name(n)}) end desc "package", "Cache the puppet modules in vendor/puppet/cache." option "quiet", :type => :boolean, :default => false option "verbose", :type => :boolean, :default => false option "line-numbers", :type => :boolean, :default => false option "clean", :type => :boolean, :default => false option "strip-dot-git", :type => :boolean option "path", :type => :string option "destructive", :type => :boolean, :default => false def package environment.vendor! install end def version say "librarian-puppet v#{Librarian::Puppet::VERSION}" end private # override the actions to use our own def install!(options = { }) Action::Install.new(environment, options).run end def resolve!(options = { }) Action::Resolve.new(environment, options).run end end end end librarian-puppet-3.0.0/lib/librarian/puppet/version.rb0000644000004100000410000000010113236362373023060 0ustar www-datawww-datamodule Librarian module Puppet VERSION = "3.0.0" end end librarian-puppet-3.0.0/lib/librarian/puppet/source/0000755000004100000410000000000013236362373022356 5ustar www-datawww-datalibrarian-puppet-3.0.0/lib/librarian/puppet/source/git.rb0000644000004100000410000000356013236362373023472 0ustar www-datawww-datarequire 'librarian/source/git' require 'librarian/puppet/source/local' module Librarian module Source class Git class Repository def hash_from(remote, reference) branch_names = remote_branch_names[remote] if branch_names.include?(reference) reference = "#{remote}/#{reference}" end command = %W(rev-parse #{reference}^{commit} --quiet) run!(command, :chdir => true).strip end end end end module Puppet module Source class Git < Librarian::Source::Git include Local include Librarian::Puppet::Util def cache! return vendor_checkout! if vendor_cached? if environment.local? raise Error, "Could not find a local copy of #{uri}#{" at #{sha}" unless sha.nil?}." end begin super rescue Librarian::Posix::CommandFailure => e raise Error, "Could not checkout #{uri}#{" at #{sha}" unless sha.nil?}: #{e}" end cache_in_vendor(repository.path) if environment.vendor? end private def vendor_tar environment.vendor_source.join("#{sha}.tar") end def vendor_tgz environment.vendor_source.join("#{sha}.tar.gz") end def vendor_cached? vendor_tgz.exist? end def vendor_checkout! repository.path.rmtree if repository.path.exist? repository.path.mkpath Librarian::Posix.run!(%W{tar xzf #{vendor_tgz}}, :chdir => repository.path.to_s) repository_cached! end def cache_in_vendor(tmp_path) Librarian::Posix.run!(%W{git archive -o #{vendor_tar} #{sha}}, :chdir => tmp_path.to_s) Librarian::Posix.run!(%W{gzip #{vendor_tar}}, :chdir => tmp_path.to_s) end end end end end librarian-puppet-3.0.0/lib/librarian/puppet/source/githubtarball/0000755000004100000410000000000013236362373025202 5ustar www-datawww-datalibrarian-puppet-3.0.0/lib/librarian/puppet/source/githubtarball/repo.rb0000644000004100000410000001276613236362373026510 0ustar www-datawww-datarequire 'uri' require 'net/https' require 'open-uri' require 'json' require 'librarian/puppet/version' require 'librarian/puppet/source/repo' module Librarian module Puppet module Source class GitHubTarball class Repo < Librarian::Puppet::Source::Repo include Librarian::Puppet::Util TOKEN_KEY = 'GITHUB_API_TOKEN' def versions return @versions if @versions data = api_call("/repos/#{source.uri}/tags") if data.nil? raise Error, "Unable to find module '#{source.uri}' on https://github.com" end all_versions = data.map { |r| r['name'].gsub(/^v/, '') }.sort.reverse all_versions.delete_if do |version| version !~ /\A\d+\.\d+(\.\d+.*)?\z/ end @versions = all_versions.compact debug { " Module #{name} found versions: #{@versions.join(", ")}" } @versions end def manifests versions.map do |version| Manifest.new(source, name, version) end end def install_version!(version, install_path) if environment.local? && !vendored?(vendored_name, version) raise Error, "Could not find a local copy of #{source.uri} at #{version}." end vendor_cache(source.uri.to_s, version) unless vendored?(vendored_name, version) cache_version_unpacked! version if install_path.exist? && rsync? != true install_path.rmtree end unpacked_path = version_unpacked_cache_path(version).children.first cp_r(unpacked_path, install_path) end def cache_version_unpacked!(version) path = version_unpacked_cache_path(version) return if path.directory? path.mkpath target = vendored?(vendored_name, version) ? vendored_path(vendored_name, version) : name Librarian::Posix.run!(%W{tar xzf #{target} -C #{path}}) end def vendor_cache(name, version) clean_up_old_cached_versions(vendored_name(name)) url = "https://api.github.com/repos/#{name}/tarball/#{version}" add_api_token_to_url(url) environment.vendor! File.open(vendored_path(vendored_name(name), version).to_s, 'wb') do |f| begin debug { "Downloading <#{url}> to <#{f.path}>" } open(url, "User-Agent" => "librarian-puppet v#{Librarian::Puppet::VERSION}") do |res| while buffer = res.read(8192) f.write(buffer) end end rescue OpenURI::HTTPError => e raise e, "Error requesting <#{url}>: #{e.to_s}" end end end def clean_up_old_cached_versions(name) Dir["#{environment.vendor_cache}/#{name}*.tar.gz"].each do |old_version| FileUtils.rm old_version end end def token_key_value ENV[TOKEN_KEY] end def token_key_nil? token_key_value.nil? || token_key_value.empty? end def add_api_token_to_url url if token_key_nil? debug { "#{TOKEN_KEY} environment value is empty or missing" } elsif url.include? "?" url << "&access_token=#{ENV[TOKEN_KEY]}" else url << "?access_token=#{ENV[TOKEN_KEY]}" end url end private def api_call(path) tags = [] url = "https://api.github.com#{path}?page=1&per_page=100" while true do debug { " Module #{name} getting tags at: #{url}" } add_api_token_to_url(url) response = http_get(url, :headers => { "User-Agent" => "librarian-puppet v#{Librarian::Puppet::VERSION}" }) code, data = response.code.to_i, response.body if code == 200 tags.concat JSON.parse(data) else begin message = JSON.parse(data)['message'] if code == 403 && message && message.include?('API rate limit exceeded') raise Error, message + " -- increase limit by authenticating via #{TOKEN_KEY}=your-token" elsif message raise Error, "Error fetching #{url}: [#{code}] #{message}" end rescue JSON::ParserError # response does not return json end raise Error, "Error fetching #{url}: [#{code}] #{response.body}" end # next page break if response["link"].nil? next_link = response["link"].split(",").select{|l| l.match /rel=.*next.*/} break if next_link.empty? url = next_link.first.match(/<(.*)>/)[1] end return tags end def http_get(url, options) uri = URI.parse(url) http = Net::HTTP.new(uri.host, uri.port) http.use_ssl = true request = Net::HTTP::Get.new(uri.request_uri) options[:headers].each { |k, v| request.add_field k, v } http.request(request) end def vendored_name(name = source.uri.to_s) name.sub('/','-') end end end end end end librarian-puppet-3.0.0/lib/librarian/puppet/source/githubtarball.rb0000644000004100000410000000567313236362373025542 0ustar www-datawww-datarequire 'uri' require 'librarian/puppet/util' require 'librarian/puppet/source/githubtarball/repo' module Librarian module Puppet module Source class GitHubTarball include Librarian::Puppet::Util class << self LOCK_NAME = 'GITHUBTARBALL' def lock_name LOCK_NAME end def from_lock_options(environment, options) new(environment, options[:remote], options.reject { |k, v| k == :remote }) end def from_spec_args(environment, uri, options) recognised_options = [] unrecognised_options = options.keys - recognised_options unless unrecognised_options.empty? raise Error, "unrecognised options: #{unrecognised_options.join(", ")}" end new(environment, uri, options) end end attr_accessor :environment private :environment= attr_reader :uri def initialize(environment, uri, options = {}) self.environment = environment @uri = URI::parse(uri) @cache_path = nil end def to_s clean_uri(uri).to_s end def ==(other) other && self.class == other.class && self.uri == other.uri end alias :eql? :== def hash self.to_s.hash end def to_spec_args [clean_uri(uri).to_s, {}] end def to_lock_options {:remote => clean_uri(uri).to_s} end def pinned? false end def unpin! end def install!(manifest) manifest.source == self or raise ArgumentError debug { "Installing #{manifest}" } name = manifest.name version = manifest.version install_path = install_path(name) repo = repo(name) repo.install_version! version, install_path end def manifest(name, version, dependencies) manifest = Manifest.new(self, name) manifest.version = version manifest.dependencies = dependencies manifest end def cache_path @cache_path ||= begin environment.cache_path.join("source/puppet/githubtarball/#{uri.host}#{uri.path}") end end def install_path(name) environment.install_path.join(module_name(name)) end def fetch_version(name, version_uri) versions = repo(name).versions if versions.include? version_uri version_uri else versions.first end end def fetch_dependencies(name, version, version_uri) {} end def manifests(name) repo(name).manifests end private def repo(name) @repo ||= {} @repo[name] ||= Repo.new(self, name) end end end end end librarian-puppet-3.0.0/lib/librarian/puppet/source/path.rb0000644000004100000410000000033113236362373023634 0ustar www-datawww-datarequire 'librarian/source/path' require 'librarian/puppet/source/local' module Librarian module Puppet module Source class Path < Librarian::Source::Path include Local end end end end librarian-puppet-3.0.0/lib/librarian/puppet/source/forge/0000755000004100000410000000000013236362373023460 5ustar www-datawww-datalibrarian-puppet-3.0.0/lib/librarian/puppet/source/forge/repo_v3.rb0000644000004100000410000000367713236362373025377 0ustar www-datawww-datarequire 'librarian/puppet/source/forge/repo' require 'puppet_forge' require 'librarian/puppet/version' module Librarian module Puppet module Source class Forge class RepoV3 < Librarian::Puppet::Source::Forge::Repo PuppetForge.user_agent = "librarian-puppet/#{Librarian::Puppet::VERSION}" def initialize(source, name) PuppetForge.host = source.uri.clone super(source, name) end def get_versions get_module.releases.select{|r| r.deleted_at.nil?}.map{|r| r.version} end def dependencies(version) array = get_release(version).metadata[:dependencies].map{|d| [d[:name], d[:version_requirement]]} Hash[*array.flatten(1)] end def url(name, version) if name == "#{get_module().owner.username}/#{get_module().name}" release = get_release(version) else # should never get here as we use one repo object for each module (to be changed in the future) debug { "Looking up url for #{name}@#{version}" } release = PuppetForge::V3::Release.find("#{name}-#{version}") end "#{source}#{release.file_uri}" end private def get_module begin @module ||= PuppetForge::V3::Module.find(name) rescue Faraday::ResourceNotFound => e raise(Error, "Unable to find module '#{name}' on #{source}") end @module end def get_release(version) release = get_module.releases.find{|r| r.version == version.to_s} if release.nil? versions = get_module.releases.map{|r| r.version} raise Error, "Unable to find version '#{version}' for module '#{name}' on #{source} amongst #{versions}" end release end end end end end end librarian-puppet-3.0.0/lib/librarian/puppet/source/forge/repo.rb0000644000004100000410000001365513236362373024764 0ustar www-datawww-datarequire 'json' require 'open-uri' require 'librarian/puppet/util' require 'librarian/puppet/source/repo' module Librarian module Puppet module Source class Forge class Repo < Librarian::Puppet::Source::Repo include Librarian::Puppet::Util def versions return @versions if @versions @versions = get_versions if @versions.empty? info { "No versions found for module #{name}" } else debug { " Module #{name} found versions: #{@versions.join(", ")}" } end @versions end # fetch list of versions ordered for newer to older def get_versions # implement in subclasses end # return map with dependencies in the form {module_name => version,...} # version: Librarian::Manifest::Version def dependencies(version) # implement in subclasses end # return the url for a specific version tarball # version: Librarian::Manifest::Version def url(name, version) # implement in subclasses end def manifests versions.map do |version| Manifest.new(source, name, version) end end def install_version!(version, install_path) if environment.local? && !vendored?(name, version) raise Error, "Could not find a local copy of #{name} at #{version}." end if environment.vendor? vendor_cache(name, version) unless vendored?(name, version) end cache_version_unpacked! version if install_path.exist? && rsync? != true install_path.rmtree end unpacked_path = version_unpacked_cache_path(version).join(module_name(name)) unless unpacked_path.exist? raise Error, "#{unpacked_path} does not exist, something went wrong. Try removing it manually" else cp_r(unpacked_path, install_path) end end def cache_version_unpacked!(version) path = version_unpacked_cache_path(version) return if path.directory? # The puppet module command is only available from puppet versions >= 2.7.13 # # Specifying the version in the gemspec would force people to upgrade puppet while it's still usable for git # So we do some more clever checking # # Executing older versions or via puppet-module tool gives an exit status = 0 . # check_puppet_module_options path.mkpath target = vendored?(name, version) ? vendored_path(name, version).to_s : name # can't pass the default v3 forge url (http://forgeapi.puppetlabs.com) # to clients that use the v1 API (https://forge.puppet.com) # nor the other way around module_repository = source.to_s if Forge.client_api_version() > 1 and module_repository =~ %r{^http(s)?://forge\.puppetlabs\.com} module_repository = "https://forgeapi.puppetlabs.com" warn { "Replacing Puppet Forge API URL to use v3 #{module_repository} as required by your client version #{Librarian::Puppet.puppet_version}" } end m = module_repository.match(%r{^http(s)?://forge(api)?\.puppetlabs\.com}) if Forge.client_api_version() == 1 and m ssl = m[1] # Puppet 2.7 can't handle the 302 returned by the https url, so stick to http if ssl and Librarian::Puppet::puppet_gem_version < Gem::Version.create('3.0.0') warn { "Using plain http as your version of Puppet #{Librarian::Puppet::puppet_gem_version} can't download from forge.puppetlabs.com using https" } ssl = nil end module_repository = "http#{ssl}://forge.puppetlabs.com" end command = %W{puppet module install --version #{version} --target-dir} command.push(*[path.to_s, "--module_repository", module_repository, "--modulepath", path.to_s, "--module_working_dir", path.to_s, "--ignore-dependencies", target]) debug { "Executing puppet module install for #{name} #{version}: #{command.join(" ")}" } begin Librarian::Posix.run!(command) rescue Posix::CommandFailure => e # Rollback the directory if the puppet module had an error begin path.unlink rescue => u debug("Unable to rollback path #{path}: #{u}") end tar = Dir[File.join(path.to_s, "**/*.tar.gz")] msg = "" if e.message =~ /Unexpected EOF in archive/ and !tar.empty? file = tar.first msg = " (looks like an incomplete download of #{file})" end raise Error, "Error executing puppet module install#{msg}. Check that this command succeeds:\n#{command.join(" ")}\nError:\n#{e.message}" end end def check_puppet_module_options min_version = Gem::Version.create('2.7.13') if Librarian::Puppet.puppet_gem_version < min_version raise Error, "To get modules from the forge, we use the puppet faces module command. For this you need at least puppet version 2.7.13 and you have #{Librarian::Puppet.puppet_version}" end end def vendor_cache(name, version) url = url(name, version) path = vendored_path(name, version).to_s debug { "Downloading #{url} into #{path}"} environment.vendor! File.open(path, 'wb') do |f| open(url, "rb") do |input| f.write(input.read) end end end end end end end end librarian-puppet-3.0.0/lib/librarian/puppet/source/forge/repo_v1.rb0000644000004100000410000000655613236362373025374 0ustar www-datawww-datarequire 'json' require 'open-uri' require 'librarian/puppet/source/forge/repo' module Librarian module Puppet module Source class Forge class RepoV1 < Librarian::Puppet::Source::Forge::Repo def initialize(source, name) super(source, name) # API returned data for this module including all versions and dependencies, indexed by module name # from https://forge.puppetlabs.com/api/v1/releases.json?module=#{name} @api_data = nil # API returned data for this module and a specific version, indexed by version # from https://forge.puppetlabs.com/api/v1/releases.json?module=#{name}&version=#{version} @api_version_data = {} end def get_versions api_data(name).map { |r| r['version'] }.reverse end def dependencies(version) api_version_data(name, version)['dependencies'] end def url(name, version) info = api_version_data(name, version) "#{source}#{info[name].first['file']}" end private # convert organization/modulename to organization-modulename def normalize_dependencies(data) return nil if data.nil? # convert organization/modulename to organization-modulename data.keys.each do |m| if m =~ %r{.*/.*} data[normalize_name(m)] = data[m] data.delete(m) end end data end # get and cache the API data for a specific module with all its versions and dependencies def api_data(module_name) return @api_data[module_name] if @api_data # call API and cache data @api_data = normalize_dependencies(api_call(module_name)) if @api_data.nil? raise Error, "Unable to find module '#{name}' on #{source}" end @api_data[module_name] end # get and cache the API data for a specific module and version def api_version_data(module_name, version) # if we already got all the versions, find in cached data return @api_data[module_name].detect{|x| x['version'] == version.to_s} if @api_data # otherwise call the api for this version if not cached already @api_version_data[version] = normalize_dependencies(api_call(name, version)) if @api_version_data[version].nil? @api_version_data[version] end def api_call(module_name, version=nil) url = source.uri.clone url.path += "#{'/' if url.path.empty? or url.path[-1] != '/'}api/v1/releases.json" url.query = "module=#{module_name.sub('-','/')}" # v1 API expects "organization/module" url.query += "&version=#{version}" unless version.nil? debug { "Querying Forge API for module #{name}#{" and version #{version}" unless version.nil?}: #{url}" } begin data = open(url) {|f| f.read} JSON.parse(data) rescue OpenURI::HTTPError => e case e.io.status[0].to_i when 404,410 nil else raise e, "Error requesting #{url}: #{e.to_s}" end end end end end end end end librarian-puppet-3.0.0/lib/librarian/puppet/source/repo.rb0000644000004100000410000000145213236362373023652 0ustar www-datawww-data# parent class for githubtarball and forge source Repos module Librarian module Puppet module Source class Repo attr_accessor :source, :name private :source=, :name= def initialize(source, name) self.source = source self.name = name end def environment source.environment end def cache_path @cache_path ||= source.cache_path.join(name) end def version_unpacked_cache_path(version) cache_path.join(version.to_s) end def vendored?(name, version) vendored_path(name, version).exist? end def vendored_path(name, version) environment.vendor_cache.join("#{name}-#{version}.tar.gz") end end end end end librarian-puppet-3.0.0/lib/librarian/puppet/source/local.rb0000644000004100000410000001270213236362373023777 0ustar www-datawww-datarequire 'librarian/puppet/util' module Librarian module Puppet module Source module Local include Librarian::Puppet::Util def install!(manifest) manifest.source == self or raise ArgumentError debug { "Installing #{manifest}" } name, version = manifest.name, manifest.version found_path = found_path(name) raise Error, "Path for #{name} doesn't contain a puppet module" if found_path.nil? unless name.include? '/' or name.include? '-' warn { "Invalid module name '#{name}', you should qualify it with 'ORGANIZATION-#{name}' for resolution to work correctly" } end install_path = environment.install_path.join(module_name(name)) if install_path.exist? && rsync? != true debug { "Deleting #{relative_path_to(install_path)}" } install_path.rmtree end install_perform_step_copy!(found_path, install_path) end def fetch_version(name, extra) cache! found_path = found_path(name) module_version end def fetch_dependencies(name, version, extra) dependencies = Set.new if specfile? spec = environment.dsl(Pathname(specfile)) dependencies.merge spec.dependencies end parsed_metadata['dependencies'].each do |d| gem_requirement = Librarian::Dependency::Requirement.new(d['version_requirement']).to_gem_requirement new_dependency = Dependency.new(d['name'], gem_requirement, forge_source) dependencies << new_dependency end dependencies end def forge_source Forge.default end private # Naming this method 'version' causes an exception to be raised. def module_version if parsed_metadata['version'] parsed_metadata['version'] else warn { "Module #{to_s} does not have version, defaulting to 0.0.1" } '0.0.1' end end def require_puppet begin require 'puppet' require 'puppet/module_tool' rescue LoadError $stderr.puts <<-EOF Unable to load puppet, the puppet gem is required for :git and :path source. Install it with: gem install puppet EOF exit 1 end true end def evaluate_modulefile(modulefile) @@require_puppet ||= require_puppet metadata = ::Puppet::ModuleTool::Metadata.new # Puppet 4 does not have the class unless defined? ::Puppet::ModuleTool::ModulefileReader warn { "Can't parse Modulefile in Puppet >= 4.0 and you are using #{Librarian::Puppet::puppet_version}. Ignoring dependencies in #{modulefile}" } return metadata end begin ::Puppet::ModuleTool::ModulefileReader.evaluate(metadata, modulefile) raise SyntaxError, "Missing version" unless metadata.version rescue ArgumentError, SyntaxError => error warn { "Unable to parse #{modulefile}, ignoring: #{error}" } if metadata.respond_to? :version= metadata.version = '0.0.1' # puppet < 3.6 else metadata.update({'version' => '0.0.1'}) # puppet >= 3.6 end end metadata end def parsed_metadata if @metadata.nil? @metadata = if metadata? begin JSON.parse(File.read(metadata)) rescue JSON::ParserError => e raise Error, "Unable to parse json file #{metadata}: #{e}" end elsif modulefile? # translate Modulefile to metadata.json evaluated = evaluate_modulefile(modulefile) { 'version' => evaluated.version, 'dependencies' => evaluated.dependencies.map do |dependency| { 'name' => dependency.instance_variable_get(:@full_module_name), 'version_requirement' => dependency.instance_variable_get(:@version_requirement) } end } else {} end @metadata['dependencies'] ||= [] end @metadata end def modulefile File.join(filesystem_path, 'Modulefile') end def modulefile? File.exists?(modulefile) end def metadata File.join(filesystem_path, 'metadata.json') end def metadata? File.exists?(metadata) end def specfile File.join(filesystem_path, environment.specfile_name) end def specfile? File.exists?(specfile) end def install_perform_step_copy!(found_path, install_path) debug { "Copying #{relative_path_to(found_path)} to #{relative_path_to(install_path)}" } cp_r(found_path, install_path) end def manifest?(name, path) return true if path.join('manifests').exist? return true if path.join('lib').join('puppet').exist? return true if path.join('lib').join('facter').exist? debug { "Could not find manifests, lib/puppet or lib/facter under #{path}, maybe it is not a puppet module" } true end end end end end librarian-puppet-3.0.0/lib/librarian/puppet/source/forge.rb0000644000004100000410000001071113236362373024005 0ustar www-datawww-datarequire 'uri' require 'librarian/puppet/util' require 'librarian/puppet/source/forge/repo_v1' require 'librarian/puppet/source/forge/repo_v3' module Librarian module Puppet module Source class Forge include Librarian::Puppet::Util class << self LOCK_NAME = 'FORGE' def default=(source) @@default = source end def default @@default end def lock_name LOCK_NAME end def from_lock_options(environment, options) new(environment, options[:remote], options.reject { |k, v| k == :remote }) end def from_spec_args(environment, uri, options) recognised_options = [] unrecognised_options = options.keys - recognised_options unless unrecognised_options.empty? raise Error, "unrecognised options: #{unrecognised_options.join(", ")}" end new(environment, uri, options) end def client_api_version() version = 1 pe_version = Librarian::Puppet.puppet_version.match(/\(Puppet Enterprise (.+)\)/) # Puppet 3.6.0+ uses api v3 if Librarian::Puppet::puppet_gem_version >= Gem::Version.create('3.6.0.a') version = 3 # Puppet enterprise 3.2.0+ uses api v3 elsif pe_version and Gem::Version.create(pe_version[1].strip) >= Gem::Version.create('3.2.0') version = 3 end return version end end attr_accessor :environment private :environment= attr_reader :uri def initialize(environment, uri, options = {}) self.environment = environment if uri =~ %r{^http(s)?://forge\.puppetlabs\.com} uri = "https://forgeapi.puppetlabs.com" warn { "Replacing Puppet Forge API URL to use v3 #{uri}. You should update your Puppetfile" } end @uri = URI::parse(uri) @cache_path = nil end def to_s clean_uri(uri).to_s end def ==(other) other && self.class == other.class && self.uri == other.uri end alias :eql? :== def hash self.to_s.hash end def to_spec_args [clean_uri(uri).to_s, {}] end def to_lock_options {:remote => clean_uri(uri).to_s} end def pinned? false end def unpin! end def install!(manifest) manifest.source == self or raise ArgumentError debug { "Installing #{manifest}" } name = manifest.name version = manifest.version install_path = install_path(name) repo = repo(name) repo.install_version! version, install_path end def manifest(name, version, dependencies) manifest = Manifest.new(self, name) manifest.version = version manifest.dependencies = dependencies manifest end def cache_path @cache_path ||= begin dir = "#{uri.host}#{uri.path}".gsub(/[^0-9a-z\-_]/i, '_') environment.cache_path.join("source/puppet/forge/#{dir}") end end def install_path(name) environment.install_path.join(module_name(name)) end def fetch_version(name, version_uri) versions = repo(name).versions if versions.include? version_uri version_uri else versions.first end end def fetch_dependencies(name, version, version_uri) repo(name).dependencies(version).map do |k, v| v = Librarian::Dependency::Requirement.new(v).to_gem_requirement Dependency.new(k, v, nil) end end def manifests(name) repo(name).manifests end private def repo(name) @repo ||= {} unless @repo[name] # If we are using the official Forge then use API v3, otherwise use the preferred api # as defined by the CLI option use_v1_api if uri.hostname =~ /\.puppetlabs\.com$/ || !environment.use_v1_api @repo[name] = RepoV3.new(self, name) else @repo[name] = RepoV1.new(self, name) end end @repo[name] end end end end end librarian-puppet-3.0.0/lib/librarian/puppet/environment.rb0000644000004100000410000000224013236362373023745 0ustar www-datawww-datarequire "librarian/environment" require "librarian/puppet/dsl" require "librarian/puppet/source" require "librarian/puppet/lockfile" module Librarian module Puppet class Environment < Librarian::Environment def adapter_name "puppet" end def lockfile Lockfile.new(self, lockfile_path) end def ephemeral_lockfile Lockfile.new(self, nil) end def tmp_path part = config_db["tmp"] || ".tmp" project_path.join(part) end def install_path part = config_db["path"] || "modules" project_path.join(part) end def vendor_path project_path.join('vendor/puppet') end def vendor_cache vendor_path.join('cache') end def vendor_source vendor_path.join('source') end def vendor! vendor_cache.mkpath unless vendor_cache.exist? vendor_source.mkpath unless vendor_source.exist? end def vendor? vendor_path.exist? end def local? config_db['mode'] == 'local' end def use_v1_api config_db['use-v1-api'] end end end end librarian-puppet-3.0.0/lib/librarian/puppet/lockfile.rb0000644000004100000410000000216013236362373023172 0ustar www-datawww-data# Extend Lockfile to normalize module names from acme/mod to acme-mod module Librarian module Puppet class Lockfile < Librarian::Lockfile # Extend the parser to normalize module names in old .lock files, converting / to - class Parser < Librarian::Lockfile::Parser include Librarian::Puppet::Util def extract_and_parse_sources(lines) sources = super sources.each do |source| source[:manifests] = Hash[source[:manifests].map do |name,manifest| [normalize_name(name), manifest] end] end sources end def extract_and_parse_dependencies(lines, manifests_index) # when looking up in manifests_index normalize the name beforehand class << manifests_index include Librarian::Puppet::Util alias_method :old_lookup, :[] define_method(:[]) { |k| self.old_lookup(normalize_name(k)) } end super(lines, manifests_index) end end def load(string) Parser.new(environment).parse(string) end end end end librarian-puppet-3.0.0/lib/librarian/puppet/extension.rb0000644000004100000410000000023013236362373023412 0ustar www-datawww-datarequire 'librarian/puppet/environment' require 'librarian/action/base' module Librarian module Puppet extend self extend Librarian end end librarian-puppet-3.0.0/lib/librarian/puppet/action.rb0000644000004100000410000000012413236362373022655 0ustar www-datawww-datarequire "librarian/puppet/action/install" require "librarian/puppet/action/resolve" librarian-puppet-3.0.0/lib/librarian/puppet/util.rb0000644000004100000410000000472613236362373022371 0ustar www-datawww-datarequire 'rsync' module Librarian module Puppet module Util def debug(*args, &block) environment.logger.debug(*args, &block) end def info(*args, &block) environment.logger.info(*args, &block) end def warn(*args, &block) environment.logger.warn(*args, &block) end def rsync? environment.config_db.local['rsync'] == 'true' end # workaround Issue #173 FileUtils.cp_r will fail if there is a symlink that points to a missing file # or when the symlink is copied before the target file when preserve is true # see also https://tickets.opscode.com/browse/CHEF-833 # # If the rsync configuration parameter is set, use rsync instead of FileUtils def cp_r(src, dest) if rsync? if Gem.win_platform? src_clean = "#{src}".gsub(/^([a-z])\:/i,'/cygdrive/\1') dest_clean = "#{dest}".gsub(/^([a-z])\:/i,'/cygdrive/\1') else src_clean = src dest_clean = dest end debug { "Copying #{src_clean}/ to #{dest_clean}/ with rsync -avz --delete" } result = Rsync.run(File.join(src_clean, "/"), File.join(dest_clean, "/"), ['-avz', '--delete']) if result.success? debug { "Rsync from #{src_clean}/ to #{dest_clean}/ successfull" } else msg = "Failed to rsync from #{src_clean}/ to #{dest_clean}/: " + result.error raise Error, msg end else begin FileUtils.cp_r(src, dest, :preserve => true) rescue Errno::ENOENT, Errno::EACCES debug { "Failed to copy from #{src} to #{dest} preserving file types, trying again without preserving them" } FileUtils.rm_rf(dest) FileUtils.cp_r(src, dest) end end end # Remove user and password from a URI object def clean_uri(uri) new_uri = uri.clone new_uri.user = nil new_uri.password = nil new_uri end # normalize module name to use organization-module instead of organization/module def normalize_name(name) name.sub('/','-') end # get the module name from organization-module def module_name(name) # module name can't have dashes, so let's assume it is everything after the last dash name.rpartition('-').last end # deprecated alias :organization_name :module_name end end end librarian-puppet-3.0.0/lib/librarian/puppet/dsl.rb0000644000004100000410000000523013236362373022165 0ustar www-datawww-datarequire 'librarian/dsl' require 'librarian/dsl/target' require 'librarian/puppet/source' require 'librarian/puppet/dependency' module Librarian module Puppet class Dsl < Librarian::Dsl FORGE_URL = "https://forgeapi.puppetlabs.com" dependency :mod source :forge => Source::Forge source :git => Source::Git source :path => Source::Path source :github_tarball => Source::GitHubTarball def default_specfile Proc.new do forge FORGE_URL metadata end end def self.dependency_type Librarian::Puppet::Dependency end def post_process_target(target) # save the default forge defined default_forge = target.sources.select {|s| s.is_a? Librarian::Puppet::Source::Forge}.first Librarian::Puppet::Source::Forge.default = default_forge || Librarian::Puppet::Source::Forge.from_lock_options(environment, :remote => FORGE_URL) end def receiver(target) Receiver.new(target) end class Receiver < Librarian::Dsl::Receiver attr_reader :specfile, :working_path # save the specfile and call librarian def run(specfile = nil) @working_path = specfile.kind_of?(Pathname) ? specfile.parent : Pathname.new(Dir.pwd) @specfile = specfile super end # implement the 'modulefile' syntax for Puppetfile def modulefile f = modulefile_path raise Error, "Modulefile file does not exist: #{f}" unless File.exists?(f) File.read(f).lines.each do |line| regexp = /\s*dependency\s+('|")([^'"]+)\1\s*(?:,\s*('|")([^'"]+)\3)?/ regexp =~ line && mod($2, $4) end end # implement the 'metadata' syntax for Puppetfile def metadata f = working_path.join('metadata.json') unless File.exists?(f) msg = "Metadata file does not exist: #{f}" # try modulefile, in case we don't have a Puppetfile and we are using the default template if File.exists?(modulefile_path) modulefile return else raise Error, msg end end begin json = JSON.parse(File.read(f)) rescue JSON::ParserError => e raise Error, "Unable to parse json file #{f}: #{e}" end dependencyList = json['dependencies'] dependencyList.each do |d| mod(d['name'], d['version_requirement']) end end private def modulefile_path working_path.join('Modulefile') end end end end end librarian-puppet-3.0.0/lib/librarian/puppet.rb0000644000004100000410000000217213236362373021405 0ustar www-datawww-datarequire 'librarian' require 'fileutils' require 'librarian/puppet/extension' require 'librarian/puppet/version' require 'librarian/action/install' module Librarian module Puppet @@puppet_version = nil # Output of puppet --version, typically x.y.z # For Puppet Enterprise it contains the PE version too, ie. 3.4.3 (Puppet Enterprise 3.2.1) def puppet_version return @@puppet_version unless @@puppet_version.nil? begin @@puppet_version = Librarian::Posix.run!(%W{puppet --version}).strip rescue Errno::ENOENT, Librarian::Posix::CommandFailure => error msg = "Unable to load puppet. Please install it using native packages for your platform (eg .deb, .rpm, .dmg, etc)." msg += "\npuppet --version returned #{error.status}" if error.respond_to? :status msg += "\n#{error.message}" unless error.message.nil? $stderr.puts msg exit 1 end return @@puppet_version end # Puppet version x.y.z translated as a Gem version def puppet_gem_version Gem::Version.create(puppet_version.split(' ').first.strip.gsub('-', '.')) end end end librarian-puppet-3.0.0/librarian-puppet.gemspec0000644000004100000410000000742313236362373021661 0ustar www-datawww-data######################################################### # This file has been automatically generated by gem2tgz # ######################################################### # -*- encoding: utf-8 -*- Gem::Specification.new do |s| s.name = "librarian-puppet" s.version = "3.0.0" s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version= s.authors = ["Tim Sharpe", "Carlos Sanchez"] s.date = "2018-01-15" s.description = "Simplify deployment of your Puppet infrastructure by\n automatically pulling in modules from the forge and git repositories with\n a single command." s.email = ["tim@sharpe.id.au", "carlos@apache.org"] s.executables = ["librarian-puppet"] s.files = [".gitignore", "LICENSE", "README.md", "bin/librarian-puppet", "lib/librarian/puppet.rb", "lib/librarian/puppet/action.rb", "lib/librarian/puppet/action/install.rb", "lib/librarian/puppet/action/resolve.rb", "lib/librarian/puppet/cli.rb", "lib/librarian/puppet/dependency.rb", "lib/librarian/puppet/dsl.rb", "lib/librarian/puppet/environment.rb", "lib/librarian/puppet/extension.rb", "lib/librarian/puppet/lockfile.rb", "lib/librarian/puppet/source.rb", "lib/librarian/puppet/source/forge.rb", "lib/librarian/puppet/source/forge/repo.rb", "lib/librarian/puppet/source/forge/repo_v1.rb", "lib/librarian/puppet/source/forge/repo_v3.rb", "lib/librarian/puppet/source/git.rb", "lib/librarian/puppet/source/githubtarball.rb", "lib/librarian/puppet/source/githubtarball/repo.rb", "lib/librarian/puppet/source/local.rb", "lib/librarian/puppet/source/path.rb", "lib/librarian/puppet/source/repo.rb", "lib/librarian/puppet/templates/Puppetfile", "lib/librarian/puppet/util.rb", "lib/librarian/puppet/version.rb"] s.homepage = "https://github.com/voxpupuli/librarian-puppet" s.licenses = ["MIT"] s.require_paths = ["lib"] s.required_ruby_version = Gem::Requirement.new(">= 2.0.0") s.rubygems_version = "1.8.23" s.summary = "Bundler for your Puppet modules" if s.respond_to? :specification_version then s.specification_version = 4 if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then s.add_development_dependency(%q, ["< 0.8.0"]) s.add_development_dependency(%q, ["< 3.0.0"]) s.add_runtime_dependency(%q, [">= 0.6.3"]) s.add_development_dependency(%q, ["~> 5"]) s.add_development_dependency(%q, [">= 0"]) s.add_development_dependency(%q, ["~> 5.2.0"]) s.add_runtime_dependency(%q, ["~> 2.1"]) s.add_development_dependency(%q, [">= 0"]) s.add_development_dependency(%q, [">= 0"]) s.add_runtime_dependency(%q, [">= 0"]) s.add_development_dependency(%q, [">= 0.9.0"]) else s.add_dependency(%q, ["< 0.8.0"]) s.add_dependency(%q, ["< 3.0.0"]) s.add_dependency(%q, [">= 0.6.3"]) s.add_dependency(%q, ["~> 5"]) s.add_dependency(%q, [">= 0"]) s.add_dependency(%q, ["~> 5.2.0"]) s.add_dependency(%q, ["~> 2.1"]) s.add_dependency(%q, [">= 0"]) s.add_dependency(%q, [">= 0"]) s.add_dependency(%q, [">= 0"]) s.add_dependency(%q, [">= 0.9.0"]) end else s.add_dependency(%q, ["< 0.8.0"]) s.add_dependency(%q, ["< 3.0.0"]) s.add_dependency(%q, [">= 0.6.3"]) s.add_dependency(%q, ["~> 5"]) s.add_dependency(%q, [">= 0"]) s.add_dependency(%q, ["~> 5.2.0"]) s.add_dependency(%q, ["~> 2.1"]) s.add_dependency(%q, [">= 0"]) s.add_dependency(%q, [">= 0"]) s.add_dependency(%q, [">= 0"]) s.add_dependency(%q, [">= 0.9.0"]) end end librarian-puppet-3.0.0/.gitignore0000644000004100000410000000004713236362373017021 0ustar www-datawww-datapkg/ Gemfile.lock tmp/ coverage/ *.gem librarian-puppet-3.0.0/LICENSE0000644000004100000410000000207613236362373016042 0ustar www-datawww-dataCopyright (c) 2012-2014 Tim Sharpe, Carlos Sanchez and others Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. librarian-puppet-3.0.0/README.md0000644000004100000410000003035113236362373016311 0ustar www-datawww-data# Librarian-puppet [![Build Status](https://travis-ci.org/voxpupuli/librarian-puppet.png?branch=master)](https://travis-ci.org/voxpupuli/librarian-puppet) ## Introduction Librarian-puppet is a bundler for your puppet infrastructure. You can use librarian-puppet to manage the puppet modules your infrastructure depends on, whether the modules come from the [Puppet Forge](https://forge.puppet.com/), Git repositories or just a path. * Librarian-puppet can reuse the dependencies listed in your `Modulefile` or `metadata.json` * Forge modules can be installed from [Puppetlabs Forge](https://forge.puppet.com/) or an internal Forge such as [Pulp](http://www.pulpproject.org/) * Git modules can be installed from a branch, tag or specific commit, optionally using a path inside the repository * Modules can be installed from GitHub using tarballs, without needing Git installed * Modules can be installed from a filesystem path * Module dependencies are resolved transitively without needing to list all the modules explicitly Librarian-puppet manages your `modules/` directory for you based on your `Puppetfile`. Your `Puppetfile` becomes the authoritative source for what modules you require and at what version, tag or branch. Once using Librarian-puppet you should not modify the contents of your `modules` directory. The individual modules' repos should be updated, tagged with a new release and the version bumped in your Puppetfile. It is based on [Librarian](https://github.com/applicationsonline/librarian), a framework for writing bundlers, which are tools that resolve, fetch, install, and isolate a project's dependencies. ## Versions Librarian-Puppet 3.0.0 and newer requires Ruby >= 2.0. Use version 2.2.4 if you need support for Puppet 3.7 or earlier, or Ruby 1.9 or earlier. Note that [Puppet 4.10 and newer require Ruby 2.1](https://puppet.com/docs/puppet/4.10/system_requirements.html#prerequisites) or newer. Librarian-Puppet 2.0.0 and newer requires Ruby >= 1.9 and uses Puppet Forge API v3. For Ruby 1.8 use 1.5.0. See the [Changelog](Changelog.md) for more details. ## The Puppetfile Every Puppet repository that uses Librarian-puppet may have a file named `Puppetfile`, `metadata.json` or `Modulefile` in the root directory of that repository. The full specification for which modules your puppet infrastructure repository depends goes in here. ### Simple usage If no Puppetfile is present, `librarian-puppet` will download all the dependencies listed in your `metadata.json` or `Modulefile` from the Puppet Forge, as if the Puppetfile contained forge "https://forgeapi.puppetlabs.com" metadata ### Example Puppetfile forge "https://forgeapi.puppetlabs.com" mod 'puppetlabs-razor' mod 'puppetlabs-ntp', "0.0.3" mod 'puppetlabs-apt', :git => "git://github.com/puppetlabs/puppetlabs-apt.git" mod 'puppetlabs-stdlib', :git => "git://github.com/puppetlabs/puppetlabs-stdlib.git" mod 'puppetlabs-apache', '0.6.0', :github_tarball => 'puppetlabs/puppetlabs-apache' mod 'acme-mymodule', :path => './some_folder' exclusion 'acme-bad_module' ### Recursive module dependency resolution When fetching a module all dependencies specified in its `Modulefile`, `metadata.json` and `Puppetfile` will be resolved and installed. ### Puppetfile Breakdown forge "https://forgeapi.puppetlabs.com" This declares that we want to use the official Puppet Labs Forge as our default source when pulling down modules. If you run your own local forge, you may want to change this. metadata Download all the dependencies listed in your `metadata.json` or `Modulefile` from the Puppet Forge. mod 'puppetlabs-razor' Pull in the latest version of the Puppet Labs Razor module from the default source. mod 'puppetlabs-ntp', "0.0.3" Pull in version 0.0.3 of the Puppet Labs NTP module from the default source. mod 'puppetlabs-apt', :git => "git://github.com/puppetlabs/puppetlabs-apt.git" Our puppet infrastructure repository depends on the `apt` module from the Puppet Labs GitHub repos and checks out the `master` branch. mod 'puppetlabs-apt', :git => "git://github.com/puppetlabs/puppetlabs-apt.git", :ref => '0.0.3' Our puppet infrastructure repository depends on the `apt` module from the Puppet Labs GitHub repos and checks out a tag of `0.0.3`. mod 'puppetlabs-apt', :git => "git://github.com/puppetlabs/puppetlabs-apt.git", :ref => 'feature/master/dans_refactor' Our puppet infrastructure repository depends on the `apt` module from the Puppet Labs GitHub repos and checks out the `dans_refactor` branch. When using a Git source, we do not have to use a `:ref =>`. If we do not, then librarian-puppet will assume we meant the `master` branch. If we use a `:ref =>`, we can use anything that Git will recognize as a ref. This includes any branch name, tag name, SHA, or SHA unique prefix. If we use a branch, we can later ask Librarian-puppet to update the module by fetching the most recent version of the module from that same branch. Note that Librarian-puppet recognizes the [r10k Puppetfile's](https://github.com/puppetlabs/r10k/blob/master/doc/puppetfile.mkd) additional options, `:tag`, `:commit`, and `:branch`, but only as aliases for `:ref`. That is, there is no implementation of r10k's optimizations around fetching these different types of git objects. The Git source also supports a `:path =>` option. If we use the path option, Librarian-puppet will navigate down into the Git repository and only use the specified subdirectory. Some people have the habit of having a single repository with many modules in it. If we need a module from such a repository, we can use the `:path =>` option here to help Librarian-puppet drill down and find the module subdirectory. mod 'puppetlabs-apt', :git => "git://github.com/fake/puppet-modules.git", :path => "modules/apt" Our puppet infrastructure repository depends on the `apt` module, which we have stored as a directory under our `puppet-modules` git repos. mod 'puppetlabs-apache', '0.6.0', :github_tarball => 'puppetlabs/puppetlabs-apache' Our puppet infrastructure repository depends on the `puppetlabs-apache` module, to be downloaded from GitHub tarball. mod 'acme-mymodule', :path => './some_folder' Our puppet infrastructure repository depends on the `acme-mymodule` module, which is already in the filesystem. exclusion 'acme-bad_module' Exclude the module `acme-bad_module` from resolution and installation. ## How to Use Install librarian-puppet: $ gem install librarian-puppet Prepare your puppet infrastructure repository: $ cd ~/path/to/puppet-inf-repos $ (git) rm -rf modules $ librarian-puppet init Librarian-puppet takes over your `modules/` directory, and will always reinstall (if missing) the modules listed the `Puppetfile.lock` into your `modules/` directory, therefore you do not need your `modules/` directory to be tracked in Git. Librarian-puppet uses a `.tmp/` directory for tempfiles and caches. You should not track this directory in Git. Running `librarian-puppet init` will create a skeleton Puppetfile for you as well as adding `tmp/` and `modules/` to your `.gitignore`. $ librarian-puppet install [--clean] [--verbose] This command looks at each `mod` declaration and fetches the module from the source specified. This command writes the complete resolution into `Puppetfile.lock` and then copies all of the fetched modules into your `modules/` directory, overwriting whatever was there before. Librarian-puppet support both v1 and v3 of the Puppet Forge API. Specify a specific API version when installing modules: $ librarian-puppet install --use-v1-api # this is default; ignored for official Puppet Forge $ librarian-puppet install --no-use-v1-api # use the v3 API; default for official Puppet Forge Please note that this does not apply for the official Puppet Forge, where v3 is used by default. Get an overview of your `Puppetfile.lock` with: $ librarian-puppet show Inspect the details of specific resolved dependencies with: $ librarian-puppet show NAME1 [NAME2, ...] Find out which dependencies are outdated and may be updated: $ librarian-puppet outdated [--verbose] Update the version of a dependency: $ librarian-puppet update apt [--verbose] $ git diff Puppetfile.lock $ git add Puppetfile.lock $ git commit -m "bumped the version of apt up to 0.0.4." ## Configuration Configuration comes from three sources with the following highest-to-lowest precedence: * The local config (`./.librarian/puppet/config`) * The environment * The global config (`~/.librarian/puppet/config`) You can inspect the final configuration with: $ librarian-puppet config You can find out where a particular key is set with: $ librarian-puppet config KEY You can set a key at the global level with: $ librarian-puppet config KEY VALUE --global And remove it with: $ librarian-puppet config KEY --global --delete You can set a key at the local level with: $ librarian-puppet config KEY VALUE --local And remove it with: $ librarian-puppet config KEY --local --delete You cannot set or delete environment-level config keys with the CLI. Configuration set at either the global or local level will affect subsequent invocations of `librarian-puppet`. Configurations set at the environment level are not saved and will not affect subsequent invocations of `librarian-puppet`. You can pass a config at the environment level by taking the original config key and transforming it: replace hyphens (`-`) with underscores (`_`) and periods (`.`) with doubled underscores (`__`), uppercase, and finally prefix with `LIBRARIAN_PUPPET_`. For example, to pass a config in the environment for the key `part-one.part-two`, set the environment variable `LIBRARIAN_PUPPET_PART_ONE__PART_TWO`. Configuration affects how various commands operate. * The `path` config sets the directory to install to. If a relative path, it is relative to the directory containing the `Puppetfile`. The equivalent environment variable is `LIBRARIAN_PUPPET_PATH`. * The `tmp` config sets the cache directory for librarian. If a relative path, it is relative to the directory containing the `Puppetfile`. The equivalent environment variable is `LIBRARIAN_PUPPET_TMP`. Configuration can be set by passing specific options to other commands. * The `path` config can be set at the local level by passing the `--path` option to the `install` command. It can be unset at the local level by passing the `--no-path` option to the `install` command. Note that if this is set at the environment or global level then, even if `--no-path` is given as an option, the environment or global config will be used. ## Rsync Option The default convergence strategy between the cache and the module directory is to execute an `rm -r` on the module directory and just `cp -r` from the cache. This causes the module to be removed from the module path every time librarian puppet updates, regardless of whether the content has changed. This can cause some problems in environments with lots of change. The problem arises when the module directory gets removed while Puppet is trying to read files inside it. The `puppet master` process will lose its CWD and the catalog will fail to compile. To avoid this, you can use `rsync` to implement a more conservative convergence strategy. This will use `rsync` with the `-avz` and `--delete` flags instead of a `rm -r` and `cp -r`. To use this feature, just set the `rsync` configuration setting to `true`. $ librarian-puppet config rsync true --global Alternatively, using an environment variable: LIBRARIAN_PUPPET_RSYNC='true' Note that the directories will still be purged if you run librarian-puppet with the --clean or --destructive flags. ## How to Contribute * Pull requests please. * Bonus points for feature branches. ## Reporting Issues Bug reports to the github issue tracker please. Please include: * Relevant `Puppetfile` and `Puppetfile.lock` files * Version of ruby, librarian-puppet, and puppet * What distro * Please run the `librarian-puppet` commands in verbose mode by using the `--verbose` flag, and include the verbose output in the bug report as well. ## License Please see the [LICENSE](https://github.com/voxpupuli/librarian-puppet/blob/master/LICENSE) file.