pax_global_header00006660000000000000000000000064132331746010014512gustar00rootroot0000000000000052 comment=c44413c57410e9a5f67d76c59663a76be5fa2d28 ratt-0.0~git20180127.c44413c/000077500000000000000000000000001323317460100150365ustar00rootroot00000000000000ratt-0.0~git20180127.c44413c/LICENSE000066400000000000000000000020771323317460100160510ustar00rootroot00000000000000Copyright (c) Michael Stapelberg , 2015 Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ratt-0.0~git20180127.c44413c/README.md000066400000000000000000000052371323317460100163240ustar00rootroot00000000000000# Description ratt (“Rebuild All The Things!”) operates on a Debian .changes file of a just-built package, identifies all reverse-build-dependencies and rebuilds them with the .debs from the .changes file. The intended use-case is, for example, to package a new snapshot of a Go library and verify that the new version does not break any other Go libraries/binaries. # Installation (from git, for hacking on ratt) Please install ratt from Debian. In case you want to hack on ratt, you can use the following commands to install Go, download ratt from git and compile/install it into your `$GOPATH`: ```bash sudo apt-get install golang-go export GOPATH=~/gocode go get -u github.com/Debian/ratt ``` Start the resulting binary in `~/gocode/bin/ratt`: ```bash ~/gocode/bin/ratt -help ``` After making changes to the code, to recompile and install it again, use: ```bash go install github.com/Debian/ratt ``` # Usage Let’s assume you build a new version of a Go library, like so: ```bash debcheckout golang-github-jacobsa-gcloud-dev cd golang-github-jacobsa-gcloud-dev dch -i -m 'dummy new version' git commit -a -m 'dummy new version' gbp buildpackage --git-pbuilder ``` Now you can use ratt to identify and rebuild all reverse-build-dependencies: ``` $ ratt golang-github-jacobsa-gcloud_0.0\~git20150709-2_amd64.changes 2015/08/16 11:48:41 Loading changes file "golang-github-jacobsa-gcloud_0.0~git20150709-2_amd64.changes" 2015/08/16 11:48:41 - 1 binary packages: golang-github-jacobsa-gcloud-dev 2015/08/16 11:48:41 - corresponding .debs (will be injected when building): 2015/08/16 11:48:41 golang-github-jacobsa-gcloud-dev_0.0~git20150709-2_all.deb 2015/08/16 11:48:41 Loading sources index "/var/lib/apt/lists/ftp.ch.debian.org_debian_dists_sid_contrib_source_Sources" 2015/08/16 11:48:41 Loading sources index "/var/lib/apt/lists/ftp.ch.debian.org_debian_dists_sid_main_source_Sources" 2015/08/16 11:48:43 Loading sources index "/var/lib/apt/lists/ftp.ch.debian.org_debian_dists_sid_non-free_source_Sources" 2015/08/16 11:48:43 Building golang-github-jacobsa-ratelimit_0.0~git20150723.0.2ca5e0c-1 (commandline: [sbuild --arch-all --dist=sid --nolog golang-github-jacobsa-ratelimit_0.0~git20150723.0.2ca5e0c-1 --extra-package=golang-github-jacobsa-gcloud-dev_0.0~git20150709-2_all.deb]) 2015/08/16 11:49:19 Build results: 2015/08/16 11:49:19 PASSED: golang-github-jacobsa-ratelimit_0.0~git20150723.0.2ca5e0c-1 ``` ratt uses `sbuild(1)` to build packages, see https://wiki.debian.org/sbuild for instructions on how to set up sbuild. Be sure to add `--components=main,contrib,non-free` to the sbuild-createchroot line in case you want to deal with packages outside of main as well. ratt-0.0~git20180127.c44413c/ratt.go000066400000000000000000000252701323317460100163450ustar00rootroot00000000000000// ratt operates on a Debian .changes file of a just-built package, identifies // all reverse-build-dependencies and rebuilds them with the .debs from the // .changes file. // // The intended use-case is, for example, to package a new snapshot of a Go // library and verify that the new version does not break any other Go // libraries/binaries. package main import ( "bufio" "bytes" "flag" "fmt" "io" "log" "os" "os/exec" "path/filepath" "regexp" "sort" "strings" "pault.ag/go/debian/control" "pault.ag/go/debian/version" ) type buildResult struct { src string version *version.Version err error recheckErr error logFile string recheckLogFile string } var ( logDir = flag.String("log_dir", "buildlogs", "Directory in which sbuild(1) logs for all reverse-build-dependencies are stored") dryRun = flag.Bool("dry_run", false, "Print sbuild command lines, but do not actually build the reverse-build-dependencies") sbuildDist = flag.String("sbuild_dist", "", "sbuild --dist= value (e.g. \"sid\"). Defaults to the Distribution: entry from the specified .changes file") dist = flag.String("dist", "", "Distribution to look up reverse-build-dependencies from. Defaults to the Distribution: entry from the specified .changes file") recheck = flag.Bool("recheck", false, "Rebuild without new changes to check if the failures are really related") listsPrefixRe = regexp.MustCompile(`/([^/]*_dists_.*)_InRelease$`) ) func dependsOn(src control.SourceIndex, binaries map[string]bool) bool { buildDepends := src.GetBuildDepends() for _, possibility := range buildDepends.GetAllPossibilities() { if binaries[possibility.Name] { return true } } return false } func addReverseBuildDeps(sourcesPath string, binaries map[string]bool, rebuild map[string][]version.Version) error { log.Printf("Loading sources index %q\n", sourcesPath) catFile := exec.Command("/usr/lib/apt/apt-helper", "cat-file", sourcesPath) var s *bufio.Reader if lines, err := catFile.Output(); err == nil { s = bufio.NewReader(bytes.NewReader(lines)) } else { // Fallback for older versions of apt-get. See // <20160111171230.GA17291@debian.org> for context. o, err := os.Open(sourcesPath) if err != nil { return err } defer o.Close() s = bufio.NewReader(o) } idx, err := control.ParseSourceIndex(s) if err != nil && err != io.EOF { return err } for _, src := range idx { if dependsOn(src, binaries) { rebuild[src.Package] = append(rebuild[src.Package], src.Version) } } return nil } func fallback(sourcesPaths []string, binaries []string) (map[string][]version.Version, error) { bins := make(map[string]bool) for _, bin := range binaries { bins[bin] = true } rebuild := make(map[string][]version.Version) for _, sourcesPath := range sourcesPaths { if err := addReverseBuildDeps(sourcesPath, bins, rebuild); err != nil { return nil, err } } return rebuild, nil } func reverseBuildDeps(packagesPaths, sourcesPaths []string, binaries []string) (map[string][]version.Version, error) { if _, err := exec.LookPath("dose-ceve"); err != nil { log.Printf("dose-ceve(1) not found. Please install the dose-extra package for more accurate results. Falling back to interpreting Sources directly") return fallback(sourcesPaths, binaries) } archOut, err := exec.Command("dpkg-architecture", "--query=DEB_BUILD_ARCH").Output() if err != nil { log.Fatal(err) } arch := strings.TrimSpace(string(archOut)) // TODO: Cache this output based on the .changes file. dose-ceve takes quite a while. ceve := exec.Command( "dose-ceve", "--deb-native-arch="+arch, "-T", "debsrc", "-r", strings.Join(binaries, ","), "-G", "pkg") for _, packagesPath := range packagesPaths { ceve.Args = append(ceve.Args, "deb://"+packagesPath) } for _, sourcesPath := range sourcesPaths { ceve.Args = append(ceve.Args, "debsrc://"+sourcesPath) } ceve.Stderr = os.Stderr log.Printf("Figuring out reverse build dependencies using dose-ceve(1). This might take a while") out, err := ceve.Output() if err != nil { log.Printf("dose-ceve(1) failed (%v), falling back to interpreting Sources directly", err) return fallback(sourcesPaths, binaries) } var doseCeves []struct { Package string Version version.Version } r := bufio.NewReader(bytes.NewReader(out)) if err := control.Unmarshal(&doseCeves, r); err != nil { return nil, err } rebuild := make(map[string][]version.Version) for _, doseCeve := range doseCeves { rebuild[doseCeve.Package] = append(rebuild[doseCeve.Package], doseCeve.Version) } return rebuild, nil } func main() { flag.Parse() if flag.NArg() == 0 { log.Fatalf("Usage: %s [options] ...\n", os.Args[0]) } var debs []string var binaries []string var changesDist string for i, changesPath := range flag.Args() { log.Printf("Loading changes file %q\n", changesPath) c, err := os.Open(changesPath) if err != nil { log.Fatal(err) } defer c.Close() changes, err := control.ParseChanges(bufio.NewReader(c), changesPath) if err != nil && err != io.EOF { log.Fatal(err) } log.Printf(" - %d binary packages: %s\n", len(changes.Binaries), strings.Join(changes.Binaries, " ")) for _, file := range changes.Files { if filepath.Ext(file.Filename) == ".deb" { debs = append(debs, filepath.Join(filepath.Dir(changesPath), file.Filename)) } } binaries = append(binaries, changes.Binaries...) if i == 0 { changesDist = changes.Distribution } else if changesDist != changes.Distribution { log.Printf("%s has different distrution, but we will only consider %s\n", changes.Filename, changesDist) } } log.Printf("Corresponding .debs (will be injected when building):\n") for _, deb := range debs { log.Printf(" %s\n", deb) } if strings.TrimSpace(*dist) == "" { *dist = changesDist // Rewrite unstable to sid, which apt-get indextargets (below) requires. if *dist == "unstable" { *dist = "sid" } log.Printf("Setting -dist=%s (from .changes file)\n", *dist) } var sourcesPaths []string var packagesPaths []string indexTargets := exec.Command("apt-get", "indextargets", "--format", "$(FILENAME)", "Codename: "+*dist, "ShortDesc: Sources") if lines, err := indexTargets.Output(); err == nil { for _, line := range strings.Split(string(lines), "\n") { trimmed := strings.TrimSpace(line) if trimmed != "" { sourcesPaths = append(sourcesPaths, line) } } binaryIndexTargets := exec.Command( "apt-get", "indextargets", "--format", "$(FILENAME)", "Codename: "+*dist, "ShortDesc: Packages") lines, err = binaryIndexTargets.Output() if err != nil { log.Fatal("Could not get packages files using %+v: %v", binaryIndexTargets.Args, err) } for _, line := range strings.Split(string(lines), "\n") { trimmed := strings.TrimSpace(line) if trimmed != "" { packagesPaths = append(packagesPaths, line) } } } else { // Fallback for older versions of apt-get. See // https://bugs.debian.org/801594 for context. releaseMatches, err := filepath.Glob("/var/lib/apt/lists/*_InRelease") if err != nil { log.Fatal(err) } for _, releasepath := range releaseMatches { r, err := os.Open(releasepath) if err != nil { log.Fatal(err) } defer r.Close() var inRelease struct { Suite string } if err := control.Unmarshal(&inRelease, bufio.NewReader(r)); err != nil { log.Fatal(err) } listsPrefix := listsPrefixRe.FindStringSubmatch(releasepath) if len(listsPrefix) != 2 { log.Fatalf("release file path %q does not match regexp %q\n", releasepath, listsPrefixRe) } sourceMatches, err := filepath.Glob(fmt.Sprintf("/var/lib/apt/lists/%s_*_Sources", listsPrefix[1])) if err != nil { log.Fatal(err) } sourcesPaths = append(sourcesPaths, sourceMatches...) packagesMatches, err := filepath.Glob(fmt.Sprintf("/var/lib/apt/lists/%s_*_Packages", listsPrefix[1])) if err != nil { log.Fatal(err) } packagesPaths = append(packagesPaths, packagesMatches...) } } if len(sourcesPaths) == 0 { log.Fatal("Could not find InRelease file for " + *dist + " . Are you missing " + *dist + " in your /etc/apt/sources.list?") } rebuild, err := reverseBuildDeps(packagesPaths, sourcesPaths, binaries) if err != nil { log.Fatal(err) } // TODO: add -recursive flag to also cover dependencies which are not DIRECT dependencies. use http://godoc.org/pault.ag/go/debian/control#OrderDSCForBuild (topsort) to build dependencies in the right order (saving CPU time). // TODO: what’s a good integration method for doing this in more setups, e.g. on a cloud provider or something? mapreri from #debian-qa says jenkins.debian.net is suitable. if strings.TrimSpace(*sbuildDist) == "" { *sbuildDist = changesDist log.Printf("Setting -sbuild_dist=%s (from .changes file)\n", *sbuildDist) } if err := os.MkdirAll(*logDir, 0755); err != nil { log.Fatal(err) } builder := &sbuild{ dist: *sbuildDist, logDir: *logDir, dryRun: *dryRun, extraDebs: debs, } cnt := 1 buildresults := make(map[string](*buildResult)) for src, versions := range rebuild { sort.Sort(sort.Reverse(version.Slice(versions))) newest := versions[0] log.Printf("Building package %d of %d: %s \n", cnt, len(rebuild), src) cnt++ result := builder.build(src, &newest) if result.err != nil { log.Printf("building %s failed: %v\n", src, result.err) } buildresults[src] = result } if *dryRun { return } if *recheck { log.Printf("Begin to rebuild all failed packages without new changes\n") recheckBuilder := &sbuild{ dist: *sbuildDist, logDir: *logDir + "_recheck", dryRun: false, } if err := os.MkdirAll(recheckBuilder.logDir, 0755); err != nil { log.Fatal(err) } for src, result := range buildresults { if result.err == nil { continue } recheckResult := recheckBuilder.build(src, result.version) result.recheckErr = recheckResult.err result.recheckLogFile = recheckResult.logFile if recheckResult.err != nil { log.Printf("rebuilding %s without new packages failed: %v\n", src, recheckResult.err) } } } log.Printf("Build results:\n") // Print all successful builds first (not as interesting), then failed ones. for src, result := range buildresults { if result.err == nil { log.Printf("PASSED: %s\n", src) } } for src, result := range buildresults { if result.err != nil && result.recheckErr != nil { log.Printf("FAILED: %s, but maybe unrelated to new changes (see %s and %s)\n", src, result.logFile, result.recheckLogFile) } } failures := false for src, result := range buildresults { if result.err != nil && result.recheckErr == nil { log.Printf("FAILED: %s (see %s)\n", src, result.logFile) failures = true } } if failures { os.Exit(1) } } ratt-0.0~git20180127.c44413c/sbuild.go000066400000000000000000000020211323317460100166420ustar00rootroot00000000000000package main import ( "fmt" "log" "os" "os/exec" "path/filepath" "pault.ag/go/debian/version" ) type sbuild struct { dist string logDir string dryRun bool extraDebs []string } func (s *sbuild) build(sourcePackage string, version *version.Version) *buildResult { result := &buildResult{ src: sourcePackage, version: version, } target := fmt.Sprintf("%s_%s", sourcePackage, version) // TODO: discard resulting package immediately? args := []string{ "--arch-all", "--dist=" + s.dist, "--nolog", target, } for _, filename := range s.extraDebs { args = append(args, fmt.Sprintf("--extra-package=%s", filename)) } cmd := exec.Command("sbuild", args...) if s.dryRun { log.Printf(" commandline: %v\n", cmd.Args) return result } buildlog, err := os.Create(filepath.Join(s.logDir, target)) defer buildlog.Close() if err != nil { result.err = err return result } cmd.Stdout = buildlog cmd.Stderr = buildlog result.err = cmd.Run() result.logFile = buildlog.Name() return result }