rebuildd/0000755000000000000000000000000012003463442007534 5ustar rebuildd/tests/0000755000000000000000000000000012003462215010673 5ustar rebuildd/tests/TestDistribution.pyc0000644000000000000000000000664712003462215014744 0ustar ó øcPc@sddlZejjddƒejjddƒddlmZmZddlZddlZddlZddl m Z ddl m Z ddl mZdd lmZd ejfd „ƒYZed krejƒjeƒZejd dƒjeƒndS(iÿÿÿÿNis..t.(trebuildd_global_test_setuptrebuildd_global_test_teardown(t Distribution(tRebuilddConfig(tRebuildd(tPackagetTestDistributioncBsGeZd„Zd„Zd„Zd„Zd„Zd„Zd„ZRS(cCsMtƒtddƒ|_tddddƒ|_tddddƒ|_dS(Ntsidtalphatnametxutilstversions 7.1.ds.3-1s 1:7.1.ds.3-1(RRtdRtpackagetpackage_dotted(tself((s;/home/jd/Debian/rebuildd/rebuildd/tests/TestDistribution.pytsetUpscCs tƒdS(N(R(R((s;/home/jd/Debian/rebuildd/rebuildd/tests/TestDistribution.pyttearDownscCs|j|jjdkƒdS(NR(tassert_R R (R((s;/home/jd/Debian/rebuildd/rebuildd/tests/TestDistribution.pyt test_namescCs|j|jjdkƒdS(NR (RR tarch(R((s;/home/jd/Debian/rebuildd/rebuildd/tests/TestDistribution.pyt test_archscCsztƒjdddƒ|jj|jƒ}|j|jj|kƒ|j|jj|kƒ|j|jj|kƒdS(Ntbuildt source_cmds/bin/true $d $a $p $v(RtsetR tget_source_cmdRRR R (Rtcmd((s;/home/jd/Debian/rebuildd/rebuildd/tests/TestDistribution.pyttest_get_source_cmds cCsÁtƒjdddƒ|jj|jƒ}|j|jj|kƒ|j|jj|kƒ|j|jj|kƒ|j|jj|kƒ|jj|j ƒ}|j|j j|kƒdS(NRt build_cmds/bin/true $d $a $p $v( RRR t get_build_cmdRRR RR R(RR((s;/home/jd/Debian/rebuildd/rebuildd/tests/TestDistribution.pyttest_get_build_cmd&scCs¸tƒjdddƒ|jj|jƒ}|j|dkƒtƒjdddƒ|jj|jƒ}|j|jj|kƒ|j|jj|kƒ|j|jj|kƒdS(NRtpost_build_cmdts/bin/true $d $a $p $v( RRR tget_post_build_cmdRRtNoneR R (RR((s;/home/jd/Debian/rebuildd/rebuildd/tests/TestDistribution.pyttest_get_post_build_cmd0s( t__name__t __module__RRRRRRR$(((s;/home/jd/Debian/rebuildd/rebuildd/tests/TestDistribution.pyRs      t__main__t verbosityi(tsystpathtinserttRebuilddTestSetupRRtunittestttypestostrebuildd.DistributionRtrebuildd.RebuilddConfigRtrebuildd.RebuilddRtrebuildd.PackageRtTestCaseRR%t TestLoadertloadTestsFromTestCasetsuitetTextTestRunnertrun(((s;/home/jd/Debian/rebuildd/rebuildd/tests/TestDistribution.pyts $, rebuildd/tests/TestJob.py0000755000000000000000000000737612003461770012644 0ustar #!/usr/bin/python import sys sys.path.insert(0, "..") sys.path.insert(0, ".") from RebuilddTestSetup import rebuildd_global_test_setup, rebuildd_global_test_teardown import unittest, types, os import sqlobject from rebuildd.RebuilddConfig import RebuilddConfig from rebuildd.Package import Package from rebuildd.Job import Job from rebuildd.JobStatus import JobStatus class TestJob(unittest.TestCase): def setUp(self): rebuildd_global_test_setup() self.job = Job(package=Package(name="bash", version="3.1dfsg-8"), arch="alpha", dist="sid") def tearDown(self): rebuildd_global_test_teardown() def test_DB_OK(self): self.assert_(os.path.isfile('/tmp/rebuildd-tests.db')) self.assert_(os.path.getsize('/tmp/rebuildd-tests.db') > 0) def test_init_job(self): self.assert_(type(self.job) is Job) def test_setattr(self): self.job.status = JobStatus.UNKNOWN self.assert_(self.job.status == JobStatus.UNKNOWN) self.job.status = JobStatus.WAIT self.assert_(self.job.status == JobStatus.WAIT) def test_open_logfile(self): file = open(self.job.logfile, "w") self.assert_(file is not None) filero = open(self.job.logfile, "r") self.assert_(filero is not None) file.close() filero.close() os.unlink(file.name) def test_status_on_doquit(self): self.job.do_quit.set() self.job.start() self.job.join() self.assert_(self.job.status == JobStatus.WAIT_LOCKED) def test_build_success(self): self.job.do_quit.clear() RebuilddConfig().set('build', 'source_cmd', '/bin/true') RebuilddConfig().set('build', 'build_cmd', '/bin/true') RebuilddConfig().set('build', 'post_build_cmd', '/bin/true') self.job.start() self.job.join() self.assert_(self.job.status == JobStatus.BUILD_OK) def test_build_failure_source(self): self.job.do_quit.clear() RebuilddConfig().set('build', 'source_cmd', '/bin/false') RebuilddConfig().set('build', 'build_cmd', '/bin/true') RebuilddConfig().set('build', 'post_build_cmd', '/bin/true') self.job.start() self.job.join() self.assert_(self.job.status == JobStatus.SOURCE_FAILED) def test_build_failure_build(self): self.job.do_quit.clear() RebuilddConfig().set('build', 'source_cmd', '/bin/true %s %s %s') RebuilddConfig().set('build', 'build_cmd', '/bin/false %s %s %s %s') RebuilddConfig().set('build', 'post_build_cmd', '/bin/true %s %s %s %s') self.job.start() self.job.join() self.assert_(self.job.status == JobStatus.BUILD_FAILED) def test_build_failure_post_build(self): RebuilddConfig().set('build', 'source_cmd', '/bin/true %s %s %s') RebuilddConfig().set('build', 'build_cmd', '/bin/true %s %s %s %s') RebuilddConfig().set('build', 'post_build_cmd', '/bin/false %s %s %s %s') self.job.start() self.job.join() self.assert_(self.job.status == JobStatus.POST_BUILD_FAILED) def test_send_build_log(self): file = open(self.job.logfile, "w") self.assert_(file is not None) file.write("Fake log file") file.close() self.assert_(self.job.send_build_log() is False) self.job.status = JobStatus.BUILD_OK self.assert_(self.job.send_build_log() is True) self.assert_(self.job.status is JobStatus.BUILD_OK) self.job.status = JobStatus.BUILD_FAILED self.assert_(self.job.send_build_log() is True) self.assert_(self.job.status is JobStatus.BUILD_FAILED) if __name__ == '__main__': suite = unittest.TestLoader().loadTestsFromTestCase(TestJob) unittest.TextTestRunner(verbosity=2).run(suite) rebuildd/tests/RebuilddTestSetup.pyc0000644000000000000000000000312612003462214015024 0ustar ó øcPc@srddlmZddlmZddlmZddlmZddlm Z ddl Z d„Z d„Z dS( iÿÿÿÿ(tRebuilddConfig(tRebuildd(tPackage(tJob(tLogNcCs tdtƒtƒjdddƒtƒjdddƒtƒjddd ƒtƒjdd d ƒtƒjdd d ƒtƒjddd ƒtƒjddd ƒddgtƒ_tƒyRtjdtƒtjdtƒtjdtƒtj ƒtj ƒtj ƒWnnXdS(Nt dontparsetlogtlogs_dirs/tmptbuildt database_urissqlite:///tmp/rebuildd-tests.dbtmax_jobst100tfiles /dev/nullt mail_failedt0tbuild_more_recenttmail_successfultalphatanytifExists( RtTruetsettarchRRt dropTableRRt createTable(((s</home/jd/Debian/rebuildd/rebuildd/tests/RebuilddTestSetup.pytrebuildd_global_test_setups&   cCsAy3tƒjjƒtƒjjƒitƒj_WnnXdS(N(Rt_sqlconnectiont dropDatabasetcloset _threadPool(((s</home/jd/Debian/rebuildd/rebuildd/tests/RebuilddTestSetup.pytrebuildd_global_test_teardowns ( trebuildd.RebuilddConfigRtrebuildd.RebuilddRtrebuildd.PackageRt rebuildd.JobRtrebuildd.RebuilddLogRtosRR(((s</home/jd/Debian/rebuildd/rebuildd/tests/RebuilddTestSetup.pyts  rebuildd/tests/TestRebuildd.py0000755000000000000000000001304712003461770013654 0ustar #!/usr/bin/python import sys sys.path.insert(0, "..") sys.path.insert(0, ".") from RebuilddTestSetup import rebuildd_global_test_setup, rebuildd_global_test_teardown import unittest, types, os, socket from rebuildd.RebuilddConfig import RebuilddConfig from rebuildd.Rebuildd import Rebuildd from rebuildd.Package import Package from rebuildd.Job import Job from rebuildd.JobStatus import JobStatus class TestRebuildd(unittest.TestCase): def setUp(self): rebuildd_global_test_setup() self.package = Package(name="zsh", version="4.3.4-10") self.r = Rebuildd() def tearDown(self): rebuildd_global_test_teardown() def test_add_job(self): ret = self.r.add_job(name="telak", version="0.5-1", priority='optional', dist="sid") self.assert_(ret is True) ret = self.r.add_job(name="telak", version="0.5-1", priority='optional', dist="sid") self.assert_(ret is False) ret = self.r.add_job(name="telak", version="0.5-1", priority='optional', dist="jaydeerulez") self.assert_(ret is False) ret = self.r.add_job(name="telak", version="0.6-1", priority='optional', dist="sid") self.assert_(ret is True) def test_clean_jobs(self): ret = self.r.clean_jobs() self.assert_(ret is True) def test_stop_all_jobs(self): ret = self.r.stop_all_jobs() self.assert_(ret is True) # def test_release_jobs(self): # self.r.add_job(name="zsh", version="4.3.4-10", priority='optional', dist="sid") # pkg = Package.selectBy(name="zsh", version="4.3.4-10")[0] # c = Job.selectBy(package=pkg)[0] # c.status = JobStatus.WAIT_LOCKED # c.host = socket.gethostname() # ret = self.r.release_jobs() # self.assert_(ret is True) def test_get_job(self): self.r.add_job(name="glibc", version="2.6-3", priority='required', dist="sid") pkg = Package.selectBy(name="glibc", version="2.6-3")[0] job = Job.selectBy(package=pkg)[0] self.assert_(self.r.get_new_jobs() > 0) self.assert_(self.r.get_job(job.id) is job) def test_get_new_jobs(self): self.r.add_job(name="xpdf", version="3.02-1", priority='optional', dist="sid") self.assert_(self.r.get_new_jobs() >= 1) def test_cancel_job(self): self.r.add_job(name="glibc", version="2.6-2", priority='required', dist="sid") self.r.get_new_jobs() pkg = Package.selectBy(name="glibc", version="2.6-2")[0] job = Job.selectBy(package=pkg)[0] self.assert_(self.r.cancel_job(job.id) is True) self.assert_(self.r.cancel_job(42) is False) def test_fix_job(self): self.r.add_job(name="glibc", version="2.6.1-3", priority='required', dist="sid") pkg = Package.selectBy(name="glibc", version="2.6.1-3")[0] a = Job.selectBy(package=pkg)[0] a.status = JobStatus.BUILDING a.host = socket.gethostname() self.r.add_job(name="xterm", version="1.2-2", priority='extra', dist="sid") pkg = Package.selectBy(name="xterm", version="1.2-2")[0] b = Job.selectBy(package=pkg)[0] b.status = JobStatus.BUILDING b.host = "whoisgonnacallaboxlikethis" self.r.add_job(name="iceweasel", version="5.0-2", priority='optional', dist="sid") pkg = Package.selectBy(name="iceweasel", version="5.0-2")[0] c = Job.selectBy(package=pkg)[0] c.status = JobStatus.WAIT_LOCKED c.host = socket.gethostname() self.assert_(self.r.fix_jobs(False) is True) self.assert_(a.status is JobStatus.WAIT) self.assert_(a.host is None) self.assert_(b.status is JobStatus.BUILDING) self.assert_(b.host is "whoisgonnacallaboxlikethis") self.assert_(c.status is JobStatus.WAIT) self.assert_(c.host is None) def test_build_more_recent(self): self.r.get_new_jobs() RebuilddConfig().set('build', 'build_more_recent', '1') RebuilddConfig().arch.append("alpha") self.r.add_job(name="recenter", version="2.6.1-3", priority='required', dist="sid", arch="alpha") pkg = Package.selectBy(name="recenter", version="2.6.1-3")[0] a = Job.selectBy(package=pkg)[0] self.r.add_job(name="recenter", version="1:2.6.1-2", priority='required', dist="sid", arch="alpha") pkg = Package.selectBy(name="recenter", version="1:2.6.1-2")[0] b = Job.selectBy(package=pkg)[0] self.r.add_job(name="recenter", version="3.6.1-4", priority='required', dist="sid", arch="alpha") pkg = Package.selectBy(name="recenter", version="3.6.1-4")[0] c = Job.selectBy(package=pkg)[0] self.r.add_job(name="recenter", version="2.6.0-2", priority='required', dist="sid", arch="any") pkg = Package.selectBy(name="recenter", version="2.6.0-2")[0] d = Job.selectBy(package=pkg)[0] self.r.add_job(name="recenter", version="4.6.0-2", priority='required', dist="sid", arch="any") pkg = Package.selectBy(name="recenter", version="4.6.0-2")[0] e = Job.selectBy(package=pkg)[0] self.assert_(self.r.get_new_jobs() > 0) self.assert_(a.status == JobStatus.GIVEUP) self.assert_(b.status == JobStatus.WAIT_LOCKED) self.assert_(c.status == JobStatus.GIVEUP) self.assert_(d.status == JobStatus.GIVEUP) self.assert_(e.status == JobStatus.WAIT_LOCKED) RebuilddConfig().set('build', 'build_more_recent', '0') if __name__ == '__main__': suite = unittest.TestLoader().loadTestsFromTestCase(TestRebuildd) unittest.TextTestRunner(verbosity=2).run(suite) rebuildd/tests/RebuilddTestSetup.py0000644000000000000000000000242412003461770014667 0ustar from rebuildd.RebuilddConfig import RebuilddConfig from rebuildd.Rebuildd import Rebuildd from rebuildd.Package import Package from rebuildd.Job import Job from rebuildd.RebuilddLog import Log import os def rebuildd_global_test_setup(): RebuilddConfig(dontparse=True) RebuilddConfig().set('log', 'logs_dir', '/tmp') RebuilddConfig().set('build', 'database_uri', 'sqlite:///tmp/rebuildd-tests.db') RebuilddConfig().set('build', 'max_jobs', '100') RebuilddConfig().set('log', 'file', '/dev/null') RebuilddConfig().set('log', 'mail_failed', '0') RebuilddConfig().set('build', 'build_more_recent', '0') RebuilddConfig().set('log', 'mail_successful', '0') RebuilddConfig().arch = ["alpha", "any"] Rebuildd() try: Package.dropTable(ifExists=True) Job.dropTable(ifExists=True) Log.dropTable(ifExists=True) Package.createTable() Job.createTable() Log.createTable() except: pass def rebuildd_global_test_teardown(): try: Rebuildd()._sqlconnection.dropDatabase() Rebuildd()._sqlconnection.close() Rebuildd()._sqlconnection._threadPool = {} except: pass rebuildd/tests/runtest.py0000755000000000000000000000100611010313021012734 0ustar #!/usr/bin/python import sys sys.path.insert(0, "..") sys.path.insert(0, ".") import unittest from TestJob import TestJob from TestDistribution import TestDistribution from TestRebuildd import TestRebuildd if __name__ == '__main__': suite = unittest.TestLoader().loadTestsFromTestCase(TestJob) suite.addTests(unittest.TestLoader().loadTestsFromTestCase(TestDistribution)) suite.addTests(unittest.TestLoader().loadTestsFromTestCase(TestRebuildd)) unittest.TextTestRunner(verbosity=2).run(suite) rebuildd/tests/TestRebuildd.pyc0000644000000000000000000001403712003462215014007 0ustar ó øcPc@s"ddlZejjddƒejjddƒddlmZmZddlZddlZddlZddl Z ddl m Z ddl m Z ddlmZdd lmZdd lmZd ejfd „ƒYZed krejƒjeƒZejddƒjeƒndS(iÿÿÿÿNis..t.(trebuildd_global_test_setuptrebuildd_global_test_teardown(tRebuilddConfig(tRebuildd(tPackage(tJob(t JobStatust TestRebuilddcBsbeZd„Zd„Zd„Zd„Zd„Zd„Zd„Zd„Z d„Z d „Z RS( cCs/tƒtddddƒ|_tƒ|_dS(Ntnametzshtversions4.3.4-10(RRtpackageRtr(tself((s7/home/jd/Debian/rebuildd/rebuildd/tests/TestRebuildd.pytsetUpscCs tƒdS(N(R(R((s7/home/jd/Debian/rebuildd/rebuildd/tests/TestRebuildd.pyttearDownsc Csì|jjddddddddƒ}|j|tkƒ|jjddddddddƒ}|j|tkƒ|jjdddddddd ƒ}|j|tkƒ|jjdddd ddddƒ}|j|tkƒdS( NR ttelakR s0.5-1tprioritytoptionaltdisttsidt jaydeerulezs0.6-1(R tadd_jobtassert_tTruetFalse(Rtret((s7/home/jd/Debian/rebuildd/rebuildd/tests/TestRebuildd.pyt test_add_jobs''''cCs&|jjƒ}|j|tkƒdS(N(R t clean_jobsRR(RR((s7/home/jd/Debian/rebuildd/rebuildd/tests/TestRebuildd.pyttest_clean_jobs#scCs&|jjƒ}|j|tkƒdS(N(R t stop_all_jobsRR(RR((s7/home/jd/Debian/rebuildd/rebuildd/tests/TestRebuildd.pyttest_stop_all_jobs'sc Cs™|jjddddddddƒtjddddƒd }tjd |ƒd }|j|jjƒd kƒ|j|jj|jƒ|kƒdS( NR tglibcR s2.6-3RtrequiredRRiR ( R RRtselectByRRt get_new_jobstget_jobtid(Rtpkgtjob((s7/home/jd/Debian/rebuildd/rebuildd/tests/TestRebuildd.pyt test_get_job4s %c CsE|jjddddddddƒ|j|jjƒd kƒdS( NR txpdfR s3.02-1RRRRi(R RRR$(R((s7/home/jd/Debian/rebuildd/rebuildd/tests/TestRebuildd.pyttest_get_new_jobs;s%c Cs©|jjddddddddƒ|jjƒtjddddƒd }tjd |ƒd }|j|jj|jƒt kƒ|j|jjd ƒt kƒdS( NR R!R s2.6-2RR"RRiR i*( R RR$RR#RRt cancel_jobR&RR(RR'R(((s7/home/jd/Debian/rebuildd/rebuildd/tests/TestRebuildd.pyttest_cancel_job?s % "c Cs|jjddddddddƒtjddddƒd }tjd |ƒd }tj|_tj ƒ|_ |jjdd dd dd ddƒtjdd dd ƒd }tjd |ƒd }tj|_d|_ |jjddddddddƒtjddddƒd }tjd |ƒd }tj |_tj ƒ|_ |j |jj tƒtkƒ|j |jtjkƒ|j |j dkƒ|j |jtjkƒ|j |j dkƒ|j |jtjkƒ|j |j dkƒdS(NR R!R s2.6.1-3RR"RRiR txterms1.2-2textratwhoisgonnacallaboxlikethist iceweasels5.0-2R(R RRR#RRtBUILDINGtstatustsockett gethostnamethostt WAIT_LOCKEDRtfix_jobsRRtWAITtNone(RR'tatbtc((s7/home/jd/Debian/rebuildd/rebuildd/tests/TestRebuildd.pyt test_fix_jobGs,% %  % c Csº|jjƒtƒjdddƒtƒjjdƒ|jjddddd d d d d dƒtjddddƒd}t jd|ƒd}|jjddddd d d d d dƒtjddddƒd}t jd|ƒd}|jjddddd d d d d dƒtjddddƒd}t jd|ƒd}|jjddddd d d d d dƒtjddddƒd}t jd|ƒd}|jjddddd d d d d dƒtjddddƒd}t jd|ƒd}|j |jjƒdkƒ|j |j t j kƒ|j |j t jkƒ|j |j t j kƒ|j |j t j kƒ|j |j t jkƒtƒjdddƒdS(Ntbuildtbuild_more_recentt1talphaR trecenterR s2.6.1-3RR"RRtarchiR s 1:2.6.1-2s3.6.1-4s2.6.0-2tanys4.6.0-2t0(R R$RtsetRDtappendRRR#RRR3RtGIVEUPR7(RR'R;R<R=tdte((s7/home/jd/Debian/rebuildd/rebuildd/tests/TestRebuildd.pyttest_build_more_recentcs2 +++++( t__name__t __module__RRRRR R)R+R-R>RL(((s7/home/jd/Debian/rebuildd/rebuildd/tests/TestRebuildd.pyRs       t__main__t verbosityi(tsystpathtinserttRebuilddTestSetupRRtunittestttypestosR4trebuildd.RebuilddConfigRtrebuildd.RebuilddRtrebuildd.PackageRt rebuildd.JobRtrebuildd.JobStatusRtTestCaseRRMt TestLoadertloadTestsFromTestCasetsuitetTextTestRunnertrun(((s7/home/jd/Debian/rebuildd/rebuildd/tests/TestRebuildd.pyts 0v rebuildd/tests/TestJob.pyc0000644000000000000000000001252612003462214012767 0ustar ó øcPc@sddlZejjddƒejjddƒddlmZmZddlZddlZddlZddl Z ddl m Z ddl m Z ddlmZdd lmZd ejfd „ƒYZed krejƒjeƒZejd dƒjeƒndS(iÿÿÿÿNis..t.(trebuildd_global_test_setuptrebuildd_global_test_teardown(tRebuilddConfig(tPackage(tJob(t JobStatustTestJobcBsteZd„Zd„Zd„Zd„Zd„Zd„Zd„Zd„Z d„Z d „Z d „Z d „Z RS( cCs8tƒtdtddddƒdddd ƒ|_dS( Ntpackagetnametbashtversions 3.1dfsg-8tarchtalphatdisttsid(RRRtjob(tself((s2/home/jd/Debian/rebuildd/rebuildd/tests/TestJob.pytsetUpscCs tƒdS(N(R(R((s2/home/jd/Debian/rebuildd/rebuildd/tests/TestJob.pyttearDownscCs<|jtjjdƒƒ|jtjjdƒdkƒdS(Ns/tmp/rebuildd-tests.dbi(tassert_tostpathtisfiletgetsize(R((s2/home/jd/Debian/rebuildd/rebuildd/tests/TestJob.pyt test_DB_OKscCs |jt|jƒtkƒdS(N(RttypeRR(R((s2/home/jd/Debian/rebuildd/rebuildd/tests/TestJob.pyt test_init_jobscCsZtj|j_|j|jjtjkƒtj|j_|j|jjtjkƒdS(N(RtUNKNOWNRtstatusRtWAIT(R((s2/home/jd/Debian/rebuildd/rebuildd/tests/TestJob.pyt test_setattrscCsxt|jjdƒ}|j|dk ƒt|jjdƒ}|j|dk ƒ|jƒ|jƒtj|jƒdS(Ntwtr( topenRtlogfileRtNonetcloseRtunlinkR (Rtfiletfilero((s2/home/jd/Debian/rebuildd/rebuildd/tests/TestJob.pyttest_open_logfile$s  cCsJ|jjjƒ|jjƒ|jjƒ|j|jjtjkƒdS(N( Rtdo_quittsettstarttjoinRRRt WAIT_LOCKED(R((s2/home/jd/Debian/rebuildd/rebuildd/tests/TestJob.pyttest_status_on_doquit-s  cCsŒ|jjjƒtƒjdddƒtƒjdddƒtƒjdddƒ|jjƒ|jjƒ|j|jjt j kƒdS(Ntbuildt source_cmds /bin/truet build_cmdtpost_build_cmd( RR*tclearRR+R,R-RRRtBUILD_OK(R((s2/home/jd/Debian/rebuildd/rebuildd/tests/TestJob.pyttest_build_success3s  cCsŒ|jjjƒtƒjdddƒtƒjdddƒtƒjdddƒ|jjƒ|jjƒ|j|jjt j kƒdS(NR0R1s /bin/falseR2s /bin/trueR3( RR*R4RR+R,R-RRRt SOURCE_FAILED(R((s2/home/jd/Debian/rebuildd/rebuildd/tests/TestJob.pyttest_build_failure_source<s  cCsŒ|jjjƒtƒjdddƒtƒjdddƒtƒjdddƒ|jjƒ|jjƒ|j|jjt j kƒdS(NR0R1s/bin/true %s %s %sR2s/bin/false %s %s %s %sR3s/bin/true %s %s %s %s( RR*R4RR+R,R-RRRt BUILD_FAILED(R((s2/home/jd/Debian/rebuildd/rebuildd/tests/TestJob.pyttest_build_failure_buildEs  cCs|tƒjdddƒtƒjdddƒtƒjdddƒ|jjƒ|jjƒ|j|jjtjkƒdS(NR0R1s/bin/true %s %s %sR2s/bin/true %s %s %s %sR3s/bin/false %s %s %s %s( RR+RR,R-RRRtPOST_BUILD_FAILED(R((s2/home/jd/Debian/rebuildd/rebuildd/tests/TestJob.pyttest_build_failure_post_buildNs   cCsít|jjdƒ}|j|dk ƒ|jdƒ|jƒ|j|jjƒtkƒt j |j_ |j|jjƒt kƒ|j|jj t j kƒt j |j_ |j|jjƒt kƒ|j|jj t j kƒdS(NR s Fake log file(R"RR#RR$twriteR%tsend_build_logtFalseRR5RtTrueR9(RR'((s2/home/jd/Debian/rebuildd/rebuildd/tests/TestJob.pyttest_send_build_logVs  (t__name__t __module__RRRRRR)R/R6R8R:R<RA(((s2/home/jd/Debian/rebuildd/rebuildd/tests/TestJob.pyRs       t__main__t verbosityi(tsysRtinserttRebuilddTestSetupRRtunittestttypesRt sqlobjecttrebuildd.RebuilddConfigRtrebuildd.PackageRt rebuildd.JobRtrebuildd.JobStatusRtTestCaseRRBt TestLoadertloadTestsFromTestCasetsuitetTextTestRunnertrun(((s2/home/jd/Debian/rebuildd/rebuildd/tests/TestJob.pyts $ U rebuildd/tests/TestDistribution.py0000755000000000000000000000433012003461770014574 0ustar #!/usr/bin/python import sys sys.path.insert(0, "..") sys.path.insert(0, ".") from RebuilddTestSetup import rebuildd_global_test_setup, rebuildd_global_test_teardown import unittest, types, os from rebuildd.Distribution import Distribution from rebuildd.RebuilddConfig import RebuilddConfig from rebuildd.Rebuildd import Rebuildd from rebuildd.Package import Package class TestDistribution(unittest.TestCase): def setUp(self): rebuildd_global_test_setup() self.d = Distribution("sid", "alpha") self.package = Package(name="xutils", version="7.1.ds.3-1") self.package_dotted = Package(name="xutils", version="1:7.1.ds.3-1") def tearDown(self): rebuildd_global_test_teardown() def test_name(self): self.assert_(self.d.name is "sid") def test_arch(self): self.assert_(self.d.arch is "alpha") def test_get_source_cmd(self): RebuilddConfig().set('build', 'source_cmd', '/bin/true $d $a $p $v') cmd = self.d.get_source_cmd(self.package) self.assert_(self.d.name in cmd) self.assert_(self.package.name in cmd) self.assert_(self.package.version in cmd) def test_get_build_cmd(self): RebuilddConfig().set('build', 'build_cmd', '/bin/true $d $a $p $v') cmd = self.d.get_build_cmd(self.package) self.assert_(self.d.name in cmd) self.assert_(self.d.arch in cmd) self.assert_(self.package.name in cmd) self.assert_(self.package.version in cmd) cmd = self.d.get_build_cmd(self.package_dotted) self.assert_(self.package_dotted.version not in cmd) def test_get_post_build_cmd(self): RebuilddConfig().set('build', 'post_build_cmd', '') cmd = self.d.get_post_build_cmd(self.package) self.assert_(cmd is None) RebuilddConfig().set('build', 'post_build_cmd', '/bin/true $d $a $p $v') cmd = self.d.get_post_build_cmd(self.package) self.assert_(self.d.name in cmd) self.assert_(self.package.name in cmd) self.assert_(self.package.version in cmd) if __name__ == '__main__': suite = unittest.TestLoader().loadTestsFromTestCase(TestDistribution) unittest.TextTestRunner(verbosity=2).run(suite) rebuildd/rebuildd/0000755000000000000000000000000012003462215011323 5ustar rebuildd/rebuildd/JobStatus.py0000644000000000000000000000267310720364127013632 0ustar # rebuildd - Debian packages rebuild tool # # (c) 2007 - Julien Danjou # # This software is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 dated June, 1991. # # This software is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this software; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA # from Enumeration import Enumeration JobStatus = Enumeration([ ("UNKNOWN", 0), ("WAIT", 100), ("WAIT_LOCKED", 150), ("BUILDING", 200), ("SOURCE_FAILED", 250), ("BUILD_FAILED", 300), ("POST_BUILD_FAILED", 350), ("CANCELED", 800), ("GIVEUP", 850), ("FAILED", 900), ("BUILD_OK", 1000) ]) FailedStatus = (JobStatus.SOURCE_FAILED, JobStatus.BUILD_FAILED, JobStatus.POST_BUILD_FAILED) rebuildd/rebuildd/RebuilddHTTPServer.py0000644000000000000000000001366312003461770015334 0ustar # rebuildd - Debian packages rebuild tool # # (c) 2007 - Julien Danjou # # This software is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 dated June, 1991. # # This software is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this software; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA # from __future__ import with_statement from RebuilddConfig import RebuilddConfig from Rebuildd import Rebuildd from Package import Package from Job import Job from JobStatus import JobStatus from JobStatus import FailedStatus import tempfile, socket, sqlobject import web import gdchart render = web.template.render(RebuilddConfig().get('http', 'templates_dir'), \ cache=RebuilddConfig().getboolean('http', 'cache')) class RequestIndex: def GET(self): return render.base(page=render.index(), \ hostname=socket.gethostname(), \ archs=RebuilddConfig().arch, \ dists=RebuilddConfig().get('build', 'dists').split(' ')) class RequestPackage: def GET(self, name=None, version=None): jobs = [] if version: pkg = Package.selectBy(name=name, version=version)[0] title = "%s %s" % (name, version) package = "%s/%s" % (name, version) else: pkg = Package.selectBy(name=name)[0] title = package = name jobs.extend(Job.selectBy(package=pkg)) return render.base(page=render.tab(jobs=jobs), \ hostname=socket.gethostname(), \ title=title, \ package=package, \ archs=RebuilddConfig().arch, \ dists=RebuilddConfig().get('build', 'dists').split(' ')) class RequestArch: def GET(self, dist, arch=None): jobs = [] jobs.extend(Job.select(sqlobject.AND(Job.q.arch == arch, Job.q.dist == dist), orderBy=sqlobject.DESC(Job.q.creation_date))[:10]) return render.base(page=render.tab(jobs=jobs), \ arch=arch, \ dist=dist, \ title="%s/%s" % (dist, arch), \ hostname=socket.gethostname(), \ archs=RebuilddConfig().arch, \ dists=RebuilddConfig().get('build', 'dists').split(' ')) class RequestJob: def GET(self, jobid=None): job = Job.selectBy(id=jobid)[0] try: with open(job.logfile, "r") as build_logfile: build_log = build_logfile.read() except IOError, error: build_log = job.log.text return render.base(page=render.job(job=job, build_log=build_log), \ hostname=socket.gethostname(), \ title="job %s" % job.id, \ archs=RebuilddConfig().arch, \ dists=RebuilddConfig().get('build', 'dists').split(' ')) class RequestGraph: GET = web.autodelegate("GET_") def graph_init(self): web.header("Content-Type","image/png") graph = gdchart.Bar3D() graph.width = 300 graph.height = 300 graph.ytitle = "Jobs" graph.xtitle = "Build status" graph.ext_color = [ "yellow", "orange", "red", "green"] graph.bg_color = "white" graph.setLabels(["WAIT", "BUILDING", "FAILED", "OK"]) return graph def compute_stats(self, jobs): jw = 0 jb = 0 jf = 0 jo = 0 for job in jobs: if job.status == JobStatus.WAIT or \ job.status == JobStatus.WAIT_LOCKED: jw += 1 elif job.status == JobStatus.BUILDING: jb += 1 elif job.status in FailedStatus: jf += 1 elif job.status == JobStatus.BUILD_OK: jo += 1 return (jw, jb, jf, jo) def GET_buildstats(self, distarch=None): graph = self.graph_init() if distarch == "/": graph.title = "Build status" jobs = Job.selectBy() else: dindex = distarch.rindex("/") graph.title = "Build status for %s" % distarch[1:] jobs = Job.selectBy(arch=distarch[dindex+1:], dist=distarch[1:dindex]) graph.setData(self.compute_stats(jobs)) tmp = tempfile.TemporaryFile() graph.draw(tmp) tmp.seek(0) return tmp.read() def GET_package(self, package=None): graph = self.graph_init() if package == "/": graph.title = "Build status" jobs = Job.selectBy() else: dindex = package.rindex("/") graph.title = "Build status for %s" % package[1:] pkg = Package.selectBy(version=package[dindex+1:], name=package[1:dindex])[0] jobs = Job.selectBy(package=pkg) graph.setData(self.compute_stats(jobs)) tmp = tempfile.TemporaryFile() graph.draw(tmp) tmp.seek(0) return tmp.read() class RebuilddHTTPServer: """Main HTTP server""" urls = ( '/', 'RequestIndex', '/dist/(.*)/arch/(.*)', 'RequestArch', '/job/(.*)', 'RequestJob', '/package/(.*)/(.*)', 'RequestPackage', '/package/(.*)', 'RequestPackage', '/graph/(.*)', 'RequestGraph', ) def __init__(self): Rebuildd() def start(self): """Run main HTTP server thread""" web.webapi.internalerror = web.debugerror import sys; sys.argv.append(RebuilddConfig().get('http', 'ip') + ":" + RebuilddConfig().get('http', 'port')) app = web.application(self.urls, globals()) app.run() rebuildd/rebuildd/Job.pyc0000644000000000000000000002101312003462215012547 0ustar ó øcPc@€s!ddlmZddlZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddlm Z m Z ddl mZddlmZddlmZddlmZddlmZmZdd lmZd Zd eje jfd „ƒYZdS( iÿÿÿÿ(twith_statementN(tPopentPIPE(tMessage(tDists(t JobStatus(t FailedStatus(t RebuilddLogtLog(tRebuilddConfigs$Rev$tJobcB€s^eZdZejdejƒZejddƒZ ej dde ƒZ ejddƒZejddƒZejdejjƒZejddƒZejddƒZejddƒZejddƒZejdddd d ƒZejd ƒZdZd „Zd „Zed„ƒZd„Z d„Z!d„Z"d„Z#d„Z$d„Z%d„Z&RS(sClass implementing a build jobtdefaulttPackagetcascadetsidtanyR t joinColumntjobat otherColumntjobbRcO€sHtjj|ƒtjj|||Žtjƒ|_tjƒ|_dS(sInit jobN( t threadingtThreadt__init__t sqlobjectt SQLObjecttEventtdo_quittLockt status_lock(tselftargstkwargs((s./rebuildd/Job.pyR3sc C€s‹|dkrqtjd|j|jj|jj|j|jtj |j ƒtj |ƒfƒt j j ƒ|_nt jj|||ƒdS(s,Override setattr to log build status changeststatuss6Job %s for %s_%s on %s/%s changed status from %s to %sN(RtinfotidtpackagetnametversiontdisttarchRtwhatisR Rt DateTimeColtnowtstatus_changedRt __setattr__(RR$tvalue((s./rebuildd/Job.pyR,<s   cC€sSdtƒjddƒ|jj|jj|j|j|jjdƒ|j f}|S(sCompute and return logfile names%s/%s_%s-%s-%s-%s.%s.logtlogtlogs_dirs %Y%m%d-%H%M%S( R tgetR#R$R%R&R't creation_datetstrftimeR"(Rtbuild_log_file((s./rebuildd/Job.pytlogfileHs   cC€stjƒdS(s0Start a new group process before executing childN(tostsetsid(R((s./rebuildd/Job.pyt preexec_childSsc C€sŸtjjƒ|_yxt|jdƒ`}|jd|jj|jj |j |j |j t fƒ|jd|jƒ|jdƒWdQXWntk rždSXt|jdƒ}|jtj|_WdQXx¾tƒj|j |j ƒj|jƒtjgtƒj|j |j ƒj|jƒtjgtƒj|j |j ƒj|jƒtjgfD]/\}}|dkryq[ny=tj|jƒddd |j d |d dd tj!ƒ}WnJt"k r}|jd ||fƒ|j||_WdQXd}PnX|j#ƒ}x<|j$j%ƒ rM|dkrM|j#ƒ}|j$j&dƒqW|j$j%ƒraPn|dkr[|j||_WdQXPq[q[W|j$j%ƒrãt'j(d|j)ƒy#t*j+t*j,|j-ƒt.j/ƒWn-t0k r}t'j1d|j)|fƒnXd}t2ƒj3ddƒ}x9|j#ƒdkrZ||krZt4j5dƒ|d7}q"W|j#ƒdkr£t'j1d|j)ƒt*j+t*j,|j-ƒt.j6ƒn|jtj7|_WdQXd|_ |jdƒ|j8ƒdS|j |dkrtj9|_nWdQXtjjƒ|_:|jdƒ|jdtj;|jƒ|j:fƒ|jd|j:|jƒ|j8ƒ|j<r‘|j<j=ƒn|j>ƒdS(s.Run job thread, download and build the packagetws8Automatic build of %s_%s on %s for %s/%s by rebuildd %s sBuild started at %s sO****************************************************************************** Ntatbufsizeit preexec_fntstdouttstdintstderrs# Unable to execute command "%s": %sisKilling job %s with SIGINTsError killing job %s: %stbuildt kill_timeouts.Killing job %s timed out, killing with SIGKILLs Build job killed on request sFinished with status %s at %s sBuild needed %s (?RR)R*t build_starttopenR4twriteR#R$R%thostR&R't __version__tIOErrortfileRRtBUILDINGR Rtget_disttget_source_cmdt SOURCE_FAILEDt get_build_cmdt BUILD_FAILEDtget_post_build_cmdtPOST_BUILD_FAILEDtNonet subprocessRtsplitR7tSTDOUTt ExceptiontpollRtisSettwaitRR!R"R5tkillpgtgetpgidtpidtsignaltSIGINTtOSErrorterrorR R0ttimetsleeptSIGKILLt WAIT_LOCKEDtclosetBUILD_OKt build_endR(tnotifytsettsend_build_log( Rt build_logtcmdt failed_statustprocR^tstatetcounterttimemax((s./rebuildd/Job.pytrunXs˜   ' $ $        #!   "       &  c C€sÒy+t|jdƒ}|jƒ}WdQXWn(tk rU}tjd|jƒtSX|jrn||j_ n|j |j t j kr|j tkrtStƒjddƒ rÉ|j t j krÉtStƒjddƒ rò|j tkròtSWdQX|j t j krd}nd}tƒ}|jr;|j|d s` rebuildd/rebuildd/RebuilddConfig.pyc0000644000000000000000000000762312003462214014727 0ustar ó øcPc@s8ddlZddlZdeejfd„ƒYZdS(iÿÿÿÿNtRebuilddConfigcBsJeZdZdZdZd„Zed„Zd„Z d„Z d„Z RS(sMain configuration singletons/etc/rebuildd/rebuilddrccOs>|jdkr7tj|ƒ|_|jj||Žn|jS(N(t _instancetNonetobjectt__new__tinit(tclstargstkwargs((srebuildd/RebuilddConfig.pyRscCsjtjj|ƒ|jdƒ|jdƒ|jdƒ|jdƒ|jdƒ|jdddƒ|jddd ƒ|jdd d ƒ|jdd d ƒ|jdddƒ|jdddƒ|jdddƒ|jdddƒ|jdddƒ|jdddƒ|jdddƒ|jdddƒ|jdddƒ|jdd d!ƒ|jdd"d!ƒ|jdd#d$ƒ|jdd%d&ƒ|jdd'd(ƒ|jdd)d*ƒ|jdd+d,ƒ|jdd-d.ƒ|jdd/d0ƒ|jdd)d1ƒ|jdd+d2ƒ|jdd3d4ƒ|jdd5d6ƒ|jdd7dƒ|jdd8d9ƒ|jdd:d;ƒ|jdd<d=ƒ|jdd>d?ƒ|jdd@dƒ|jddAdƒ|s×|jƒng|_|jddƒdBkr0tjdCƒ}|jj |j ƒj ƒƒ|j ƒnx3|j ddƒjdDƒD]}|jj |ƒqLWdS(ENtbuildtmailttelnetthttptlogt check_everyt300t max_threadst2tmax_jobst5t kill_timeoutt90t source_cmds3apt-get -q --download-only -t ${d} source ${p}=${v}t build_cmdsHpbuilder build --basetgz /var/cache/pbuilder/${d}-${a}.tgz ${p}_${v}.dsctpost_build_cmdttdistsssqueeze wheezy sidtwork_dirs/var/cache/rebuildd/buildt database_uris&sqlite:///var/lib/rebuildd/rebuildd.dbtbuild_more_recentt1t more_archstanytno_system_archt0tfromsrebuildd@localhosttmailtotsubject_prefixs [rebuildd]t smtp_hostt localhostt smtp_portt25tportt9999tips 127.0.0.1tpromptsrebuildd@localhost->tmotds"Connected on rebuildd on localhostt9998s0.0.0.0t log_lines_nbt30t templates_dirs/usr/share/rebuildd/templatestcachetlogfiles/var/log/rebuildd/httpd.logtfiles/var/log/rebuildd/rebuildd.logt time_formats%Y-%m-%d %H:%M:%Stlogs_dirs/var/log/rebuildd/build_logst mail_failedtmail_successfulisdpkg --print-architecturet (t ConfigParsert__init__t add_sectiontsettreloadtarchtgetinttostpopentappendtreadlinetstriptclosetgettsplit(tselft dontparsetparchta((srebuildd/RebuilddConfig.pyR"s`        "cCs|j|jƒS(sReload configuration file(treadt config_file(RJ((srebuildd/RebuilddConfig.pyR?_scCsqd}xd|jƒD]V}|d|d7}x1|j|ƒD] \}}|d||f7}q;W|d7}qW|S(sDump running configurationRt[s] s%s = %s s (tsectionstitems(RJtconftsectiontitemtvalue((srebuildd/RebuilddConfig.pytdumpdscCs8y|jt|jdƒƒWntk r3}tSXtS(sSave configuration filetw(twriteR5ROt ExceptiontFalsetTrue(RJterror((srebuildd/RebuilddConfig.pytsaveos N( t__name__t __module__t__doc__RORRRR[RR?RWR^(((srebuildd/RebuilddConfig.pyRs  =  (R;RBRR(((srebuildd/RebuilddConfig.pyts  rebuildd/rebuildd/Distribution.pyc0000644000000000000000000000517512003462214014526 0ustar ó \ÎGc@sJddlmZddlmZddlmZdefd„ƒYZdS(iÿÿÿÿ(tRebuilddConfig(t RebuilddLog(tTemplatet DistributioncBs2eZdZd„Zd„Zd„Zd„ZRS(såClass implementing a Debian distribution Substitutions are done in the command strings: $d => The distro's name $a => the target architecture $p => the package's name $v => the package's version cCs||_||_dS(N(tnametarch(tselfRR((s./rebuildd/Distribution.pyt__init__"s cCs†yZi|jd6|jd6|jd6|jd6}ttƒjddƒƒ}|j|SWn%tk r}tj d|ƒdSXdS( s<Return command used for grabing source for this distributiontdtatvtptbuildt source_cmds%get_source_cmd has invalid format: %sN( RRtversionRRtgettsafe_substitutet TypeErrorRterrortNone(RtpackagetargsttR((s./rebuildd/Distribution.pytget_source_cmd&s! cCsyt|jjdƒ}i|jd6|jd6|j|dd6|jd6}ttƒjddƒƒ}|j|SWntk r‡nXyZi|jd6|jd6|jd6|jd6}ttƒjddƒƒ}|j|SWn%t k r }t j d |ƒd SXd S( s=Return command used for building source for this distributiont:RR iR R R t build_cmds$get_build_cmd has invalid format: %sN( RtindexRRRRRRt ValueErrorRRRR(RRRRRR((s./rebuildd/Distribution.pyt get_build_cmd2s  cCsœtƒjddƒ}|dkr%d SyKi|jd6|jd6|jd6|jd6}t|ƒ}|j|SWn%tk r—}t j d|ƒd SXd S( s?Return command used after building source for this distributionR tpost_build_cmdtRR R R s%post_build_cmd has invalid format: %sN( RRRRRRRRRRR(RRtcmdRRR((s./rebuildd/Distribution.pytget_post_build_cmdHs  (t__name__t __module__t__doc__RRRR (((s./rebuildd/Distribution.pyRs    N(RRtstringRtobjectR(((s./rebuildd/Distribution.pytsrebuildd/rebuildd/Package.py0000644000000000000000000000222411423073051013231 0ustar # rebuildd - Debian packages rebuild tool # # (c) 2007 - Julien Danjou # # This software is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 dated June, 1991. # # This software is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this software; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA # import sqlobject import apt_pkg apt_pkg.init_system() class Package(sqlobject.SQLObject): """Class implemeting a Debian package""" name = sqlobject.StringCol() version = sqlobject.StringCol(default=None) priority = sqlobject.StringCol(default=None) @staticmethod def version_compare(a, b): return apt_pkg.version_compare(a.version, b.version) rebuildd/rebuildd/Dists.pyc0000644000000000000000000000172312003462215013131 0ustar ó ÎÞSGc@sdefd„ƒYZdS(tDistscBs5eZdZdZgZd„Zd„Zd„ZRS(s4Singleton implementing a set of Debian distributionscCs+|jdkr$tj|ƒ|_n|jS(N(t _instancetNonetobjectt__new__(tcls((s./rebuildd/Dists.pyRscCs|jj|ƒdS(N(tdiststappend(tselftdist((s./rebuildd/Dists.pytadd_distscCs:x3|jD](}|j|kr |j|kr |Sq WdS(N(RtnametarchR(RR R td((s./rebuildd/Dists.pytget_dist!sN( t__name__t __module__t__doc__RRRRR R(((s./rebuildd/Dists.pyRs   N(RR(((s./rebuildd/Dists.pytsrebuildd/rebuildd/Distribution.py0000644000000000000000000000610710763510134014365 0ustar # rebuildd - Debian packages rebuild tool # # (c) 2007 - Julien Danjou # # This software is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 dated June, 1991. # # This software is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this software; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA # from RebuilddConfig import RebuilddConfig from RebuilddLog import RebuilddLog from string import Template class Distribution(object): """Class implementing a Debian distribution Substitutions are done in the command strings: $d => The distro's name $a => the target architecture $p => the package's name $v => the package's version """ def __init__(self, name, arch): self.name = name self.arch = arch def get_source_cmd(self, package): """Return command used for grabing source for this distribution""" try: args = { 'd': self.name, 'a': self.arch, 'v': package.version, \ 'p': package.name } t = Template(RebuilddConfig().get('build', 'source_cmd')) return t.safe_substitute(**args) except TypeError, error: RebuilddLog.error("get_source_cmd has invalid format: %s" % error) return None def get_build_cmd(self, package): """Return command used for building source for this distribution""" # Strip epochs (x:) away try: index = package.version.index(":") args = { 'd': self.name, 'a': self.arch, \ 'v': package.version[index+1:], 'p': package.name } t = Template(RebuilddConfig().get('build', 'build_cmd')) return t.safe_substitute(**args) except ValueError: pass try: args = { 'd': self.name, 'a': self.arch, \ 'v': package.version, 'p': package.name } t = Template(RebuilddConfig().get('build', 'build_cmd')) return t.safe_substitute(**args) except TypeError, error: RebuilddLog.error("get_build_cmd has invalid format: %s" % error) return None def get_post_build_cmd(self, package): """Return command used after building source for this distribution""" cmd = RebuilddConfig().get('build', 'post_build_cmd') if cmd == '': return None try: args = { 'd': self.name, 'a': self.arch, \ 'v': package.version, 'p': package.name } t = Template(cmd) return t.safe_substitute(**args) except TypeError, error: RebuilddLog.error("post_build_cmd has invalid format: %s" % error) return None rebuildd/rebuildd/Enumeration.py0000644000000000000000000000362511423073051014172 0ustar # rebuildd - Debian packages rebuild tool # # (c) 2007 - Julien Danjou # # This software is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 dated June, 1991. # # This software is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this software; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA # import types class Enumeration: """Simple enumeration class with reverse lookup""" def __init__(self, enumlist): self.lookup = { } self.reverse_lookup = { } val = 0 for elem in enumlist: if type(elem) == types.TupleType: elem, val = elem if type(elem) != types.StringType: raise ValueError("enum name is not a string: " + elem) if type(val) != types.IntType: raise ValueError("enum value is not an integer: " + val) if self.lookup.has_key(elem): raise ValueError("enum name is not unique: " + elem) if val in self.lookup.values(): raise ValueError("enum value is not unique for " + val) self.lookup[elem] = val self.reverse_lookup[val] = elem val += 1 def __getattr__(self, attr): if not self.lookup.has_key(attr): raise AttributeError return self.lookup[attr] def whatis(self, value): """Return element name for a value""" return self.reverse_lookup[value] rebuildd/rebuildd/JobStatus.pyc0000644000000000000000000000140712003462215013760 0ustar ó WèAGc @sYddlmZedddddddd d!d"d#g ƒZejejejfZdS($iÿÿÿÿ(t EnumerationtUNKNOWNitWAITidt WAIT_LOCKEDi–tBUILDINGiÈt SOURCE_FAILEDiút BUILD_FAILEDi,tPOST_BUILD_FAILEDi^tCANCELEDi tGIVEUPiRtFAILEDi„tBUILD_OKièN(sUNKNOWNi(sWAITid(s WAIT_LOCKEDi–(sBUILDINGiÈ(s SOURCE_FAILEDiú(s BUILD_FAILEDi,(sPOST_BUILD_FAILEDi^(sCANCELEDi (sGIVEUPiR(sFAILEDi„(sBUILD_OKiè(Rt JobStatusRRRt FailedStatus(((s./rebuildd/JobStatus.pyts rebuildd/rebuildd/RebuilddNetworkClient.pyc0000644000000000000000000001564412003462215016315 0ustar ó øcPc@€s[ddlmZddlmZddlZddlZdZdejfd„ƒYZdS(iÿÿÿÿ(twith_statement(tRebuilddConfigNs$Rev$tRebuilddNetworkClientcB€s•eZdZd„Zd„Zd„Zd„Zd„Zd„Zd„Z d„Z d „Z d „Z d „Z d „Zd „Zd„Zd„ZRS(s'Network client used for each connectioncC€s&tjj|ƒ||_||_dS(N(t threadingtThreadt__init__trebuilddtsocket(tselfRR((s#./rebuildd/RebuilddNetworkClient.pyRs cC€s|jjdƒ|jjtƒjddƒdƒtƒjddƒd}d}t}x³|dkr |jjjƒ r |sÅy-|jj|j |ƒƒ|jj|ƒWqÅt k rÁPqÅXny(d}|jj d ƒj ƒ}t}Wq[tj k r t}q[Xq[W|jjƒd S( sRun client threadittelnettmotds tpromptt ttexitiN(Rt settimeouttsendRtgettFalseRtdo_quittisSettexec_cmdt ExceptiontrecvtstripttimeouttTruetclose(RR tlinet has_timeout((s#./rebuildd/RebuilddNetworkClient.pytrun"s&#"  cC€sŠ|dkrdS|jdƒ}|d}|jƒs9dS|j|dƒy!t|d|ƒt|ƒŒSWntk r…}d|SXdS(s#Execute a command asked by a clientR R it exec_cmd_sE: command error: %s N(tsplittisalnumtremovetgetattrttupleR(Rtcmdtoptcmd_fctterror((s#./rebuildd/RebuilddNetworkClient.pyR:s   !cG€scd}xVttƒD]H}|jdƒr|d|djddƒtt|ƒjf7}qqW|S(s Show helpR Rs %s -- %s i t_R (tdirRt startswithtreplaceR#t__doc__(Rtargsthelptattr((s#./rebuildd/RebuilddNetworkClient.pyt exec_cmd_helpLscG€s…t|ƒdkrdS|ddkr=tƒjƒr9dSdS|ddkrZtƒjƒS|ddkrtƒjƒr}d Sd SdS( sManipulate configuration fileis$E: usage: config [reload|dump|save] itreloadsI: config reloaded sE: config not reloded tdumptsavesI: config saved sE: config not saved (tlenRR2R3R4(RR.((s#./rebuildd/RebuilddNetworkClient.pytexec_cmd_configXs cG€s |jjƒS(sShow current jobs status(Rt dump_jobs(RR.((s#./rebuildd/RebuilddNetworkClient.pytexec_cmd_statusmscG€stdS(s Show versions (t __version__(RR.((s#./rebuildd/RebuilddNetworkClient.pytexec_cmd_versionqsc G€sMt|ƒdkr/|ddkr/|j|ŒSt|ƒdkr^|ddkr^|j|ŒSt|ƒdkr|ddkr|j|ŒSt|ƒdkr¼|ddkr¼|j|ŒSt|ƒdkrë|ddkrë|j|ŒSt|ƒdkr|ddkr|j|ŒSt|ƒdkrI|ddkrI|j|ŒSd S( sManipulate jobsitaddtdepstcanceltstartR2tstatustrequeuesE: usage: job [args] (R5texec_cmd_job_addtexec_cmd_job_depstexec_cmd_job_canceltexec_cmd_job_starttexec_cmd_job_reloadtexec_cmd_job_statustexec_cmd_job_requeue(RR.((s#./rebuildd/RebuilddNetworkClient.pyt exec_cmd_jobus" " " " " " " cG€s,t}t|ƒdkrdSt|ƒdkrh|jjd|dd|dd|d d |dƒ}nt|ƒd kr¾|jjd|dd|dd|d d |dd |dƒ}nt|ƒd kr|jjd|dd|dd|d d |dd |dd|d ƒ}n|r(dSdS(sAdd jobisAE: usage: job add [arch] [mailto] itnameitversionitpriorityitdistitarchitmailtos I: job added sE: error adding job (RR5Rtadd_job(RR.tret((s#./rebuildd/RebuilddNetworkClient.pyRAs0         cG€s=t|ƒdkrdS|jjdt|dƒƒr9dSdS(s Requeue jobisE: usage: job requeue tjob_idisI: job requeued sE: error requeuing the job (R5Rt requeue_jobtint(RR.((s#./rebuildd/RebuilddNetworkClient.pyRG¯s cG€sMt}t|ƒdkrdS|jjd|dd|dƒ}|rIdSdS( sAdd dependencyisJE: usage: job deps [dependency_job_id] [...] RQitdependency_idsisI: Dependency added sE: error adding deps(RR5Rtadd_deps(RR.RP((s#./rebuildd/RebuilddNetworkClient.pyRB¹s cG€s:t|ƒdkrdS|jjt|dƒƒr6dSdS(s Cancel jobisE: usage: job cancel isI: job canceled sE: unknown job (R5Rt cancel_jobRS(RR.((s#./rebuildd/RebuilddNetworkClient.pyRCÈs cG€sAt|ƒdkr0d|jjt|dƒƒSd|jjƒS(s Start jobsisI: %s jobs started i(R5Rt start_jobsRS(RR.((s#./rebuildd/RebuilddNetworkClient.pyRDÑs cG€sd|jjƒS(s Load new jobssI: %s new jobs added (Rt get_new_jobs(RR.((s#./rebuildd/RebuilddNetworkClient.pyREÚsc G€s,t|ƒdks$t|ƒdkr(dSt|ƒdkrV|jjd|dƒ}nÆt|ƒdkrŽ|jjd|dd|dƒ}nŽt|ƒdkrÐ|jjd|dd|dd |dƒ}nLt|ƒdkr|jjd|dd|dd |dd |dƒ}n|jj|ƒS( sDump job statusiis0E: usage: job status [arch] RIiiRJiRLRM(R5Rtget_jobsR7(RR.tjobs((s#./rebuildd/RebuilddNetworkClient.pyRFßs"$   (t__name__t __module__R-RRRR1R6R8R:RHRARGRBRCRDRERF(((s#./rebuildd/RebuilddNetworkClient.pyRs          (t __future__RRRRR9RR(((s#./rebuildd/RebuilddNetworkClient.pytsrebuildd/rebuildd/RebuilddConfig.py0000644000000000000000000001045212003461770014564 0ustar # rebuildd - Debian packages rebuild tool # # (c) 2007 - Julien Danjou # # This software is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 dated June, 1991. # # This software is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this software; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA # import ConfigParser import os class RebuilddConfig(object, ConfigParser.ConfigParser): """Main configuration singleton""" config_file = "/etc/rebuildd/rebuilddrc" _instance = None def __new__(cls, *args, **kwargs): if cls._instance is None: cls._instance = object.__new__(cls) cls._instance.init(*args, **kwargs) return cls._instance def init(self, dontparse=False): ConfigParser.ConfigParser.__init__(self) # add default sections self.add_section('build') self.add_section('mail') self.add_section('telnet') self.add_section('http') self.add_section('log') # add default values self.set('build', 'check_every', '300') self.set('build', 'max_threads', '2') self.set('build', 'max_jobs', '5') self.set('build', 'kill_timeout', '90') self.set('build', 'source_cmd', 'apt-get -q --download-only -t ${d} source ${p}=${v}') self.set('build', 'build_cmd', 'pbuilder build --basetgz /var/cache/pbuilder/${d}-${a}.tgz ${p}_${v}.dsc') self.set('build', 'post_build_cmd', '') self.set('build', 'dists', 'squeeze wheezy sid') self.set('build', 'work_dir', '/var/cache/rebuildd/build') self.set('build', 'database_uri', 'sqlite:///var/lib/rebuildd/rebuildd.db') self.set('build', 'build_more_recent', '1') self.set('build', 'more_archs', 'any') self.set('build', 'no_system_arch', '0') self.set('mail', 'from', 'rebuildd@localhost') self.set('mail', 'mailto', 'rebuildd@localhost') self.set('mail', 'subject_prefix', '[rebuildd]') self.set('mail', 'smtp_host', 'localhost') self.set('mail', 'smtp_port', '25') self.set('telnet', 'port', '9999') self.set('telnet', 'ip', '127.0.0.1') self.set('telnet', 'prompt', 'rebuildd@localhost->') self.set('telnet', 'motd', 'Connected on rebuildd on localhost') self.set('http', 'port', '9998') self.set('http', 'ip', '0.0.0.0') # This is dedicated to MadCoder self.set('http', 'log_lines_nb', '30') self.set('http', 'templates_dir', '/usr/share/rebuildd/templates') self.set('http', 'cache', '1') self.set('http', 'logfile', '/var/log/rebuildd/httpd.log') self.set('log', 'file', "/var/log/rebuildd/rebuildd.log") self.set('log', 'time_format', "%Y-%m-%d %H:%M:%S") self.set('log', 'logs_dir', "/var/log/rebuildd/build_logs") self.set('log', 'mail_failed', '1') self.set('log', 'mail_successful', '0') if not dontparse: self.reload() self.arch = [] if self.getint('build', 'no_system_arch') == 0: parch = os.popen("dpkg --print-architecture") self.arch.append(parch.readline().strip()) parch.close() for a in self.get('build', 'more_archs').split(' '): self.arch.append(a) def reload(self): """Reload configuration file""" return self.read(self.config_file) def dump(self): """Dump running configuration""" conf = "" for section in self.sections(): conf += "[" + section + "]\n" for item, value in self.items(section): conf += "%s = %s\n" % (item, value) conf += "\n" return conf def save(self): """Save configuration file""" try: self.write(file(self.config_file, 'w')) except Exception, error: return False return True rebuildd/rebuildd/Rebuildd.pyc0000644000000000000000000002671512003462214013604 0ustar ó øcPc@€s0dZddlmZddlmZddlmZddlmZddlmZmZddlmZddl m Z dd l m Z dd l m Z dd l m Z dd l m Z dd lZdd lZdd lZdd lZdd lZdd lZdd lZdZdefd„ƒYZd S(s'rebuildd - Debian packages rebuild tooliÿÿÿÿ(twith_statement(t Enumeration(t Distribution(tDists(t RebuilddLogtLog(tRebuilddConfig(tRebuilddNetworkServer(tPackage(tJob(t JobStatus(t FailedStatusNs$Rev$tRebuilddcB€séeZgZdZd„Zd„Zd„Zd„Zd„Z d„Z d„Z d„Z dd „Z dddd „Zdd „Zd „Zd „Zddd„Zd„Zd„Zd„Zd„Zed„Zd„Zd„ZRS(cC€s8|jdkr1tj|ƒ|_|jjƒn|jS(N(t _instancetNonetobjectt__new__tinit(tcls((s./rebuildd/Rebuildd.pyR*scC€sÍtƒ|_tƒtj|jjddƒƒ|_|jtj_xV|jjddƒj dƒD]6}x-|jj D]}t ƒj t ||ƒƒquWqbWtjƒ|_tjƒ|_tjƒ|_dS(Ntbuildt database_uritdistst (RtcfgRt sqlobjecttconnectionForURItgett_sqlconnectiontsqlhubtprocessConnectiontsplittarchRtadd_distRt threadingtEventtdo_quittLockt jobs_lockert job_finished(tselftdistR((s./rebuildd/Rebuildd.pyR0s !%!cC€sÑtjdtƒ|jƒtjdƒt|ƒ|_|jjtƒ|jjƒtjdƒ|j ƒtjdƒ|j ƒtjdƒ|j ƒtjdƒ|j ƒ|jj dƒtjdƒdS( NsStarting rebuildd %ssLaunching network serversRunning main loops#Cleaning finished and canceled jobssStopping all jobssReleasing wait-locked jobsi sExiting rebuildd(Rtinfot __version__t daemonizeRtnetservt setDaemontTruetstarttloopt clean_jobst stop_all_jobst release_jobstjoin(R'((s./rebuildd/Rebuildd.pytdaemonBs            cC€sÇtjtj|jƒtjtj|jƒy tj|jjddƒƒWn)tk rw}d|GHt j dƒnXy,t |jjddƒdƒt _ t _ Wntk rÂ}d|GHnXd S( sDo daemon stuffRtwork_dirs"E: unable to chdir to work_dir: %sitlogtfiletasE: unable to open logfile: %sN(tsignaltSIGTERMthandle_sigtermtSIGINTtostchdirRRt ExceptiontsystexitR8tstdouttstderr(R'terror((s./rebuildd/Rebuildd.pyR+Zs  ,cC€s+x$|jD]}|j|kr |Sq WdS(N(tjobstidR(R'tjobidtjob((s./rebuildd/Rebuildd.pytget_jobkscK€s g}|jtj|ƒ|S(N(textendR tselectBy(R'tkwargsRF((s./rebuildd/Rebuildd.pyt get_all_jobsrsc C€s;|jjddƒ}t|jƒ}|j||kr>dSg}xFtƒjD]8}|jtj dt j d|j d|j ƒ| ƒqQWd}x›|D]“}|jjddƒr­tj d|jj ƒ}g}|j|ƒ|jd tjƒ|jƒd } xž|D]–} g} | jtj d | d|jd|j ƒƒx\| D]T} | r| | kr| jt j krt j| _qH| jt j krH| } qHqHWq W| }n| sš|jt j krÌqšn|jƒsÞqšnt j|_tjƒ|_|jj|ƒ|d 7}|d 7}||kršPqšqšWWd QX|S( s.Feed jobs list with waiting jobs and lock themRtmax_jobsitstatusR(Rtbuild_more_recenttnametcmptpackageiN(RtgetinttlenRFR%RRRKR RLR tWAITRRRt getbooleanRRTtsorttversion_comparetreverseRR(RPtGIVEUPtis_allowed_to_buildt WAIT_LOCKEDtsockett gethostnamethosttappend( R'tmax_newt count_currentRFR(t count_newRItpackagestcandidate_packagestnewjobtcpackagetcandidate_jobstcjob((s./rebuildd/Rebuildd.pyt get_new_jobswsL  6    + $      cC€sYd}|jDx<|jD]1}|jtjkr|jƒr|d7}qqWWdQX|S(sCount running jobsiiN(R%RFRPR tBUILDINGtisAlive(R'tcountRI((s./rebuildd/Rebuildd.pytcount_running_jobs²s  ic C€s|jƒ}t||jjddƒƒ}d}|j¦xž|jD]“}||krZPn|jr|jtj krÑ|j ƒ rÑt j d|j ƒ|j|_|jtƒ|jƒ|d7}|d}nWdQXqDWWdQXt j d||tjƒfƒ|S(sStart waiting jobsRt max_threadsisStarting new thread for job %siNs,Running threads: [ build %s/%s ] [ real %s ](RptmaxRRUR%RFt status_lockRPR R^RnRR)RGR&tnotifyR-R.R/R!t activeCount(R'toverruntrunning_threadsRqt jobs_startedRI((s./rebuildd/Rebuildd.pyt start_jobs½s$         c C€s.|r!tjd|d|ƒ}ntjd|ƒ}|jƒsCgSg}|rŽ|rŽxÒ|D]+}|jtjd|d|d|ƒƒq\Wnœ|rÇx“|D]%}|jtjd|d|ƒƒq›Wnc|rxZ|D]%}|jtjd|d|ƒƒqÔWn*x'|D]}|jtjd|ƒƒqW|S(sDump a job statusRRtversionRTR(R(RRLRoRKR (R'RRRzR(Rtpkgstretjobstpkg((s./rebuildd/Rebuildd.pytget_jobs×s$   , & & cC€sCd}|s|j}nx$|D]}d|t|ƒf}qW|S(sDump all jobs statusts%s%s (RFtstr(R'tjoblisttretRI((s./rebuildd/Rebuildd.pyt dump_jobsòs   c C€sÀ|j±|j|ƒ}|dkr¶|jƒrK|jjƒ|jƒn|jtj |_ WdQX|j j |ƒt jd|j|jj|jj|j|j|jfƒtSWdQXtS(s Cancel a jobNs)Canceled job %s for %s_%s on %s/%s for %s(R%RJRRnR#tsetR4RsR tCANCELEDRPRFtremoveRR)RGRTRRRzR(RtmailtoR.tFalse(R'RHRI((s./rebuildd/Rebuildd.pyt cancel_jobÿs        c C€s®|jŸxS|jD]H}|jtjkr|jƒr|jjƒtj d|j ƒqqWxA|jD]6}|jƒrjtj d|j ƒ|j dƒqjqjWWdQXt S(sStop all running jobssSending stop to job %ssWaiting for job %s to terminatei<N( R%RFRPR RmRnR#R„RR)RGR4R.(R'RI((s./rebuildd/Rebuildd.pyR2s   c C€sg|s|jjd}ntƒj||ƒsOtjd||||fƒtStjd|d|ƒ}|j ƒr€|d}ntd|d|d|ƒ}t jd|d|d|d |d t j ƒj ƒ} | rûtjd |j |j||fƒtSt d|d|d|ƒ} t j | _|| _|| _td | ƒ} tjd |||||fƒtS(s Add a jobisLCouldn't find dist/arch in the config file for %s_%s on %s/%s, not adding itRRRztpriorityRTR(RR‡RPs6Job already existing for %s_%s on %s/%s, not adding itRIs#Added job for %s_%s on %s/%s for %s(RRRtget_distRRERˆRRLRoR R RWRRRzRPR‡RR)R.( R'RRRzRŠR(R‡RR{R}t jobs_countRIR7((s./rebuildd/Rebuildd.pytadd_job!s.   3     cC€sttjd|ƒjƒdkr3tjd|ƒtStjd|ƒd}|jtkrptj |_d|_ nt S(sRequeue a failed jobRGis5There is no job related to %s that is in the job listR( R RLRoRRERˆRPR R RWRaR.(R'tjob_idRI((s./rebuildd/Rebuildd.pyt requeue_jobEs  cC€sÇtjd|ƒjƒdkr3tjd|ƒtStjd|ƒd}g}xd|D]\}tjd|ƒjƒdkrtjd|ƒtStjd|ƒd}|j|ƒqVW|j|ƒtS(NRGis5There is no job related to %s that is in the job list( R RLRoRRERˆRbtadd_depsR.(R'RŽtdependency_idsRItdepstdeptdep_job((s./rebuildd/Rebuildd.pyRSs  c C€sn|j_xW|jD]L}|jtjksM|jtksM|jtjkr|jj|ƒqqWWdQXtS(sClean finished or canceled jobsN( R%RFRPR tBUILD_OKR R…R†R.(R'RI((s./rebuildd/Rebuildd.pyR1es c C€sb|jSxK|jD]@}|j/|jtjkrNtj|_d|_nWdQXqWWdQXtS(s Release jobsRN( R%RFRsRPR R^RWRaR.(R'RI((s./rebuildd/Rebuildd.pyR3qs   cC€s¹g}|jtjdtjƒdtjƒƒ|jtjdtjƒdtjƒƒx\|D]T}|rŠd|jtj |j ƒfGHnd|_ tj |_ d|_d|_q]WtS(s0If rebuildd crashed, reset jobs to a valid stateRaRPsI: Fixing job %s (was %s)N(RKR RLR_R`R R^RmRGtwhatisRPRRaRWt build_startt build_endR.(R't print_resultRFRI((s./rebuildd/Rebuildd.pytfix_jobs}s(( !    cC€s"tjd|ƒ|jjƒdS(NsBReceiving transmission... it's a signal %s capt'ain! EVERYONE OUT!(RR)R#R„(R'tsignumtstack((s./rebuildd/Rebuildd.pyR<ŽscC€s§|jjddƒ}x‹|jjƒs¢||jjddƒksQ|jjƒr…|jƒ|jƒ|jƒd}|jjƒn|jj dƒ|d7}qWdS(sRebuildd main loopRt check_everyiiN( RRUR#tisSetR&RlRyR1tcleartwait(R'tcounter((s./rebuildd/Rebuildd.pyR0’s   N(t__name__t __module__RFRR RRR5R+RJRNRlRpRyR~RƒR‰R2RRRR1R3R.RšR<R0(((s./rebuildd/Rebuildd.pyR &s.       ;   $    (t__doc__t __future__RRRRRRRRRR R R R!R>ttimeRAR:R_RR*RR (((s./rebuildd/Rebuildd.pytsH rebuildd/rebuildd/Enumeration.pyc0000644000000000000000000000273112003462214014330 0ustar ó )vLLc@s#ddlZddd„ƒYZdS(iÿÿÿÿNt EnumerationcBs)eZdZd„Zd„Zd„ZRS(s,Simple enumeration class with reverse lookupcCsi|_i|_d}xó|D]ë}t|ƒtjkrI|\}}nt|ƒtjkrqtd|ƒ‚nt|ƒtjkr™td|ƒ‚n|jj|ƒr¾td|ƒ‚n||jj ƒkrætd|ƒ‚n||j|<||j|<|d7}qWdS(Nisenum name is not a string: senum value is not an integer: senum name is not unique: senum value is not unique for i( tlookuptreverse_lookupttypettypest TupleTypet StringTypet ValueErrortIntTypethas_keytvalues(tselftenumlisttvaltelem((s./rebuildd/Enumeration.pyt__init__s"     cCs&|jj|ƒst‚n|j|S(N(RR tAttributeError(R tattr((s./rebuildd/Enumeration.pyt __getattr__+s cCs |j|S(sReturn element name for a value(R(R tvalue((s./rebuildd/Enumeration.pytwhatis0s(t__name__t __module__t__doc__RRR(((s./rebuildd/Enumeration.pyRs  ((RR(((s./rebuildd/Enumeration.pyts rebuildd/rebuildd/Package.pyc0000644000000000000000000000136312003462215013376 0ustar ó )vLLc@s?ddlZddlZejƒdejfd„ƒYZdS(iÿÿÿÿNtPackagecBsMeZdZejƒZejddƒZejddƒZe d„ƒZ RS(s"Class implemeting a Debian packagetdefaultcCstj|j|jƒS(N(tapt_pkgtversion_comparetversion(tatb((s./rebuildd/Package.pyRsN( t__name__t __module__t__doc__t sqlobjectt StringColtnametNoneRtpriorityt staticmethodR(((s./rebuildd/Package.pyRs  (R Rt init_systemt SQLObjectR(((s./rebuildd/Package.pyts   rebuildd/rebuildd/RebuilddNetworkServer.py0000644000000000000000000000370610720364127016205 0ustar # rebuildd - Debian packages rebuild tool # # (c) 2007 - Julien Danjou # # This software is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 dated June, 1991. # # This software is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this software; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA # import threading, socket from RebuilddConfig import RebuilddConfig from RebuilddNetworkClient import RebuilddNetworkClient class RebuilddNetworkServer(threading.Thread): """Main network server listening for connection""" def __init__(self, rebuildd): threading.Thread.__init__(self) self.rebuildd = rebuildd def run(self): """Run main network server thread""" self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) self.socket.settimeout(1) self.socket.bind((RebuilddConfig().get('telnet', 'ip'), RebuilddConfig().getint('telnet', 'port'))) self.socket.listen(2) while not self.rebuildd.do_quit.isSet(): try: (client_socket, client_info) = self.socket.accept() if client_socket: interface = RebuilddNetworkClient(client_socket, self.rebuildd) interface.setDaemon(True) interface.start() except socket.timeout: pass self.socket.close() rebuildd/rebuildd/Rebuildd.py0000644000000000000000000003300412003461770013434 0ustar # rebuildd - Debian packages rebuild tool # # (c) 2007 - Julien Danjou # # This software is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 dated June, 1991. # # This software is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this software; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA # """rebuildd - Debian packages rebuild tool""" from __future__ import with_statement from Enumeration import Enumeration from Distribution import Distribution from Dists import Dists from RebuilddLog import RebuilddLog, Log from RebuilddConfig import RebuilddConfig from RebuilddNetworkServer import RebuilddNetworkServer from Package import Package from Job import Job from JobStatus import JobStatus from JobStatus import FailedStatus import threading, os, time, sys, signal, socket import sqlobject __version__ = "$Rev$" class Rebuildd(object): jobs = [] _instance = None def __new__(cls): if cls._instance is None: cls._instance = object.__new__(cls) cls._instance.init() return cls._instance def init(self): self.cfg = RebuilddConfig() # Init log system RebuilddLog() self._sqlconnection = sqlobject.connectionForURI(self.cfg.get('build', 'database_uri')) sqlobject.sqlhub.processConnection = self._sqlconnection # Create distributions for dist in self.cfg.get('build', 'dists').split(' '): for arch in self.cfg.arch: Dists().add_dist(Distribution(dist, arch)) self.do_quit = threading.Event() self.jobs_locker = threading.Lock() self.job_finished = threading.Event() def daemon(self): RebuilddLog.info("Starting rebuildd %s" % __version__) self.daemonize() # Run the network server thread RebuilddLog.info("Launching network server") self.netserv = RebuilddNetworkServer(self) self.netserv.setDaemon(True) self.netserv.start() # Run main loop RebuilddLog.info("Running main loop") self.loop() # On exit RebuilddLog.info("Cleaning finished and canceled jobs") self.clean_jobs() RebuilddLog.info("Stopping all jobs") self.stop_all_jobs() RebuilddLog.info("Releasing wait-locked jobs") self.release_jobs() self.netserv.join(10) RebuilddLog.info("Exiting rebuildd") def daemonize(self): """Do daemon stuff""" signal.signal(signal.SIGTERM, self.handle_sigterm) signal.signal(signal.SIGINT, self.handle_sigterm) try: os.chdir(self.cfg.get('build', 'work_dir')) except Exception, error: print "E: unable to chdir to work_dir: %s" % error sys.exit(1) try: sys.stdout = sys.stderr = file(self.cfg.get('log', 'file'), "a") except Exception, error: print "E: unable to open logfile: %s" % error def get_job(self, jobid): for job in self.jobs: if job.id == jobid: return job return None def get_all_jobs(self, **kwargs): jobs = [] jobs.extend(Job.selectBy(**kwargs)) return jobs def get_new_jobs(self): """Feed jobs list with waiting jobs and lock them""" max_new = self.cfg.getint('build', 'max_jobs') count_current = len(self.jobs) with self.jobs_locker: if count_current >= max_new: return 0 jobs = [] for dist in Dists().dists: jobs.extend(Job.selectBy(status=JobStatus.WAIT, dist=dist.name, arch=dist.arch)[:max_new]) count_new = 0 for job in jobs: # Look for higher versions ? if self.cfg.getboolean('build', 'build_more_recent'): packages = Package.selectBy(name=job.package.name) candidate_packages = [] candidate_packages.extend(packages) candidate_packages.sort(cmp=Package.version_compare) candidate_packages.reverse() newjob = None # so there are packages with higher version number # try to see if there's a job for us for cpackage in candidate_packages: candidate_jobs = [] candidate_jobs.extend(Job.selectBy(package=cpackage, dist=job.dist, arch=job.arch)) for cjob in candidate_jobs: if newjob and newjob != cjob and cjob.status == JobStatus.WAIT: cjob.status = JobStatus.GIVEUP elif cjob.status == JobStatus.WAIT: newjob = cjob job = newjob # We have to check because it might have changed # between our first select and the build_more_recent stuffs if not job or job.status != JobStatus.WAIT: continue # Check dependencies if not job.is_allowed_to_build(): continue job.status = JobStatus.WAIT_LOCKED job.host = socket.gethostname() self.jobs.append(job) count_new += 1 count_current += 1 if count_current >= max_new: break return count_new def count_running_jobs(self): """Count running jobs""" count = 0 with self.jobs_locker: for job in self.jobs: if job.status == JobStatus.BUILDING and job.isAlive(): count += 1 return count def start_jobs(self, overrun=0): """Start waiting jobs""" running_threads = self.count_running_jobs() max_threads = max(overrun, self.cfg.getint('build', 'max_threads')) jobs_started = 0 with self.jobs_locker: for job in self.jobs: if running_threads >= max_threads: break with job.status_lock: if job.status == JobStatus.WAIT_LOCKED and not job.isAlive(): RebuilddLog.info("Starting new thread for job %s" % job.id) job.notify = self.job_finished job.setDaemon(True) job.start() jobs_started += 1 running_threads = running_threads + 1 RebuilddLog.info("Running threads: [ build %s/%s ] [ real %s ]" % (running_threads, max_threads, threading.activeCount())) return jobs_started def get_jobs(self, name, version=None, dist=None, arch=None): """Dump a job status""" if version: pkgs = Package.selectBy(name=name, version=version) else: pkgs = Package.selectBy(name=name) if not pkgs.count(): return [] retjobs = [] if dist and arch: for pkg in pkgs: retjobs.extend(Job.selectBy(package=pkg, dist=dist, arch=arch)) elif dist: for pkg in pkgs: retjobs.extend(Job.selectBy(package=pkg, dist=dist)) elif arch: for pkg in pkgs: retjobs.extend(Job.selectBy(package=pkg, arch=arch)) else: for pkg in pkgs: retjobs.extend(Job.selectBy(package=pkg)) return retjobs def dump_jobs(self, joblist=None): """Dump all jobs status""" ret = "" if not joblist: joblist = self.jobs for job in joblist: ret = "%s%s\n" % (ret, str(job)) return ret def cancel_job(self, jobid): """Cancel a job""" with self.jobs_locker: job = self.get_job(jobid) if job != None: if job.isAlive(): job.do_quit.set() job.join() with job.status_lock: job.status = JobStatus.CANCELED self.jobs.remove(job) RebuilddLog.info("Canceled job %s for %s_%s on %s/%s for %s" \ % (job.id, job.package.name, job.package.version, job.dist, job.arch, job.mailto)) return True return False def stop_all_jobs(self): """Stop all running jobs""" with self.jobs_locker: for job in self.jobs: if job.status == JobStatus.BUILDING and job.isAlive(): job.do_quit.set() RebuilddLog.info("Sending stop to job %s" % job.id) for job in self.jobs: if job.isAlive(): RebuilddLog.info("Waiting for job %s to terminate" % job.id) job.join(60) return True def add_job(self, name, version, priority, dist, mailto=None, arch=None): """Add a job""" if not arch: arch = self.cfg.arch[0] if not Dists().get_dist(dist, arch): RebuilddLog.error("Couldn't find dist/arch in the config file for %s_%s on %s/%s, not adding it" \ % (name, version, dist, arch)) return False pkgs = Package.selectBy(name=name, version=version) if pkgs.count(): # If several packages exists, just take the first pkg = pkgs[0] else: # Maybe we found no packages, so create a brand new one! pkg = Package(name=name, version=version, priority=priority) jobs_count = Job.selectBy(package=pkg, dist=dist, arch=arch, mailto=mailto, status=JobStatus.WAIT).count() if jobs_count: RebuilddLog.error("Job already existing for %s_%s on %s/%s, not adding it" \ % (pkg.name, pkg.version, dist, arch)) return False job = Job(package=pkg, dist=dist, arch=arch) job.status = JobStatus.WAIT job.arch = arch job.mailto = mailto log = Log(job=job) RebuilddLog.info("Added job for %s_%s on %s/%s for %s" \ % (name, version, dist, arch, mailto)) return True def requeue_job(self, job_id): """Requeue a failed job""" if Job.selectBy(id=job_id).count() == 0: RebuilddLog.error("There is no job related to %s that is in the job list" % job_id) return False job = Job.selectBy(id=job_id)[0] if job.status in FailedStatus: job.status = JobStatus.WAIT job.host = "" return True def add_deps(self, job_id, dependency_ids): if Job.selectBy(id=job_id).count() == 0: RebuilddLog.error("There is no job related to %s that is in the job list" % job_id) return False job = Job.selectBy(id=job_id)[0] deps = [] for dep in dependency_ids: if Job.selectBy(id=dep).count() == 0: RebuilddLog.error("There is no job related to %s that is in the job list" % dep) return False dep_job = Job.selectBy(id=dep)[0] deps.append(dep_job) job.add_deps(deps) return True def clean_jobs(self): """Clean finished or canceled jobs""" with self.jobs_locker: for job in self.jobs: if job.status == JobStatus.BUILD_OK \ or job.status in FailedStatus \ or job.status == JobStatus.CANCELED: self.jobs.remove(job) return True def release_jobs(self): """Release jobs""" with self.jobs_locker: for job in self.jobs: with job.status_lock: if job.status == JobStatus.WAIT_LOCKED: job.status = JobStatus.WAIT job.host = "" return True def fix_jobs(self, print_result=True): """If rebuildd crashed, reset jobs to a valid state""" jobs = [] jobs.extend(Job.selectBy(host=socket.gethostname(), status=JobStatus.WAIT_LOCKED)) jobs.extend(Job.selectBy(host=socket.gethostname(), status=JobStatus.BUILDING)) for job in jobs: if print_result: print "I: Fixing job %s (was %s)" % (job.id, JobStatus.whatis(job.status)) job.host = None job.status = JobStatus.WAIT job.build_start = None job.build_end = None return True def handle_sigterm(self, signum, stack): RebuilddLog.info("Receiving transmission... it's a signal %s capt'ain! EVERYONE OUT!" % signum) self.do_quit.set() def loop(self): """Rebuildd main loop""" counter = self.cfg.getint('build', 'check_every') while not self.do_quit.isSet(): if counter == self.cfg.getint('build', 'check_every') \ or self.job_finished.isSet(): self.get_new_jobs() # Start jobs self.start_jobs() # Clean finished jobs self.clean_jobs() counter = 0 self.job_finished.clear() self.do_quit.wait(1) counter += 1 rebuildd/rebuildd/Dists.py0000644000000000000000000000236710724757316013014 0ustar # rebuildd - Debian packages rebuild tool # # (c) 2007 - Julien Danjou # # This software is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 dated June, 1991. # # This software is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this software; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA # class Dists(object): """Singleton implementing a set of Debian distributions""" _instance = None dists = [] def __new__(cls): if cls._instance is None: cls._instance = object.__new__(cls) return cls._instance def add_dist(self, dist): self.dists.append(dist) def get_dist(self, name, arch): for d in self.dists: if d.name == name and d.arch == arch: return d return None rebuildd/rebuildd/__init__.pyc0000644000000000000000000000075312003462214013603 0ustar ó øcPc @s7dZdZdddddddd d d d g Zd S(s'rebuildd - Debian packages rebuild toolsJulien Danjou t Distributiont EnumerationtJobt JobStatustPackagetRebuilddtRebuilddConfigt RebuilddLogtRebuilddNetworkClienttRebuilddNetworkServertRebuilddHTTPServerN(t__doc__t __author__t__all__(((srebuildd/__init__.pytsrebuildd/rebuildd/RebuilddNetworkClient.py0000644000000000000000000001740212003461770016151 0ustar # rebuildd - Debian packages rebuild tool # # (c) 2007 - Julien Danjou # # This software is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 dated June, 1991. # # This software is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this software; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA # from __future__ import with_statement from RebuilddConfig import RebuilddConfig import threading, socket __version__ = "$Rev$" class RebuilddNetworkClient(threading.Thread): """Network client used for each connection""" def __init__(self, socket, rebuildd): threading.Thread.__init__(self) self.rebuildd = rebuildd self.socket = socket def run(self): """Run client thread""" self.socket.settimeout(1) self.socket.send(RebuilddConfig().get('telnet', 'motd') + "\n") prompt = RebuilddConfig().get('telnet', 'prompt') + " " line = "" has_timeout = False while line != "exit" and not self.rebuildd.do_quit.isSet(): if not has_timeout: try: self.socket.send(self.exec_cmd(line)) self.socket.send(prompt) except Exception: break try: line = "" line = self.socket.recv(512).strip() has_timeout = False except socket.timeout: has_timeout = True self.socket.close() def exec_cmd(self, cmd): """Execute a command asked by a client""" # Return empty if empty if cmd == "": return "" # Split words op = cmd.split(' ') cmd_fct = op[0] if not cmd_fct.isalnum(): return "" op.remove(op[0]) try: return getattr(self, "exec_cmd_" + cmd_fct)(*tuple(op)) except Exception, error: return "E: command error: %s\n" % error def exec_cmd_help(self, *args): """Show help""" help = "" for attr in dir(RebuilddNetworkClient): if attr.startswith("exec_cmd_"): help += "%s -- %s\n" \ % (attr[9:].replace("_", " "), getattr(RebuilddNetworkClient, attr).__doc__) return help def exec_cmd_config(self, *args): """Manipulate configuration file""" if len(args) < 1: return "E: usage: config [reload|dump|save]\n" if args[0] == "reload": if RebuilddConfig().reload(): return "I: config reloaded\n" return "E: config not reloded\n" if args[0] == "dump": return RebuilddConfig().dump() if args[0] == "save": if RebuilddConfig().save(): return "I: config saved\n" return "E: config not saved\n" return "E: usage: config [reload|dump|save]\n" def exec_cmd_status(self, *args): """Show current jobs status""" return self.rebuildd.dump_jobs() def exec_cmd_version(self, *args): """Show version""" return __version__ + "\n" def exec_cmd_job(self, *args): """Manipulate jobs""" if len(args) > 0 and args[0] == "add": return self.exec_cmd_job_add(*args) if len(args) > 0 and args[0] == "deps": return self.exec_cmd_job_deps(*args) if len(args) > 0 and args[0] == "cancel": return self.exec_cmd_job_cancel(*args) if len(args) > 0 and args[0] == "start": return self.exec_cmd_job_start(*args) if len(args) > 0 and args[0] == "reload": return self.exec_cmd_job_reload(*args) if len(args) > 0 and args[0] == "status": return self.exec_cmd_job_status(*args) if len(args) > 0 and args[0] == "requeue": return self.exec_cmd_job_requeue(*args) return "E: usage: job [args]\n" def exec_cmd_job_add(self, *args): """Add job""" ret = False if len(args) < 4: return "E: usage: job add [arch] [mailto]\n" if len(args) == 5: ret = self.rebuildd.add_job(name=args[1], version=args[2], priority=args[3], dist=args[4]) if len(args) == 6: ret = self.rebuildd.add_job(name=args[1], version=args[2], priority=args[3], dist=args[4], arch=args[5]) if len(args) == 7: ret = self.rebuildd.add_job(name=args[1], version=args[2], priority=args[3], dist=args[4], arch=args[5], mailto=args[6]) if ret: return "I: job added\n" return "E: error adding job\n" def exec_cmd_job_requeue(self, *args): """Requeue job""" if len(args) != 2: return "E: usage: job requeue \n" if self.rebuildd.requeue_job(job_id=int(args[1])): return "I: job requeued\n" return "E: error requeuing the job\n" def exec_cmd_job_deps(self, *args): """Add dependency""" ret = False if len(args) < 3: return "E: usage: job deps [dependency_job_id] [...]\n" ret = self.rebuildd.add_deps(job_id=args[1], dependency_ids=args[2:]) if ret: return "I: Dependency added\n" return "E: error adding deps" def exec_cmd_job_cancel(self, *args): """Cancel job""" if len(args) < 2: return "E: usage: job cancel \n" if self.rebuildd.cancel_job(int(args[1])): return "I: job canceled\n" return "E: unknown job\n" def exec_cmd_job_start(self, *args): """Start jobs""" if len(args) == 2: return "I: %s jobs started\n" \ % self.rebuildd.start_jobs(int(args[1])) return "I: %s jobs started\n" \ % self.rebuildd.start_jobs() def exec_cmd_job_reload(self, *args): """Load new jobs""" return "I: %s new jobs added\n" % self.rebuildd.get_new_jobs() def exec_cmd_job_status(self, *args): """Dump job status""" if len(args) < 2 or len(args) > 5: return "E: usage: job status [arch]\n" elif len(args) == 2: jobs = self.rebuildd.get_jobs(name=args[1]) elif len(args) == 3: jobs = self.rebuildd.get_jobs(name=args[1], version=args[2]) elif len(args) == 4: jobs = self.rebuildd.get_jobs(name=args[1], version=args[2], dist=args[3]) elif len(args) == 5: jobs = self.rebuildd.get_jobs(name=args[1], version=args[2], dist=args[3], arch=args[4]) return self.rebuildd.dump_jobs(jobs) rebuildd/rebuildd/Job.py0000644000000000000000000002531212003461770012417 0ustar # rebuildd - Debian packages rebuild tool # # (c) 2007 - Julien Danjou # # This software is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 dated June, 1991. # # This software is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this software; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA # from __future__ import with_statement import threading, subprocess, smtplib, time, os, signal, socket, select import sqlobject from subprocess import Popen,PIPE from email.Message import Message from Dists import Dists from JobStatus import JobStatus from JobStatus import FailedStatus from RebuilddLog import RebuilddLog, Log from RebuilddConfig import RebuilddConfig __version__ = "$Rev$" class Job(threading.Thread, sqlobject.SQLObject): """Class implementing a build job""" status = sqlobject.IntCol(default=JobStatus.UNKNOWN) mailto = sqlobject.StringCol(default=None) package = sqlobject.ForeignKey('Package', cascade=True) dist = sqlobject.StringCol(default='sid') arch = sqlobject.StringCol(default='any') creation_date = sqlobject.DateTimeCol(default=sqlobject.DateTimeCol.now) status_changed = sqlobject.DateTimeCol(default=None) build_start = sqlobject.DateTimeCol(default=None) build_end = sqlobject.DateTimeCol(default=None) host = sqlobject.StringCol(default=None) deps = sqlobject.RelatedJoin('Job', joinColumn='joba', otherColumn='jobb') log = sqlobject.SingleJoin('Log') notify = None def __init__(self, *args, **kwargs): """Init job""" threading.Thread.__init__(self) sqlobject.SQLObject.__init__(self, *args, **kwargs) self.do_quit = threading.Event() self.status_lock = threading.Lock() def __setattr__(self, name, value): """Override setattr to log build status changes""" if name == "status": RebuilddLog.info("Job %s for %s_%s on %s/%s changed status from %s to %s"\ % (self.id, self.package.name, self.package.version, self.dist, self.arch, JobStatus.whatis(self.status), JobStatus.whatis(value))) self.status_changed = sqlobject.DateTimeCol.now() sqlobject.SQLObject.__setattr__(self, name, value) @property def logfile(self): """Compute and return logfile name""" build_log_file = "%s/%s_%s-%s-%s-%s.%s.log" % (RebuilddConfig().get('log', 'logs_dir'), self.package.name, self.package.version, self.dist, self.arch, self.creation_date.strftime("%Y%m%d-%H%M%S"), self.id) return build_log_file def preexec_child(self): """Start a new group process before executing child""" os.setsid() def run(self): """Run job thread, download and build the package""" self.build_start = sqlobject.DateTimeCol.now() try: with open(self.logfile, "w") as build_log: build_log.write("Automatic build of %s_%s on %s for %s/%s by rebuildd %s\n" % \ (self.package.name, self.package.version, self.host, self.dist, self.arch, __version__)) build_log.write("Build started at %s\n" % self.build_start) build_log.write("******************************************************************************\n") except IOError: return build_log = file(self.logfile, "a") # we are building with self.status_lock: self.status = JobStatus.BUILDING # execute commands for cmd, failed_status in ([Dists().get_dist(self.dist, self.arch).get_source_cmd(self.package), JobStatus.SOURCE_FAILED], [Dists().get_dist(self.dist, self.arch).get_build_cmd(self.package), JobStatus.BUILD_FAILED], [Dists().get_dist(self.dist, self.arch).get_post_build_cmd(self.package), JobStatus.POST_BUILD_FAILED]): if cmd is None: continue try: proc = subprocess.Popen(cmd.split(), bufsize=0, preexec_fn=self.preexec_child, stdout=build_log, stdin=None, stderr=subprocess.STDOUT) except Exception, error: build_log.write("\nUnable to execute command \"%s\": %s" %\ (cmd, error)) with self.status_lock: self.status = failed_status state = 1 break state = proc.poll() while not self.do_quit.isSet() and state == None: state = proc.poll() self.do_quit.wait(1) if self.do_quit.isSet(): break if state != 0: with self.status_lock: self.status = failed_status break if self.do_quit.isSet(): # Kill gently the process RebuilddLog.info("Killing job %s with SIGINT" % self.id) try: os.killpg(os.getpgid(proc.pid), signal.SIGINT) except OSError, error: RebuilddLog.error("Error killing job %s: %s" % (self.id, error)) # If after 60s it's not dead, KILL HIM counter = 0 timemax = RebuilddConfig().get('build', 'kill_timeout') while proc.poll() == None and counter < timemax: time.sleep(1) counter += 1 if proc.poll() == None: RebuilddLog.error("Killing job %s timed out, killing with SIGKILL" \ % self.id) os.killpg(os.getpgid(proc.pid), signal.SIGKILL) with self.status_lock: self.status = JobStatus.WAIT_LOCKED # Reset host self.host = None build_log.write("\nBuild job killed on request\n") build_log.close() return # build is finished with self.status_lock: if state == 0: self.status = JobStatus.BUILD_OK self.build_end = sqlobject.DateTimeCol.now() build_log.write("******************************************************************************\n") build_log.write("Finished with status %s at %s\n" % (JobStatus.whatis(self.status), self.build_end)) build_log.write("Build needed %s\n" % (self.build_end - self.build_start)) build_log.close() # Send event to Rebuildd to inform it that it can # run a brand new job! if self.notify: self.notify.set() self.send_build_log() def send_build_log(self): """When job is built, send logs by mail""" try: with open(self.logfile, "r") as build_log: log = build_log.read() except IOError, error: RebuilddLog.error("Unable to open logfile for job %d" % self.id) return False # Store in database if self.log: self.log.text = log with self.status_lock: if self.status != JobStatus.BUILD_OK and \ not self.status in FailedStatus: return False if not RebuilddConfig().getboolean('log', 'mail_successful') \ and self.status == JobStatus.BUILD_OK: return True elif not RebuilddConfig().getboolean('log', 'mail_failed') \ and self.status in FailedStatus: return True if self.status == JobStatus.BUILD_OK: bstatus = "successful" else: bstatus = "failed" msg = Message() if self.mailto: msg['To'] = self.mailto else: msg['To'] = RebuilddConfig().get('mail', 'mailto') msg['From'] = RebuilddConfig().get('mail', 'from') msg['Subject'] = RebuilddConfig().get('mail', 'subject_prefix') + \ " Log for %s build of %s_%s on %s/%s" % \ (bstatus, self.package.name, self.package.version, self.dist, self.arch) msg['X-Rebuildd-Version'] = __version__ msg['X-Rebuildd-Host'] = socket.getfqdn() msg.set_payload(log) try: smtp = smtplib.SMTP() smtp.connect(RebuilddConfig().get('mail', 'smtp_host'), RebuilddConfig().get('mail', 'smtp_port')) smtp.sendmail(RebuilddConfig().get('mail', 'from'), [m.strip() for m in msg['To'].split(",")], msg.as_string()) except Exception, error: try: process = Popen("sendmail", shell=True, stdin=PIPE) process.communicate(input=msg.as_string()) except: RebuilddLog.error("Unable to send build log mail for job %d: %s" % (self.id, error)) return True def __str__(self): return "I: Job %s for %s_%s is status %s on %s for %s/%s" % \ (self.id, self.package.name, self.package.version, self.host, JobStatus.whatis(self.status), self.dist, self.arch) def is_allowed_to_build(self): """ Check if job is allowed to build """ for dep in Job.selectBy(id=self)[0].deps: if Job.selectBy(id=dep)[0].status != JobStatus.BUILD_OK: return False return True def add_dep(self, dep): """Add a job dependency on another job""" for existing_dep in self.deps: if existing_dep.id == dep.id: RebuilddLog.error("Already existing dependency between job %s and job %s" % (self.id, dep.id)) return self.addJob(dep) RebuilddLog.info("Dependency added between job %s and job %s" % (self.id, dep.id)) def add_deps(self,deps): """ Add several job dependency on another job""" for dep in deps: self.add_dep(dep) rebuildd/rebuildd/RebuilddLog.pyc0000644000000000000000000000402712003462214014236 0ustar ó øcPc@s[ddlZddlZddlmZdejfd„ƒYZdefd„ƒYZdS(iÿÿÿÿN(tRebuilddConfigtLogcBs5eZdZejddeƒZejddƒZRS(sClass implementing a LogtJobtcascadetdefaultsNo build log available( t__name__t __module__t__doc__t sqlobjectt ForeignKeytTruetjobtBLOBColttext(((s./rebuildd/RebuilddLog.pyRst RebuilddLogcBsSeZdZdZd„Zd„Zed„ƒZed„ƒZ ed„ƒZ RS(sSingleton used for loggingcCs8|jdkr1tj|ƒ|_|jjƒn|jS(N(t _instancetNonetobjectt__new__tinit(tcls((s./rebuildd/RebuilddLog.pyR"sc CsPtƒ}tjdtjddd|jddƒd|jddƒd d ƒdS( Ntleveltformats%%(asctime)s %(levelname)s %(message)stfilenametlogtfiletdatefmtt time_formattfilemodeta(Rtloggingt basicConfigtDEBUGtget(tselftcfg((s./rebuildd/RebuilddLog.pyR(s  cCstj|ƒdS(sLog a string with info priorityN(Rtinfo(R"tstr((s./rebuildd/RebuilddLog.pyR$0scCstj|ƒdS(sLog a string with warn priorityN(Rtwarning(R"R%((s./rebuildd/RebuilddLog.pytwarn6scCstj|ƒdS(s Log a string with error priorityN(Rterror(R"R%((s./rebuildd/RebuilddLog.pyR(<sN( RRRRRRRt classmethodR$R'R((((s./rebuildd/RebuilddLog.pyRs  (RRRt SQLObjectRRR(((s./rebuildd/RebuilddLog.pyts  rebuildd/rebuildd/RebuilddNetworkServer.pyc0000644000000000000000000000301512003462215016332 0ustar ó WèAGc@sUddlZddlZddlmZddlmZdejfd„ƒYZdS(iÿÿÿÿN(tRebuilddConfig(tRebuilddNetworkClienttRebuilddNetworkServercBs eZdZd„Zd„ZRS(s,Main network server listening for connectioncCstjj|ƒ||_dS(N(t threadingtThreadt__init__trebuildd(tselfR((s#./rebuildd/RebuilddNetworkServer.pyRscCstjtjtjƒ|_|jjtjtjdƒ|jjdƒ|jjtƒj ddƒtƒj ddƒfƒ|jj dƒxx|j j jƒsyK|jjƒ\}}|rêt||j ƒ}|jtƒ|jƒnWqŽtjk rqŽXqŽW|jjƒdS(sRun main network server threadittelnettiptportiN(tsockettAF_INETt SOCK_STREAMt setsockoptt SOL_SOCKETt SO_REUSEADDRt settimeouttbindRtgettgetinttlistenRtdo_quittisSettacceptRt setDaemontTruetstartttimeouttclose(Rt client_sockett client_infot interface((s#./rebuildd/RebuilddNetworkServer.pytruns"  (t__name__t __module__t__doc__RR!(((s#./rebuildd/RebuilddNetworkServer.pyRs (RR RRRR(((s#./rebuildd/RebuilddNetworkServer.pytsrebuildd/rebuildd/__init__.py0000644000000000000000000000220212003461770013435 0ustar # rebuildd - Debian packages rebuild tool # # (c) 2007 - Julien Danjou # # This software is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 dated June, 1991. # # This software is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this software; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA # """rebuildd - Debian packages rebuild tool""" __author__ = "Julien Danjou " __all__ = [ "Distribution", "Enumeration", "Job", "JobStatus", "Package", "Rebuildd", "RebuilddConfig", "RebuilddLog", "RebuilddNetworkClient", "RebuilddNetworkServer", "RebuilddHTTPServer" ] rebuildd/rebuildd/RebuilddLog.py0000644000000000000000000000372112003461770014101 0ustar # rebuildd - Debian packages rebuild tool # # (c) 2007 - Julien Danjou # # This software is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 dated June, 1991. # # This software is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this software; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA # import logging import sqlobject from RebuilddConfig import RebuilddConfig class Log(sqlobject.SQLObject): """Class implementing a Log""" job = sqlobject.ForeignKey('Job', cascade=True) text = sqlobject.BLOBCol(default="No build log available") class RebuilddLog(object): """Singleton used for logging""" _instance = None def __new__(cls): if cls._instance is None: cls._instance = object.__new__(cls) cls._instance.init() return cls._instance def init(self): cfg = RebuilddConfig() logging.basicConfig(level=logging.DEBUG, format='%(asctime)s %(levelname)s %(message)s', filename=cfg.get('log', 'file'), datefmt=cfg.get('log', 'time_format'), filemode='a') @classmethod def info(self, str): """Log a string with info priority""" logging.info(str) @classmethod def warn(self, str): """Log a string with warn priority""" logging.warning(str) @classmethod def error(self, str): """Log a string with error priority""" logging.error(str) rebuildd/maintenance/0000755000000000000000000000000011604070403012013 5ustar rebuildd/maintenance/00_update_build_system0000755000000000000000000000230311604070403016303 0ustar #! /bin/bash # Exit on errors set -e # Use pbuilder by default PBUILDER_BIN="/usr/sbin/pbuilder" PBUILDER_CACHE_PATH="/var/cache/pbuilder" PBUILDER_MIRROR="http://ftp.debian.org/debian" if [ -f /etc/default/rebuildd ] then unset PBUILDER_BIN . /etc/default/rebuildd fi test "$ENABLE_BUILDER_MAINT" = 0 && exit 0 pbuilder_update() { if [ ! -x "$PBUILDER_BIN" ] then echo "E: pbuilder not found: $PBUILDER_BIN" exit 1 fi echo "I: Updating pbuilder for $DISTS" for d in $DISTS do for a in $ARCHS do $PBUILDER_BIN update --basetgz $PBUILDER_CACHE_PATH/${d}-${a}.tgz done done } cowbuilder_update() { if [ ! -x "$COWBUILDER_BIN" ] then echo "E: cowbuilder not found: $COWBUILDER_BIN" exit 1 fi echo "I: Updating cowbuilder for $DISTS" for d in $DISTS do for a in $ARCHS do $COWBUILDER_BIN --update --basepath $PBUILDER_CACHE_PATH/${d}-${a}.cow done done } if [ ! -z "$COWBUILDER_BIN" ] then cowbuilder_update else pbuilder_update fi rebuildd/rebuildd-init-build-system0000755000000000000000000000347111423073051014637 0ustar #! /bin/bash # Exit on errors set -e # Use pbuilder by default PBUILDER_BIN="/usr/sbin/pbuilder" PBUILDER_CACHE_PATH="/var/cache/pbuilder" PBUILDER_MIRROR="http://ftp.debian.org/debian" PBUILDER_OTHER_OPTIONS=() if [ -f /etc/default/rebuildd ] then unset PBUILDER_BIN . /etc/default/rebuildd fi pbuilder_create () { if [ ! -x "$PBUILDER_BIN" ] then echo "E: pbuilder not found: $PBUILDER_BIN" exit 1 fi for d in $DISTS do for a in $ARCHS do echo "I: Initializing pbuilder for ${d}-${a}" $PBUILDER_BIN create --debootstrapopts --arch=$a --basetgz $PBUILDER_CACHE_PATH/${d}-${a}.tgz --distribution $d --mirror $PBUILDER_MIRROR "${PBUILDER_OTHER_OPTIONS[@]}" done done echo "I: Init done, please remember to set rebuildd configuration to the correct value:" echo "build_cmd = pbuilder build --basetgz $PBUILDER_CACHE_PATH/\${d}-\${a}.tgz \${p}_\${v}.dsc" } cowbuilder_create () { if [ ! -x "$COWBUILDER_BIN" ] then echo "E: cowbuilder not found: $COWBUILDER_BIN" exit 1 fi for d in $DISTS do for a in $ARCHS do echo "I: Initializing cowbuilder for ${d}-${a}" $COWBUILDER_BIN --create --debootstrapopts --arch=$a --basepath $PBUILDER_CACHE_PATH/${d}-${a}.cow --distribution $d --mirror $PBUILDER_MIRROR "${PBUILDER_OTHER_OPTIONS[@]}" done done echo "I: Init done, please remember to set rebuildd configuration to the correct value:" echo "build_cmd = cowbuilder --build --basepath $PBUILDER_CACHE_PATH/\${d}-\${a}.cow \${p}_\${v}.dsc" } if [ ! -z "$COWBUILDER_BIN" ] then cowbuilder_create else pbuilder_create fi rebuildd/rebuildd-httpd0000755000000000000000000000241211010312742012365 0ustar #!/usr/bin/python # # rebuildd-httpd - Debian packages rebuild tool Web server # # (c) 2007 - Julien Danjou # # This software is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 dated June, 1991. # # This software is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this software; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA # from rebuildd.RebuilddHTTPServer import RebuilddHTTPServer from rebuildd.RebuilddConfig import RebuilddConfig import sys, os #try: # sys.stdout = sys.stderr = file(RebuilddConfig().get('http', 'logfile'), "a") #except Exception, error: # print "E: cannot open logfile: %s" % error try: os.chdir(RebuilddConfig().get('http', 'templates_dir')) except Exception, error: print "E: cannot chdir to templates_dir: %s" % error sys.exit(1) RebuilddHTTPServer().start() rebuildd/rebuildd.py0000755000000000000000000000317212003461770011710 0ustar #!/usr/bin/python # # rebuildd - Debian packages rebuild tool # # (c) 2007 - Julien Danjou # # This software is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 dated June, 1991. # # This software is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this software; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA # from rebuildd.Rebuildd import Rebuildd from rebuildd.RebuilddConfig import RebuilddConfig import sqlobject, sys # Create database def create_db(): try: sqlobject.sqlhub.processConnection = \ sqlobject.connectionForURI(RebuilddConfig().get('build', 'database_uri')) from rebuildd.Package import Package from rebuildd.Job import Job from rebuildd.RebuilddLog import Log Package.createTable() Job.createTable() Log.createTable() except Exception, error: print "E: %s" % error return 1 return 0 if len(sys.argv) == 2: if sys.argv[1] == "init": sys.exit(create_db()) if sys.argv[1] == "dumpconfig": print RebuilddConfig().dump() sys.exit(0) if sys.argv[1] == "fix": Rebuildd().fix_jobs() sys.exit(0) Rebuildd().daemon() rebuildd/rebuildd-job0000755000000000000000000001712712003461770012036 0ustar #!/usr/bin/python # # rebuildd-job - Manage jobs for rebuildd # # (c) 2007 - Julien Danjou # # This software is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 dated June, 1991. # # This software is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this software; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA # from rebuildd.Rebuildd import Rebuildd from rebuildd.RebuilddConfig import RebuilddConfig from rebuildd.Package import Package from rebuildd.JobStatus import JobStatus from rebuildd.Job import Job import sys, os.path def usage(): print "%s -- Manage job for rebuildd" % os.path.basename(sys.argv[0]) print "Usage: %s [command] \n" % os.path.basename(sys.argv[0]) print "Commands:" print " add - add jobs, reading from stdin" print " add-deps - add build-depends, reading from stdin" print " add-quinn-diff - add package reading quinn-diff input from stdin" print " requeue - requeue job for rebuild" print " delete - delete job" print " list-deps - list the whole depends table" print " list = - list jobs matching criteria" print " stats - view a nice graph of build status" sys.exit(1) def add_deps(): empty = True for line in sys.stdin.readlines(): empty = False args = line.strip().split(' ') Rebuildd().add_deps(job_id=args[0], dependency_ids=args[1:]) if empty: print "E: usage: %s\n read [dependency_job_id] [...] from stdin" % sys.argv[0] sys.exit(1) def dep_list_to_string(job_list): output_string = "" for job in job_list: output_string = output_string + " " + str(job.id) return output_string def display_dep(): jobs = Job.selectBy() print " id | dependencies " print "------+----------------------" for job in jobs: print "%04.4s | %021.21s" % (job.id, dep_list_to_string(job.deps)) def add(): empty = True for line in sys.stdin.readlines(): empty = False args = line.strip().split(' ') if len(args) == 4: Rebuildd().add_job(args[0], args[1], args[2], args[3]) if len(args) == 5: Rebuildd().add_job(args[0], args[1], args[2], args[3], mailto=None, arch=args[4]) if len(args) == 6: Rebuildd().add_job(args[0], args[1], args[2], args[3], mailto=args[5], arch=args[4]) if empty: print "E: usage: %s\n read [arch] [mailto] from stdin" % sys.argv[0] sys.exit(1) def add_quinn_diff(dist): for line in sys.stdin.readlines(): name = line.split('/')[1].split('_')[0] version = line.split('/')[1].split('_')[1].split('.dsc')[0] priority = line.split('/')[1].split('_')[1].split('.dsc')[1].split(':')[0][2:] Rebuildd().add_job(name, version, priority, dist) def requeue_job(jobid): if not Rebuildd().requeue_job(jobid): print "E: rebuild not accepted" sys.exit(1) def list(): if len(sys.argv) == 3: try: (critname, critvalue) = sys.argv[2].split('=') except ValueError: print "E: usage: %s list criteria=value" % os.path.basename(sys.argv[0]) return False else: print_jobs(Job.selectBy()) return True if critname == "package": critvaluepkg = critvalue.split('_') pkgname = critvaluepkg[0] pkgver = None if len(critvaluepkg) > 1: pkgver = critvaluepkg[1] if pkgver: pkgs = Package.selectBy(name=pkgname, version=pkgver) else: pkgs = Package.selectBy(name=pkgname) for pkg in pkgs: print_jobs(Job.selectBy(package=pkg)) if critname == "arch": print_jobs(Job.selectBy(arch=critvalue)) if critname == "dist": print_jobs(Job.selectBy(dist=critvalue)) if critname == "host": print_jobs(Job.selectBy(host=critvalue)) if critname == "status": try: print_jobs(Job.selectBy(status=getattr(JobStatus, critvalue))) except AttributeError: print "E: unknown status" sys.exit(1) if critname == "id": print_jobs(Job.selectBy(id=critvalue)) def stats(): nbjobs = Job.selectBy().count() wait = Job.selectBy(status=JobStatus.WAIT).count() wait += Job.selectBy(status=JobStatus.WAIT_LOCKED).count() building = Job.selectBy(status=JobStatus.BUILDING).count() ok = Job.selectBy(status=JobStatus.BUILD_OK).count() failed = Job.selectBy(status=JobStatus.SOURCE_FAILED).count() failed += Job.selectBy(status=JobStatus.BUILD_FAILED).count() failed += Job.selectBy(status=JobStatus.POST_BUILD_FAILED).count() if nbjobs > 0: wait *= 10.0 / nbjobs building *= 10.0 / nbjobs failed *= 10.0 / nbjobs ok *= 10.0 / nbjobs else: wait = 0.0 building = 0.0 failed = 0.0 ok = 0.0 lines = 10 while lines > 0: print "%3.0d %% |" % (lines * 10), if int(round(wait)) >= lines: print " ##0 ", else: print " ", if int(round(building)) >= lines: print " ##0 ", else: print " ", if int(round(failed)) >= lines: print " ##0 ", else: print " ", if int(round(ok)) >= lines: print " ##0 ", else: print " ", print "" lines -= 1 print " +----------------------" print " WA BU FA OK" def delete(jobid): job = Job.get(jobid) if job.status == JobStatus.WAIT: if job.destroySelf(): print "I: job %s deleted" % jobid else: print "E: unable to delete job" else: print "E: can't delete job, build status is %s and should be WAIT" % JobStatus.whatis(job.status) def print_headers(): print " id | package name | version | status | host | dist | arch " print "------+----------------------+----------------+--------------+--------+-------+--------" def print_jobs(jobs): try: print_headers() for job in jobs: print "%05.5s | %020.20s | %014.14s | %012.12s | %06.6s | %05.5s | %06.6s" % \ (job.id, job.package.name, job.package.version, JobStatus.whatis(job.status), job.host, job.dist, job.arch) except IOError: pass if len(sys.argv) > 1: # Init system Rebuildd() if sys.argv[1] == "add": add() if sys.argv[1] == "add-deps": add_deps() if sys.argv[1] == "list-deps": display_dep() if sys.argv[1] == "add-quinn-diff" and len(sys.argv) > 2: add_quinn_diff(sys.argv[2]) if sys.argv[1] == "list": list() if sys.argv[1] == "stats": stats() if sys.argv[1] == "delete" and len(sys.argv) > 2: delete(int(sys.argv[2])) if sys.argv[1] == "requeue" and len(sys.argv) == 3: requeue_job(int(sys.argv[2])) else: usage() rebuildd/build-stamp-rebuildd0000644000000000000000000000000011516235001013452 0ustar rebuildd/build/0000755000000000000000000000000012003462241010627 5ustar rebuildd/build/lib.linux-x86_64-2.6/0000755000000000000000000000000011516234765014012 5ustar rebuildd/build/lib.linux-x86_64-2.6/rebuildd/0000755000000000000000000000000012003462241015564 5ustar rebuildd/build/lib.linux-x86_64-2.6/rebuildd/JobStatus.py0000644000000000000000000000267310720364127020074 0ustar # rebuildd - Debian packages rebuild tool # # (c) 2007 - Julien Danjou # # This software is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 dated June, 1991. # # This software is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this software; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA # from Enumeration import Enumeration JobStatus = Enumeration([ ("UNKNOWN", 0), ("WAIT", 100), ("WAIT_LOCKED", 150), ("BUILDING", 200), ("SOURCE_FAILED", 250), ("BUILD_FAILED", 300), ("POST_BUILD_FAILED", 350), ("CANCELED", 800), ("GIVEUP", 850), ("FAILED", 900), ("BUILD_OK", 1000) ]) FailedStatus = (JobStatus.SOURCE_FAILED, JobStatus.BUILD_FAILED, JobStatus.POST_BUILD_FAILED) rebuildd/build/lib.linux-x86_64-2.6/rebuildd/RebuilddHTTPServer.py0000644000000000000000000001366312003461770021576 0ustar # rebuildd - Debian packages rebuild tool # # (c) 2007 - Julien Danjou # # This software is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 dated June, 1991. # # This software is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this software; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA # from __future__ import with_statement from RebuilddConfig import RebuilddConfig from Rebuildd import Rebuildd from Package import Package from Job import Job from JobStatus import JobStatus from JobStatus import FailedStatus import tempfile, socket, sqlobject import web import gdchart render = web.template.render(RebuilddConfig().get('http', 'templates_dir'), \ cache=RebuilddConfig().getboolean('http', 'cache')) class RequestIndex: def GET(self): return render.base(page=render.index(), \ hostname=socket.gethostname(), \ archs=RebuilddConfig().arch, \ dists=RebuilddConfig().get('build', 'dists').split(' ')) class RequestPackage: def GET(self, name=None, version=None): jobs = [] if version: pkg = Package.selectBy(name=name, version=version)[0] title = "%s %s" % (name, version) package = "%s/%s" % (name, version) else: pkg = Package.selectBy(name=name)[0] title = package = name jobs.extend(Job.selectBy(package=pkg)) return render.base(page=render.tab(jobs=jobs), \ hostname=socket.gethostname(), \ title=title, \ package=package, \ archs=RebuilddConfig().arch, \ dists=RebuilddConfig().get('build', 'dists').split(' ')) class RequestArch: def GET(self, dist, arch=None): jobs = [] jobs.extend(Job.select(sqlobject.AND(Job.q.arch == arch, Job.q.dist == dist), orderBy=sqlobject.DESC(Job.q.creation_date))[:10]) return render.base(page=render.tab(jobs=jobs), \ arch=arch, \ dist=dist, \ title="%s/%s" % (dist, arch), \ hostname=socket.gethostname(), \ archs=RebuilddConfig().arch, \ dists=RebuilddConfig().get('build', 'dists').split(' ')) class RequestJob: def GET(self, jobid=None): job = Job.selectBy(id=jobid)[0] try: with open(job.logfile, "r") as build_logfile: build_log = build_logfile.read() except IOError, error: build_log = job.log.text return render.base(page=render.job(job=job, build_log=build_log), \ hostname=socket.gethostname(), \ title="job %s" % job.id, \ archs=RebuilddConfig().arch, \ dists=RebuilddConfig().get('build', 'dists').split(' ')) class RequestGraph: GET = web.autodelegate("GET_") def graph_init(self): web.header("Content-Type","image/png") graph = gdchart.Bar3D() graph.width = 300 graph.height = 300 graph.ytitle = "Jobs" graph.xtitle = "Build status" graph.ext_color = [ "yellow", "orange", "red", "green"] graph.bg_color = "white" graph.setLabels(["WAIT", "BUILDING", "FAILED", "OK"]) return graph def compute_stats(self, jobs): jw = 0 jb = 0 jf = 0 jo = 0 for job in jobs: if job.status == JobStatus.WAIT or \ job.status == JobStatus.WAIT_LOCKED: jw += 1 elif job.status == JobStatus.BUILDING: jb += 1 elif job.status in FailedStatus: jf += 1 elif job.status == JobStatus.BUILD_OK: jo += 1 return (jw, jb, jf, jo) def GET_buildstats(self, distarch=None): graph = self.graph_init() if distarch == "/": graph.title = "Build status" jobs = Job.selectBy() else: dindex = distarch.rindex("/") graph.title = "Build status for %s" % distarch[1:] jobs = Job.selectBy(arch=distarch[dindex+1:], dist=distarch[1:dindex]) graph.setData(self.compute_stats(jobs)) tmp = tempfile.TemporaryFile() graph.draw(tmp) tmp.seek(0) return tmp.read() def GET_package(self, package=None): graph = self.graph_init() if package == "/": graph.title = "Build status" jobs = Job.selectBy() else: dindex = package.rindex("/") graph.title = "Build status for %s" % package[1:] pkg = Package.selectBy(version=package[dindex+1:], name=package[1:dindex])[0] jobs = Job.selectBy(package=pkg) graph.setData(self.compute_stats(jobs)) tmp = tempfile.TemporaryFile() graph.draw(tmp) tmp.seek(0) return tmp.read() class RebuilddHTTPServer: """Main HTTP server""" urls = ( '/', 'RequestIndex', '/dist/(.*)/arch/(.*)', 'RequestArch', '/job/(.*)', 'RequestJob', '/package/(.*)/(.*)', 'RequestPackage', '/package/(.*)', 'RequestPackage', '/graph/(.*)', 'RequestGraph', ) def __init__(self): Rebuildd() def start(self): """Run main HTTP server thread""" web.webapi.internalerror = web.debugerror import sys; sys.argv.append(RebuilddConfig().get('http', 'ip') + ":" + RebuilddConfig().get('http', 'port')) app = web.application(self.urls, globals()) app.run() rebuildd/build/lib.linux-x86_64-2.6/rebuildd/Package.py0000644000000000000000000000222411423073051017473 0ustar # rebuildd - Debian packages rebuild tool # # (c) 2007 - Julien Danjou # # This software is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 dated June, 1991. # # This software is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this software; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA # import sqlobject import apt_pkg apt_pkg.init_system() class Package(sqlobject.SQLObject): """Class implemeting a Debian package""" name = sqlobject.StringCol() version = sqlobject.StringCol(default=None) priority = sqlobject.StringCol(default=None) @staticmethod def version_compare(a, b): return apt_pkg.version_compare(a.version, b.version) rebuildd/build/lib.linux-x86_64-2.6/rebuildd/Distribution.py0000644000000000000000000000610710763510134020627 0ustar # rebuildd - Debian packages rebuild tool # # (c) 2007 - Julien Danjou # # This software is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 dated June, 1991. # # This software is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this software; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA # from RebuilddConfig import RebuilddConfig from RebuilddLog import RebuilddLog from string import Template class Distribution(object): """Class implementing a Debian distribution Substitutions are done in the command strings: $d => The distro's name $a => the target architecture $p => the package's name $v => the package's version """ def __init__(self, name, arch): self.name = name self.arch = arch def get_source_cmd(self, package): """Return command used for grabing source for this distribution""" try: args = { 'd': self.name, 'a': self.arch, 'v': package.version, \ 'p': package.name } t = Template(RebuilddConfig().get('build', 'source_cmd')) return t.safe_substitute(**args) except TypeError, error: RebuilddLog.error("get_source_cmd has invalid format: %s" % error) return None def get_build_cmd(self, package): """Return command used for building source for this distribution""" # Strip epochs (x:) away try: index = package.version.index(":") args = { 'd': self.name, 'a': self.arch, \ 'v': package.version[index+1:], 'p': package.name } t = Template(RebuilddConfig().get('build', 'build_cmd')) return t.safe_substitute(**args) except ValueError: pass try: args = { 'd': self.name, 'a': self.arch, \ 'v': package.version, 'p': package.name } t = Template(RebuilddConfig().get('build', 'build_cmd')) return t.safe_substitute(**args) except TypeError, error: RebuilddLog.error("get_build_cmd has invalid format: %s" % error) return None def get_post_build_cmd(self, package): """Return command used after building source for this distribution""" cmd = RebuilddConfig().get('build', 'post_build_cmd') if cmd == '': return None try: args = { 'd': self.name, 'a': self.arch, \ 'v': package.version, 'p': package.name } t = Template(cmd) return t.safe_substitute(**args) except TypeError, error: RebuilddLog.error("post_build_cmd has invalid format: %s" % error) return None rebuildd/build/lib.linux-x86_64-2.6/rebuildd/Enumeration.py0000644000000000000000000000362511423073051020434 0ustar # rebuildd - Debian packages rebuild tool # # (c) 2007 - Julien Danjou # # This software is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 dated June, 1991. # # This software is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this software; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA # import types class Enumeration: """Simple enumeration class with reverse lookup""" def __init__(self, enumlist): self.lookup = { } self.reverse_lookup = { } val = 0 for elem in enumlist: if type(elem) == types.TupleType: elem, val = elem if type(elem) != types.StringType: raise ValueError("enum name is not a string: " + elem) if type(val) != types.IntType: raise ValueError("enum value is not an integer: " + val) if self.lookup.has_key(elem): raise ValueError("enum name is not unique: " + elem) if val in self.lookup.values(): raise ValueError("enum value is not unique for " + val) self.lookup[elem] = val self.reverse_lookup[val] = elem val += 1 def __getattr__(self, attr): if not self.lookup.has_key(attr): raise AttributeError return self.lookup[attr] def whatis(self, value): """Return element name for a value""" return self.reverse_lookup[value] rebuildd/build/lib.linux-x86_64-2.6/rebuildd/RebuilddConfig.py0000644000000000000000000001045212003461770021026 0ustar # rebuildd - Debian packages rebuild tool # # (c) 2007 - Julien Danjou # # This software is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 dated June, 1991. # # This software is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this software; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA # import ConfigParser import os class RebuilddConfig(object, ConfigParser.ConfigParser): """Main configuration singleton""" config_file = "/etc/rebuildd/rebuilddrc" _instance = None def __new__(cls, *args, **kwargs): if cls._instance is None: cls._instance = object.__new__(cls) cls._instance.init(*args, **kwargs) return cls._instance def init(self, dontparse=False): ConfigParser.ConfigParser.__init__(self) # add default sections self.add_section('build') self.add_section('mail') self.add_section('telnet') self.add_section('http') self.add_section('log') # add default values self.set('build', 'check_every', '300') self.set('build', 'max_threads', '2') self.set('build', 'max_jobs', '5') self.set('build', 'kill_timeout', '90') self.set('build', 'source_cmd', 'apt-get -q --download-only -t ${d} source ${p}=${v}') self.set('build', 'build_cmd', 'pbuilder build --basetgz /var/cache/pbuilder/${d}-${a}.tgz ${p}_${v}.dsc') self.set('build', 'post_build_cmd', '') self.set('build', 'dists', 'squeeze wheezy sid') self.set('build', 'work_dir', '/var/cache/rebuildd/build') self.set('build', 'database_uri', 'sqlite:///var/lib/rebuildd/rebuildd.db') self.set('build', 'build_more_recent', '1') self.set('build', 'more_archs', 'any') self.set('build', 'no_system_arch', '0') self.set('mail', 'from', 'rebuildd@localhost') self.set('mail', 'mailto', 'rebuildd@localhost') self.set('mail', 'subject_prefix', '[rebuildd]') self.set('mail', 'smtp_host', 'localhost') self.set('mail', 'smtp_port', '25') self.set('telnet', 'port', '9999') self.set('telnet', 'ip', '127.0.0.1') self.set('telnet', 'prompt', 'rebuildd@localhost->') self.set('telnet', 'motd', 'Connected on rebuildd on localhost') self.set('http', 'port', '9998') self.set('http', 'ip', '0.0.0.0') # This is dedicated to MadCoder self.set('http', 'log_lines_nb', '30') self.set('http', 'templates_dir', '/usr/share/rebuildd/templates') self.set('http', 'cache', '1') self.set('http', 'logfile', '/var/log/rebuildd/httpd.log') self.set('log', 'file', "/var/log/rebuildd/rebuildd.log") self.set('log', 'time_format', "%Y-%m-%d %H:%M:%S") self.set('log', 'logs_dir', "/var/log/rebuildd/build_logs") self.set('log', 'mail_failed', '1') self.set('log', 'mail_successful', '0') if not dontparse: self.reload() self.arch = [] if self.getint('build', 'no_system_arch') == 0: parch = os.popen("dpkg --print-architecture") self.arch.append(parch.readline().strip()) parch.close() for a in self.get('build', 'more_archs').split(' '): self.arch.append(a) def reload(self): """Reload configuration file""" return self.read(self.config_file) def dump(self): """Dump running configuration""" conf = "" for section in self.sections(): conf += "[" + section + "]\n" for item, value in self.items(section): conf += "%s = %s\n" % (item, value) conf += "\n" return conf def save(self): """Save configuration file""" try: self.write(file(self.config_file, 'w')) except Exception, error: return False return True rebuildd/build/lib.linux-x86_64-2.6/rebuildd/RebuilddNetworkServer.py0000644000000000000000000000370610720364127022447 0ustar # rebuildd - Debian packages rebuild tool # # (c) 2007 - Julien Danjou # # This software is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 dated June, 1991. # # This software is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this software; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA # import threading, socket from RebuilddConfig import RebuilddConfig from RebuilddNetworkClient import RebuilddNetworkClient class RebuilddNetworkServer(threading.Thread): """Main network server listening for connection""" def __init__(self, rebuildd): threading.Thread.__init__(self) self.rebuildd = rebuildd def run(self): """Run main network server thread""" self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) self.socket.settimeout(1) self.socket.bind((RebuilddConfig().get('telnet', 'ip'), RebuilddConfig().getint('telnet', 'port'))) self.socket.listen(2) while not self.rebuildd.do_quit.isSet(): try: (client_socket, client_info) = self.socket.accept() if client_socket: interface = RebuilddNetworkClient(client_socket, self.rebuildd) interface.setDaemon(True) interface.start() except socket.timeout: pass self.socket.close() rebuildd/build/lib.linux-x86_64-2.6/rebuildd/Rebuildd.py0000644000000000000000000003300412003461770017676 0ustar # rebuildd - Debian packages rebuild tool # # (c) 2007 - Julien Danjou # # This software is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 dated June, 1991. # # This software is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this software; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA # """rebuildd - Debian packages rebuild tool""" from __future__ import with_statement from Enumeration import Enumeration from Distribution import Distribution from Dists import Dists from RebuilddLog import RebuilddLog, Log from RebuilddConfig import RebuilddConfig from RebuilddNetworkServer import RebuilddNetworkServer from Package import Package from Job import Job from JobStatus import JobStatus from JobStatus import FailedStatus import threading, os, time, sys, signal, socket import sqlobject __version__ = "$Rev$" class Rebuildd(object): jobs = [] _instance = None def __new__(cls): if cls._instance is None: cls._instance = object.__new__(cls) cls._instance.init() return cls._instance def init(self): self.cfg = RebuilddConfig() # Init log system RebuilddLog() self._sqlconnection = sqlobject.connectionForURI(self.cfg.get('build', 'database_uri')) sqlobject.sqlhub.processConnection = self._sqlconnection # Create distributions for dist in self.cfg.get('build', 'dists').split(' '): for arch in self.cfg.arch: Dists().add_dist(Distribution(dist, arch)) self.do_quit = threading.Event() self.jobs_locker = threading.Lock() self.job_finished = threading.Event() def daemon(self): RebuilddLog.info("Starting rebuildd %s" % __version__) self.daemonize() # Run the network server thread RebuilddLog.info("Launching network server") self.netserv = RebuilddNetworkServer(self) self.netserv.setDaemon(True) self.netserv.start() # Run main loop RebuilddLog.info("Running main loop") self.loop() # On exit RebuilddLog.info("Cleaning finished and canceled jobs") self.clean_jobs() RebuilddLog.info("Stopping all jobs") self.stop_all_jobs() RebuilddLog.info("Releasing wait-locked jobs") self.release_jobs() self.netserv.join(10) RebuilddLog.info("Exiting rebuildd") def daemonize(self): """Do daemon stuff""" signal.signal(signal.SIGTERM, self.handle_sigterm) signal.signal(signal.SIGINT, self.handle_sigterm) try: os.chdir(self.cfg.get('build', 'work_dir')) except Exception, error: print "E: unable to chdir to work_dir: %s" % error sys.exit(1) try: sys.stdout = sys.stderr = file(self.cfg.get('log', 'file'), "a") except Exception, error: print "E: unable to open logfile: %s" % error def get_job(self, jobid): for job in self.jobs: if job.id == jobid: return job return None def get_all_jobs(self, **kwargs): jobs = [] jobs.extend(Job.selectBy(**kwargs)) return jobs def get_new_jobs(self): """Feed jobs list with waiting jobs and lock them""" max_new = self.cfg.getint('build', 'max_jobs') count_current = len(self.jobs) with self.jobs_locker: if count_current >= max_new: return 0 jobs = [] for dist in Dists().dists: jobs.extend(Job.selectBy(status=JobStatus.WAIT, dist=dist.name, arch=dist.arch)[:max_new]) count_new = 0 for job in jobs: # Look for higher versions ? if self.cfg.getboolean('build', 'build_more_recent'): packages = Package.selectBy(name=job.package.name) candidate_packages = [] candidate_packages.extend(packages) candidate_packages.sort(cmp=Package.version_compare) candidate_packages.reverse() newjob = None # so there are packages with higher version number # try to see if there's a job for us for cpackage in candidate_packages: candidate_jobs = [] candidate_jobs.extend(Job.selectBy(package=cpackage, dist=job.dist, arch=job.arch)) for cjob in candidate_jobs: if newjob and newjob != cjob and cjob.status == JobStatus.WAIT: cjob.status = JobStatus.GIVEUP elif cjob.status == JobStatus.WAIT: newjob = cjob job = newjob # We have to check because it might have changed # between our first select and the build_more_recent stuffs if not job or job.status != JobStatus.WAIT: continue # Check dependencies if not job.is_allowed_to_build(): continue job.status = JobStatus.WAIT_LOCKED job.host = socket.gethostname() self.jobs.append(job) count_new += 1 count_current += 1 if count_current >= max_new: break return count_new def count_running_jobs(self): """Count running jobs""" count = 0 with self.jobs_locker: for job in self.jobs: if job.status == JobStatus.BUILDING and job.isAlive(): count += 1 return count def start_jobs(self, overrun=0): """Start waiting jobs""" running_threads = self.count_running_jobs() max_threads = max(overrun, self.cfg.getint('build', 'max_threads')) jobs_started = 0 with self.jobs_locker: for job in self.jobs: if running_threads >= max_threads: break with job.status_lock: if job.status == JobStatus.WAIT_LOCKED and not job.isAlive(): RebuilddLog.info("Starting new thread for job %s" % job.id) job.notify = self.job_finished job.setDaemon(True) job.start() jobs_started += 1 running_threads = running_threads + 1 RebuilddLog.info("Running threads: [ build %s/%s ] [ real %s ]" % (running_threads, max_threads, threading.activeCount())) return jobs_started def get_jobs(self, name, version=None, dist=None, arch=None): """Dump a job status""" if version: pkgs = Package.selectBy(name=name, version=version) else: pkgs = Package.selectBy(name=name) if not pkgs.count(): return [] retjobs = [] if dist and arch: for pkg in pkgs: retjobs.extend(Job.selectBy(package=pkg, dist=dist, arch=arch)) elif dist: for pkg in pkgs: retjobs.extend(Job.selectBy(package=pkg, dist=dist)) elif arch: for pkg in pkgs: retjobs.extend(Job.selectBy(package=pkg, arch=arch)) else: for pkg in pkgs: retjobs.extend(Job.selectBy(package=pkg)) return retjobs def dump_jobs(self, joblist=None): """Dump all jobs status""" ret = "" if not joblist: joblist = self.jobs for job in joblist: ret = "%s%s\n" % (ret, str(job)) return ret def cancel_job(self, jobid): """Cancel a job""" with self.jobs_locker: job = self.get_job(jobid) if job != None: if job.isAlive(): job.do_quit.set() job.join() with job.status_lock: job.status = JobStatus.CANCELED self.jobs.remove(job) RebuilddLog.info("Canceled job %s for %s_%s on %s/%s for %s" \ % (job.id, job.package.name, job.package.version, job.dist, job.arch, job.mailto)) return True return False def stop_all_jobs(self): """Stop all running jobs""" with self.jobs_locker: for job in self.jobs: if job.status == JobStatus.BUILDING and job.isAlive(): job.do_quit.set() RebuilddLog.info("Sending stop to job %s" % job.id) for job in self.jobs: if job.isAlive(): RebuilddLog.info("Waiting for job %s to terminate" % job.id) job.join(60) return True def add_job(self, name, version, priority, dist, mailto=None, arch=None): """Add a job""" if not arch: arch = self.cfg.arch[0] if not Dists().get_dist(dist, arch): RebuilddLog.error("Couldn't find dist/arch in the config file for %s_%s on %s/%s, not adding it" \ % (name, version, dist, arch)) return False pkgs = Package.selectBy(name=name, version=version) if pkgs.count(): # If several packages exists, just take the first pkg = pkgs[0] else: # Maybe we found no packages, so create a brand new one! pkg = Package(name=name, version=version, priority=priority) jobs_count = Job.selectBy(package=pkg, dist=dist, arch=arch, mailto=mailto, status=JobStatus.WAIT).count() if jobs_count: RebuilddLog.error("Job already existing for %s_%s on %s/%s, not adding it" \ % (pkg.name, pkg.version, dist, arch)) return False job = Job(package=pkg, dist=dist, arch=arch) job.status = JobStatus.WAIT job.arch = arch job.mailto = mailto log = Log(job=job) RebuilddLog.info("Added job for %s_%s on %s/%s for %s" \ % (name, version, dist, arch, mailto)) return True def requeue_job(self, job_id): """Requeue a failed job""" if Job.selectBy(id=job_id).count() == 0: RebuilddLog.error("There is no job related to %s that is in the job list" % job_id) return False job = Job.selectBy(id=job_id)[0] if job.status in FailedStatus: job.status = JobStatus.WAIT job.host = "" return True def add_deps(self, job_id, dependency_ids): if Job.selectBy(id=job_id).count() == 0: RebuilddLog.error("There is no job related to %s that is in the job list" % job_id) return False job = Job.selectBy(id=job_id)[0] deps = [] for dep in dependency_ids: if Job.selectBy(id=dep).count() == 0: RebuilddLog.error("There is no job related to %s that is in the job list" % dep) return False dep_job = Job.selectBy(id=dep)[0] deps.append(dep_job) job.add_deps(deps) return True def clean_jobs(self): """Clean finished or canceled jobs""" with self.jobs_locker: for job in self.jobs: if job.status == JobStatus.BUILD_OK \ or job.status in FailedStatus \ or job.status == JobStatus.CANCELED: self.jobs.remove(job) return True def release_jobs(self): """Release jobs""" with self.jobs_locker: for job in self.jobs: with job.status_lock: if job.status == JobStatus.WAIT_LOCKED: job.status = JobStatus.WAIT job.host = "" return True def fix_jobs(self, print_result=True): """If rebuildd crashed, reset jobs to a valid state""" jobs = [] jobs.extend(Job.selectBy(host=socket.gethostname(), status=JobStatus.WAIT_LOCKED)) jobs.extend(Job.selectBy(host=socket.gethostname(), status=JobStatus.BUILDING)) for job in jobs: if print_result: print "I: Fixing job %s (was %s)" % (job.id, JobStatus.whatis(job.status)) job.host = None job.status = JobStatus.WAIT job.build_start = None job.build_end = None return True def handle_sigterm(self, signum, stack): RebuilddLog.info("Receiving transmission... it's a signal %s capt'ain! EVERYONE OUT!" % signum) self.do_quit.set() def loop(self): """Rebuildd main loop""" counter = self.cfg.getint('build', 'check_every') while not self.do_quit.isSet(): if counter == self.cfg.getint('build', 'check_every') \ or self.job_finished.isSet(): self.get_new_jobs() # Start jobs self.start_jobs() # Clean finished jobs self.clean_jobs() counter = 0 self.job_finished.clear() self.do_quit.wait(1) counter += 1 rebuildd/build/lib.linux-x86_64-2.6/rebuildd/Dists.py0000644000000000000000000000236710724757316017256 0ustar # rebuildd - Debian packages rebuild tool # # (c) 2007 - Julien Danjou # # This software is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 dated June, 1991. # # This software is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this software; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA # class Dists(object): """Singleton implementing a set of Debian distributions""" _instance = None dists = [] def __new__(cls): if cls._instance is None: cls._instance = object.__new__(cls) return cls._instance def add_dist(self, dist): self.dists.append(dist) def get_dist(self, name, arch): for d in self.dists: if d.name == name and d.arch == arch: return d return None rebuildd/build/lib.linux-x86_64-2.6/rebuildd/RebuilddNetworkClient.py0000644000000000000000000001740212003461770022413 0ustar # rebuildd - Debian packages rebuild tool # # (c) 2007 - Julien Danjou # # This software is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 dated June, 1991. # # This software is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this software; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA # from __future__ import with_statement from RebuilddConfig import RebuilddConfig import threading, socket __version__ = "$Rev$" class RebuilddNetworkClient(threading.Thread): """Network client used for each connection""" def __init__(self, socket, rebuildd): threading.Thread.__init__(self) self.rebuildd = rebuildd self.socket = socket def run(self): """Run client thread""" self.socket.settimeout(1) self.socket.send(RebuilddConfig().get('telnet', 'motd') + "\n") prompt = RebuilddConfig().get('telnet', 'prompt') + " " line = "" has_timeout = False while line != "exit" and not self.rebuildd.do_quit.isSet(): if not has_timeout: try: self.socket.send(self.exec_cmd(line)) self.socket.send(prompt) except Exception: break try: line = "" line = self.socket.recv(512).strip() has_timeout = False except socket.timeout: has_timeout = True self.socket.close() def exec_cmd(self, cmd): """Execute a command asked by a client""" # Return empty if empty if cmd == "": return "" # Split words op = cmd.split(' ') cmd_fct = op[0] if not cmd_fct.isalnum(): return "" op.remove(op[0]) try: return getattr(self, "exec_cmd_" + cmd_fct)(*tuple(op)) except Exception, error: return "E: command error: %s\n" % error def exec_cmd_help(self, *args): """Show help""" help = "" for attr in dir(RebuilddNetworkClient): if attr.startswith("exec_cmd_"): help += "%s -- %s\n" \ % (attr[9:].replace("_", " "), getattr(RebuilddNetworkClient, attr).__doc__) return help def exec_cmd_config(self, *args): """Manipulate configuration file""" if len(args) < 1: return "E: usage: config [reload|dump|save]\n" if args[0] == "reload": if RebuilddConfig().reload(): return "I: config reloaded\n" return "E: config not reloded\n" if args[0] == "dump": return RebuilddConfig().dump() if args[0] == "save": if RebuilddConfig().save(): return "I: config saved\n" return "E: config not saved\n" return "E: usage: config [reload|dump|save]\n" def exec_cmd_status(self, *args): """Show current jobs status""" return self.rebuildd.dump_jobs() def exec_cmd_version(self, *args): """Show version""" return __version__ + "\n" def exec_cmd_job(self, *args): """Manipulate jobs""" if len(args) > 0 and args[0] == "add": return self.exec_cmd_job_add(*args) if len(args) > 0 and args[0] == "deps": return self.exec_cmd_job_deps(*args) if len(args) > 0 and args[0] == "cancel": return self.exec_cmd_job_cancel(*args) if len(args) > 0 and args[0] == "start": return self.exec_cmd_job_start(*args) if len(args) > 0 and args[0] == "reload": return self.exec_cmd_job_reload(*args) if len(args) > 0 and args[0] == "status": return self.exec_cmd_job_status(*args) if len(args) > 0 and args[0] == "requeue": return self.exec_cmd_job_requeue(*args) return "E: usage: job [args]\n" def exec_cmd_job_add(self, *args): """Add job""" ret = False if len(args) < 4: return "E: usage: job add [arch] [mailto]\n" if len(args) == 5: ret = self.rebuildd.add_job(name=args[1], version=args[2], priority=args[3], dist=args[4]) if len(args) == 6: ret = self.rebuildd.add_job(name=args[1], version=args[2], priority=args[3], dist=args[4], arch=args[5]) if len(args) == 7: ret = self.rebuildd.add_job(name=args[1], version=args[2], priority=args[3], dist=args[4], arch=args[5], mailto=args[6]) if ret: return "I: job added\n" return "E: error adding job\n" def exec_cmd_job_requeue(self, *args): """Requeue job""" if len(args) != 2: return "E: usage: job requeue \n" if self.rebuildd.requeue_job(job_id=int(args[1])): return "I: job requeued\n" return "E: error requeuing the job\n" def exec_cmd_job_deps(self, *args): """Add dependency""" ret = False if len(args) < 3: return "E: usage: job deps [dependency_job_id] [...]\n" ret = self.rebuildd.add_deps(job_id=args[1], dependency_ids=args[2:]) if ret: return "I: Dependency added\n" return "E: error adding deps" def exec_cmd_job_cancel(self, *args): """Cancel job""" if len(args) < 2: return "E: usage: job cancel \n" if self.rebuildd.cancel_job(int(args[1])): return "I: job canceled\n" return "E: unknown job\n" def exec_cmd_job_start(self, *args): """Start jobs""" if len(args) == 2: return "I: %s jobs started\n" \ % self.rebuildd.start_jobs(int(args[1])) return "I: %s jobs started\n" \ % self.rebuildd.start_jobs() def exec_cmd_job_reload(self, *args): """Load new jobs""" return "I: %s new jobs added\n" % self.rebuildd.get_new_jobs() def exec_cmd_job_status(self, *args): """Dump job status""" if len(args) < 2 or len(args) > 5: return "E: usage: job status [arch]\n" elif len(args) == 2: jobs = self.rebuildd.get_jobs(name=args[1]) elif len(args) == 3: jobs = self.rebuildd.get_jobs(name=args[1], version=args[2]) elif len(args) == 4: jobs = self.rebuildd.get_jobs(name=args[1], version=args[2], dist=args[3]) elif len(args) == 5: jobs = self.rebuildd.get_jobs(name=args[1], version=args[2], dist=args[3], arch=args[4]) return self.rebuildd.dump_jobs(jobs) rebuildd/build/lib.linux-x86_64-2.6/rebuildd/Job.py0000644000000000000000000002531212003461770016661 0ustar # rebuildd - Debian packages rebuild tool # # (c) 2007 - Julien Danjou # # This software is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 dated June, 1991. # # This software is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this software; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA # from __future__ import with_statement import threading, subprocess, smtplib, time, os, signal, socket, select import sqlobject from subprocess import Popen,PIPE from email.Message import Message from Dists import Dists from JobStatus import JobStatus from JobStatus import FailedStatus from RebuilddLog import RebuilddLog, Log from RebuilddConfig import RebuilddConfig __version__ = "$Rev$" class Job(threading.Thread, sqlobject.SQLObject): """Class implementing a build job""" status = sqlobject.IntCol(default=JobStatus.UNKNOWN) mailto = sqlobject.StringCol(default=None) package = sqlobject.ForeignKey('Package', cascade=True) dist = sqlobject.StringCol(default='sid') arch = sqlobject.StringCol(default='any') creation_date = sqlobject.DateTimeCol(default=sqlobject.DateTimeCol.now) status_changed = sqlobject.DateTimeCol(default=None) build_start = sqlobject.DateTimeCol(default=None) build_end = sqlobject.DateTimeCol(default=None) host = sqlobject.StringCol(default=None) deps = sqlobject.RelatedJoin('Job', joinColumn='joba', otherColumn='jobb') log = sqlobject.SingleJoin('Log') notify = None def __init__(self, *args, **kwargs): """Init job""" threading.Thread.__init__(self) sqlobject.SQLObject.__init__(self, *args, **kwargs) self.do_quit = threading.Event() self.status_lock = threading.Lock() def __setattr__(self, name, value): """Override setattr to log build status changes""" if name == "status": RebuilddLog.info("Job %s for %s_%s on %s/%s changed status from %s to %s"\ % (self.id, self.package.name, self.package.version, self.dist, self.arch, JobStatus.whatis(self.status), JobStatus.whatis(value))) self.status_changed = sqlobject.DateTimeCol.now() sqlobject.SQLObject.__setattr__(self, name, value) @property def logfile(self): """Compute and return logfile name""" build_log_file = "%s/%s_%s-%s-%s-%s.%s.log" % (RebuilddConfig().get('log', 'logs_dir'), self.package.name, self.package.version, self.dist, self.arch, self.creation_date.strftime("%Y%m%d-%H%M%S"), self.id) return build_log_file def preexec_child(self): """Start a new group process before executing child""" os.setsid() def run(self): """Run job thread, download and build the package""" self.build_start = sqlobject.DateTimeCol.now() try: with open(self.logfile, "w") as build_log: build_log.write("Automatic build of %s_%s on %s for %s/%s by rebuildd %s\n" % \ (self.package.name, self.package.version, self.host, self.dist, self.arch, __version__)) build_log.write("Build started at %s\n" % self.build_start) build_log.write("******************************************************************************\n") except IOError: return build_log = file(self.logfile, "a") # we are building with self.status_lock: self.status = JobStatus.BUILDING # execute commands for cmd, failed_status in ([Dists().get_dist(self.dist, self.arch).get_source_cmd(self.package), JobStatus.SOURCE_FAILED], [Dists().get_dist(self.dist, self.arch).get_build_cmd(self.package), JobStatus.BUILD_FAILED], [Dists().get_dist(self.dist, self.arch).get_post_build_cmd(self.package), JobStatus.POST_BUILD_FAILED]): if cmd is None: continue try: proc = subprocess.Popen(cmd.split(), bufsize=0, preexec_fn=self.preexec_child, stdout=build_log, stdin=None, stderr=subprocess.STDOUT) except Exception, error: build_log.write("\nUnable to execute command \"%s\": %s" %\ (cmd, error)) with self.status_lock: self.status = failed_status state = 1 break state = proc.poll() while not self.do_quit.isSet() and state == None: state = proc.poll() self.do_quit.wait(1) if self.do_quit.isSet(): break if state != 0: with self.status_lock: self.status = failed_status break if self.do_quit.isSet(): # Kill gently the process RebuilddLog.info("Killing job %s with SIGINT" % self.id) try: os.killpg(os.getpgid(proc.pid), signal.SIGINT) except OSError, error: RebuilddLog.error("Error killing job %s: %s" % (self.id, error)) # If after 60s it's not dead, KILL HIM counter = 0 timemax = RebuilddConfig().get('build', 'kill_timeout') while proc.poll() == None and counter < timemax: time.sleep(1) counter += 1 if proc.poll() == None: RebuilddLog.error("Killing job %s timed out, killing with SIGKILL" \ % self.id) os.killpg(os.getpgid(proc.pid), signal.SIGKILL) with self.status_lock: self.status = JobStatus.WAIT_LOCKED # Reset host self.host = None build_log.write("\nBuild job killed on request\n") build_log.close() return # build is finished with self.status_lock: if state == 0: self.status = JobStatus.BUILD_OK self.build_end = sqlobject.DateTimeCol.now() build_log.write("******************************************************************************\n") build_log.write("Finished with status %s at %s\n" % (JobStatus.whatis(self.status), self.build_end)) build_log.write("Build needed %s\n" % (self.build_end - self.build_start)) build_log.close() # Send event to Rebuildd to inform it that it can # run a brand new job! if self.notify: self.notify.set() self.send_build_log() def send_build_log(self): """When job is built, send logs by mail""" try: with open(self.logfile, "r") as build_log: log = build_log.read() except IOError, error: RebuilddLog.error("Unable to open logfile for job %d" % self.id) return False # Store in database if self.log: self.log.text = log with self.status_lock: if self.status != JobStatus.BUILD_OK and \ not self.status in FailedStatus: return False if not RebuilddConfig().getboolean('log', 'mail_successful') \ and self.status == JobStatus.BUILD_OK: return True elif not RebuilddConfig().getboolean('log', 'mail_failed') \ and self.status in FailedStatus: return True if self.status == JobStatus.BUILD_OK: bstatus = "successful" else: bstatus = "failed" msg = Message() if self.mailto: msg['To'] = self.mailto else: msg['To'] = RebuilddConfig().get('mail', 'mailto') msg['From'] = RebuilddConfig().get('mail', 'from') msg['Subject'] = RebuilddConfig().get('mail', 'subject_prefix') + \ " Log for %s build of %s_%s on %s/%s" % \ (bstatus, self.package.name, self.package.version, self.dist, self.arch) msg['X-Rebuildd-Version'] = __version__ msg['X-Rebuildd-Host'] = socket.getfqdn() msg.set_payload(log) try: smtp = smtplib.SMTP() smtp.connect(RebuilddConfig().get('mail', 'smtp_host'), RebuilddConfig().get('mail', 'smtp_port')) smtp.sendmail(RebuilddConfig().get('mail', 'from'), [m.strip() for m in msg['To'].split(",")], msg.as_string()) except Exception, error: try: process = Popen("sendmail", shell=True, stdin=PIPE) process.communicate(input=msg.as_string()) except: RebuilddLog.error("Unable to send build log mail for job %d: %s" % (self.id, error)) return True def __str__(self): return "I: Job %s for %s_%s is status %s on %s for %s/%s" % \ (self.id, self.package.name, self.package.version, self.host, JobStatus.whatis(self.status), self.dist, self.arch) def is_allowed_to_build(self): """ Check if job is allowed to build """ for dep in Job.selectBy(id=self)[0].deps: if Job.selectBy(id=dep)[0].status != JobStatus.BUILD_OK: return False return True def add_dep(self, dep): """Add a job dependency on another job""" for existing_dep in self.deps: if existing_dep.id == dep.id: RebuilddLog.error("Already existing dependency between job %s and job %s" % (self.id, dep.id)) return self.addJob(dep) RebuilddLog.info("Dependency added between job %s and job %s" % (self.id, dep.id)) def add_deps(self,deps): """ Add several job dependency on another job""" for dep in deps: self.add_dep(dep) rebuildd/build/lib.linux-x86_64-2.6/rebuildd/__init__.py0000644000000000000000000000220212003461770017677 0ustar # rebuildd - Debian packages rebuild tool # # (c) 2007 - Julien Danjou # # This software is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 dated June, 1991. # # This software is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this software; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA # """rebuildd - Debian packages rebuild tool""" __author__ = "Julien Danjou " __all__ = [ "Distribution", "Enumeration", "Job", "JobStatus", "Package", "Rebuildd", "RebuilddConfig", "RebuilddLog", "RebuilddNetworkClient", "RebuilddNetworkServer", "RebuilddHTTPServer" ] rebuildd/build/lib.linux-x86_64-2.6/rebuildd/RebuilddLog.py0000644000000000000000000000372112003461770020343 0ustar # rebuildd - Debian packages rebuild tool # # (c) 2007 - Julien Danjou # # This software is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 dated June, 1991. # # This software is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this software; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA # import logging import sqlobject from RebuilddConfig import RebuilddConfig class Log(sqlobject.SQLObject): """Class implementing a Log""" job = sqlobject.ForeignKey('Job', cascade=True) text = sqlobject.BLOBCol(default="No build log available") class RebuilddLog(object): """Singleton used for logging""" _instance = None def __new__(cls): if cls._instance is None: cls._instance = object.__new__(cls) cls._instance.init() return cls._instance def init(self): cfg = RebuilddConfig() logging.basicConfig(level=logging.DEBUG, format='%(asctime)s %(levelname)s %(message)s', filename=cfg.get('log', 'file'), datefmt=cfg.get('log', 'time_format'), filemode='a') @classmethod def info(self, str): """Log a string with info priority""" logging.info(str) @classmethod def warn(self, str): """Log a string with warn priority""" logging.warning(str) @classmethod def error(self, str): """Log a string with error priority""" logging.error(str) rebuildd/build/lib.linux-x86_64-2.7/0000755000000000000000000000000012003462241013773 5ustar rebuildd/build/lib.linux-x86_64-2.7/rebuildd/0000755000000000000000000000000012003462241015565 5ustar rebuildd/build/lib.linux-x86_64-2.7/rebuildd/JobStatus.py0000644000000000000000000000267310720364127020075 0ustar # rebuildd - Debian packages rebuild tool # # (c) 2007 - Julien Danjou # # This software is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 dated June, 1991. # # This software is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this software; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA # from Enumeration import Enumeration JobStatus = Enumeration([ ("UNKNOWN", 0), ("WAIT", 100), ("WAIT_LOCKED", 150), ("BUILDING", 200), ("SOURCE_FAILED", 250), ("BUILD_FAILED", 300), ("POST_BUILD_FAILED", 350), ("CANCELED", 800), ("GIVEUP", 850), ("FAILED", 900), ("BUILD_OK", 1000) ]) FailedStatus = (JobStatus.SOURCE_FAILED, JobStatus.BUILD_FAILED, JobStatus.POST_BUILD_FAILED) rebuildd/build/lib.linux-x86_64-2.7/rebuildd/RebuilddHTTPServer.py0000644000000000000000000001366312003461770021577 0ustar # rebuildd - Debian packages rebuild tool # # (c) 2007 - Julien Danjou # # This software is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 dated June, 1991. # # This software is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this software; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA # from __future__ import with_statement from RebuilddConfig import RebuilddConfig from Rebuildd import Rebuildd from Package import Package from Job import Job from JobStatus import JobStatus from JobStatus import FailedStatus import tempfile, socket, sqlobject import web import gdchart render = web.template.render(RebuilddConfig().get('http', 'templates_dir'), \ cache=RebuilddConfig().getboolean('http', 'cache')) class RequestIndex: def GET(self): return render.base(page=render.index(), \ hostname=socket.gethostname(), \ archs=RebuilddConfig().arch, \ dists=RebuilddConfig().get('build', 'dists').split(' ')) class RequestPackage: def GET(self, name=None, version=None): jobs = [] if version: pkg = Package.selectBy(name=name, version=version)[0] title = "%s %s" % (name, version) package = "%s/%s" % (name, version) else: pkg = Package.selectBy(name=name)[0] title = package = name jobs.extend(Job.selectBy(package=pkg)) return render.base(page=render.tab(jobs=jobs), \ hostname=socket.gethostname(), \ title=title, \ package=package, \ archs=RebuilddConfig().arch, \ dists=RebuilddConfig().get('build', 'dists').split(' ')) class RequestArch: def GET(self, dist, arch=None): jobs = [] jobs.extend(Job.select(sqlobject.AND(Job.q.arch == arch, Job.q.dist == dist), orderBy=sqlobject.DESC(Job.q.creation_date))[:10]) return render.base(page=render.tab(jobs=jobs), \ arch=arch, \ dist=dist, \ title="%s/%s" % (dist, arch), \ hostname=socket.gethostname(), \ archs=RebuilddConfig().arch, \ dists=RebuilddConfig().get('build', 'dists').split(' ')) class RequestJob: def GET(self, jobid=None): job = Job.selectBy(id=jobid)[0] try: with open(job.logfile, "r") as build_logfile: build_log = build_logfile.read() except IOError, error: build_log = job.log.text return render.base(page=render.job(job=job, build_log=build_log), \ hostname=socket.gethostname(), \ title="job %s" % job.id, \ archs=RebuilddConfig().arch, \ dists=RebuilddConfig().get('build', 'dists').split(' ')) class RequestGraph: GET = web.autodelegate("GET_") def graph_init(self): web.header("Content-Type","image/png") graph = gdchart.Bar3D() graph.width = 300 graph.height = 300 graph.ytitle = "Jobs" graph.xtitle = "Build status" graph.ext_color = [ "yellow", "orange", "red", "green"] graph.bg_color = "white" graph.setLabels(["WAIT", "BUILDING", "FAILED", "OK"]) return graph def compute_stats(self, jobs): jw = 0 jb = 0 jf = 0 jo = 0 for job in jobs: if job.status == JobStatus.WAIT or \ job.status == JobStatus.WAIT_LOCKED: jw += 1 elif job.status == JobStatus.BUILDING: jb += 1 elif job.status in FailedStatus: jf += 1 elif job.status == JobStatus.BUILD_OK: jo += 1 return (jw, jb, jf, jo) def GET_buildstats(self, distarch=None): graph = self.graph_init() if distarch == "/": graph.title = "Build status" jobs = Job.selectBy() else: dindex = distarch.rindex("/") graph.title = "Build status for %s" % distarch[1:] jobs = Job.selectBy(arch=distarch[dindex+1:], dist=distarch[1:dindex]) graph.setData(self.compute_stats(jobs)) tmp = tempfile.TemporaryFile() graph.draw(tmp) tmp.seek(0) return tmp.read() def GET_package(self, package=None): graph = self.graph_init() if package == "/": graph.title = "Build status" jobs = Job.selectBy() else: dindex = package.rindex("/") graph.title = "Build status for %s" % package[1:] pkg = Package.selectBy(version=package[dindex+1:], name=package[1:dindex])[0] jobs = Job.selectBy(package=pkg) graph.setData(self.compute_stats(jobs)) tmp = tempfile.TemporaryFile() graph.draw(tmp) tmp.seek(0) return tmp.read() class RebuilddHTTPServer: """Main HTTP server""" urls = ( '/', 'RequestIndex', '/dist/(.*)/arch/(.*)', 'RequestArch', '/job/(.*)', 'RequestJob', '/package/(.*)/(.*)', 'RequestPackage', '/package/(.*)', 'RequestPackage', '/graph/(.*)', 'RequestGraph', ) def __init__(self): Rebuildd() def start(self): """Run main HTTP server thread""" web.webapi.internalerror = web.debugerror import sys; sys.argv.append(RebuilddConfig().get('http', 'ip') + ":" + RebuilddConfig().get('http', 'port')) app = web.application(self.urls, globals()) app.run() rebuildd/build/lib.linux-x86_64-2.7/rebuildd/Package.py0000644000000000000000000000222411423073051017474 0ustar # rebuildd - Debian packages rebuild tool # # (c) 2007 - Julien Danjou # # This software is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 dated June, 1991. # # This software is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this software; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA # import sqlobject import apt_pkg apt_pkg.init_system() class Package(sqlobject.SQLObject): """Class implemeting a Debian package""" name = sqlobject.StringCol() version = sqlobject.StringCol(default=None) priority = sqlobject.StringCol(default=None) @staticmethod def version_compare(a, b): return apt_pkg.version_compare(a.version, b.version) rebuildd/build/lib.linux-x86_64-2.7/rebuildd/Distribution.py0000644000000000000000000000610710763510134020630 0ustar # rebuildd - Debian packages rebuild tool # # (c) 2007 - Julien Danjou # # This software is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 dated June, 1991. # # This software is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this software; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA # from RebuilddConfig import RebuilddConfig from RebuilddLog import RebuilddLog from string import Template class Distribution(object): """Class implementing a Debian distribution Substitutions are done in the command strings: $d => The distro's name $a => the target architecture $p => the package's name $v => the package's version """ def __init__(self, name, arch): self.name = name self.arch = arch def get_source_cmd(self, package): """Return command used for grabing source for this distribution""" try: args = { 'd': self.name, 'a': self.arch, 'v': package.version, \ 'p': package.name } t = Template(RebuilddConfig().get('build', 'source_cmd')) return t.safe_substitute(**args) except TypeError, error: RebuilddLog.error("get_source_cmd has invalid format: %s" % error) return None def get_build_cmd(self, package): """Return command used for building source for this distribution""" # Strip epochs (x:) away try: index = package.version.index(":") args = { 'd': self.name, 'a': self.arch, \ 'v': package.version[index+1:], 'p': package.name } t = Template(RebuilddConfig().get('build', 'build_cmd')) return t.safe_substitute(**args) except ValueError: pass try: args = { 'd': self.name, 'a': self.arch, \ 'v': package.version, 'p': package.name } t = Template(RebuilddConfig().get('build', 'build_cmd')) return t.safe_substitute(**args) except TypeError, error: RebuilddLog.error("get_build_cmd has invalid format: %s" % error) return None def get_post_build_cmd(self, package): """Return command used after building source for this distribution""" cmd = RebuilddConfig().get('build', 'post_build_cmd') if cmd == '': return None try: args = { 'd': self.name, 'a': self.arch, \ 'v': package.version, 'p': package.name } t = Template(cmd) return t.safe_substitute(**args) except TypeError, error: RebuilddLog.error("post_build_cmd has invalid format: %s" % error) return None rebuildd/build/lib.linux-x86_64-2.7/rebuildd/Enumeration.py0000644000000000000000000000362511423073051020435 0ustar # rebuildd - Debian packages rebuild tool # # (c) 2007 - Julien Danjou # # This software is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 dated June, 1991. # # This software is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this software; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA # import types class Enumeration: """Simple enumeration class with reverse lookup""" def __init__(self, enumlist): self.lookup = { } self.reverse_lookup = { } val = 0 for elem in enumlist: if type(elem) == types.TupleType: elem, val = elem if type(elem) != types.StringType: raise ValueError("enum name is not a string: " + elem) if type(val) != types.IntType: raise ValueError("enum value is not an integer: " + val) if self.lookup.has_key(elem): raise ValueError("enum name is not unique: " + elem) if val in self.lookup.values(): raise ValueError("enum value is not unique for " + val) self.lookup[elem] = val self.reverse_lookup[val] = elem val += 1 def __getattr__(self, attr): if not self.lookup.has_key(attr): raise AttributeError return self.lookup[attr] def whatis(self, value): """Return element name for a value""" return self.reverse_lookup[value] rebuildd/build/lib.linux-x86_64-2.7/rebuildd/RebuilddConfig.py0000644000000000000000000001045212003461770021027 0ustar # rebuildd - Debian packages rebuild tool # # (c) 2007 - Julien Danjou # # This software is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 dated June, 1991. # # This software is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this software; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA # import ConfigParser import os class RebuilddConfig(object, ConfigParser.ConfigParser): """Main configuration singleton""" config_file = "/etc/rebuildd/rebuilddrc" _instance = None def __new__(cls, *args, **kwargs): if cls._instance is None: cls._instance = object.__new__(cls) cls._instance.init(*args, **kwargs) return cls._instance def init(self, dontparse=False): ConfigParser.ConfigParser.__init__(self) # add default sections self.add_section('build') self.add_section('mail') self.add_section('telnet') self.add_section('http') self.add_section('log') # add default values self.set('build', 'check_every', '300') self.set('build', 'max_threads', '2') self.set('build', 'max_jobs', '5') self.set('build', 'kill_timeout', '90') self.set('build', 'source_cmd', 'apt-get -q --download-only -t ${d} source ${p}=${v}') self.set('build', 'build_cmd', 'pbuilder build --basetgz /var/cache/pbuilder/${d}-${a}.tgz ${p}_${v}.dsc') self.set('build', 'post_build_cmd', '') self.set('build', 'dists', 'squeeze wheezy sid') self.set('build', 'work_dir', '/var/cache/rebuildd/build') self.set('build', 'database_uri', 'sqlite:///var/lib/rebuildd/rebuildd.db') self.set('build', 'build_more_recent', '1') self.set('build', 'more_archs', 'any') self.set('build', 'no_system_arch', '0') self.set('mail', 'from', 'rebuildd@localhost') self.set('mail', 'mailto', 'rebuildd@localhost') self.set('mail', 'subject_prefix', '[rebuildd]') self.set('mail', 'smtp_host', 'localhost') self.set('mail', 'smtp_port', '25') self.set('telnet', 'port', '9999') self.set('telnet', 'ip', '127.0.0.1') self.set('telnet', 'prompt', 'rebuildd@localhost->') self.set('telnet', 'motd', 'Connected on rebuildd on localhost') self.set('http', 'port', '9998') self.set('http', 'ip', '0.0.0.0') # This is dedicated to MadCoder self.set('http', 'log_lines_nb', '30') self.set('http', 'templates_dir', '/usr/share/rebuildd/templates') self.set('http', 'cache', '1') self.set('http', 'logfile', '/var/log/rebuildd/httpd.log') self.set('log', 'file', "/var/log/rebuildd/rebuildd.log") self.set('log', 'time_format', "%Y-%m-%d %H:%M:%S") self.set('log', 'logs_dir', "/var/log/rebuildd/build_logs") self.set('log', 'mail_failed', '1') self.set('log', 'mail_successful', '0') if not dontparse: self.reload() self.arch = [] if self.getint('build', 'no_system_arch') == 0: parch = os.popen("dpkg --print-architecture") self.arch.append(parch.readline().strip()) parch.close() for a in self.get('build', 'more_archs').split(' '): self.arch.append(a) def reload(self): """Reload configuration file""" return self.read(self.config_file) def dump(self): """Dump running configuration""" conf = "" for section in self.sections(): conf += "[" + section + "]\n" for item, value in self.items(section): conf += "%s = %s\n" % (item, value) conf += "\n" return conf def save(self): """Save configuration file""" try: self.write(file(self.config_file, 'w')) except Exception, error: return False return True rebuildd/build/lib.linux-x86_64-2.7/rebuildd/RebuilddNetworkServer.py0000644000000000000000000000370610720364127022450 0ustar # rebuildd - Debian packages rebuild tool # # (c) 2007 - Julien Danjou # # This software is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 dated June, 1991. # # This software is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this software; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA # import threading, socket from RebuilddConfig import RebuilddConfig from RebuilddNetworkClient import RebuilddNetworkClient class RebuilddNetworkServer(threading.Thread): """Main network server listening for connection""" def __init__(self, rebuildd): threading.Thread.__init__(self) self.rebuildd = rebuildd def run(self): """Run main network server thread""" self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) self.socket.settimeout(1) self.socket.bind((RebuilddConfig().get('telnet', 'ip'), RebuilddConfig().getint('telnet', 'port'))) self.socket.listen(2) while not self.rebuildd.do_quit.isSet(): try: (client_socket, client_info) = self.socket.accept() if client_socket: interface = RebuilddNetworkClient(client_socket, self.rebuildd) interface.setDaemon(True) interface.start() except socket.timeout: pass self.socket.close() rebuildd/build/lib.linux-x86_64-2.7/rebuildd/Rebuildd.py0000644000000000000000000003300412003461770017677 0ustar # rebuildd - Debian packages rebuild tool # # (c) 2007 - Julien Danjou # # This software is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 dated June, 1991. # # This software is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this software; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA # """rebuildd - Debian packages rebuild tool""" from __future__ import with_statement from Enumeration import Enumeration from Distribution import Distribution from Dists import Dists from RebuilddLog import RebuilddLog, Log from RebuilddConfig import RebuilddConfig from RebuilddNetworkServer import RebuilddNetworkServer from Package import Package from Job import Job from JobStatus import JobStatus from JobStatus import FailedStatus import threading, os, time, sys, signal, socket import sqlobject __version__ = "$Rev$" class Rebuildd(object): jobs = [] _instance = None def __new__(cls): if cls._instance is None: cls._instance = object.__new__(cls) cls._instance.init() return cls._instance def init(self): self.cfg = RebuilddConfig() # Init log system RebuilddLog() self._sqlconnection = sqlobject.connectionForURI(self.cfg.get('build', 'database_uri')) sqlobject.sqlhub.processConnection = self._sqlconnection # Create distributions for dist in self.cfg.get('build', 'dists').split(' '): for arch in self.cfg.arch: Dists().add_dist(Distribution(dist, arch)) self.do_quit = threading.Event() self.jobs_locker = threading.Lock() self.job_finished = threading.Event() def daemon(self): RebuilddLog.info("Starting rebuildd %s" % __version__) self.daemonize() # Run the network server thread RebuilddLog.info("Launching network server") self.netserv = RebuilddNetworkServer(self) self.netserv.setDaemon(True) self.netserv.start() # Run main loop RebuilddLog.info("Running main loop") self.loop() # On exit RebuilddLog.info("Cleaning finished and canceled jobs") self.clean_jobs() RebuilddLog.info("Stopping all jobs") self.stop_all_jobs() RebuilddLog.info("Releasing wait-locked jobs") self.release_jobs() self.netserv.join(10) RebuilddLog.info("Exiting rebuildd") def daemonize(self): """Do daemon stuff""" signal.signal(signal.SIGTERM, self.handle_sigterm) signal.signal(signal.SIGINT, self.handle_sigterm) try: os.chdir(self.cfg.get('build', 'work_dir')) except Exception, error: print "E: unable to chdir to work_dir: %s" % error sys.exit(1) try: sys.stdout = sys.stderr = file(self.cfg.get('log', 'file'), "a") except Exception, error: print "E: unable to open logfile: %s" % error def get_job(self, jobid): for job in self.jobs: if job.id == jobid: return job return None def get_all_jobs(self, **kwargs): jobs = [] jobs.extend(Job.selectBy(**kwargs)) return jobs def get_new_jobs(self): """Feed jobs list with waiting jobs and lock them""" max_new = self.cfg.getint('build', 'max_jobs') count_current = len(self.jobs) with self.jobs_locker: if count_current >= max_new: return 0 jobs = [] for dist in Dists().dists: jobs.extend(Job.selectBy(status=JobStatus.WAIT, dist=dist.name, arch=dist.arch)[:max_new]) count_new = 0 for job in jobs: # Look for higher versions ? if self.cfg.getboolean('build', 'build_more_recent'): packages = Package.selectBy(name=job.package.name) candidate_packages = [] candidate_packages.extend(packages) candidate_packages.sort(cmp=Package.version_compare) candidate_packages.reverse() newjob = None # so there are packages with higher version number # try to see if there's a job for us for cpackage in candidate_packages: candidate_jobs = [] candidate_jobs.extend(Job.selectBy(package=cpackage, dist=job.dist, arch=job.arch)) for cjob in candidate_jobs: if newjob and newjob != cjob and cjob.status == JobStatus.WAIT: cjob.status = JobStatus.GIVEUP elif cjob.status == JobStatus.WAIT: newjob = cjob job = newjob # We have to check because it might have changed # between our first select and the build_more_recent stuffs if not job or job.status != JobStatus.WAIT: continue # Check dependencies if not job.is_allowed_to_build(): continue job.status = JobStatus.WAIT_LOCKED job.host = socket.gethostname() self.jobs.append(job) count_new += 1 count_current += 1 if count_current >= max_new: break return count_new def count_running_jobs(self): """Count running jobs""" count = 0 with self.jobs_locker: for job in self.jobs: if job.status == JobStatus.BUILDING and job.isAlive(): count += 1 return count def start_jobs(self, overrun=0): """Start waiting jobs""" running_threads = self.count_running_jobs() max_threads = max(overrun, self.cfg.getint('build', 'max_threads')) jobs_started = 0 with self.jobs_locker: for job in self.jobs: if running_threads >= max_threads: break with job.status_lock: if job.status == JobStatus.WAIT_LOCKED and not job.isAlive(): RebuilddLog.info("Starting new thread for job %s" % job.id) job.notify = self.job_finished job.setDaemon(True) job.start() jobs_started += 1 running_threads = running_threads + 1 RebuilddLog.info("Running threads: [ build %s/%s ] [ real %s ]" % (running_threads, max_threads, threading.activeCount())) return jobs_started def get_jobs(self, name, version=None, dist=None, arch=None): """Dump a job status""" if version: pkgs = Package.selectBy(name=name, version=version) else: pkgs = Package.selectBy(name=name) if not pkgs.count(): return [] retjobs = [] if dist and arch: for pkg in pkgs: retjobs.extend(Job.selectBy(package=pkg, dist=dist, arch=arch)) elif dist: for pkg in pkgs: retjobs.extend(Job.selectBy(package=pkg, dist=dist)) elif arch: for pkg in pkgs: retjobs.extend(Job.selectBy(package=pkg, arch=arch)) else: for pkg in pkgs: retjobs.extend(Job.selectBy(package=pkg)) return retjobs def dump_jobs(self, joblist=None): """Dump all jobs status""" ret = "" if not joblist: joblist = self.jobs for job in joblist: ret = "%s%s\n" % (ret, str(job)) return ret def cancel_job(self, jobid): """Cancel a job""" with self.jobs_locker: job = self.get_job(jobid) if job != None: if job.isAlive(): job.do_quit.set() job.join() with job.status_lock: job.status = JobStatus.CANCELED self.jobs.remove(job) RebuilddLog.info("Canceled job %s for %s_%s on %s/%s for %s" \ % (job.id, job.package.name, job.package.version, job.dist, job.arch, job.mailto)) return True return False def stop_all_jobs(self): """Stop all running jobs""" with self.jobs_locker: for job in self.jobs: if job.status == JobStatus.BUILDING and job.isAlive(): job.do_quit.set() RebuilddLog.info("Sending stop to job %s" % job.id) for job in self.jobs: if job.isAlive(): RebuilddLog.info("Waiting for job %s to terminate" % job.id) job.join(60) return True def add_job(self, name, version, priority, dist, mailto=None, arch=None): """Add a job""" if not arch: arch = self.cfg.arch[0] if not Dists().get_dist(dist, arch): RebuilddLog.error("Couldn't find dist/arch in the config file for %s_%s on %s/%s, not adding it" \ % (name, version, dist, arch)) return False pkgs = Package.selectBy(name=name, version=version) if pkgs.count(): # If several packages exists, just take the first pkg = pkgs[0] else: # Maybe we found no packages, so create a brand new one! pkg = Package(name=name, version=version, priority=priority) jobs_count = Job.selectBy(package=pkg, dist=dist, arch=arch, mailto=mailto, status=JobStatus.WAIT).count() if jobs_count: RebuilddLog.error("Job already existing for %s_%s on %s/%s, not adding it" \ % (pkg.name, pkg.version, dist, arch)) return False job = Job(package=pkg, dist=dist, arch=arch) job.status = JobStatus.WAIT job.arch = arch job.mailto = mailto log = Log(job=job) RebuilddLog.info("Added job for %s_%s on %s/%s for %s" \ % (name, version, dist, arch, mailto)) return True def requeue_job(self, job_id): """Requeue a failed job""" if Job.selectBy(id=job_id).count() == 0: RebuilddLog.error("There is no job related to %s that is in the job list" % job_id) return False job = Job.selectBy(id=job_id)[0] if job.status in FailedStatus: job.status = JobStatus.WAIT job.host = "" return True def add_deps(self, job_id, dependency_ids): if Job.selectBy(id=job_id).count() == 0: RebuilddLog.error("There is no job related to %s that is in the job list" % job_id) return False job = Job.selectBy(id=job_id)[0] deps = [] for dep in dependency_ids: if Job.selectBy(id=dep).count() == 0: RebuilddLog.error("There is no job related to %s that is in the job list" % dep) return False dep_job = Job.selectBy(id=dep)[0] deps.append(dep_job) job.add_deps(deps) return True def clean_jobs(self): """Clean finished or canceled jobs""" with self.jobs_locker: for job in self.jobs: if job.status == JobStatus.BUILD_OK \ or job.status in FailedStatus \ or job.status == JobStatus.CANCELED: self.jobs.remove(job) return True def release_jobs(self): """Release jobs""" with self.jobs_locker: for job in self.jobs: with job.status_lock: if job.status == JobStatus.WAIT_LOCKED: job.status = JobStatus.WAIT job.host = "" return True def fix_jobs(self, print_result=True): """If rebuildd crashed, reset jobs to a valid state""" jobs = [] jobs.extend(Job.selectBy(host=socket.gethostname(), status=JobStatus.WAIT_LOCKED)) jobs.extend(Job.selectBy(host=socket.gethostname(), status=JobStatus.BUILDING)) for job in jobs: if print_result: print "I: Fixing job %s (was %s)" % (job.id, JobStatus.whatis(job.status)) job.host = None job.status = JobStatus.WAIT job.build_start = None job.build_end = None return True def handle_sigterm(self, signum, stack): RebuilddLog.info("Receiving transmission... it's a signal %s capt'ain! EVERYONE OUT!" % signum) self.do_quit.set() def loop(self): """Rebuildd main loop""" counter = self.cfg.getint('build', 'check_every') while not self.do_quit.isSet(): if counter == self.cfg.getint('build', 'check_every') \ or self.job_finished.isSet(): self.get_new_jobs() # Start jobs self.start_jobs() # Clean finished jobs self.clean_jobs() counter = 0 self.job_finished.clear() self.do_quit.wait(1) counter += 1 rebuildd/build/lib.linux-x86_64-2.7/rebuildd/Dists.py0000644000000000000000000000236710724757316017257 0ustar # rebuildd - Debian packages rebuild tool # # (c) 2007 - Julien Danjou # # This software is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 dated June, 1991. # # This software is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this software; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA # class Dists(object): """Singleton implementing a set of Debian distributions""" _instance = None dists = [] def __new__(cls): if cls._instance is None: cls._instance = object.__new__(cls) return cls._instance def add_dist(self, dist): self.dists.append(dist) def get_dist(self, name, arch): for d in self.dists: if d.name == name and d.arch == arch: return d return None rebuildd/build/lib.linux-x86_64-2.7/rebuildd/RebuilddNetworkClient.py0000644000000000000000000001740212003461770022414 0ustar # rebuildd - Debian packages rebuild tool # # (c) 2007 - Julien Danjou # # This software is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 dated June, 1991. # # This software is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this software; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA # from __future__ import with_statement from RebuilddConfig import RebuilddConfig import threading, socket __version__ = "$Rev$" class RebuilddNetworkClient(threading.Thread): """Network client used for each connection""" def __init__(self, socket, rebuildd): threading.Thread.__init__(self) self.rebuildd = rebuildd self.socket = socket def run(self): """Run client thread""" self.socket.settimeout(1) self.socket.send(RebuilddConfig().get('telnet', 'motd') + "\n") prompt = RebuilddConfig().get('telnet', 'prompt') + " " line = "" has_timeout = False while line != "exit" and not self.rebuildd.do_quit.isSet(): if not has_timeout: try: self.socket.send(self.exec_cmd(line)) self.socket.send(prompt) except Exception: break try: line = "" line = self.socket.recv(512).strip() has_timeout = False except socket.timeout: has_timeout = True self.socket.close() def exec_cmd(self, cmd): """Execute a command asked by a client""" # Return empty if empty if cmd == "": return "" # Split words op = cmd.split(' ') cmd_fct = op[0] if not cmd_fct.isalnum(): return "" op.remove(op[0]) try: return getattr(self, "exec_cmd_" + cmd_fct)(*tuple(op)) except Exception, error: return "E: command error: %s\n" % error def exec_cmd_help(self, *args): """Show help""" help = "" for attr in dir(RebuilddNetworkClient): if attr.startswith("exec_cmd_"): help += "%s -- %s\n" \ % (attr[9:].replace("_", " "), getattr(RebuilddNetworkClient, attr).__doc__) return help def exec_cmd_config(self, *args): """Manipulate configuration file""" if len(args) < 1: return "E: usage: config [reload|dump|save]\n" if args[0] == "reload": if RebuilddConfig().reload(): return "I: config reloaded\n" return "E: config not reloded\n" if args[0] == "dump": return RebuilddConfig().dump() if args[0] == "save": if RebuilddConfig().save(): return "I: config saved\n" return "E: config not saved\n" return "E: usage: config [reload|dump|save]\n" def exec_cmd_status(self, *args): """Show current jobs status""" return self.rebuildd.dump_jobs() def exec_cmd_version(self, *args): """Show version""" return __version__ + "\n" def exec_cmd_job(self, *args): """Manipulate jobs""" if len(args) > 0 and args[0] == "add": return self.exec_cmd_job_add(*args) if len(args) > 0 and args[0] == "deps": return self.exec_cmd_job_deps(*args) if len(args) > 0 and args[0] == "cancel": return self.exec_cmd_job_cancel(*args) if len(args) > 0 and args[0] == "start": return self.exec_cmd_job_start(*args) if len(args) > 0 and args[0] == "reload": return self.exec_cmd_job_reload(*args) if len(args) > 0 and args[0] == "status": return self.exec_cmd_job_status(*args) if len(args) > 0 and args[0] == "requeue": return self.exec_cmd_job_requeue(*args) return "E: usage: job [args]\n" def exec_cmd_job_add(self, *args): """Add job""" ret = False if len(args) < 4: return "E: usage: job add [arch] [mailto]\n" if len(args) == 5: ret = self.rebuildd.add_job(name=args[1], version=args[2], priority=args[3], dist=args[4]) if len(args) == 6: ret = self.rebuildd.add_job(name=args[1], version=args[2], priority=args[3], dist=args[4], arch=args[5]) if len(args) == 7: ret = self.rebuildd.add_job(name=args[1], version=args[2], priority=args[3], dist=args[4], arch=args[5], mailto=args[6]) if ret: return "I: job added\n" return "E: error adding job\n" def exec_cmd_job_requeue(self, *args): """Requeue job""" if len(args) != 2: return "E: usage: job requeue \n" if self.rebuildd.requeue_job(job_id=int(args[1])): return "I: job requeued\n" return "E: error requeuing the job\n" def exec_cmd_job_deps(self, *args): """Add dependency""" ret = False if len(args) < 3: return "E: usage: job deps [dependency_job_id] [...]\n" ret = self.rebuildd.add_deps(job_id=args[1], dependency_ids=args[2:]) if ret: return "I: Dependency added\n" return "E: error adding deps" def exec_cmd_job_cancel(self, *args): """Cancel job""" if len(args) < 2: return "E: usage: job cancel \n" if self.rebuildd.cancel_job(int(args[1])): return "I: job canceled\n" return "E: unknown job\n" def exec_cmd_job_start(self, *args): """Start jobs""" if len(args) == 2: return "I: %s jobs started\n" \ % self.rebuildd.start_jobs(int(args[1])) return "I: %s jobs started\n" \ % self.rebuildd.start_jobs() def exec_cmd_job_reload(self, *args): """Load new jobs""" return "I: %s new jobs added\n" % self.rebuildd.get_new_jobs() def exec_cmd_job_status(self, *args): """Dump job status""" if len(args) < 2 or len(args) > 5: return "E: usage: job status [arch]\n" elif len(args) == 2: jobs = self.rebuildd.get_jobs(name=args[1]) elif len(args) == 3: jobs = self.rebuildd.get_jobs(name=args[1], version=args[2]) elif len(args) == 4: jobs = self.rebuildd.get_jobs(name=args[1], version=args[2], dist=args[3]) elif len(args) == 5: jobs = self.rebuildd.get_jobs(name=args[1], version=args[2], dist=args[3], arch=args[4]) return self.rebuildd.dump_jobs(jobs) rebuildd/build/lib.linux-x86_64-2.7/rebuildd/Job.py0000644000000000000000000002531212003461770016662 0ustar # rebuildd - Debian packages rebuild tool # # (c) 2007 - Julien Danjou # # This software is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 dated June, 1991. # # This software is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this software; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA # from __future__ import with_statement import threading, subprocess, smtplib, time, os, signal, socket, select import sqlobject from subprocess import Popen,PIPE from email.Message import Message from Dists import Dists from JobStatus import JobStatus from JobStatus import FailedStatus from RebuilddLog import RebuilddLog, Log from RebuilddConfig import RebuilddConfig __version__ = "$Rev$" class Job(threading.Thread, sqlobject.SQLObject): """Class implementing a build job""" status = sqlobject.IntCol(default=JobStatus.UNKNOWN) mailto = sqlobject.StringCol(default=None) package = sqlobject.ForeignKey('Package', cascade=True) dist = sqlobject.StringCol(default='sid') arch = sqlobject.StringCol(default='any') creation_date = sqlobject.DateTimeCol(default=sqlobject.DateTimeCol.now) status_changed = sqlobject.DateTimeCol(default=None) build_start = sqlobject.DateTimeCol(default=None) build_end = sqlobject.DateTimeCol(default=None) host = sqlobject.StringCol(default=None) deps = sqlobject.RelatedJoin('Job', joinColumn='joba', otherColumn='jobb') log = sqlobject.SingleJoin('Log') notify = None def __init__(self, *args, **kwargs): """Init job""" threading.Thread.__init__(self) sqlobject.SQLObject.__init__(self, *args, **kwargs) self.do_quit = threading.Event() self.status_lock = threading.Lock() def __setattr__(self, name, value): """Override setattr to log build status changes""" if name == "status": RebuilddLog.info("Job %s for %s_%s on %s/%s changed status from %s to %s"\ % (self.id, self.package.name, self.package.version, self.dist, self.arch, JobStatus.whatis(self.status), JobStatus.whatis(value))) self.status_changed = sqlobject.DateTimeCol.now() sqlobject.SQLObject.__setattr__(self, name, value) @property def logfile(self): """Compute and return logfile name""" build_log_file = "%s/%s_%s-%s-%s-%s.%s.log" % (RebuilddConfig().get('log', 'logs_dir'), self.package.name, self.package.version, self.dist, self.arch, self.creation_date.strftime("%Y%m%d-%H%M%S"), self.id) return build_log_file def preexec_child(self): """Start a new group process before executing child""" os.setsid() def run(self): """Run job thread, download and build the package""" self.build_start = sqlobject.DateTimeCol.now() try: with open(self.logfile, "w") as build_log: build_log.write("Automatic build of %s_%s on %s for %s/%s by rebuildd %s\n" % \ (self.package.name, self.package.version, self.host, self.dist, self.arch, __version__)) build_log.write("Build started at %s\n" % self.build_start) build_log.write("******************************************************************************\n") except IOError: return build_log = file(self.logfile, "a") # we are building with self.status_lock: self.status = JobStatus.BUILDING # execute commands for cmd, failed_status in ([Dists().get_dist(self.dist, self.arch).get_source_cmd(self.package), JobStatus.SOURCE_FAILED], [Dists().get_dist(self.dist, self.arch).get_build_cmd(self.package), JobStatus.BUILD_FAILED], [Dists().get_dist(self.dist, self.arch).get_post_build_cmd(self.package), JobStatus.POST_BUILD_FAILED]): if cmd is None: continue try: proc = subprocess.Popen(cmd.split(), bufsize=0, preexec_fn=self.preexec_child, stdout=build_log, stdin=None, stderr=subprocess.STDOUT) except Exception, error: build_log.write("\nUnable to execute command \"%s\": %s" %\ (cmd, error)) with self.status_lock: self.status = failed_status state = 1 break state = proc.poll() while not self.do_quit.isSet() and state == None: state = proc.poll() self.do_quit.wait(1) if self.do_quit.isSet(): break if state != 0: with self.status_lock: self.status = failed_status break if self.do_quit.isSet(): # Kill gently the process RebuilddLog.info("Killing job %s with SIGINT" % self.id) try: os.killpg(os.getpgid(proc.pid), signal.SIGINT) except OSError, error: RebuilddLog.error("Error killing job %s: %s" % (self.id, error)) # If after 60s it's not dead, KILL HIM counter = 0 timemax = RebuilddConfig().get('build', 'kill_timeout') while proc.poll() == None and counter < timemax: time.sleep(1) counter += 1 if proc.poll() == None: RebuilddLog.error("Killing job %s timed out, killing with SIGKILL" \ % self.id) os.killpg(os.getpgid(proc.pid), signal.SIGKILL) with self.status_lock: self.status = JobStatus.WAIT_LOCKED # Reset host self.host = None build_log.write("\nBuild job killed on request\n") build_log.close() return # build is finished with self.status_lock: if state == 0: self.status = JobStatus.BUILD_OK self.build_end = sqlobject.DateTimeCol.now() build_log.write("******************************************************************************\n") build_log.write("Finished with status %s at %s\n" % (JobStatus.whatis(self.status), self.build_end)) build_log.write("Build needed %s\n" % (self.build_end - self.build_start)) build_log.close() # Send event to Rebuildd to inform it that it can # run a brand new job! if self.notify: self.notify.set() self.send_build_log() def send_build_log(self): """When job is built, send logs by mail""" try: with open(self.logfile, "r") as build_log: log = build_log.read() except IOError, error: RebuilddLog.error("Unable to open logfile for job %d" % self.id) return False # Store in database if self.log: self.log.text = log with self.status_lock: if self.status != JobStatus.BUILD_OK and \ not self.status in FailedStatus: return False if not RebuilddConfig().getboolean('log', 'mail_successful') \ and self.status == JobStatus.BUILD_OK: return True elif not RebuilddConfig().getboolean('log', 'mail_failed') \ and self.status in FailedStatus: return True if self.status == JobStatus.BUILD_OK: bstatus = "successful" else: bstatus = "failed" msg = Message() if self.mailto: msg['To'] = self.mailto else: msg['To'] = RebuilddConfig().get('mail', 'mailto') msg['From'] = RebuilddConfig().get('mail', 'from') msg['Subject'] = RebuilddConfig().get('mail', 'subject_prefix') + \ " Log for %s build of %s_%s on %s/%s" % \ (bstatus, self.package.name, self.package.version, self.dist, self.arch) msg['X-Rebuildd-Version'] = __version__ msg['X-Rebuildd-Host'] = socket.getfqdn() msg.set_payload(log) try: smtp = smtplib.SMTP() smtp.connect(RebuilddConfig().get('mail', 'smtp_host'), RebuilddConfig().get('mail', 'smtp_port')) smtp.sendmail(RebuilddConfig().get('mail', 'from'), [m.strip() for m in msg['To'].split(",")], msg.as_string()) except Exception, error: try: process = Popen("sendmail", shell=True, stdin=PIPE) process.communicate(input=msg.as_string()) except: RebuilddLog.error("Unable to send build log mail for job %d: %s" % (self.id, error)) return True def __str__(self): return "I: Job %s for %s_%s is status %s on %s for %s/%s" % \ (self.id, self.package.name, self.package.version, self.host, JobStatus.whatis(self.status), self.dist, self.arch) def is_allowed_to_build(self): """ Check if job is allowed to build """ for dep in Job.selectBy(id=self)[0].deps: if Job.selectBy(id=dep)[0].status != JobStatus.BUILD_OK: return False return True def add_dep(self, dep): """Add a job dependency on another job""" for existing_dep in self.deps: if existing_dep.id == dep.id: RebuilddLog.error("Already existing dependency between job %s and job %s" % (self.id, dep.id)) return self.addJob(dep) RebuilddLog.info("Dependency added between job %s and job %s" % (self.id, dep.id)) def add_deps(self,deps): """ Add several job dependency on another job""" for dep in deps: self.add_dep(dep) rebuildd/build/lib.linux-x86_64-2.7/rebuildd/__init__.py0000644000000000000000000000220212003461770017700 0ustar # rebuildd - Debian packages rebuild tool # # (c) 2007 - Julien Danjou # # This software is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 dated June, 1991. # # This software is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this software; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA # """rebuildd - Debian packages rebuild tool""" __author__ = "Julien Danjou " __all__ = [ "Distribution", "Enumeration", "Job", "JobStatus", "Package", "Rebuildd", "RebuilddConfig", "RebuilddLog", "RebuilddNetworkClient", "RebuilddNetworkServer", "RebuilddHTTPServer" ] rebuildd/build/lib.linux-x86_64-2.7/rebuildd/RebuilddLog.py0000644000000000000000000000372112003461770020344 0ustar # rebuildd - Debian packages rebuild tool # # (c) 2007 - Julien Danjou # # This software is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 dated June, 1991. # # This software is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this software; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA # import logging import sqlobject from RebuilddConfig import RebuilddConfig class Log(sqlobject.SQLObject): """Class implementing a Log""" job = sqlobject.ForeignKey('Job', cascade=True) text = sqlobject.BLOBCol(default="No build log available") class RebuilddLog(object): """Singleton used for logging""" _instance = None def __new__(cls): if cls._instance is None: cls._instance = object.__new__(cls) cls._instance.init() return cls._instance def init(self): cfg = RebuilddConfig() logging.basicConfig(level=logging.DEBUG, format='%(asctime)s %(levelname)s %(message)s', filename=cfg.get('log', 'file'), datefmt=cfg.get('log', 'time_format'), filemode='a') @classmethod def info(self, str): """Log a string with info priority""" logging.info(str) @classmethod def warn(self, str): """Log a string with warn priority""" logging.warning(str) @classmethod def error(self, str): """Log a string with error priority""" logging.error(str) rebuildd/build/lib/0000755000000000000000000000000011301044714011376 5ustar rebuildd/build/lib/rebuildd/0000755000000000000000000000000011301044714013170 5ustar rebuildd/build/lib/rebuildd/JobStatus.py0000644000000000000000000000267310720364127015477 0ustar # rebuildd - Debian packages rebuild tool # # (c) 2007 - Julien Danjou # # This software is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 dated June, 1991. # # This software is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this software; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA # from Enumeration import Enumeration JobStatus = Enumeration([ ("UNKNOWN", 0), ("WAIT", 100), ("WAIT_LOCKED", 150), ("BUILDING", 200), ("SOURCE_FAILED", 250), ("BUILD_FAILED", 300), ("POST_BUILD_FAILED", 350), ("CANCELED", 800), ("GIVEUP", 850), ("FAILED", 900), ("BUILD_OK", 1000) ]) FailedStatus = (JobStatus.SOURCE_FAILED, JobStatus.BUILD_FAILED, JobStatus.POST_BUILD_FAILED) rebuildd/build/lib/rebuildd/RebuilddHTTPServer.py0000644000000000000000000001367711233330313017176 0ustar # rebuildd - Debian packages rebuild tool # # (c) 2007 - Julien Danjou # # This software is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 dated June, 1991. # # This software is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this software; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA # from __future__ import with_statement from RebuilddConfig import RebuilddConfig from Rebuildd import Rebuildd from Package import Package from Job import Job from JobStatus import JobStatus from JobStatus import FailedStatus import tempfile, socket, sqlobject import web import gdchart render = web.template.render(RebuilddConfig().get('http', 'templates_dir'), \ cache=RebuilddConfig().getboolean('http', 'cache')) class RequestIndex: def GET(self): return render.base(page=render.index(), \ hostname=socket.gethostname(), \ archs=RebuilddConfig().arch, \ dists=RebuilddConfig().get('build', 'dists').split(' ')) class RequestPackage: def GET(self, name=None, version=None): jobs = [] if version: pkg = Package.selectBy(name=name, version=version)[0] title = "%s %s" % (name, version) package = "%s/%s" % (name, version) else: pkg = Package.selectBy(name=name)[0] title = package = name jobs.extend(Job.selectBy(package=pkg)) return render.base(page=render.tab(jobs=jobs), \ hostname=socket.gethostname(), \ title=title, \ package=package, \ archs=RebuilddConfig().arch, \ dists=RebuilddConfig().get('build', 'dists').split(' ')) class RequestArch: def GET(self, dist, arch=None): jobs = [] jobs.extend(Job.select(sqlobject.AND(Job.q.arch == arch, Job.q.dist == dist), orderBy=sqlobject.DESC(Job.q.creation_date))[:10]) return render.base(page=render.tab(jobs=jobs), \ arch=arch, \ dist=dist, \ title="%s/%s" % (dist, arch), \ hostname=socket.gethostname(), \ archs=RebuilddConfig().arch, \ dists=RebuilddConfig().get('build', 'dists').split(' ')) class RequestJob: def GET(self, jobid=None): job = Job.selectBy(id=jobid)[0] try: with open(job.logfile, "r") as build_logfile: build_log = build_logfile.read() except IOError, error: build_log = "No build log available" return render.base(page=render.job(job=job, build_log=build_log), \ hostname=socket.gethostname(), \ title="job %s" % job.id, \ archs=RebuilddConfig().arch, \ dists=RebuilddConfig().get('build', 'dists').split(' ')) class RequestGraph: GET = web.autodelegate("GET_") def graph_init(self): web.header("Content-Type","image/png") graph = gdchart.Bar3D() graph.width = 300 graph.height = 300 graph.ytitle = "Jobs" graph.xtitle = "Build status" graph.ext_color = [ "yellow", "orange", "red", "green"] graph.bg_color = "white" graph.setLabels(["WAIT", "BUILDING", "FAILED", "OK"]) return graph def compute_stats(self, jobs): jw = 0 jb = 0 jf = 0 jo = 0 for job in jobs: if job.status == JobStatus.WAIT or \ job.status == JobStatus.WAIT_LOCKED: jw += 1 elif job.status == JobStatus.BUILDING: jb += 1 elif job.status in FailedStatus: jf += 1 elif job.status == JobStatus.BUILD_OK: jo += 1 return (jw, jb, jf, jo) def GET_buildstats(self, distarch=None): graph = self.graph_init() if distarch == "/": graph.title = "Build status" jobs = Job.selectBy() else: dindex = distarch.rindex("/") graph.title = "Build status for %s" % distarch[1:] jobs = Job.selectBy(arch=distarch[dindex+1:], dist=distarch[1:dindex]) graph.setData(self.compute_stats(jobs)) tmp = tempfile.TemporaryFile() graph.draw(tmp) tmp.seek(0) return tmp.read() def GET_package(self, package=None): graph = self.graph_init() if package == "/": graph.title = "Build status" jobs = Job.selectBy() else: dindex = package.rindex("/") graph.title = "Build status for %s" % package[1:] pkg = Package.selectBy(version=package[dindex+1:], name=package[1:dindex])[0] jobs = Job.selectBy(package=pkg) graph.setData(self.compute_stats(jobs)) tmp = tempfile.TemporaryFile() graph.draw(tmp) tmp.seek(0) return tmp.read() class RebuilddHTTPServer: """Main HTTP server""" urls = ( '/', 'RequestIndex', '/dist/(.*)/arch/(.*)', 'RequestArch', '/job/(.*)', 'RequestJob', '/package/(.*)/(.*)', 'RequestPackage', '/package/(.*)', 'RequestPackage', '/graph/(.*)', 'RequestGraph', ) def __init__(self): Rebuildd() def start(self): """Run main HTTP server thread""" web.webapi.internalerror = web.debugerror import sys; sys.argv.append(RebuilddConfig().get('http', 'ip') + ":" + RebuilddConfig().get('http', 'port')) app = web.application(self.urls, globals()) app.run() rebuildd/build/lib/rebuildd/Package.py0000644000000000000000000000234110724757316015116 0ustar # rebuildd - Debian packages rebuild tool # # (c) 2007 - Julien Danjou # # This software is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 dated June, 1991. # # This software is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this software; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA # import sqlobject # apt warning make me crazzzyyy import warnings warnings.filterwarnings("ignore", module='apt') del warnings import apt class Package(sqlobject.SQLObject): """Class implemeting a Debian package""" name = sqlobject.StringCol() version = sqlobject.StringCol(default=None) priority = sqlobject.StringCol(default=None) @staticmethod def VersionCompare(a, b): return apt.VersionCompare(a.version, b.version) rebuildd/build/lib/rebuildd/Distribution.py0000644000000000000000000000610710763510134016232 0ustar # rebuildd - Debian packages rebuild tool # # (c) 2007 - Julien Danjou # # This software is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 dated June, 1991. # # This software is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this software; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA # from RebuilddConfig import RebuilddConfig from RebuilddLog import RebuilddLog from string import Template class Distribution(object): """Class implementing a Debian distribution Substitutions are done in the command strings: $d => The distro's name $a => the target architecture $p => the package's name $v => the package's version """ def __init__(self, name, arch): self.name = name self.arch = arch def get_source_cmd(self, package): """Return command used for grabing source for this distribution""" try: args = { 'd': self.name, 'a': self.arch, 'v': package.version, \ 'p': package.name } t = Template(RebuilddConfig().get('build', 'source_cmd')) return t.safe_substitute(**args) except TypeError, error: RebuilddLog.error("get_source_cmd has invalid format: %s" % error) return None def get_build_cmd(self, package): """Return command used for building source for this distribution""" # Strip epochs (x:) away try: index = package.version.index(":") args = { 'd': self.name, 'a': self.arch, \ 'v': package.version[index+1:], 'p': package.name } t = Template(RebuilddConfig().get('build', 'build_cmd')) return t.safe_substitute(**args) except ValueError: pass try: args = { 'd': self.name, 'a': self.arch, \ 'v': package.version, 'p': package.name } t = Template(RebuilddConfig().get('build', 'build_cmd')) return t.safe_substitute(**args) except TypeError, error: RebuilddLog.error("get_build_cmd has invalid format: %s" % error) return None def get_post_build_cmd(self, package): """Return command used after building source for this distribution""" cmd = RebuilddConfig().get('build', 'post_build_cmd') if cmd == '': return None try: args = { 'd': self.name, 'a': self.arch, \ 'v': package.version, 'p': package.name } t = Template(cmd) return t.safe_substitute(**args) except TypeError, error: RebuilddLog.error("post_build_cmd has invalid format: %s" % error) return None rebuildd/build/lib/rebuildd/Enumeration.py0000644000000000000000000000355510720364127016047 0ustar # rebuildd - Debian packages rebuild tool # # (c) 2007 - Julien Danjou # # This software is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 dated June, 1991. # # This software is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this software; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA # import types class Enumeration: """Simple enumeration class with reverse lookup""" def __init__(self, enumlist): self.lookup = { } self.reverse_lookup = { } val = 0 for elem in enumlist: if type(elem) == types.TupleType: elem, val = elem if type(elem) != types.StringType: raise "enum name is not a string: " + elem if type(val) != types.IntType: raise "enum value is not an integer: " + val if self.lookup.has_key(elem): raise "enum name is not unique: " + elem if val in self.lookup.values(): raise "enum value is not unique for " + val self.lookup[elem] = val self.reverse_lookup[val] = elem val += 1 def __getattr__(self, attr): if not self.lookup.has_key(attr): raise AttributeError return self.lookup[attr] def whatis(self, value): """Return element name for a value""" return self.reverse_lookup[value] rebuildd/build/lib/rebuildd/RebuilddConfig.py0000644000000000000000000001012211024533504016420 0ustar # rebuildd - Debian packages rebuild tool # # (c) 2007 - Julien Danjou # # This software is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 dated June, 1991. # # This software is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this software; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA # import ConfigParser import os class RebuilddConfig(object, ConfigParser.ConfigParser): """Main configuration singleton""" config_file = "/etc/rebuildd/rebuilddrc" _instance = None def __new__(cls, *args, **kwargs): if cls._instance is None: cls._instance = object.__new__(cls) cls._instance.init(*args, **kwargs) return cls._instance def init(self, dontparse=False): ConfigParser.ConfigParser.__init__(self) # add default sections self.add_section('build') self.add_section('mail') self.add_section('telnet') self.add_section('http') self.add_section('log') # add default values self.set('build', 'check_every', '300') self.set('build', 'max_threads', '2') self.set('build', 'max_jobs', '5') self.set('build', 'kill_timeout', '90') self.set('build', 'source_cmd', 'apt-get -q --download-only -t ${d} source ${p}=${v}') self.set('build', 'build_cmd', 'pbuilder build --basetgz /var/cache/pbuilder/${d}-${a}.tgz ${p}_${v}.dsc') self.set('build', 'post_build_cmd', '') self.set('build', 'dists', 'etch lenny sid') self.set('build', 'work_dir', '/var/cache/rebuildd/build') self.set('build', 'database_uri', 'sqlite:///var/lib/rebuildd/rebuildd.db') self.set('build', 'build_more_recent', '1') self.set('build', 'more_archs', 'any') self.set('mail', 'from', 'rebuildd@localhost') self.set('mail', 'mailto', 'rebuildd@localhost') self.set('mail', 'subject_prefix', '[rebuildd]') self.set('telnet', 'port', '9999') self.set('telnet', 'ip', '127.0.0.1') self.set('telnet', 'prompt', 'rebuildd@localhost->') self.set('telnet', 'motd', 'Connected on rebuildd on localhost') self.set('http', 'port', '9998') self.set('http', 'ip', '0.0.0.0') # This is dedicated to MadCoder self.set('http', 'log_lines_nb', '30') self.set('http', 'templates_dir', '/usr/share/rebuildd/templates') self.set('http', 'cache', '1') self.set('http', 'logfile', '/var/log/rebuildd/httpd.log') self.set('log', 'file', "/var/log/rebuildd/rebuildd.log") self.set('log', 'time_format', "%d-%m-%Y %H:%M:%S") self.set('log', 'logs_dir', "/var/log/rebuildd/build_logs") self.set('log', 'mail_failed', '1') self.set('log', 'mail_successful', '0') self.arch = [] parch = os.popen("dpkg --print-architecture") self.arch.append(parch.readline().strip()) parch.close() if not dontparse: self.reload() for a in self.get('build', 'more_archs').split(' '): self.arch.append(a) def reload(self): """Reload configuration file""" return self.read(self.config_file) def dump(self): """Dump running configuration""" conf = "" for section in self.sections(): conf += "[" + section + "]\n" for item, value in self.items(section): conf += "%s = %s\n" % (item, value) conf += "\n" return conf def save(self): """Save configuration file""" try: self.write(file(self.config_file, 'w')) except Exception, error: return False return True rebuildd/build/lib/rebuildd/RebuilddNetworkServer.py0000644000000000000000000000370610720364127020052 0ustar # rebuildd - Debian packages rebuild tool # # (c) 2007 - Julien Danjou # # This software is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 dated June, 1991. # # This software is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this software; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA # import threading, socket from RebuilddConfig import RebuilddConfig from RebuilddNetworkClient import RebuilddNetworkClient class RebuilddNetworkServer(threading.Thread): """Main network server listening for connection""" def __init__(self, rebuildd): threading.Thread.__init__(self) self.rebuildd = rebuildd def run(self): """Run main network server thread""" self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) self.socket.settimeout(1) self.socket.bind((RebuilddConfig().get('telnet', 'ip'), RebuilddConfig().getint('telnet', 'port'))) self.socket.listen(2) while not self.rebuildd.do_quit.isSet(): try: (client_socket, client_info) = self.socket.accept() if client_socket: interface = RebuilddNetworkClient(client_socket, self.rebuildd) interface.setDaemon(True) interface.start() except socket.timeout: pass self.socket.close() rebuildd/build/lib/rebuildd/Rebuildd.py0000644000000000000000000003045510724757316015324 0ustar # rebuildd - Debian packages rebuild tool # # (c) 2007 - Julien Danjou # # This software is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 dated June, 1991. # # This software is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this software; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA # """rebuildd - Debian packages rebuild tool""" from __future__ import with_statement from Enumeration import Enumeration from Distribution import Distribution from Dists import Dists from RebuilddLog import RebuilddLog from RebuilddConfig import RebuilddConfig from RebuilddNetworkServer import RebuilddNetworkServer from Package import Package from Job import Job from JobStatus import JobStatus from JobStatus import FailedStatus import threading, os, time, sys, signal, socket import sqlobject __version__ = "$Rev$" class Rebuildd(object): jobs = [] _instance = None def __new__(cls): if cls._instance is None: cls._instance = object.__new__(cls) cls._instance.init() return cls._instance def init(self): self.cfg = RebuilddConfig() # Init log system RebuilddLog() sqlobject.sqlhub.processConnection = \ sqlobject.connectionForURI(self.cfg.get('build', 'database_uri')) # Create distributions for dist in self.cfg.get('build', 'dists').split(' '): for arch in self.cfg.arch: Dists().add_dist(Distribution(dist, arch)) self.do_quit = threading.Event() self.jobs_locker = threading.Lock() self.job_finished = threading.Event() def daemon(self): RebuilddLog.info("Starting rebuildd %s" % __version__) self.daemonize() # Run the network server thread RebuilddLog.info("Launching network server") self.netserv = RebuilddNetworkServer(self) self.netserv.setDaemon(True) self.netserv.start() # Run main loop RebuilddLog.info("Running main loop") self.loop() # On exit RebuilddLog.info("Cleaning finished and canceled jobs") self.clean_jobs() RebuilddLog.info("Stopping all jobs") self.stop_all_jobs() RebuilddLog.info("Releasing wait-locked jobs") self.release_jobs() self.netserv.join(10) RebuilddLog.info("Exiting rebuildd") def daemonize(self): """Do daemon stuff""" signal.signal(signal.SIGTERM, self.handle_sigterm) signal.signal(signal.SIGINT, self.handle_sigterm) try: os.chdir(self.cfg.get('build', 'work_dir')) except Exception, error: print "E: unable to chdir to work_dir: %s" % error sys.exit(1) try: sys.stdout = sys.stderr = file(self.cfg.get('log', 'file'), "a") except Exception, error: print "E: unable to open logfile: %s" % error def get_job(self, jobid): for job in self.jobs: if job.id == jobid: return job return None def get_all_jobs(self, **kwargs): jobs = [] jobs.extend(Job.selectBy(**kwargs)) return jobs def get_new_jobs(self): """Feed jobs list with waiting jobs and lock them""" max_new = self.cfg.getint('build', 'max_jobs') count_current = len(self.jobs) with self.jobs_locker: if count_current >= max_new: return 0 jobs = [] for dist in Dists().dists: jobs.extend(Job.selectBy(status=JobStatus.WAIT, dist=dist.name, arch=dist.arch)[:max_new]) count_new = 0 for job in jobs: # Look for higher versions ? if self.cfg.getboolean('build', 'build_more_recent'): packages = Package.selectBy(name=job.package.name) candidate_packages = [] candidate_packages.extend(packages) candidate_packages.sort(cmp=Package.VersionCompare) candidate_packages.reverse() newjob = None # so there are packages with higher version number # try to see if there's a job for us for cpackage in candidate_packages: candidate_jobs = [] candidate_jobs.extend(Job.selectBy(package=cpackage, dist=job.dist, arch=job.arch)) for cjob in candidate_jobs: if newjob and newjob != cjob and cjob.status == JobStatus.WAIT: cjob.status = JobStatus.GIVEUP elif cjob.status == JobStatus.WAIT: newjob = cjob job = newjob # We have to check because it might have changed # between our first select and the build_more_recent stuffs if not job or job.status != JobStatus.WAIT: continue job.status = JobStatus.WAIT_LOCKED job.host = socket.gethostname() self.jobs.append(job) count_new += 1 count_current += 1 if count_current >= max_new: break return count_new def count_running_jobs(self): """Count running jobs""" count = 0 with self.jobs_locker: for job in self.jobs: if job.status == JobStatus.BUILDING and job.isAlive(): count += 1 return count def start_jobs(self, overrun=0): """Start waiting jobs""" running_threads = self.count_running_jobs() max_threads = max(overrun, self.cfg.getint('build', 'max_threads')) jobs_started = 0 with self.jobs_locker: for job in self.jobs: if running_threads >= max_threads: break with job.status_lock: if job.status == JobStatus.WAIT_LOCKED and not job.isAlive(): RebuilddLog.info("Starting new thread for job %s" % job.id) job.notify = self.job_finished job.setDaemon(True) job.start() jobs_started += 1 running_threads = running_threads + 1 RebuilddLog.info("Running threads: [ build %s/%s ] [ real %s ]" % (running_threads, max_threads, threading.activeCount())) return jobs_started def get_jobs(self, name, version=None, dist=None, arch=None): """Dump a job status""" if version: pkgs = Package.selectBy(name=name, version=version) else: pkgs = Package.selectBy(name=name) if not pkgs.count(): return [] retjobs = [] if dist and arch: for pkg in pkgs: retjobs.extend(Job.selectBy(package=pkg, dist=dist, arch=arch)) elif dist: for pkg in pkgs: retjobs.extend(Job.selectBy(package=pkg, dist=dist)) elif arch: for pkg in pkgs: retjobs.extend(Job.selectBy(package=pkg, arch=arch)) else: for pkg in pkgs: retjobs.extend(Job.selectBy(package=pkg)) return retjobs def dump_jobs(self, joblist=None): """Dump all jobs status""" ret = "" if not joblist: joblist = self.jobs for job in joblist: ret = "%s%s\n" % (ret, str(job)) return ret def cancel_job(self, jobid): """Cancel a job""" with self.jobs_locker: job = self.get_job(jobid) if job != None: if job.isAlive(): job.do_quit.set() job.join() with job.status_lock: job.status = JobStatus.CANCELED self.jobs.remove(job) RebuilddLog.info("Canceled job %s for %s_%s on %s/%s for %s" \ % (job.id, job.package.name, job.package.version, job.dist, job.arch, job.mailto)) return True return False def stop_all_jobs(self): """Stop all running jobs""" with self.jobs_locker: for job in self.jobs: if job.status == JobStatus.BUILDING and job.isAlive(): job.do_quit.set() RebuilddLog.info("Sending stop to job %s" % job.id) for job in self.jobs: if job.isAlive(): RebuilddLog.info("Waiting for job %s to terminate" % job.id) job.join(60) return True def add_job(self, name, version, priority, dist, mailto=None, arch=None): """Add a job""" if not arch: arch = self.cfg.arch[0] if not Dists().get_dist(dist, arch): return False pkgs = Package.selectBy(name=name, version=version) if pkgs.count(): # If several packages exists, just take the first pkg = pkgs[0] else: # Maybe we found no packages, so create a brand new one! pkg = Package(name=name, version=version, priority=priority) jobs_count = Job.selectBy(package=pkg, dist=dist, arch=arch, mailto=mailto, status=JobStatus.WAIT).count() if jobs_count: RebuilddLog.error("Job already existing for %s_%s on %s/%s, don't adding it" \ % (pkg.name, pkg.version, dist, arch)) return False job = Job(package=pkg, dist=dist, arch=arch) job.status = JobStatus.WAIT job.arch = arch job.mailto = mailto RebuilddLog.info("Added job for %s_%s on %s/%s for %s" \ % (name, version, dist, arch, mailto)) return True def clean_jobs(self): """Clean finished or canceled jobs""" with self.jobs_locker: for job in self.jobs: if job.status == JobStatus.BUILD_OK \ or job.status in FailedStatus \ or job.status == JobStatus.CANCELED: self.jobs.remove(job) return True def release_jobs(self): """Release jobs""" with self.jobs_locker: for job in self.jobs: with job.status_lock: if job.status == JobStatus.WAIT_LOCKED: job.status = JobStatus.WAIT job.host = "" return True def fix_jobs(self, print_result=True): """If rebuildd crashed, reset jobs to a valid state""" jobs = [] jobs.extend(Job.selectBy(host=socket.gethostname(), status=JobStatus.WAIT_LOCKED)) jobs.extend(Job.selectBy(host=socket.gethostname(), status=JobStatus.BUILDING)) for job in jobs: if print_result: print "I: Fixing job %s (was %s)" % (job.id, JobStatus.whatis(job.status)) job.host = None job.status = JobStatus.WAIT job.build_start = None job.build_end = None return True def handle_sigterm(self, signum, stack): RebuilddLog.info("Receiving transmission... it's a signal %s capt'ain! EVERYONE OUT!" % signum) self.do_quit.set() def loop(self): """Rebuildd main loop""" counter = self.cfg.getint('build', 'check_every') while not self.do_quit.isSet(): if counter == self.cfg.getint('build', 'check_every') \ or self.job_finished.isSet(): self.get_new_jobs() # Start jobs self.start_jobs() # Clean finished jobs self.clean_jobs() counter = 0 self.job_finished.clear() self.do_quit.wait(1) counter += 1 rebuildd/build/lib/rebuildd/Dists.py0000644000000000000000000000236710724757316014661 0ustar # rebuildd - Debian packages rebuild tool # # (c) 2007 - Julien Danjou # # This software is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 dated June, 1991. # # This software is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this software; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA # class Dists(object): """Singleton implementing a set of Debian distributions""" _instance = None dists = [] def __new__(cls): if cls._instance is None: cls._instance = object.__new__(cls) return cls._instance def add_dist(self, dist): self.dists.append(dist) def get_dist(self, name, arch): for d in self.dists: if d.name == name and d.arch == arch: return d return None rebuildd/build/lib/rebuildd/RebuilddNetworkClient.py0000644000000000000000000001563310724757316020036 0ustar # rebuildd - Debian packages rebuild tool # # (c) 2007 - Julien Danjou # # This software is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 dated June, 1991. # # This software is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this software; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA # from __future__ import with_statement from RebuilddConfig import RebuilddConfig import threading, socket __version__ = "$Rev$" class RebuilddNetworkClient(threading.Thread): """Network client used for each connection""" def __init__(self, socket, rebuildd): threading.Thread.__init__(self) self.rebuildd = rebuildd self.socket = socket def run(self): """Run client thread""" self.socket.settimeout(1) self.socket.send(RebuilddConfig().get('telnet', 'motd') + "\n") prompt = RebuilddConfig().get('telnet', 'prompt') + " " line = "" has_timeout = False while line != "exit" and not self.rebuildd.do_quit.isSet(): if not has_timeout: try: self.socket.send(self.exec_cmd(line)) self.socket.send(prompt) except Exception: break try: line = "" line = self.socket.recv(512).strip() has_timeout = False except socket.timeout: has_timeout = True self.socket.close() def exec_cmd(self, cmd): """Execute a command asked by a client""" # Return empty if empty if cmd == "": return "" # Split words op = cmd.split(' ') cmd_fct = op[0] if not cmd_fct.isalnum(): return "" op.remove(op[0]) try: return getattr(self, "exec_cmd_" + cmd_fct)(*tuple(op)) except Exception, error: return "E: command error: %s\n" % error def exec_cmd_help(self, *args): """Show help""" help = "" for attr in dir(RebuilddNetworkClient): if attr.startswith("exec_cmd_"): help += "%s -- %s\n" \ % (attr[9:].replace("_", " "), getattr(RebuilddNetworkClient, attr).__doc__) return help def exec_cmd_config(self, *args): """Manipulate configuration file""" if len(args) < 1: return "E: usage: config [reload|dump|save]\n" if args[0] == "reload": if RebuilddConfig().reload(): return "I: config reloaded\n" return "E: config not reloded\n" if args[0] == "dump": return RebuilddConfig().dump() if args[0] == "save": if RebuilddConfig().save(): return "I: config saved\n" return "E: config not saved\n" return "E: usage: config [reload|dump|save]\n" def exec_cmd_status(self, *args): """Show current jobs status""" return self.rebuildd.dump_jobs() def exec_cmd_version(self, *args): """Show version""" return __version__ + "\n" def exec_cmd_job(self, *args): """Manipulate jobs""" if len(args) > 0 and args[0] == "add": return self.exec_cmd_job_add(*args) if len(args) > 0 and args[0] == "cancel": return self.exec_cmd_job_cancel(*args) if len(args) > 0 and args[0] == "start": return self.exec_cmd_job_start(*args) if len(args) > 0 and args[0] == "reload": return self.exec_cmd_job_reload(*args) if len(args) > 0 and args[0] == "status": return self.exec_cmd_job_status(*args) return "E: usage: job [args]\n" def exec_cmd_job_add(self, *args): """Add job""" ret = False if len(args) < 4: return "E: usage: job add [arch] [mailto]\n" if len(args) == 5: ret = self.rebuildd.add_job(name=args[1], version=args[2], priority=args[3], dist=args[4]) if len(args) == 6: ret = self.rebuildd.add_job(name=args[1], version=args[2], priority=args[3], dist=args[4], arch=args[5]) if len(args) == 7: ret = self.rebuildd.add_job(name=args[1], version=args[2], priority=args[3], dist=args[4], arch=args[5], mailto=args[6]) if ret: return "I: job added\n" return "E: error adding job\n" def exec_cmd_job_cancel(self, *args): """Cancel job""" if len(args) < 2: return "E: usage: job cancel \n" if self.rebuildd.cancel_job(int(args[1])): return "I: job canceled\n" return "E: unknown job\n" def exec_cmd_job_start(self, *args): """Start jobs""" if len(args) == 2: return "I: %s jobs started\n" \ % self.rebuildd.start_jobs(int(args[1])) return "I: %s jobs started\n" \ % self.rebuildd.start_jobs() def exec_cmd_job_reload(self, *args): """Load new jobs""" return "I: %s new jobs added\n" % self.rebuildd.get_new_jobs() def exec_cmd_job_status(self, *args): """Dump job status""" if len(args) < 2 or len(args) > 5: return "E: usage: job status [arch]\n" elif len(args) == 2: jobs = self.rebuildd.get_jobs(name=args[1]) elif len(args) == 3: jobs = self.rebuildd.get_jobs(name=args[1], version=args[2]) elif len(args) == 4: jobs = self.rebuildd.get_jobs(name=args[1], version=args[2], dist=args[3]) elif len(args) == 5: jobs = self.rebuildd.get_jobs(name=args[1], version=args[2], dist=args[3], arch=args[4]) return self.rebuildd.dump_jobs(jobs) rebuildd/build/lib/rebuildd/Job.py0000644000000000000000000002315210724757316014300 0ustar # rebuildd - Debian packages rebuild tool # # (c) 2007 - Julien Danjou # # This software is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 dated June, 1991. # # This software is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this software; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA # from __future__ import with_statement import threading, subprocess, smtplib, time, os, signal, socket, select import sqlobject from email.Message import Message from Dists import Dists from JobStatus import JobStatus from JobStatus import FailedStatus from RebuilddLog import RebuilddLog from RebuilddConfig import RebuilddConfig __version__ = "$Rev$" class Job(threading.Thread, sqlobject.SQLObject): """Class implementing a build job""" status = sqlobject.IntCol(default=JobStatus.UNKNOWN) mailto = sqlobject.StringCol(default=None) package = sqlobject.ForeignKey('Package', cascade=True) dist = sqlobject.StringCol(default='sid') arch = sqlobject.StringCol(default='any') creation_date = sqlobject.DateTimeCol(default=sqlobject.DateTimeCol.now) status_changed = sqlobject.DateTimeCol(default=None) build_start = sqlobject.DateTimeCol(default=None) build_end = sqlobject.DateTimeCol(default=None) host = sqlobject.StringCol(default=None) notify = None def __init__(self, *args, **kwargs): """Init job""" threading.Thread.__init__(self) sqlobject.SQLObject.__init__(self, *args, **kwargs) self.do_quit = threading.Event() self.status_lock = threading.Lock() def __setattr__(self, name, value): """Override setattr to log build status changes""" if name == "status": RebuilddLog.info("Job %s for %s_%s on %s/%s changed status from %s to %s"\ % (self.id, self.package.name, self.package.version, self.dist, self.arch, JobStatus.whatis(self.status), JobStatus.whatis(value))) self.status_changed = sqlobject.DateTimeCol.now() sqlobject.SQLObject.__setattr__(self, name, value) @property def logfile(self): """Compute and return logfile name""" build_log_file = "%s/%s_%s-%s-%s-%s.%s.log" % (RebuilddConfig().get('log', 'logs_dir'), self.package.name, self.package.version, self.dist, self.arch, self.creation_date.strftime("%Y%m%d-%H%M%S"), self.id) return build_log_file def preexec_child(self): """Start a new group process before executing child""" os.setsid() def run(self): """Run job thread, download and build the package""" self.build_start = sqlobject.DateTimeCol.now() try: with open(self.logfile, "w") as build_log: build_log.write("Automatic build of %s_%s on %s for %s/%s by rebuildd %s\n" % \ (self.package.name, self.package.version, self.host, self.dist, self.arch, __version__)) build_log.write("Build started at %s\n" % self.build_start) build_log.write("******************************************************************************\n") except IOError: return build_log = file(self.logfile, "a") # we are building with self.status_lock: self.status = JobStatus.BUILDING # execute commands for cmd, failed_status in ([Dists().get_dist(self.dist, self.arch).get_source_cmd(self.package), JobStatus.SOURCE_FAILED], [Dists().get_dist(self.dist, self.arch).get_build_cmd(self.package), JobStatus.BUILD_FAILED], [Dists().get_dist(self.dist, self.arch).get_post_build_cmd(self.package), JobStatus.POST_BUILD_FAILED]): if cmd is None: continue try: proc = subprocess.Popen(cmd.split(), bufsize=0, preexec_fn=self.preexec_child, stdout=build_log, stdin=None, stderr=subprocess.STDOUT) except Exception, error: state = 1 break state = proc.poll() while not self.do_quit.isSet() and state == None: state = proc.poll() self.do_quit.wait(1) if self.do_quit.isSet(): break if state != 0: with self.status_lock: self.status = failed_status break if self.do_quit.isSet(): # Kill gently the process RebuilddLog.info("Killing job %s with SIGINT" % self.id) try: os.killpg(os.getpgid(proc.pid), signal.SIGINT) except OSError, error: RebuilddLog.error("Error killing job %s: %s" % (self.id, error)) # If after 60s it's not dead, KILL HIM counter = 0 timemax = RebuilddConfig().get('build', 'kill_timeout') while proc.poll() == None and counter < timemax: time.sleep(1) counter += 1 if proc.poll() == None: RebuilddLog.error("Killing job %s timed out, killing with SIGKILL" \ % self.id) os.killpg(os.getpgid(proc.pid), signal.SIGKILL) with self.status_lock: self.status = JobStatus.WAIT_LOCKED # Reset host self.host = None build_log.write("\nBuild job killed on request\n") build_log.close() return # build is finished with self.status_lock: if state == 0: self.status = JobStatus.BUILD_OK self.build_end = sqlobject.DateTimeCol.now() build_log.write("******************************************************************************\n") build_log.write("Finished with status %s at %s\n" % (JobStatus.whatis(self.status), self.build_end)) build_log.write("Build needed %s\n" % (self.build_end - self.build_start)) build_log.close() # Send event to Rebuildd to inform it that it can # run a brand new job! if self.notify: self.notify.set() self.send_build_log() def send_build_log(self): """When job is built, send logs by mail""" with self.status_lock: if self.status != JobStatus.BUILD_OK and \ not self.status in FailedStatus: return False if not RebuilddConfig().getboolean('log', 'mail_successful') \ and self.status == JobStatus.BUILD_OK: return True elif not RebuilddConfig().getboolean('log', 'mail_failed') \ and self.status in FailedStatus: return True if self.status == JobStatus.BUILD_OK: bstatus = "successful" else: bstatus = "failed" msg = Message() if self.mailto: msg['To'] = self.mailto else: msg['To'] = RebuilddConfig().get('mail', 'mailto') msg['From'] = RebuilddConfig().get('mail', 'from') msg['Subject'] = RebuilddConfig().get('mail', 'subject_prefix') + \ " Log for %s build of %s_%s on %s/%s" % \ (bstatus, self.package.name, self.package.version, self.dist, self.arch) msg['X-Rebuildd-Version'] = __version__ msg['X-Rebuildd-Host'] = socket.getfqdn() try: with open(self.logfile, "r") as build_log: log = "" for line in build_log.readlines(): log += line except IOError, error: RebuilddLog.error("Unable to open logfile for job %d" % self.id) return False msg.set_payload(log) try: smtp = smtplib.SMTP() smtp.connect() if self.mailto: smtp.sendmail(RebuilddConfig().get('mail', 'from'), self.mailto, msg.as_string()) else: smtp.sendmail(RebuilddConfig().get('mail', 'from'), RebuilddConfig().get('mail', 'mailto'), msg.as_string()) except Exception, error: RebuilddLog.error("Unable to send build log mail for job %d: %s" % (self.id, error)) return True def __str__(self): return "I: Job %s for %s_%s is status %s on %s for %s/%s" % \ (self.id, self.package.name, self.package.version, self.host, JobStatus.whatis(self.status), self.dist, self.arch) rebuildd/build/lib/rebuildd/__init__.py0000644000000000000000000000226310720364127015313 0ustar # rebuildd - Debian packages rebuild tool # # (c) 2007 - Julien Danjou # # This software is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 dated June, 1991. # # This software is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this software; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA # """rebuildd - Debian packages rebuild tool""" __author__ = "Julien Danjou " __cvsid__ = "$Id$" __version__ = "$Rev$"[6:-2] __all__ = [ "Distribution", "Enumeration", "Job", "JobStatus", "Package", "Rebuildd", "RebuilddConfig", "RebuilddLog", "RebuilddNetworkClient", "RebuilddNetworkServer", "RebuilddHTTPServer" ] rebuildd/build/lib/rebuildd/RebuilddLog.py0000644000000000000000000000341010720364127015743 0ustar # rebuildd - Debian packages rebuild tool # # (c) 2007 - Julien Danjou # # This software is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 dated June, 1991. # # This software is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this software; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA # import logging from RebuilddConfig import RebuilddConfig class RebuilddLog(object): """Singleton used for logging""" _instance = None def __new__(cls): if cls._instance is None: cls._instance = object.__new__(cls) cls._instance.init() return cls._instance def init(self): cfg = RebuilddConfig() logging.basicConfig(level=logging.DEBUG, format='%(asctime)s %(levelname)s %(message)s', filename=cfg.get('log', 'file'), datefmt=cfg.get('log', 'time_format'), filemode='a') @classmethod def info(self, str): """Log a string with info priority""" logging.info(str) @classmethod def warn(self, str): """Log a string with warn priority""" logging.warning(str) @classmethod def error(self, str): """Log a string with error priority""" logging.error(str) rebuildd/templates/0000755000000000000000000000000012003461770011534 5ustar rebuildd/templates/static/0000755000000000000000000000000010720364127013025 5ustar rebuildd/templates/static/rebuildd.css0000644000000000000000000000122510720364127015331 0ustar body { font-size: 0.9em; font-family: Helvetica, Verdana, Arial; } h1 { text-align: center; } h2 { text-align: center; background-color: black; color: white; size: 0%; } #graph { position: fixed; top: 110px; right: 10px; width: 300px; height: 300px; } #content { margin-right: 350px; } table#list { border-collapse: collapse; margin: 0px; padding: 0px; border: 2px solid black; } table #list tr,th,td { padding: 5px; border: 1px solid black; } #links { margin-bottom: 30px; text-align: center; } .log { overflow: auto; width: 100%; height: 400px; } rebuildd/templates/index.html0000644000000000000000000000002610720364127013531 0ustar Welcome on rebuildd ! rebuildd/templates/tab.html0000644000000000000000000000351112003461770013170 0ustar $def with (jobs) $for job in jobs: $if job.status == 0: $if job.status == 100: $if job.status == 150: $if job.status == 200: $if job.status == 250: $if job.status == 300: $if job.status == 350: $if job.status == 400: $if job.status == 800: $if job.status == 850: $if job.status == 900: $if job.status == 1000:
Id Package Version Distribution/Architecture Creation date Mail to Build status Host Build start Build end
$job.id $job.package.name $job.package.version $job.dist/$job.arch $job.creation_date $job.mailtoDuh?WAITWAIT_LOCKEDBUILDINGSOURCE_FAILEDBUILD_FAILEDPOST_BUILD_FAILEDBUILD_OKCANCELEDGIVEUPFAILEDOK $job.host $job.build_start $job.build_end
rebuildd/templates/base.html0000600000000000000000000000172211233330313013316 0ustar $def with (page=None, title=None, hostname="a yellow submarine", dists=None, archs=None, dist=None, arch=None, package=None) rebuildd\ $if title: - $title\
$if dist and arch: $elif package: $else:

rebuildd on $hostname

$if title:

Status for $title

$:page
rebuildd/templates/job.html0000644000000000000000000000357012003461770013201 0ustar $def with (job, build_log) $if job.status == 0: $if job.status == 100: $if job.status == 150: $if job.status == 200: $if job.status == 250: $if job.status == 300: $if job.status == 350: $if job.status == 400: $if job.status == 800: $if job.status == 850: $if job.status == 900: $if job.status == 1000:
Id Package Version Distribution/Architecture Creation date Mail to Build status Host Build start Build end
$job.id $job.package.name $job.package.version $job.dist/$job.arch $job.creation_date $job.mailtoDuh?WAITWAIT_LOCKEDBUILDINGSOURCE_FAILEDBUILD_FAILEDPOST_BUILD_FAILEDBUILD_OKCANCELEDGIVEUPFAILEDOK $job.host $job.build_start $job.build_end

Build log

$build_log
rebuildd/setup.py0000755000000000000000000000047212003461770011256 0ustar #!/usr/bin/python # Setup script for rebuildd from distutils.core import setup setup(name="rebuildd", description="Debian packages rebuild tool", author="Julien Danjou", author_email="acid@debian.org", url="http://julien.danjou.info/software/rebuildd.html", packages=['rebuildd']) rebuildd/debian/0000755000000000000000000000000012003463442010756 5ustar rebuildd/debian/rebuildd.manpage.xml0000644000000000000000000000653610720364127014717 0ustar Julien"> Danjou"> July 7, 2007"> 1"> acid@debian.org"> REBUILDD"> Debian"> GNU"> GPL"> ]>
&dhemail;
2007 &dhusername; &dhdate; &dhucpackage; &dhsection; &dhpackage; rebuild daemon &dhpackage; DESCRIPTION &dhpackage; allows you to manage a set of jobs. Each job is a package rebuilding task. Rebuilding is done by pbuilder (or cowbuilder if you want, or anything else), since everything is cutomizable via configuration file. OPTIONS Initialize database: create tables. Dump configuration. Fix database. If rebuildd crashed, some job can be still in a inconsistent state. This will reset all this jobs for this host to a WAIT status SEE ALSO rebuildd-job (1), rebuildd-init-build-system (1). FILES /etc/rebuildd/rebuilddrc AUTHOR This manual page was written by &dhusername; &dhemail; for the &debian; system (but may be used by others). Permission is granted to copy, distribute and/or modify this document under the terms of the &gnu; General Public License, Version 2 any later version published by the Free Software Foundation. On Debian systems, the complete text of the GNU General Public License can be found in /usr/share/common-licenses/GPL. rebuildd/debian/rebuildd.init.d0000644000000000000000000000447511301043155013664 0ustar #! /bin/bash # rebuildd init script # ### BEGIN INIT INFO # Provides: rebuildd # Required-Start: $remote_fs # Required-Stop: $remote_fs # Should-Start: $network # Should-Stop: $network # Default-Start: 2 3 4 5 # Default-Stop: 0 1 6 # Short-Description: rebuild daemon # Description: daemon providing rebuild system # for Debian packages ### END INIT INFO PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin DAEMON=/usr/sbin/rebuildd NAME=rebuildd DESC="rebuild daemon" test -x $DAEMON || exit 0 # Include rebuildd defaults if available if [ -f /etc/default/rebuildd ] ; then . /etc/default/rebuildd fi test "$START_REBUILDD" = 1 || exit 0 . /lib/lsb/init-functions set -e case "$1" in start) log_daemon_msg "Starting $DESC" "$NAME" start-stop-daemon --start --quiet --pidfile /var/run/$NAME.pid \ --background --make-pidfile --exec $DAEMON -- $DAEMON_OPTS log_end_msg $? ;; stop) log_daemon_msg "Stopping $DESC" "$NAME" start-stop-daemon --stop --quiet --oknodo --retry 120 --pidfile /var/run/$NAME.pid log_end_msg $? ;; #reload) # # If the daemon can reload its config files on the fly # for example by sending it SIGHUP, do it here. # # If the daemon responds to changes in its config file # directly anyway, make this a do-nothing entry. # # echo "Reloading $DESC configuration files." # start-stop-daemon --stop --signal 1 --quiet --pidfile \ # /var/run/$NAME.pid --exec $DAEMON #;; force-reload) # # If the "reload" option is implemented, move the "force-reload" # option to the "reload" entry above. If not, "force-reload" is # just the same as "restart" except that it does nothing if the # daemon isn't already running. # check wether $DAEMON is running. If so, restart start-stop-daemon --stop --test --quiet --pidfile \ /var/run/$NAME.pid --exec $DAEMON \ && $0 restart \ || exit 0 ;; restart) log_daemon_msg "Restarting $DESC" "$NAME" start-stop-daemon --stop --quiet --pidfile \ /var/run/$NAME.pid --exec $DAEMON sleep 1 start-stop-daemon --start --quiet --pidfile \ /var/run/$NAME.pid --exec $DAEMON -- $DAEMON_OPTS log_end_msg $? ;; *) N=/etc/init.d/$NAME # echo "Usage: $N {start|stop|restart|reload|force-reload}" >&2 echo "Usage: $N {start|stop|restart|force-reload}" >&2 exit 1 ;; esac exit 0 rebuildd/debian/rebuildd-job.manpage.xml0000644000000000000000000001076512003461770015464 0ustar Julien"> Danjou"> August 8, 2007"> 1"> acid@debian.org"> REBUILDD-JOB"> Debian"> GNU"> GPL"> ]>
&dhemail;
2007 &dhusername; &dhdate; &dhucpackage; &dhsection; &dhpackage; rebuildd job database management &dhpackage; DESCRIPTION &dhpackage; allows you to manage rebuildd database. You can list or add jobs. OPTIONS Add a job to the database. Job list is read from stdin, with one package per line. Format is: package_name package_version priority dist_name arch mail Add dependencies between jobs. Dependencies are read from stdin, with the following format: job_id dependency_job_id [dependency_job_id] [...] Add jobs to the database. Job list is read from stdin. The input format is expected to be the same as quinn-diff. Distribution has to be specified as argument. Reschedule a job. The job id has to be provided as argument. The job must be in an error state (previous build failed). Delete a job from the database. The job id has to be provided as argument. List jobs. A criteria can be added, with its value to filter output. Valid criteria are: package, arch, dist, status Dump a nice ASCII art graphic about jobs. SEE ALSO rebuildd (1), rebuildd-init-build-system (1). AUTHOR This manual page was written by &dhusername; &dhemail; for the &debian; system (but may be used by others). Permission is granted to copy, distribute and/or modify this document under the terms of the &gnu; General Public License, Version 2 any later version published by the Free Software Foundation. On Debian systems, the complete text of the GNU General Public License can be found in /usr/share/common-licenses/GPL. rebuildd/debian/cron.d0000644000000000000000000000006510720364127012071 0ustar 0 */4 * * * root run-parts /etc/rebuildd/maintenance rebuildd/debian/control0000644000000000000000000000256612003461770012374 0ustar Source: rebuildd Section: devel Priority: extra Maintainer: Julien Danjou Build-Depends: debhelper (>= 7.0.50~), python, xsltproc, docbook-xsl, python-sqlobject, python-apt (>= 0.7.91~) Standards-Version: 3.9.3 Homepage: http://julien.danjou.info/software/rebuildd Vcs-Git: git://anonscm.debian.org/rebuildd/rebuildd.git Vcs-Browser: http://anonscm.debian.org/gitweb/?p=rebuildd/rebuildd.git Package: rebuildd Architecture: all Suggests: cowdancer Recommends: pbuilder, python-gdchart2, python-webpy Depends: ${shlibs:Depends}, ${misc:Depends}, ${python:Depends}, lsb-base, python-sqlobject, python-apt (>= 0.7.91~) Description: build daemon aiming at rebuilding Debian packages This software allows you to manage a set of jobs. Each job is a package rebuilding task. Rebuilding is done by pbuilder (or cowbuilder if you want), or anything else, since everything is customizable via configuration file. It can also send build logs by email, event each log can be sent to a different email address. . rebuildd is multi-threaded, so you can run multiple build jobs in parallel. It is also administrable via a telnet interface. A Web interface is also embedded so you can see your jobs queue and watch log file in real-time in your browser. . rebuildd is designed to be run on multiple hosts even with different architecture set, and to parallelize the rebuild tasks. rebuildd/debian/NEWS.Debian0000644000000000000000000000052712003461770012644 0ustar rebuildd (0.3.11-1) unstable; urgency=low The schema changed between 0.3.10 and 0.3.11. You will need to either manually convert the existing build status database to the new schema (no documentation provided, read the source) or start fresh with "rebuildd init". -- Julien Danjou Wed, 05 Oct 2011 18:50:08 +0100 rebuildd/debian/dirs0000644000000000000000000000015010720364127011642 0ustar usr/sbin etc/rebuildd/maintenance var/lib/rebuildd var/cache/rebuildd/build var/log/rebuildd/build_logs rebuildd/debian/copyright0000644000000000000000000000203512003461770012713 0ustar Format: http://dep.debian.net/deps/dep5 Upstream-Name: rebuildd Source: http://julien.danjou.info/software/rebuildd.html Files: * Copyright: 2007, 2008, 2009, 2010, 2011 Julien Danjou License: GPL-2+ License: GPL-2+ This package is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. . This package is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. . You should have received a copy of the GNU General Public License along with this package; if not, write to the Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA . On Debian systems, the complete text of the GNU General Public License can be found in `/usr/share/common-licenses/GPL'. rebuildd/debian/rebuildd.default0000644000000000000000000000171112003463104014111 0ustar # Defaults for rebuildd initscript # sourced by /etc/init.d/rebuildd # Set to 1 to enable rebuildd START_REBUILDD=0 # Set to 1 to enable rebuildd Web server START_REBUILDD_HTTPD=0 # The variables below are used by various shell scripts # to manage your pbuilder instances. # Set this to your pbuilder path # You can also use cowbuilder instead PBUILDER_BIN=/usr/sbin/pbuilder #COWBUILDER_BIN=/usr/sbin/cowbuilder # Specify cache path # PBUILDER_CACHE_PATH="/var/cache/pbuilder" # Default Debian mirror # PBUILDER_MIRROR=http://ftp.debian.org/debian # Pass other options to pbuilder # PBUILDER_OTHER_OPTIONS[0]="--components" # PBUILDER_OTHER_OPTIONS[1]="main contrib" # PBUILDER_OTHER_OPTIONS[2]="--othermirror" # PBUILDER_OTHER_OPTIONS[3]="deb http://my.apt.repo sid main" # Distributions to generate and manage ARCHS="$(dpkg --print-architecture)" DISTS="stable testing unstable" # Set to 1 to enable pbuilder/cowbuilder update in cron ENABLE_BUILDER_MAINT=0 rebuildd/debian/source/0000755000000000000000000000000011423073051012254 5ustar rebuildd/debian/source/format0000644000000000000000000000001511423073051013463 0ustar 3.0 (native) rebuildd/debian/rebuildd.rebuildd-httpd.init.d0000644000000000000000000000455411301043155016574 0ustar #! /bin/bash # rebuildd-httpd init script # ### BEGIN INIT INFO # Provides: rebuildd-httpd # Required-Start: $remote_fs $network # Required-Stop: $remote_fs $network # Should-Start: # Should-Stop: # Default-Start: 2 3 4 5 # Default-Stop: 0 1 6 # Short-Description: rebuild daemon Web server # Description: daemon providing Web interface to # rebuild system for Debian packages ### END INIT INFO PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin DAEMON=/usr/sbin/rebuildd-httpd NAME=rebuildd-httpd DESC="rebuild daemon HTTP server" test -x $DAEMON || exit 0 # Include rebuildd defaults if available if [ -f /etc/default/rebuildd ] ; then . /etc/default/rebuildd fi test "$START_REBUILDD_HTTPD" = 1 || exit 0 . /lib/lsb/init-functions set -e case "$1" in start) log_daemon_msg "Starting $DESC" "$NAME" start-stop-daemon --start --quiet --pidfile /var/run/$NAME.pid \ --background --make-pidfile --exec $DAEMON -- $DAEMON_OPTS log_end_msg $? ;; stop) log_daemon_msg "Stopping $DESC" "$NAME" start-stop-daemon --stop --quiet --oknodo --pidfile /var/run/$NAME.pid log_end_msg $? ;; #reload) # # If the daemon can reload its config files on the fly # for example by sending it SIGHUP, do it here. # # If the daemon responds to changes in its config file # directly anyway, make this a do-nothing entry. # # echo "Reloading $DESC configuration files." # start-stop-daemon --stop --signal 1 --quiet --pidfile \ # /var/run/$NAME.pid --exec $DAEMON #;; force-reload) # # If the "reload" option is implemented, move the "force-reload" # option to the "reload" entry above. If not, "force-reload" is # just the same as "restart" except that it does nothing if the # daemon isn't already running. # check wether $DAEMON is running. If so, restart start-stop-daemon --stop --test --quiet --pidfile \ /var/run/$NAME.pid --exec $DAEMON \ && $0 restart \ || exit 0 ;; restart) log_daemon_msg "Restarting $DESC" "$NAME" start-stop-daemon --stop --quiet --pidfile \ /var/run/$NAME.pid --exec $DAEMON sleep 1 start-stop-daemon --start --quiet --pidfile \ /var/run/$NAME.pid --exec $DAEMON -- $DAEMON_OPTS log_end_msg $? ;; *) N=/etc/init.d/$NAME # echo "Usage: $N {start|stop|restart|reload|force-reload}" >&2 echo "Usage: $N {start|stop|restart|force-reload}" >&2 exit 1 ;; esac exit 0 rebuildd/debian/python-module-stampdir/0000755000000000000000000000000011516234765015417 5ustar rebuildd/debian/python-module-stampdir/rebuildd0000644000000000000000000000000011516235002017103 0ustar rebuildd/debian/rebuildd.manpages0000644000000000000000000000003210720364127014264 0ustar rebuildd.1 rebuildd-job.1 rebuildd/debian/changelog0000644000000000000000000002106512003463412012631 0ustar rebuildd (0.4.2) unstable; urgency=low [ Gauvain Pocentek ] * Remove unused code (Closes: #671635) * Support multiple recipients * Improve some error messages * Add a requeue command * Fix the number of arguments for jobs deps * Allow to use sendmail * Change the default date format * Add no_system_arch option * Reload before adding arch [ Julien Danjou ] * Update default dists to symbolic names * Fix PBUILDER_OTHER_OPTIONS[] examples in rebuildd.default (Closes: #682422) -- Julien Danjou Thu, 24 May 2012 16:50:03 +0200 rebuildd (0.4.1) unstable; urgency=low [ Daniel Dehennin ] * [10b50e1] Fix unit tests: Clean database drop and disable TestRebuildd.test_release_jobs(). * rebuildd/Rebuildd.py (Rebuildd.init): Add sqlobject connection as "hidden" attribute to clear and drop the database in tests. * rebuildd/Job.py (Job.__init__): Auto vivification of the log, required when the job is added through Rebuildd().add_job(). * tests/RebuilddTestSetup.py: Import Log class to create the corresponding table. (rebuildd_global_test_setup): Drop tables before creating them if they exist. (rebuildd_global_test_teardown): Clear the database as much as possible. * tests/TestDistribution.py: Import new teardown helper. (TestDistribution.tearDown): Use the new teardown helper. * tests/TestJob.py: Import new teardown helper. (TestJob.tearDown): Use the new teardown helper. (TestJob.test_DB_OK): Test if the database is created. (TestJob.test_send_build_log): Create a fake log file or send_build_log will raise an OSError. * tests/TestRebuildd.py: Import new teardown helper. (TestRebuildd.tearDown): Use the new teardown helper. (TestRebuildd.test_release_jobs): Disable because is does not work with database cleanup. (TestRebuildd.test_get_new_jobs): Fix distribution to create the job with the default configuration. (TestRebuildd.test_fix_job): Ditoo. (TestRebuildd): Remove useless cleanup since database is dropped on tearDown. (Closes: #657887) * [037e391] Switch from CDBS to debhelper and from python-support to dh_python2. * debian/control (Build-Depends): Remove cdbs. Replace python (>= 2.5) by python-all-dev (>= 2.6.6-3) for dh_python2. Remove deprecated python-support. * debian/rebuildd.install: Install rebuildd.py binary * debian/rules: Remove CDBS directives. (override_dh_clean): Remove debian/rules generated files. (override_dh_auto_test): Run test suite. (override_dh_auto_build): Generate man pages and default configuration file. (override_dh_installinit): Need two calls to dh_install init for both services. (override_dh_install): Remove rebuildd.py extension. Replace version in python libraries. * debian/pyversion: Useless with dh_python2. [ Julien Danjou ] * Fix silents error in rebuildd-job and jobs sticking in BUILDING state on error (Closes: #570425) * Fix Vcs-Browser * Bump standard version -- Julien Danjou Fri, 09 Mar 2012 11:46:21 +0100 rebuildd (0.4) unstable; urgency=low [ Julien Danjou ] * Bump version to indicate schema change and add a note in NEWS.Debian (Closes: #632525) * Bump standard version [ Daniel Dehennin ] * Fix homepage URL * Make copyright format DEP5 valid * Store log in database (Closes: #657635) * Add missing status in the Web interface (Closes: #657636) -- Julien Danjou Fri, 27 Jan 2012 17:47:37 +0100 rebuildd (0.3.11) unstable; urgency=low * Make SMTP host configurable (Closes: #570426) * Use bash to call 00_update_build_system (Closes: #632055) -- Julien Danjou Wed, 29 Jun 2011 16:13:22 +0200 rebuildd (0.3.10) unstable; urgency=low * Do not raise string exceptions (Closes: #585348) -- Julien Danjou Thu, 10 Jun 2010 12:32:01 +0200 rebuildd (0.3.9) unstable; urgency=low * Update for python-apt API transition (Closes: #571767) -- Julien Danjou Tue, 02 Mar 2010 13:56:27 +0100 rebuildd (0.3.8) unstable; urgency=low * Fix rebuildd-init-build-system advices on build_cmd (Closes: #561926) * Bump standard version * Change format source to 3.0 -- Julien Danjou Mon, 08 Feb 2010 14:50:51 +0100 rebuildd (0.3.7) unstable; urgency=low * Fix incorrect dependencies on init script (Closes: #541546) * Add missing entry in rebuildd-job manpage (Closes: #553421) * Bump standard version -- Julien Danjou Wed, 18 Nov 2009 20:17:47 +0100 rebuildd (0.3.6) unstable; urgency=low * Fix bug with new webpy in rebuildd-httpd (Closes: #536372) * Fix bashisms in maintenance script (Closes: #530179) * Add homepage (Closes: #529714) * Bump standard version -- Julien Danjou Tue, 04 Aug 2009 12:34:53 +0200 rebuildd (0.3.5) unstable; urgency=low * Bump standard version * Fix arch usage in httpd (Closes: #486146) * Fix typo in rebuildd.default (Closes: #497275) * Support passing other pbuilder options to rebuildd-init-build-system (Closes: #497295) -- Julien Danjou Sun, 31 Aug 2008 20:51:29 +0200 rebuildd (0.3.4) unstable; urgency=low * Add {} around variable names (Closes: #486143) -- Julien Danjou Fri, 13 Jun 2008 20:07:21 +0200 rebuildd (0.3.3) unstable; urgency=low * Switch to python2.5 which is now default -- Julien Danjou Wed, 07 May 2008 13:42:28 +0200 rebuildd (0.3.2) unstable; urgency=low * Set ip to 127.0.0.1 by default for obvious security reason * Fix bashisms in rebuildd-init-build-system (Closes: #455371) * Bump standard version * Fix dependency since python 2.5 is not default -- Julien Danjou Thu, 17 Apr 2008 11:35:22 +0200 rebuildd (0.3.1) unstable; urgency=low * Fix maintenance script for $arch support -- Julien Danjou Wed, 05 Dec 2007 08:17:29 +0100 rebuildd (0.3.0) unstable; urgency=low * Store timestamp of changed status * Add 'build_more_recent' option, to build only the more recent version of a package and set others to GIVEUP * Add a 'build_arch_any' to build (or not) package we want to build anywhere This works like another arch * Add 'priority' to package * Add quinn-diff support to rebuildd-job * Fix bashim in debian/rules * Add python-apt to build-deps * Add some doc in README.Debian (Closes: #443636) * Fix Web templates (Closes: #443668) * Add support for building more than one arch on a node (Closes: #447759) -- Julien Danjou Fri, 30 Nov 2007 14:29:48 +0000 rebuildd (0.2.2) unstable; urgency=low * Add rebuildd-job manpage * Actually we shouldn't add jobs if they are already in database If needed, status should be changed * Add delete to rebuildd-job * Add a fix option to fix database state if rebuildd crashed * Remove send_build_logs, not used anymore, cleaning up code * Do not restart on upgrade * Print all active thread count in logs * Now we handle 3 types of build failures, more clear -- Julien Danjou Sat, 11 Aug 2007 02:35:22 +0200 rebuildd (0.2.1) unstable; urgency=low * Add --download-only to apt-get source by default and put only one -q * If a job is building, don't allow to add it * Rewrite rebuildd-job stats function with SQL, very very very very very very very very faster * Add headers and footers to build logs * Dont't try to resend old mails, otherwise we send it twice sometimes. * Fix a bug when database is empty in rebuildd-job stats * Limit job searching to max_jobs to be more faster * Fix a bug with event/lock, which cancel all jobs at once -- Julien Danjou Tue, 07 Aug 2007 20:17:14 +0200 rebuildd (0.2.0) unstable; urgency=low * Upload to Debian (Closes: #432302) * Stock builder hostname in job table * Add new max_jobs config var, used to load max_jobs new jobs This adds a new state WAIT_LOCKED for locking jobs handled by a build. * Handle SIGINT like SIGTERM * Don't stock build_time, stock build_start and build_end * Move http server outside, and standalone * Create unit tests * Init scripts now use LSB * Rewrite Web server and interface with web.py With new templates, thanks Clément Danjou (my little bro) for them. :-) I still suck with HTML. -- Julien Danjou Tue, 10 Jul 2007 10:36:33 +0200 rebuildd (0.1.0) unstable; urgency=low * Initial release -- Julien Danjou Sat, 07 Jul 2007 14:44:58 +0200 rebuildd/debian/rules0000755000000000000000000000236012003461770012041 0ustar #!/usr/bin/make -f # export DH_VERBOSE=1 ifneq (,$(findstring UNRELEASED,$(shell head -n 1 debian/changelog))) VERSION="$(shell git describe 2>/dev/null || echo devel)" else VERSION=$(shell head -n1 debian/changelog | cut -d\( -f2 | cut -d\) -f1) endif %: dh --with python2 $@ override_dh_clean: dh_clean rebuildd.1 rebuildd-job.1 debian/rebuilddrc override_dh_auto_test: ./tests/runtest.py override_dh_auto_build: xsltproc -''-nonet /usr/share/sgml/docbook/stylesheet/xsl/nwalsh/manpages/docbook.xsl debian/rebuildd.manpage.xml xsltproc -''-nonet /usr/share/sgml/docbook/stylesheet/xsl/nwalsh/manpages/docbook.xsl debian/rebuildd-job.manpage.xml printf 'from rebuildd.RebuilddConfig import RebuilddConfig\nprint RebuilddConfig(True).dump()\n' | python > debian/rebuilddrc override_dh_installinit: dh_installinit --no-restart-on-upgrade dh_installinit --name=rebuildd-httpd --no-restart-on-upgrade override_dh_install: dh_install mv debian/rebuildd/usr/sbin/rebuildd.py debian/rebuildd/usr/sbin/rebuildd sed -i "s,^__version__.*$=,__version__ = \"$(VERSION)\"," debian/rebuildd/usr/lib/python*/dist-packages/rebuildd/*.py .PHONY: override_dh_auto_clean override_dh_auto_test override_dh_auto_build override_dh_installinit override_dh_install rebuildd/debian/compat0000644000000000000000000000000210720364127012160 0ustar 5 rebuildd/debian/README.Debian0000644000000000000000000000042110724757316013032 0ustar rebuildd ======== In order to use rebuildd you need to: 1. Edit configuration files; 2. Run rebuildd-init-build-system to create pbuilder archives; 3. Run rebuildd init to create database tables. -- Julien Danjou Tue, 27 Nov 2007 15:07:10 +0100 rebuildd/debian/pycompat0000644000000000000000000000000210720364127012531 0ustar 2 rebuildd/debian/rebuildd.install0000644000000000000000000000027512003461770014146 0ustar maintenance etc/rebuildd rebuildd.py usr/sbin rebuildd-init-build-system usr/sbin rebuildd-httpd usr/sbin rebuildd-job usr/sbin debian/rebuilddrc /etc/rebuildd templates usr/share/rebuildd