pax_global_header00006660000000000000000000000064115764107250014522gustar00rootroot0000000000000052 comment=a498b7528e0ea773ec0d37db2c9e907618de58b1 jigit-1.19/000077500000000000000000000000001157641072500125625ustar00rootroot00000000000000jigit-1.19/COPYING000066400000000000000000000431061157641072500136210ustar00rootroot00000000000000 GNU GENERAL PUBLIC LICENSE Version 2, June 1991 Copyright (C) 1989, 1991 Free Software Foundation, Inc. 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed. Preamble The licenses for most software are designed to take away your freedom to share and change it. By contrast, the GNU General Public License is intended to guarantee your freedom to share and change free software--to make sure the software is free for all its users. This General Public License applies to most of the Free Software Foundation's software and to any other program whose authors commit to using it. (Some other Free Software Foundation software is covered by the GNU Library General Public License instead.) You can apply it to your programs, too. When we speak of free software, we are referring to freedom, not price. Our General Public Licenses are designed to make sure that you have the freedom to distribute copies of free software (and charge for this service if you wish), that you receive source code or can get it if you want it, that you can change the software or use pieces of it in new free programs; and that you know you can do these things. To protect your rights, we need to make restrictions that forbid anyone to deny you these rights or to ask you to surrender the rights. These restrictions translate to certain responsibilities for you if you distribute copies of the software, or if you modify it. For example, if you distribute copies of such a program, whether gratis or for a fee, you must give the recipients all the rights that you have. You must make sure that they, too, receive or can get the source code. And you must show them these terms so they know their rights. We protect your rights with two steps: (1) copyright the software, and (2) offer you this license which gives you legal permission to copy, distribute and/or modify the software. Also, for each author's protection and ours, we want to make certain that everyone understands that there is no warranty for this free software. If the software is modified by someone else and passed on, we want its recipients to know that what they have is not the original, so that any problems introduced by others will not reflect on the original authors' reputations. Finally, any free program is threatened constantly by software patents. We wish to avoid the danger that redistributors of a free program will individually obtain patent licenses, in effect making the program proprietary. To prevent this, we have made it clear that any patent must be licensed for everyone's free use or not licensed at all. The precise terms and conditions for copying, distribution and modification follow. GNU GENERAL PUBLIC LICENSE TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION 0. This License applies to any program or other work which contains a notice placed by the copyright holder saying it may be distributed under the terms of this General Public License. The "Program", below, refers to any such program or work, and a "work based on the Program" means either the Program or any derivative work under copyright law: that is to say, a work containing the Program or a portion of it, either verbatim or with modifications and/or translated into another language. (Hereinafter, translation is included without limitation in the term "modification".) Each licensee is addressed as "you". Activities other than copying, distribution and modification are not covered by this License; they are outside its scope. The act of running the Program is not restricted, and the output from the Program is covered only if its contents constitute a work based on the Program (independent of having been made by running the Program). Whether that is true depends on what the Program does. 1. You may copy and distribute verbatim copies of the Program's source code as you receive it, in any medium, provided that you conspicuously and appropriately publish on each copy an appropriate copyright notice and disclaimer of warranty; keep intact all the notices that refer to this License and to the absence of any warranty; and give any other recipients of the Program a copy of this License along with the Program. You may charge a fee for the physical act of transferring a copy, and you may at your option offer warranty protection in exchange for a fee. 2. You may modify your copy or copies of the Program or any portion of it, thus forming a work based on the Program, and copy and distribute such modifications or work under the terms of Section 1 above, provided that you also meet all of these conditions: a) You must cause the modified files to carry prominent notices stating that you changed the files and the date of any change. b) You must cause any work that you distribute or publish, that in whole or in part contains or is derived from the Program or any part thereof, to be licensed as a whole at no charge to all third parties under the terms of this License. c) If the modified program normally reads commands interactively when run, you must cause it, when started running for such interactive use in the most ordinary way, to print or display an announcement including an appropriate copyright notice and a notice that there is no warranty (or else, saying that you provide a warranty) and that users may redistribute the program under these conditions, and telling the user how to view a copy of this License. (Exception: if the Program itself is interactive but does not normally print such an announcement, your work based on the Program is not required to print an announcement.) These requirements apply to the modified work as a whole. If identifiable sections of that work are not derived from the Program, and can be reasonably considered independent and separate works in themselves, then this License, and its terms, do not apply to those sections when you distribute them as separate works. But when you distribute the same sections as part of a whole which is a work based on the Program, the distribution of the whole must be on the terms of this License, whose permissions for other licensees extend to the entire whole, and thus to each and every part regardless of who wrote it. Thus, it is not the intent of this section to claim rights or contest your rights to work written entirely by you; rather, the intent is to exercise the right to control the distribution of derivative or collective works based on the Program. In addition, mere aggregation of another work not based on the Program with the Program (or with a work based on the Program) on a volume of a storage or distribution medium does not bring the other work under the scope of this License. 3. You may copy and distribute the Program (or a work based on it, under Section 2) in object code or executable form under the terms of Sections 1 and 2 above provided that you also do one of the following: a) Accompany it with the complete corresponding machine-readable source code, which must be distributed under the terms of Sections 1 and 2 above on a medium customarily used for software interchange; or, b) Accompany it with a written offer, valid for at least three years, to give any third party, for a charge no more than your cost of physically performing source distribution, a complete machine-readable copy of the corresponding source code, to be distributed under the terms of Sections 1 and 2 above on a medium customarily used for software interchange; or, c) Accompany it with the information you received as to the offer to distribute corresponding source code. (This alternative is allowed only for noncommercial distribution and only if you received the program in object code or executable form with such an offer, in accord with Subsection b above.) The source code for a work means the preferred form of the work for making modifications to it. For an executable work, complete source code means all the source code for all modules it contains, plus any associated interface definition files, plus the scripts used to control compilation and installation of the executable. However, as a special exception, the source code distributed need not include anything that is normally distributed (in either source or binary form) with the major components (compiler, kernel, and so on) of the operating system on which the executable runs, unless that component itself accompanies the executable. If distribution of executable or object code is made by offering access to copy from a designated place, then offering equivalent access to copy the source code from the same place counts as distribution of the source code, even though third parties are not compelled to copy the source along with the object code. 4. You may not copy, modify, sublicense, or distribute the Program except as expressly provided under this License. Any attempt otherwise to copy, modify, sublicense or distribute the Program is void, and will automatically terminate your rights under this License. However, parties who have received copies, or rights, from you under this License will not have their licenses terminated so long as such parties remain in full compliance. 5. You are not required to accept this License, since you have not signed it. However, nothing else grants you permission to modify or distribute the Program or its derivative works. These actions are prohibited by law if you do not accept this License. Therefore, by modifying or distributing the Program (or any work based on the Program), you indicate your acceptance of this License to do so, and all its terms and conditions for copying, distributing or modifying the Program or works based on it. 6. Each time you redistribute the Program (or any work based on the Program), the recipient automatically receives a license from the original licensor to copy, distribute or modify the Program subject to these terms and conditions. You may not impose any further restrictions on the recipients' exercise of the rights granted herein. You are not responsible for enforcing compliance by third parties to this License. 7. If, as a consequence of a court judgment or allegation of patent infringement or for any other reason (not limited to patent issues), conditions are imposed on you (whether by court order, agreement or otherwise) that contradict the conditions of this License, they do not excuse you from the conditions of this License. If you cannot distribute so as to satisfy simultaneously your obligations under this License and any other pertinent obligations, then as a consequence you may not distribute the Program at all. For example, if a patent license would not permit royalty-free redistribution of the Program by all those who receive copies directly or indirectly through you, then the only way you could satisfy both it and this License would be to refrain entirely from distribution of the Program. If any portion of this section is held invalid or unenforceable under any particular circumstance, the balance of the section is intended to apply and the section as a whole is intended to apply in other circumstances. It is not the purpose of this section to induce you to infringe any patents or other property right claims or to contest validity of any such claims; this section has the sole purpose of protecting the integrity of the free software distribution system, which is implemented by public license practices. Many people have made generous contributions to the wide range of software distributed through that system in reliance on consistent application of that system; it is up to the author/donor to decide if he or she is willing to distribute software through any other system and a licensee cannot impose that choice. This section is intended to make thoroughly clear what is believed to be a consequence of the rest of this License. 8. If the distribution and/or use of the Program is restricted in certain countries either by patents or by copyrighted interfaces, the original copyright holder who places the Program under this License may add an explicit geographical distribution limitation excluding those countries, so that distribution is permitted only in or among countries not thus excluded. In such case, this License incorporates the limitation as if written in the body of this License. 9. The Free Software Foundation may publish revised and/or new versions of the General Public License from time to time. Such new versions will be similar in spirit to the present version, but may differ in detail to address new problems or concerns. Each version is given a distinguishing version number. If the Program specifies a version number of this License which applies to it and "any later version", you have the option of following the terms and conditions either of that version or of any later version published by the Free Software Foundation. If the Program does not specify a version number of this License, you may choose any version ever published by the Free Software Foundation. 10. If you wish to incorporate parts of the Program into other free programs whose distribution conditions are different, write to the author to ask for permission. For software which is copyrighted by the Free Software Foundation, write to the Free Software Foundation; we sometimes make exceptions for this. Our decision will be guided by the two goals of preserving the free status of all derivatives of our free software and of promoting the sharing and reuse of software generally. NO WARRANTY 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. END OF TERMS AND CONDITIONS Appendix: How to Apply These Terms to Your New Programs If you develop a new program, and you want it to be of the greatest possible use to the public, the best way to achieve this is to make it free software which everyone can redistribute and change under these terms. To do so, attach the following notices to the program. It is safest to attach them to the start of each source file to most effectively convey the exclusion of warranty; and each file should have at least the "copyright" line and a pointer to where the full notice is found. Copyright (C) 19yy This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA Also add information on how to contact you by electronic and paper mail. If the program is interactive, make it output a short notice like this when it starts in an interactive mode: Gnomovision version 69, Copyright (C) 19yy name of author Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. This is free software, and you are welcome to redistribute it under certain conditions; type `show c' for details. The hypothetical commands `show w' and `show c' should show the appropriate parts of the General Public License. Of course, the commands you use may be called something other than `show w' and `show c'; they could even be mouse-clicks or menu items--whatever suits your program. You should also get your employer (if you work as a programmer) or your school, if any, to sign a "copyright disclaimer" for the program, if necessary. Here is a sample; alter the names: Yoyodyne, Inc., hereby disclaims all copyright interest in the program `Gnomovision' (which makes passes at compilers) written by James Hacker. , 1 April 1989 Ty Coon, President of Vice This General Public License does not permit incorporating your program into proprietary programs. If your program is a subroutine library, you may consider it more useful to permit linking proprietary applications with the library. If this is what you want to do, use the GNU Library General Public License instead of this License. jigit-1.19/ChangeLog000066400000000000000000000077761157641072500143550ustar00rootroot00000000000000Version 1.19 (2011-06-16) * Improved error-handling in mkimage and libjte. * mkjigsnap rewrite: + merge two old implementations + switch to perl: cleaner code, much faster snapshot generation + added -v (verbose) and -N (dry-run) options * Put AC_CONFIG_MACRO_DIR in a better place in the libjte configure script. * Remove -Werror from the Makefile - gcc is getting more and more pedantic with every release, and this may cause package builds to fail. Version 1.18 (2011-04-30) * More dynamic memory handling in mkimage, instead of allocating fixed-size strings on the stack. Should now work on Hurd too. * Fix man page syntax errors. * s,config.h,../config.h, in libjte for easier use by the xorriso folks Version 1.17 (2011-03-16) * Rename mkimage to jigit-mkimage, to avoid a naming clash. * Sanity-check jigdo files - require a valid JigsawDownload header by default. * Multiple fixes for how MD5 comparisons are dealt with. * Add libjte - jigdo generation code that used to be embedded in mkisofs/genisoimage. Now a shared library useable in xorriso etc. Version 1.16 (2007-10-18) * Remove build-stamp from the release tarball, to make debian package builds work better * Fix wget options typo * Add proper working bzip2-compressed template support in jigdump and mkimage * In mkimage, don't stat every file mentioned in the md5 list until we need to. * Other small cleanups in jigdump and mkimage Version 1.15 (2005-01-17) * Minor compilation fixes * Cope with missing files better * Minor jigdo file-handling cleanups * mkimage code refactored to make it easier to maintain * Cleaned up licensing - added explicit GPL v2 headers all over * Fix image reconstruction with jigit Version 1.14 (2004-10-12) * Fixed generation of rsyncsums * Added -d option to mkjigsnap to specify DIRNAME * Minor prettiness tweaks to output UI of mkimage and jigit * 64-bit printf fixes Version 1.13 (2004-09-05) * Improved the documentation, listing new options to mkimage * Fixed/updated the debian-cd patch Version 1.12 (2004-09-05) * Improved the documentation, listing new options to mkimage * Fixed/updated the debian-cd patch Version 1.11 (2004-09-05) * Added rsync sums into JTE output * Added jigit script * Added extra options to mkimage to support jigit Version 1.10 (2004-09-01) * Removed the last md5 check; no longer needed * Added new -jigdo-force-md5 option; any files matching specified paths MUST match an MD5 supplied, or fail * Fixed a couple of typos. * Added initial work for bz2 support. * debian-cd diff now includes updated grab_md5 and jigdo_cleanup scripts that must be used. * mkisofs now leaves a blank [Servers] section in the .jigdo file - jigdo_cleanup does the work. Version 1.9 (2004-08-30) * Added MD5 reverse lookups to pick up boot files; helps reduce CD/DVD #1 size substantially, BUT -md5-list option now mandatory. * Fixed sizing bug in HFS hybrid template code. Version 1.8 (2004-08-27) * Tweaked the -exclude option to reduce template file sizes. Version 1.7 (2004-08-19) * Fixed a couple of silly bugs introduced; don't assume template creation. * Added README files and man page sections for the new boot support. Version 1.6 (2004-07-14) * Added code to check MD5 sums in mkisofs; should no longer need to run mirror_check any more. See the -md5-list option. Version 1.5 (2004-07-05) * Minor bug-fixes and updates to mkimage * Added iso-image.pl wrapper script Version 1.4 (2004-06-26) * Added mkimage * Moved utils and patch into a tarball Version 1.3 (2004-06-18) * Fixed crash without jigdo options * Cleaned up mipsel boot code * Make it easier to specify the boot command line for hppa * Minor cosmetic .jigdo cleanup Version 1.2 (2004-06-14) * Added boot support for alpha, hppa, mips, mipsel * Cosmetic changes to the .jigdo file Version 1.1 (2004-06-10) * Removed .jte file code * Large cleanup to reduce patch size Version 1.0 (2004-06-07) * Initial release * Produced .jigdo, .template and .jte files jigit-1.19/JTE-patch1.gz000066400000000000000000000173051157641072500147320ustar00rootroot00000000000000? kAJTE-patch1\{wH|^¬A869̌$a!;;3R Iz y=w9'QWWwWXd´x0MM'88b W3-^Wjg8xU g"ZrmWՎ4:[f֨ԇjz+J/qhf|3~T9!`x {EA_FcټFGh;u"/ޝ{/e:{Y-2 λg:J)e^F6.;:!?fNX~HCEYovx絪`f]ݏ4 #b㔙/;zsϢgqz̞S^A&:tW\{ьfdFdRiX=e8xS]3mw7f2 l73pec^`퐙7"nи!s(d87, +bciNCNw6,"v7lp# #f }x KA3(Ìba ˘70DEƵ+0Hs&l]:q:a9lh_.-kܰ e"Ɇ! Ny)kGNvnp(?3Ϗl }E MEHB6 ,3X >L|`#b48 gNvh .d=Xyyq\@=-dUDU­f<_0V m,m=˴o\.1q5AaEK| i4ʍ:IDwћp~thgQ䷪Usi&|v .l?ԽH̦vVUohXMo5%39:>.<&KoO\'[5x r$Le9-]8xy6XF( ՌC4%_& 80^ǰ%`_F`Aܒ2+\IbPFQ;=-Ъ ˺ ZX1'|b2Ɣ-lp00CuFrMX v֍M͹40`uBՂz ρbLͅΰZcgb<>Ȧ!['R{ssasDž_ /;=A`qyDF‱H @<˃yYB9M Icl[ِ?}&~5Đŗz)q]ogpO9;.j uBW8b0gZ0a{R ]R{BKG1*A$)}@WzJĊ'uK4H+̛ S#`V0"Ng8j??9zMDe5"H0\ϧuLV+\DrɊ`Ϣ|!t_D:E`&!;HC0RȎZ}xFW&v< 2dͳB{r64acPfNG#0-'G -i[1"/IgaKJ-£;qJ֊Hyp>2Tg~b*LOFCF Tq2KE3,`Hʔ܀ XCP)8Qd0r-91B(H!TEt{- h?`Hn}/fo//m10w9ԣj՟+ju!r0Xf>a(8p׆0LJo7޴TuD^ d1 kJX «i9EܻǍA ɗXU5폆FtTdڏн@Q"`8 tK ,N)S  $޶tAoSC'cORM]q~M> /D_Yp?K@TAZSc}]ԗPfj ~K>i/2U"uV@@6)t+= t&`vI`(۪ W&(mem E)z"1/ !|V2+ۆw0@д{>A#$kX}}xחQ&kD54'LVd2n@PO/ 5qOQŠyమW Zn=+v86;Gj k-Bsv[ \),OV,+c.:*l!@2pOgT`ށcQhļ8*W1@V9kQe(F5+ طmW,סTBEAuxh`fۊ(MK_I89gK[$ky%f@΁,$>?*ױ`y]J1G mݺ3X䴱(Q+k.⊌H[Ti=ޝx9-Ji/,Tݥ;]M5} O\V$kXfRRU8zrF)W "ؿL%C#ݤu?@#uGܽi7VN$SkRx7qX-I5!ZiV8*F={xGTbܵt!#]Flot,j6٪5і̛XQ.. #XAl6N#:qgG\q^7!PgG:$&dIhniɶa$ Tn3́o,CJfGLgNo7a7a;xɆbԋH^ҫH,,V'zAJ+t>85K4Av`НwL|{ .AEӧr,DÕcӘBe/S8HU|15Iv\Gr NEwJݵH<Ϻ>>zz< N::Z54;Z6q0W2]5GM^?߿O?_ez${>yV\Y[U%s*˦DtWk+Bn~>[xq2QKo~xoŸgc䔀RMx$ɠ]sL8v.(jOI|9_rڻA:2l<^=еh6mW3llB1uHS 2% T20߻,dS=Anow9)gUq@N&b..m=DH_EspÝmjBB-zW$~p&Ԙ~w-$*-5*҈kl@~9xvڮTL&Redw{,` CKE c3s =W#G)mW 5Z E,Bjj ]/YsY{WP:#U{? \}]Ͽv]]u/>:A8_9%r?cnY#c1\3N7 c'! 9Bdk}P.vCX"Q6PCV)jI6JiIDLVeHIe ]euCi&l 4,7eMjA\0B._H30'?U.nm %:>P-!=Uf?SpEIyW8Rv: G*o*;t |«Jʢj!]=( FMbðzȨ, h歗R/kmwOIVĭ6n~<{6:ϓ iL)Z5]--Òy֦*oAbݲ1Nƞ1Ʋj h,X㰾QN{5[wT1Eqb_R ڌōXR Hp is4L9r&#s++O 6k/TW}@3q(Z3UvK8d 4K)6E,Cn+UcI0 Z6O]j7fաBrW#J7*k#,l>KxjVVC TG:oF Q0 Vsɑ#C2=dVUa]n/0z F(8SuP~K.7N/>f2rRpaEM|1o96Yt-M9˽$# D^e&%a=bKWz&w[ۀ^S/29~^^spx=]!~2 p >ЦQ26ŪQ |6Jv/]3YMo"/BETϹT ]-1 [oYٚ% 1opXkBPUK7u]qsT{3:u˔&SPz X4i-TaQ!^I`80,E9Ty YEY:~aW*)*obZVʚ |YzSr': N6 $Zgq\*6ݥ{=TmfMgIwD"sdr`CxX!W*el2=vIf ݚ61E''­n$Z=7,ZfHU(2b/%_Zen9 -QufoE#MtZtsOiQ.CXV }׼\pxb2TCu,=DFwȽ3ln<8^1<^N;|Lެ֛B. b)NKk˩r<^l".O,Mp*OlZx#?D-sK~mjigit-1.19/Makefile000066400000000000000000000010311157641072500142150ustar00rootroot00000000000000BINS = jigdump jigit-mkimage jigsum rsyncsum lib CFLAGS = -g -Wall -D_FILE_OFFSET_BITS=64 -D_LARGEFILE_SOURCE -D_LARGEFILE64_SOURCE #CC = gcc all: $(BINS) jigit-mkimage: mkimage.o endian.o md5.o $(CC) -o $@ $+ -lz -lbz2 jigsum: jigsum.o md5.o $(CC) -o $@ $+ rsyncsum: rsync.o md5.o $(CC) -o $@ $+ jigdump: jigdump.o md5.o $(CC) -o $@ $+ lib: libjte/Makefile make -C libjte libjte/Makefile: cd libjte && ./configure clean: rm -f *.o $(BINS) *~ build-stamp -make -C libjte clean distclean: clean -make -C libjte distcleanjigit-1.19/README000066400000000000000000000342701157641072500134500ustar00rootroot00000000000000README for JTE 1.19 Steve McIntyre 16 June 2011 License - GPL v2. See the file COPYING for more details. JTE - Jigdo Template Export =========================== Introduction Jigdo is a useful tool to help in the distribution of large files like CD and DVD images. See Richard Atterer's site for more details. Debian CDs and DVD ISO images are published on the web in jigdo format to allow end users to download them more efficiently. Jigdo is generic and powerful - it can be used for any large files that are made up of smaller files. However, to be this generic is costly. Creating jigdo files from ISO images is quite inefficient - to work out which files are included in the ISO image, jigdo has to calculate and compare checksums of every possible file and every extent in the image. Essentially it has to brute-force the image. On my home system, it can take several hours to do this for a 4.5GB DVD image. There are a few ways to improve this that I can see: 1. Modify jigdo so it knows about the internals of ISO images and can efficiently scan them (bad, not very generic for jigdo) 2. Write a helper tool to dump extra information for jigdo to use alongside the ISO image (helper tool written, but modifying jigdo to use this looks HARD) 3. Patch ISO creation tools to write .jigdo and .template files alongside the ISO image I've now done the third of these, and called the code JTE (or Jigdo Template Export). The code works fine, and runs in a very small fraction of the time taken to run genisoimage and jigdo separately. The output .jigdo and .template files work correctly, i.e. jigdo-file and the wrapper script jigdo-mirror accept them and will generate an ISO image that exactly matches the original. However, cdrkit / genisoimage is now deprecated in my opinion. The nice people working on xorriso (George Danchev and Thomas Schmitt) have helped a huge amount by cleaning up some of my code and wrapping it properly in a shared library. The code is now included here as libjte, and re-licensed slightly as LGPL v2.1 or later. Tools ===== JTE also comes with some extra tools: jigit-mkimage, a simple and very fast tool to reconstruct image files from .jigdo and .template files. It doesn't have any logic to cope with downloading missing files, but will list the missing files that are needed. It is also much faster for people (like me!) who already have full local mirrors. iso-image.pl is a CGI script to wrap around jigit-mkimage if you'd like to be able to offer images for HTTP download without using up multiple gigabytes of disk space. And for added network efficiency the perl CGI also supports HTTP v1.1 byte ranges so clients can resume aborted downloads. jigit is a first attempt at a user-friendly wrapper for jigit-mkimage on a user's machine. ---------------------------------------------------------------------- How to use JTE in genisoimage To use the jigdo creation code, specify the location of the output .jigdo and .template files alongside the ISO image. You can also specify the minimum size beneath which files will just be dropped into the binary template file data rather than listed as separate files to be found on the mirror, and exclude patterns to ignore certain files in the same way. And paths in the original filesystem can be mapped onto more global namespaces using the [Servers] section in the .jigdo file. For example: genisoimage -J -r -o /home/steve/test1.iso \ -jigdo-jigdo /home/steve/test1.jigdo \ -jigdo-template /home/steve/test1.template \ -jigdo-min-file-size 16384 \ -jigdo-ignore "README*" \ -jigdo-force-md5 /pool/ \ -jigdo-map Debian=/mirror/debian \ -md5-list /home/steve/md5.list \ /mirror/jigdo-test If the -jigdo-* options are not used, the normal genisoimage execution path is not affected at all. The above invocation will create 3 output files (.iso, .jigdo and .template). Multiple -jigdo-ignore and -jigdo-map options are accepted, for multiple ignore and map patterns. Use the -md5-list option to specify the location of a list of files and their md5sums in normal md5sum format. genisoimage will then compare the checksum of each file it is asked to write against the checksum of that file in the list. It will abort on any mismatches. The MD5 list file must list all the files that are expected to be found and listed in the output .jigdo file. The -jigdo-force-md5 option specifies a path where all files are expected to have an MD5 entry (e.g. /pool/). Then if any files do not have a match, they must have been corrupted and genisoimage will abort. More options have now been added in version 1.2 so that you can specify the location of boot files within the ISO image. Previously the four architectures alpha, hppa, mips and mipsel needed separate tools to make an ISO image bootable. This also made life very hard when trying to produce jigdo files. Instead, I've folded boot support for those architectures into this patch so that genisoimage will do all the work: +-----------------------------------------------------------------------------+ |Alpha | |-----------------------------------------------------------------------------| |-alpha-boot |Specify the location of the boot image (relative to the root| | |of the ISO image) | |-----------------------------------------------------------------------------| |Hppa | |-----------------------------------------------------------------------------| |-hppa-cmdline |Specify the hppa boot command line. Separate elements with | | |commas or spaces. | |----------------+------------------------------------------------------------| |-hppa-kernel-32 |Specify the location of the 32-bit boot image (relative to | | |the root of the ISO image) | |----------------+------------------------------------------------------------| |-hppa-kernel-64 |Specify the location of the 64-bit boot image (relative to | | |the root of the ISO image) | |----------------+------------------------------------------------------------| |-hppa-bootloader|Specify the location of the bootloader code (iplboot, | | |relative to the root of the ISO image) | |----------------+------------------------------------------------------------| |-hppa-ramdisk |Specify the location of the ramdisk (relative to the root of| | |the ISO image) | |-----------------------------------------------------------------------------| |Mips | |-----------------------------------------------------------------------------| |-mips-boot |Specify the location of the boot image (relative to the root| | |of the ISO image) | |-----------------------------------------------------------------------------| |Mipsel | |Mipsel is awkward because we have to parse the ELF header of the boot loader | |and write some locations from it into the boot sector. Ick! | |-----------------------------------------------------------------------------| |-mipsel-boot |Specify the location of the boot image (relative to the root| | |of the ISO image) | +-----------------------------------------------------------------------------+ ---------------------------------------------------------------------- How it works I've hooked all the places in genisoimage where it will normally write image data. All the normal data write calls (directory entries etc.) I simply pass through and build into the template file. Any file data entries are passed through with information about the original file. If that file is large enough, I grab the filename and the MD5 of the file's data so I can just write a file match record into the template file (and then the jigdo file). ---------------------------------------------------------------------- How fast is it? On my laptop (600MHz P3, slow laptop disk) I can make a template file in parallel with the ISO image from a typical 500MB data set in about 2 minutes. By simply not creating the ISO (-o /dev/null), this time halves again. The data set I'm using here is a copy of the woody i386 r2 update CD, as it's a handy image I had lying around. On my faster home server machine (1.7GHz Athlon, 512MB RAM, fast SCSI disks), I can produce a 7GB DVD iso image with the jigdo and template files in about 8 minutes. A debian-cd run from start to finish to create DVD images takes about 25 minutes per architecture. Genisoimage is normally I/O-bound on this system, but when running the jigdo creation code it's now CPU bound - it's now running 2 MD5 checksums on each data block that it sees. To boost performance when creating images, running 2 copies of debian-cd in parallel on a single system should parallelise nicely - ideally run the CD and DVD versions of each architecture together to get maximum benefit from the dentry and page cache. ---------------------------------------------------------------------- How to use jigit-mkimage jigit-mkimage is a faster, local-only version of "jigdo-file make-image", again written in portable C. It takes a few options: +-----------------------------------------------------------------------------+ |-f |use when rebuilding the image | |---------+-------------------------------------------------------------------| |-j | | |---------+-------------------------------------------------------------------| |-t | | | | | |---------+-------------------------------------------------------------------| |-m to to find the files in the mirror | |path> | | |---------+-------------------------------------------------------------------| |-M |Don't attempt to build the image; just verify that all the | | |the specified file. | |---------+-------------------------------------------------------------------| |-v |Make the output logging more verbose. | |---------+-------------------------------------------------------------------| |-l |genisoimage | |---------+-------------------------------------------------------------------| | |Don't bother checking md5sums of the input files, or of the output | |-q |image. | | |WARNING: this may lead to corrupt images, but is much faster. | |---------+-------------------------------------------------------------------| |-s |will start at the beginning (offset 0). Added for iso-image.pl use | |---------+-------------------------------------------------------------------| |-e |will run all the way to the end of the image. Added for | | |iso-image.pl use | |---------+-------------------------------------------------------------------| | |Don't attempt to reassemble the image; simply parse the image | |-z |descriptor in the template file and print the image size. Added for| | |iso-image.pl use | +-----------------------------------------------------------------------------+ Specifying a start or end offset implies -q - it's difficult to check MD5 sums if the full image is not generated! ---------------------------------------------------------------------- How to use iso-image.pl iso-image.pl is a small perl wrapper script written to drive jigit-mkimage and turn it into a CGI. It will parse the incoming request (including byte-ranges) and call jigit-mkimage to actually generate the image pieces wanted. Configuration is simple: place iso-image.pl in a cgi-bin directory and set various paths in the script: * The path to jigit-mkimage * A log file location * The path to the template and jigdo files * Other options to jigit-mkimage (e.g. -q and match locations) ---------------------------------------------------------------------- How to use jigit Jigit is a small, user-friendly wrapper around jigit-mkimage to make life easier for end users to download or upgrade existing CDs and CD images. Configure it by editing /etc/jigit.conf or ~/.jigit.conf, then run jigit with just one command line option, the name of the distribution CD. Answer the questions it asks and it will do the rest. ---------------------------------------------------------------------- External integration The current released version of debian-cd in etch supports JTE out of the box. Wodim ships with integrated JTE code too. ---------------------------------------------------------------------- What's left to do? 1. Testing! :-) This is where you lot come in! Please play with this some more and let me know if you have any problems, especially with data corruption. 2. More documentation. 3. Support for non-local mirrors in jigit-mkimage. jigit-1.19/endian.c000066400000000000000000000114041157641072500141640ustar00rootroot00000000000000/* * endian.c * * Simple helper functions for reading and writing BE and LE numbers * * Copyright (c) 2004 Steve McIntyre * * GPL v2 - see COPYING */ #include "endian.h" /* Write a 64-bit quantity out into memory in BIG ENDIAN order */ void write_be64(in, out) unsigned long long in; unsigned char *out; { out[0] = (in >> 56) & 0xFF; out[1] = (in >> 48) & 0xFF; out[2] = (in >> 40) & 0xFF; out[3] = (in >> 32) & 0xFF; out[4] = (in >> 24) & 0xFF; out[5] = (in >> 16) & 0xFF; out[6] = (in >> 8) & 0xFF; out[7] = in & 0xFF; } /* Read in a 64-bit BIG ENDIAN quantity */ unsigned long long read_be64(in) unsigned char *in; { unsigned long long result = 0; result |= (unsigned long long)in[0] << 56; result |= (unsigned long long)in[1] << 48; result |= (unsigned long long)in[2] << 40; result |= (unsigned long long)in[3] << 32; result |= (unsigned long long)in[4] << 24; result |= (unsigned long long)in[5] << 16; result |= (unsigned long long)in[6] << 8; result |= (unsigned long long)in[7]; return result; } /* Write a 64-bit quantity out into memory in LITTLE ENDIAN order */ void write_le64(in, out) unsigned long long in; unsigned char *out; { out[0] = in & 0xFF; out[1] = (in >> 8) & 0xFF; out[2] = (in >> 16) & 0xFF; out[3] = (in >> 24) & 0xFF; out[4] = (in >> 32) & 0xFF; out[5] = (in >> 40) & 0xFF; out[6] = (in >> 48) & 0xFF; out[7] = (in >> 56) & 0xFF; } /* Read in a 64-bit LITTLE ENDIAN quantity */ unsigned long long read_le64(in) unsigned char *in; { unsigned long long result = 0; result |= (unsigned long long)in[0]; result |= (unsigned long long)in[1] << 8; result |= (unsigned long long)in[2] << 16; result |= (unsigned long long)in[3] << 24; result |= (unsigned long long)in[4] << 32; result |= (unsigned long long)in[5] << 40; result |= (unsigned long long)in[6] << 48; result |= (unsigned long long)in[7] << 56; return result; } /* Write a 48-bit quantity out into memory in LITTLE ENDIAN order */ void write_le48(in, out) unsigned long long in; unsigned char *out; { out[0] = in & 0xFF; out[1] = (in >> 8) & 0xFF; out[2] = (in >> 16) & 0xFF; out[3] = (in >> 24) & 0xFF; out[4] = (in >> 32) & 0xFF; out[5] = (in >> 40) & 0xFF; } /* Read in a 48-bit LITTLE ENDIAN quantity */ unsigned long long read_le48(in) unsigned char *in; { unsigned long long result = 0; result |= (unsigned long long)in[0]; result |= (unsigned long long)in[1] << 8; result |= (unsigned long long)in[2] << 16; result |= (unsigned long long)in[3] << 24; result |= (unsigned long long)in[4] << 32; result |= (unsigned long long)in[5] << 40; return result; } /* Write a 32-bit quantity out into memory in BIG ENDIAN order */ void write_be32(in, out) unsigned long in; unsigned char *out; { out[0] = (in >> 24) & 0xFF; out[1] = (in >> 16) & 0xFF; out[2] = (in >> 8) & 0xFF; out[3] = in & 0xFF; } /* Read in a 32-bit BIG ENDIAN quantity */ unsigned long read_be32(in) unsigned char *in; { unsigned long result = 0; result |= (unsigned long)in[0] << 24; result |= (unsigned long)in[1] << 16; result |= (unsigned long)in[2] << 8; result |= (unsigned long)in[3]; return result; } /* Write a 32-bit quantity out into memory in LITTLE ENDIAN order */ void write_le32(in, out) unsigned long in; unsigned char *out; { out[0] = in & 0xFF; out[1] = (in >> 8) & 0xFF; out[2] = (in >> 16) & 0xFF; out[3] = (in >> 24) & 0xFF; } /* Read in a 32-bit LITTLE ENDIAN quantity */ unsigned long read_le32(in) unsigned char *in; { unsigned long result = 0; result |= (unsigned long)in[0]; result |= (unsigned long)in[1] << 8; result |= (unsigned long)in[2] << 16; result |= (unsigned long)in[3] << 24; return result; } /* Write a 16-bit quantity out into memory in BIG ENDIAN order */ void write_be16(in, out) unsigned short in; unsigned char *out; { out[0] = (in >> 8) & 0xFF; out[1] = in & 0xFF; } /* Read in a 16-bit BIG ENDIAN quantity */ unsigned short read_be16(in) unsigned char *in; { unsigned short result = 0; result |= (unsigned short)in[0] << 8; result |= (unsigned short)in[1]; return result; } /* Write a 16-bit quantity out into memory in LITTLE ENDIAN order */ void write_le16(in, out) unsigned short in; unsigned char *out; { out[0] = in & 0xFF; out[1] = in & 0xFF >> 8; } /* Read in a 16-bit LITTLE ENDIAN quantity */ unsigned short read_le16(in) unsigned char *in; { unsigned short result = 0; result |= (unsigned short)in[0]; result |= (unsigned short)in[1] << 8; return result; } jigit-1.19/endian.h000066400000000000000000000021151157641072500141700ustar00rootroot00000000000000/* * endian.h * * Simple helper functions for reading and writing BE and LE numbers * * Copyright (c) 2004 Steve McIntyre * * GPL v2 - see COPYING */ void write_be64 (unsigned long long in, unsigned char *out); unsigned long long read_be64 (unsigned char *in); void write_le64 (unsigned long long in, unsigned char *out); unsigned long long read_le64 (unsigned char *in); void write_le48 (unsigned long long in, unsigned char *out); unsigned long long read_le48 (unsigned char *in); void write_be32 (unsigned long in, unsigned char *out); unsigned long read_be32 (unsigned char *in); void write_le32 (unsigned long in, unsigned char *out); unsigned long read_le32 (unsigned char *in); void write_be16 (unsigned short in, unsigned char *out); unsigned short read_be16 (unsigned char *in); void write_le16 (unsigned short in, unsigned char *out); unsigned short read_le16 (unsigned char *in); jigit-1.19/iso-image.pl000077500000000000000000000166131157641072500150030ustar00rootroot00000000000000#!/usr/bin/perl -w # # iso-image.pl # # (C) 2004 Steve McIntyre # # CGI wrapper to work with mkimage # # Parse the parameters and call mkimage as appropriate, paying # particular attention to byte ranges # # GPL v2 # # v 0.1 use strict; use File::Basename; use Socket; # Configure these for your system my $mkimage = "/usr/local/bin/jigit-mkimage"; my $logfile = "/var/log/jigdo-logs/log.$$"; my $template_dir = "/mirror/jigdo"; my $matches = "-q -m Debian=/mirror/debian -m Non-US=/mirror/debian-non-US"; my $size; my $image_name; open LOGFILE, ">> $logfile" || die "Unable to open logfile!\n"; # Log an error to the user sub user_error ($) { my $my_name = "iso_image.pl"; print "Status: 400 Invalid Request\n"; print "Content-type: text/html\n\n"; print "$my_name: @_

Abort.\n"; } # Log a message to the logfile and stop sub log_die ($) { print LOGFILE @_; die @_; } # Convert the .iso filename into a .jigdo sub jigdo_name ($) { my ($jigdo_name) = @_; $jigdo_name =~ s/\.iso$/\.jigdo/g; $jigdo_name = $template_dir . "/" . $jigdo_name; return $jigdo_name; } # Convert the .iso filename into a .template sub template_name ($) { my ($template_name) = @_; $template_name =~ s/\.iso$/\.template/g; $template_name = $template_dir . "/" . $template_name; return $template_name; } # Grab the image size out of the template file sub image_size ($) { my ($image_name) = @_; my $image_size; my $cmdline = "$mkimage -l " . $logfile . " -z -t " . template_name($image_name); open (FH, '-|', $cmdline); $image_size = ; close FH; return $image_size; } # We have no range headers; simply generate the full image sub produce_full_image ($$) { my ($image_name, $size) = @_; my $output_name = basename($image_name); my $cmdline; my $err = 0; $cmdline = "$mkimage -l " . $logfile; $cmdline = $cmdline . " -t " . template_name($image_name); $cmdline = $cmdline . " -j " . jigdo_name($image_name); $cmdline = $cmdline . " " . $matches; print "Status: 200 OK\n"; print "Content-Type: application/octet-stream\n"; print "Content-Disposition: inline;filename=$output_name\n"; print "Content-Length: $size\n\n"; $err = system($cmdline) >> 8; if ($err) { log_die ("Failed to rebuild image; error $err\n"); } } # Generate the desired range sub produce_range ($$$) { my ($image_name, $start, $end) = @_; my $output_name = basename($image_name); my $cmdline; my $err = 0; $cmdline = "$mkimage -l " . $logfile; $cmdline = $cmdline . " -t " . template_name($image_name); $cmdline = $cmdline . " -j " . jigdo_name($image_name); $cmdline = $cmdline . " -s " . $start; $cmdline = $cmdline . " -e " . $end; $cmdline = $cmdline . " " . $matches; # print "X-output: cmdline $cmdline\n"; $err = system($cmdline) >> 8; if ($err) { log_die ("Failed to rebuild image; error $err\n"); } } # Calculate start, end and length from the supplied range sub parse_range ($$) { my ($range, $size) = @_; my @offsets; my $content_length = 0; if (length($range) == 1 && $range =~ m/-/g ) { $offsets[0] = 0; $offsets[1] = $size - 1; } else { @offsets = split(/-/, $range, 2); } if (!defined($offsets[0]) || !length($offsets[0])) { $offsets[0] = -1; } if (!defined($offsets[1]) || !length($offsets[1])) { $offsets[1] = $size - 1; } if ($offsets[0] == -1) { $offsets[0] = $size - $offsets[1]; $offsets[1] = $size - 1; } # Make sure we have been given numbers $offsets[0] = int($offsets[0]); $offsets[1] = int($offsets[1]); # Check they're valid if ($offsets[0] < 0 || $offsets[0] >= $size) { print "Range start $offsets[0] invalid!\n"; log_die "Range start $offsets[0] invalid!\n"; } if ($offsets[1] < 0 || $offsets[1] >= $size) { print "Range end $offsets[1] invalid!\n"; log_die "Range end $offsets[1] invalid!\n"; } if ($offsets[0] > $offsets[1]) { print "Range start $offsets[0] after end $offsets[1]!\n"; log_die "Range start $offsets[0] after end $offsets[1]!\n"; } $content_length = $offsets[1] + 1 - $offsets[0]; return ($offsets[0], $offsets[1], $content_length); } # We've been asked for ranges. Calculate which ones, then start generating them sub produce_ranges ($$$) { my ($image_name, $size, $ranges) = @_; my $output_name = basename($image_name); my $cmdline; my $err = 0; my @range_array; my ($start, $end, $range, $length); chomp $ranges; $ranges =~ s/^.*\=//g; @range_array = split (/,/, $ranges); if (scalar(@range_array) == 1) { ($start, $end, $length) = parse_range($range_array[0], $size); print "Status: 206 Partial content\n"; print "Content-Type: application/octet-stream; filename=$output_name\n"; print "Content-Range: bytes $start-$end/$size\n"; print "Content-Length: $length\n\n"; $err = produce_range($image_name, $start, $end); } else { print "Status: 206 Partial content\n"; print "Content-Type: multipart/byteranges; boundary=THIS_STRING_SEPARATES\n\n"; for my $range (@range_array) { ($start, $end, $length) = parse_range($range, $size); print "--THIS_STRING_SEPARATES\n"; print "Content-Type: application/octet-stream; filename=$output_name\n"; print "Content-Range: bytes $start-$end/$size\n\n"; $err = produce_range($image_name, $start, $end); } print "--THIS_STRING_SEPARATES\n"; } } ################################################################################ # # All starts here # my $remote_host = $ENV{'REMOTE_HOST'}; my $remote_addr = $ENV{'REMOTE_ADDR'}; my $ultimate = $ENV{'HTTP_X_FORWARDED_FOR'}; my $iaddr; my $ult_name; # Sanity checking if (!defined($ARGV[0])) { user_error("You must specify an image name to download."); log_die "No file specified...!\n"; } if (!defined($remote_host) || !length($remote_host)) { $iaddr = inet_aton($remote_addr); $remote_host = gethostbyaddr($iaddr, AF_INET); } print LOGFILE "Connection made from $remote_addr ($remote_host)\n"; if (defined($ultimate) && length($ultimate)) { $iaddr = inet_aton($ultimate); $ult_name = gethostbyaddr($iaddr, AF_INET); print LOGFILE "Proxy for: $ultimate ($ult_name)\n"; } print LOGFILE "Asking for $ENV{SCRIPT_NAME}/$ENV{QUERY_STRING}\n"; print LOGFILE "Generating $ARGV[0]:\n"; print LOGFILE scalar localtime; print LOGFILE "\n\n\n"; $image_name = $ARGV[0]; if (! -f template_name($image_name)) { user_error("No template file found to match image name \"$image_name\""); log_die ("Couldn't find template file for image $image_name\n"); } if (! -f jigdo_name($image_name)) { user_error("No jigdo file file found to match image name \"$image_name\""); log_die ("Couldn't find jigdo file for image $image_name\n"); } $size = image_size($image_name); chomp $size; if (defined($ENV{HTTP_RANGE})) { # We have range(s) specified. Parse what we've been given and call # mkimage for each range produce_ranges($image_name, $size, $ENV{HTTP_RANGE}); } else { # If we don't have a Range: header, simply return the whole image produce_full_image($image_name, $size); } print LOGFILE "Done\n"; print LOGFILE scalar localtime; print LOGFILE "\n\n\n"; exit 0; jigit-1.19/jigdo.h000066400000000000000000000014451157641072500140330ustar00rootroot00000000000000/* * jigdo.h * * Common prototypes/macros/definitions * * Copyright (c) 2007 Steve McIntyre * * GPL v2 - see COPYING */ /* Compression algorithms supported in the compressed data blocks * inside a template file */ #define COMP_GZIP 1 #define COMP_BZIP2 2 /* Block types within the template file */ #define BLOCK_DATA 2 #define BLOCK_IMAGE 5 #define BLOCK_MATCH 6 /* Useful types and macros */ typedef long long INT64; typedef unsigned long long UINT64; typedef unsigned long UINT32; #ifndef LLONG_MAX # define LLONG_MAX (INT64)INT_MAX * INT_MAX #endif #ifndef MIN #define MIN(x,y) ( ((x) < (y)) ? (x) : (y)) #endif #define BUF_SIZE 65536 typedef enum state_ { STARTING, IN_DATA, IN_DESC, DUMP_DESC, DONE, ERROR } e_state; jigit-1.19/jigdump.1000066400000000000000000000012311157641072500143000ustar00rootroot00000000000000.TH JIGDUMP 1 "September 2004" "Jigit jigdo tools" .SH NAME jigdump \- Dump the contents of a template file .SH SYNOPSIS .B jigdump