debian/0000775000000000000000000000000012661070604007172 5ustar debian/control0000664000000000000000000000132412656405431010601 0ustar Source: xdelta3 Section: utils Priority: optional XS-Python-Version: all Maintainer: Ubuntu Developers XSBC-Original-Maintainer: A Mennucc1 Build-Depends: cdbs, debhelper, liblzma-dev Standards-Version: 3.9.4 Package: xdelta3 Architecture: any Depends: ${shlibs:Depends}, ${misc:Depends} Description: Diff utility which works with binary files Xdelta3 is a set of tools designed to compute changes between binary files. These changes (delta files) are similar to the output of the "diff" program, in that they may be used to store and transmit only the changes between files. The "delta files" that Xdelta3 manages are stored in RFC3284 (VCDIFF) format. debian/changelog0000664000000000000000000000516712661070604011055 0ustar xdelta3 (3.0.7-dfsg-2ubuntu0.2) trusty-security; urgency=medium * SECURITY UPDATE: buffer overflow in main_get_appheader - debian/patches/CVE-2014-9765.patch: add check to xdelta3-main.h, add test to xdelta3-test.h. - CVE-2014-9765 * debian/patches/fix_lzma_test.patch: fix lzma test so we can run the builtin tests. -- Marc Deslauriers Wed, 17 Feb 2016 08:10:41 -0500 xdelta3 (3.0.7-dfsg-2) unstable; urgency=low * fix FTBFS in testing/regtest.cc * add liblzma to build dependencies * fix compiling warnings -- A Mennucc1 Mon, 17 Jun 2013 08:56:43 +0200 xdelta3 (3.0.7-dfsg-1) unstable; urgency=low * New upstream release (Closes: #691337). * Switch build system to CDBS * Drop 'python-xdelta3' package, currently broken. * Bump standards version. -- A Mennucc1 Sun, 09 Jun 2013 12:08:02 +0200 xdelta3 (3.0.0.dfsg-1) unstable; urgency=low * New upstream release, "3.0 is a stable release series" * Remove unused build-dependency 'cdbs' * Thanks Luca for NMU -- A Mennucc1 Sun, 29 May 2011 22:59:32 +0200 xdelta3 (0y.dfsg-1.1) unstable; urgency=low * Non-maintainer upload. * Convert package to dh_python2 (Closes: #617151). + debian/control: - Build-depend on python-all-dev (>= 2.6.6-3~). - Drop python-central from Build-Depends. - Define X-Python-Version field. - Drop XB-Python-Version field. + debian/pycompat: - Useless, removed. + debian/rules: - Replace dh_pycentral with dh_python2. * debian/patches/unversioned_shebangs.diff: - Do not use versioned shebangs (Closes: #625751). -- Luca Falavigna Sun, 08 May 2011 13:28:08 +0200 xdelta3 (0y.dfsg-1) unstable; urgency=low * New upstream release. - Can read source from pipe/fifo , new -B option. - Has manpage (thanks Leo (costela) Antunes) * Convert to 3.0 (quilt) source format * Correct lintian warnings: - Update standards version, - add misc:Depends -- A Mennucc1 Mon, 22 Feb 2010 21:59:03 +0100 xdelta3 (0s.dfsg-1) unstable; urgency=low * New upstream release -- A Mennucc1 Mon, 19 Nov 2007 13:37:26 +0100 xdelta3 (0q.dfsg-2) unstable; urgency=low * Bug fix: "xdelta3: FTBFS: Not using -fPIC to create shared object.", thanks to Kurt Roeckx (Closes: #439656). -- A Mennucc1 Sun, 26 Aug 2007 16:06:58 +0200 xdelta3 (0q.dfsg-1) unstable; urgency=low * Initial Release (Closes: #433774) -- A Mennucc1 Thu, 26 Jul 2007 14:43:12 +0200 debian/control_px30000664000000000000000000000103012155074752011367 0ustar Package: python-xdelta3 Architecture: any Section: python Depends: ${python:Depends}, ${shlibs:Depends}, ${misc:Depends} Provides: ${python:Provides} Description: Xdelta3 python module Xdelta3 is a set of tools designed to compute changes between binary files. These changes (delta files) are similar to the output of the "diff" program, in that they may be used to store and transmit only the changes between files. The "delta files" that Xdelta3 manages are stored in RFC3284 (VCDIFF) format. . This is the python module. debian/xdelta3.install0000664000000000000000000000001711570526645012135 0ustar xdelta3 usr/bindebian/source/0000775000000000000000000000000011576404233010475 5ustar debian/source/format0000664000000000000000000000001411570526631011704 0ustar 3.0 (quilt) debian/compat0000664000000000000000000000000211570527422010373 0ustar 5 debian/python-xdelta3.substvars0000664000000000000000000000001612155060210014016 0ustar misc:Depends= debian/patches/0000775000000000000000000000000012661070640010621 5ustar debian/patches/Q_not_u0000664000000000000000000005012712157400001012141 0ustar --- a/testing/delta.h +++ b/testing/delta.h @@ -53,13 +53,13 @@ public: xd3_winst &winst = stream_.whole_target.inst[i]; switch (winst.type) { case XD3_RUN: - DP(RINT "%"Q"u run %u\n", winst.position, winst.size); + DP(RINT "%"Q" run %u\n", winst.position, winst.size); break; case XD3_ADD: - DP(RINT "%"Q"u add %u\n", winst.position, winst.size); + DP(RINT "%"Q" add %u\n", winst.position, winst.size); break; default: - DP(RINT "%"Q"u copy %u @ %"Q"u (mode %u)\n", + DP(RINT "%"Q" copy %u @ %"Q" (mode %u)\n", winst.position, winst.size, winst.addr, winst.mode); break; } --- a/testing/regtest.cc +++ b/testing/regtest.cc @@ -92,7 +92,7 @@ public: bool done = false; bool done_after_input = false; - IF_DEBUG1 (XPR(NTR "source %"Q"u[%"Q"u] target %"Q"u[%lu] winsize %lu\n", + IF_DEBUG1 (XPR(NTR "source %"Q"[%"Q"] target %"Q"[%lu] winsize %lu\n", source_file.Size(), options.block_size, target_file.Size(), Constants::READ_SIZE, Constants::WINDOW_SIZE)); @@ -102,7 +102,7 @@ public: xoff_t blks = target_iterator.Blocks(); - IF_DEBUG2(XPR(NTR "target in %s: %llu..%llu %"Q"u(%"Q"u) verified %"Q"u\n", + IF_DEBUG2(XPR(NTR "target in %s: %llu..%llu %"Q"(%"Q") verified %"Q"\n", encoding ? "encoding" : "decoding", target_iterator.Offset(), target_iterator.Offset() + target_block.Size(), @@ -152,8 +152,8 @@ public: xd3_source *src = (encoding ? &encode_source : &decode_source); Block *block = (encoding ? &encode_source_block : &decode_source_block); if (encoding) { - IF_DEBUG1(XPR(NTR "[srcblock] %"Q"u last srcpos %"Q"u " - "encodepos %"Q"u\n", + IF_DEBUG1(XPR(NTR "[srcblock] %"Q" last srcpos %"Q" " + "encodepos %"Q"\n", encode_source.getblkno, encode_stream.match_last_srcpos, encode_stream.input_position + encode_stream.total_in)); @@ -230,7 +230,7 @@ public: const Options &options) { vector ecmd; char buf[16]; - snprintf(buf, sizeof(buf), "-B%"Q"u", options.encode_srcwin_maxsz); + snprintf(buf, sizeof(buf), "-B%"Q"", options.encode_srcwin_maxsz); ecmd.push_back("xdelta3"); ecmd.push_back(buf); ecmd.push_back("-s"); @@ -975,7 +975,7 @@ void UnitTest() { // These are Xdelta tests. template void MainTest() { - XPR(NT "Blocksize %"Q"u readsize %"Q"u windowsize %"Q"u\n", + XPR(NT "Blocksize %"Q" readsize %"Q" windowsize %"Q"\n", T::BLOCK_SIZE, T::READ_SIZE, T::WINDOW_SIZE); Regtest regtest; TEST(TestEmptyInMemory); --- a/xdelta3-test.h +++ b/xdelta3-test.h @@ -409,7 +409,7 @@ test_compare_files (const char* tgt, con { if (obuf[i] != rbuf[i]) { - XPR(NT "byte %u (read %u @ %"Q"u) %d != %d\n", + XPR(NT "byte %u (read %u @ %"Q") %d != %d\n", (int)i, (int)oc, offset, obuf[i], rbuf[i]); diffs++; return XD3_INTERNAL; @@ -1787,7 +1787,7 @@ test_command_line_arguments (xd3_stream if (ratio >= TEST_ADD_RATIO + TEST_EPSILON) { XPR(NT "test encode with size ratio %.4f, " - "expected < %.4f (%"Q"u, %"Q"u)\n", + "expected < %.4f (%"Q", %"Q")\n", ratio, TEST_ADD_RATIO + TEST_EPSILON, dsize, tsize); stream->msg = "strange encoding"; return XD3_INTERNAL; @@ -2610,7 +2610,7 @@ test_string_matching (xd3_stream *stream if (inst->type == XD3_CPY) { *rptr++ = '@'; - snprintf_func (rptr, rbuf+TESTBUFSIZE-rptr, "%"Q"d", inst->addr); + snprintf_func (rptr, rbuf+TESTBUFSIZE-rptr, "%"Qd, inst->addr); rptr += strlen (rptr); } --- a/xdelta3.c +++ b/xdelta3.c @@ -2590,7 +2590,7 @@ xd3_getblk (xd3_stream *stream, xoff_t b ret = stream->getblk (stream, source, blkno); if (ret != 0) { - IF_DEBUG1 (DP(RINT "[getblk] app error blkno %"Q"u: %s\n", + IF_DEBUG1 (DP(RINT "[getblk] app error blkno %"Q": %s\n", blkno, xd3_strerror (ret))); return ret; } @@ -2608,8 +2608,8 @@ xd3_getblk (xd3_stream *stream, xoff_t b { source->frontier_blkno = blkno + 1; - IF_DEBUG2 (DP(RINT "[getblk] full source blkno %"Q"u: " - "source length unknown %"Q"u\n", + IF_DEBUG2 (DP(RINT "[getblk] full source blkno %"Q": " + "source length unknown %"Q"\n", blkno, xd3_source_eof (source))); } @@ -2618,7 +2618,7 @@ xd3_getblk (xd3_stream *stream, xoff_t b if (!source->eof_known) { IF_DEBUG2 (DP(RINT "[getblk] eof block has %d bytes; " - "source length known %"Q"u\n", + "source length known %"Q"\n", xd3_bytes_on_srcblk (source, blkno), xd3_source_eof (source))); source->eof_known = 1; @@ -2629,7 +2629,7 @@ xd3_getblk (xd3_stream *stream, xoff_t b } XD3_ASSERT (source->curblk != NULL); - IF_DEBUG2 (DP(RINT "[getblk] read source block %"Q"u onblk %u blksize %u\n", + IF_DEBUG2 (DP(RINT "[getblk] read source block %"Q" onblk %u blksize %u\n", blkno, source->onblk, source->blksize)); if (blkno == source->max_blkno) @@ -2690,7 +2690,7 @@ xd3_set_source_and_size (xd3_stream *str if (ret == 0) { stream->src->eof_known = 1; - IF_DEBUG2 (DP(RINT "[set source] size known %"Q"u\n", + IF_DEBUG2 (DP(RINT "[set source] size known %"Q"\n", source_size)); xd3_blksize_div(source_size, @@ -2900,7 +2900,7 @@ xd3_iopt_finish_encoding (xd3_stream *st IF_DEBUG2 ({ static int cnt; - DP(RINT "[iopt copy:%d] pos %"Q"u-%"Q"u addr %"Q"u-%"Q"u size %u\n", + DP(RINT "[iopt copy:%d] pos %"Q"-%"Q" addr %"Q"-%"Q" size %u\n", cnt++, stream->total_in + inst->pos, stream->total_in + inst->pos + inst->size, @@ -2919,7 +2919,7 @@ xd3_iopt_finish_encoding (xd3_stream *st IF_DEBUG2 ({ static int cnt; - DP(RINT "[iopt run:%d] pos %"Q"u size %u\n", cnt++, stream->total_in + inst->pos, inst->size); + DP(RINT "[iopt run:%d] pos %"Q" size %u\n", cnt++, stream->total_in + inst->pos, inst->size); }); break; } @@ -2933,7 +2933,7 @@ xd3_iopt_finish_encoding (xd3_stream *st IF_DEBUG2 ({ static int cnt; - DP(RINT "[iopt add:%d] pos %"Q"u size %u\n", cnt++, stream->total_in + inst->pos, inst->size); + DP(RINT "[iopt add:%d] pos %"Q" size %u\n", cnt++, stream->total_in + inst->pos, inst->size); }); break; @@ -3863,7 +3863,7 @@ xd3_encode_input (xd3_stream *stream) stream->enc_state = ENC_SEARCH; - IF_DEBUG2 (DP(RINT "[WINSTART:%"Q"u] input bytes %u offset %"Q"u\n", + IF_DEBUG2 (DP(RINT "[WINSTART:%"Q"] input bytes %u offset %"Q"\n", stream->current_window, stream->avail_in, stream->total_in)); return XD3_WINSTART; @@ -4004,7 +4004,7 @@ xd3_encode_input (xd3_stream *stream) stream->total_in += (xoff_t) stream->avail_in; stream->enc_state = ENC_POSTWIN; - IF_DEBUG2 (DP(RINT "[WINFINISH:%"Q"u] in=%"Q"u\n", + IF_DEBUG2 (DP(RINT "[WINFINISH:%"Q"] in=%"Q"\n", stream->current_window, stream->total_in)); return XD3_WINFINISH; @@ -4442,13 +4442,13 @@ xd3_source_match_setup (xd3_stream *stre * back further than the LRU cache maintaining FIFO discipline, (to * avoid seeking). */ frontier_pos = stream->src->frontier_blkno * stream->src->blksize; - IF_DEBUG2(DP(RINT "[match_setup] frontier_pos %"Q"u, srcpos %"Q"u, " - "src->max_winsize %"Q"u\n", + IF_DEBUG2(DP(RINT "[match_setup] frontier_pos %"Q", srcpos %"Q", " + "src->max_winsize %"Q"\n", frontier_pos, srcpos, stream->src->max_winsize)); if (srcpos < frontier_pos && frontier_pos - srcpos > stream->src->max_winsize) { IF_DEBUG1(DP(RINT "[match_setup] rejected due to src->max_winsize " - "distance eof=%"Q"u srcpos=%"Q"u maxsz=%"Q"u\n", + "distance eof=%"Q" srcpos=%"Q" maxsz=%"Q"\n", xd3_source_eof (stream->src), srcpos, stream->src->max_winsize)); goto bad; @@ -4505,7 +4505,7 @@ xd3_source_match_setup (xd3_stream *stre } IF_DEBUG2(DP(RINT - "[match_setup] srcpos %"Q"u (tgtpos %"Q"u) " + "[match_setup] srcpos %"Q" (tgtpos %"Q") " "unrestricted maxback %u maxfwd %u\n", srcpos, stream->total_in + stream->input_position, @@ -4541,7 +4541,7 @@ xd3_source_match_setup (xd3_stream *stre } IF_DEBUG1(DP(RINT - "[match_setup] srcpos %"Q"u (tgtpos %"Q"u) " + "[match_setup] srcpos %"Q" (tgtpos %"Q") " "restricted maxback %u maxfwd %u\n", srcpos, stream->total_in + stream->input_position, @@ -4620,7 +4620,7 @@ xd3_source_extend_match (xd3_stream *str usize_t tryrem; /* tryrem is the number of matchable bytes */ usize_t matched; - IF_DEBUG2(DP(RINT "[extend match] srcpos %"Q"u\n", + IF_DEBUG2(DP(RINT "[extend match] srcpos %"Q"\n", stream->match_srcpos)); XD3_ASSERT (src != NULL); @@ -4658,7 +4658,7 @@ xd3_source_extend_match (xd3_stream *str tryrem = min (tryoff, stream->match_maxback - stream->match_back); - IF_DEBUG2(DP(RINT "[maxback] maxback %u trysrc %"Q"u/%u tgt %u tryrem %u\n", + IF_DEBUG2(DP(RINT "[maxback] maxback %u trysrc %"Q"/%u tgt %u tryrem %u\n", stream->match_maxback, tryblk, tryoff, streamoff, tryrem)); /* TODO: This code can be optimized similar to xd3_match_forward() */ @@ -4783,7 +4783,7 @@ xd3_source_extend_match (xd3_stream *str IF_DEBUG2 ({ static int x = 0; - DP(RINT "[source match:%d] (%s) [ %u bytes ]\n", + DP(RINT "[source match:%d] (%s) [ %u bytes ]\n", x++, stream->total_in + target_position, stream->total_in + target_position + match_length, @@ -5110,13 +5110,13 @@ xd3_srcwin_move_point (xd3_stream *strea ret = XD3_INTERNAL; } IF_DEBUG1 (DP(RINT - "[srcwin_move_point] async getblk return for %"Q"u\n", + "[srcwin_move_point] async getblk return for %"Q"\n", blkno)); return ret; } IF_DEBUG1 (DP(RINT - "[srcwin_move_point] T=%"Q"u{%"Q"u} S=%"Q"u EOF=%"Q"u %s\n", + "[srcwin_move_point] T=%"Q"{%"Q"} S=%"Q" EOF=%"Q" %s\n", stream->total_in + stream->input_position, logical_input_cksum_pos, stream->srcwin_cksum_pos, @@ -5164,8 +5164,8 @@ xd3_srcwin_move_point (xd3_stream *strea } IF_DEBUG1 (DP(RINT - "[srcwin_move_point] exited loop T=%"Q"u{%"Q"u} " - "S=%"Q"u EOF=%"Q"u %s\n", + "[srcwin_move_point] exited loop T=%"Q"{%"Q"} " + "S=%"Q" EOF=%"Q" %s\n", stream->total_in + stream->input_position, logical_input_cksum_pos, stream->srcwin_cksum_pos, --- a/xdelta3-blkcache.h +++ b/xdelta3-blkcache.h @@ -246,7 +246,7 @@ main_set_source (xd3_stream *stream, xd3 if (sfile->size_known) { - short_sprintf (srcszbuf, "source size %s [%"Q"u]", + short_sprintf (srcszbuf, "source size %s [%"Q"]", main_format_bcnt (source_size, &srccntbuf), source_size); } @@ -366,7 +366,7 @@ main_read_seek_source (xd3_stream *strea if (!option_quiet) { XPR(NT "source can't seek backwards; requested block offset " - "%"Q"u source position is %"Q"u\n", + "%"Q" source position is %"Q"\n", pos, sfile->source_position); } @@ -386,7 +386,7 @@ main_read_seek_source (xd3_stream *strea if (option_verbose > 1) { - XPR(NT "seek error at offset %"Q"u: %s\n", + XPR(NT "seek error at offset %"Q": %s\n", pos, xd3_mainerror (ret)); } } @@ -395,7 +395,7 @@ main_read_seek_source (xd3_stream *strea if (option_verbose > 1 && pos != sfile->source_position) { - XPR(NT "non-seekable source skipping %"Q"u bytes @ %"Q"u\n", + XPR(NT "non-seekable source skipping %"Q" bytes @ %"Q"\n", pos - sfile->source_position, sfile->source_position); } @@ -440,7 +440,7 @@ main_read_seek_source (xd3_stream *strea sfile->source_position += nread; blru->size = nread; - IF_DEBUG1 (DP(RINT "[getblk] skip blkno %"Q"u size %u\n", + IF_DEBUG1 (DP(RINT "[getblk] skip blkno %"Q" size %u\n", skip_blkno, blru->size)); XD3_ASSERT (sfile->source_position <= pos); @@ -536,20 +536,20 @@ main_getblk_func (xd3_stream *stream, { if (blru->blkno != blkno) { - XPR(NT "source block %"Q"u read %zu ejects %"Q"u (lru_hits=%u, " + XPR(NT "source block %"Q" read %zu ejects %"Q" (lru_hits=%u, " "lru_misses=%u, lru_filled=%u)\n", blkno, nread, blru->blkno, lru_hits, lru_misses, lru_filled); } else { - XPR(NT "source block %"Q"u read %zu (lru_hits=%u, " + XPR(NT "source block %"Q" read %zu (lru_hits=%u, " "lru_misses=%u, lru_filled=%u)\n", blkno, nread, lru_hits, lru_misses, lru_filled); } } else { - XPR(NT "source block %"Q"u read %zu (lru_hits=%u, lru_misses=%u, " + XPR(NT "source block %"Q" read %zu (lru_hits=%u, lru_misses=%u, " "lru_filled=%u)\n", blkno, nread, lru_hits, lru_misses, lru_filled); } @@ -561,8 +561,8 @@ main_getblk_func (xd3_stream *stream, blru->size = nread; blru->blkno = blkno; - IF_DEBUG1 (DP(RINT "[main_getblk] blkno %"Q"u onblk %zu pos %"Q"u " - "srcpos %"Q"u\n", + IF_DEBUG1 (DP(RINT "[main_getblk] blkno %"Q" onblk %zu pos %"Q" " + "srcpos %"Q"\n", blkno, nread, pos, sfile->source_position)); return 0; --- a/xdelta3-decode.h +++ b/xdelta3-decode.h @@ -224,7 +224,7 @@ xd3_decode_parse_halfinst (xd3_stream *s { IF_DEBUG2 ({ static int cnt = 0; - XPR(NT "DECODE:%u: COPY at %"Q"u (winoffset %u) size %u winaddr %u\n", + XPR(NT "DECODE:%u: COPY at %"Q" (winoffset %u) size %u winaddr %u\n", cnt++, stream->total_out + (stream->dec_position - stream->dec_cpylen), @@ -265,7 +265,7 @@ xd3_decode_parse_halfinst (xd3_stream *s if (inst->type == XD3_ADD) { static int cnt; - XPR(NT "DECODE:%d: ADD at %"Q"u (winoffset %u) size %u\n", + XPR(NT "DECODE:%d: ADD at %"Q" (winoffset %u) size %u\n", cnt++, (stream->total_out + stream->dec_position - stream->dec_cpylen), stream->dec_position - stream->dec_cpylen, @@ -275,7 +275,7 @@ xd3_decode_parse_halfinst (xd3_stream *s { static int cnt; XD3_ASSERT (inst->type == XD3_RUN); - XPR(NT "DECODE:%d: RUN at %"Q"u (winoffset %u) size %u\n", + XPR(NT "DECODE:%d: RUN at %"Q" (winoffset %u) size %u\n", cnt++, stream->total_out + stream->dec_position - stream->dec_cpylen, stream->dec_position - stream->dec_cpylen, @@ -453,7 +453,7 @@ xd3_decode_output_halfinst (xd3_stream * if ((source->onblk != blksize) && (blkoff + take > source->onblk)) { - IF_DEBUG1 (XPR(NT "[srcfile] short at blkno %"Q"u onblk " + IF_DEBUG1 (XPR(NT "[srcfile] short at blkno %"Q" onblk " "%u blksize %u blkoff %u take %u\n", block, source->onblk, @@ -936,7 +936,7 @@ xd3_decode_input (xd3_stream *stream) stream->dec_state = DEC_CPYLEN; - IF_DEBUG2 (DP(RINT "--------- TARGET WINDOW %"Q"u -----------\n", + IF_DEBUG2 (DP(RINT "--------- TARGET WINDOW %"Q" -----------\n", stream->current_window)); } @@ -1093,8 +1093,8 @@ xd3_decode_input (xd3_stream *stream) &src->cpyoff_blkoff); IF_DEBUG1(DP(RINT - "decode cpyoff %"Q"u " - "cpyblkno %"Q"u " + "decode cpyoff %"Q" " + "cpyblkno %"Q" " "cpyblkoff %u " "blksize %u\n", stream->dec_cpyoff, --- a/xdelta3-main.h +++ b/xdelta3-main.h @@ -616,7 +616,7 @@ main_format_bcnt (xoff_t r, shortbuf *bu if (r >= 100 && r < 1000) { - short_sprintf (*buf, "%"Q"u %s", r, fmts[i]); + short_sprintf (*buf, "%"Q" %s", r, fmts[i]); return buf->buf; } @@ -716,12 +716,12 @@ main_atoux (const char* arg, xoff_t *xo, if (x < low) { - XPR(NT "-%c: minimum value: %"Q"u\n", which, low); + XPR(NT "-%c: minimum value: %"Q"\n", which, low); return EXIT_FAILURE; } if (high != 0 && x > high) { - XPR(NT "-%c: maximum value: %"Q"u\n", which, high); + XPR(NT "-%c: maximum value: %"Q"\n", which, high); return EXIT_FAILURE; } (*xo) = x; @@ -1296,7 +1296,7 @@ main_print_window (xd3_stream* stream, m if ((ret = xd3_decode_instruction (stream))) { - XPR(NT "instruction decode error at %"Q"u: %s\n", + XPR(NT "instruction decode error at %"Q": %s\n", stream->dec_winstart + size, stream->msg); return ret; } @@ -1487,7 +1487,7 @@ main_print_func (xd3_stream* stream, mai VC(UT "\n")VE; } - VC(UT "VCDIFF window number: %"Q"u\n", stream->current_window)VE; + VC(UT "VCDIFF window number: %"Q"\n", stream->current_window)VE; VC(UT "VCDIFF window indicator: ")VE; if ((stream->dec_win_ind & VCD_SOURCE) != 0) VC(UT "VCD_SOURCE ")VE; if ((stream->dec_win_ind & VCD_TARGET) != 0) VC(UT "VCD_TARGET ")VE; @@ -1513,14 +1513,14 @@ main_print_func (xd3_stream* stream, mai if (stream->dec_winstart != 0) { - VC(UT "VCDIFF window at offset: %"Q"u\n", stream->dec_winstart)VE; + VC(UT "VCDIFF window at offset: %"Q"\n", stream->dec_winstart)VE; } if (SRCORTGT (stream->dec_win_ind)) { VC(UT "VCDIFF copy window length: %u\n", (usize_t)stream->dec_cpylen)VE; - VC(UT "VCDIFF copy window offset: %"Q"u\n", + VC(UT "VCDIFF copy window offset: %"Q"\n", stream->dec_cpyoff)VE; } @@ -1973,7 +1973,7 @@ main_merge_output (xd3_stream *stream, m XD3_ASSERT (inst->addr >= window_start); addr = inst->addr - window_start; } - IF_DEBUG2 (XPR(NTR "[merge copy] winpos %u take %u addr %"Q"u mode %u\n", + IF_DEBUG2 (XPR(NTR "[merge copy] winpos %u take %u addr %"Q" mode %u\n", window_pos, take, addr, inst->mode)); if ((ret = xd3_found_match (recode_stream, window_pos, take, addr, inst->mode != 0))) @@ -2249,7 +2249,7 @@ main_pipe_copier (uint8_t *pipe_buf, if (option_verbose && skipped != 0) { - XPR(NT "skipping %"Q"u bytes in %s\n", + XPR(NT "skipping %"Q" bytes in %s\n", skipped, ifile->filename); } return 0; @@ -3290,7 +3290,7 @@ main_input (xd3_cmd cmd, /* Warn when no source copies are found */ if (option_verbose && ! xd3_encoder_used_source (& stream)) { - XPR(NT "warning: input window %"Q"u..%"Q"u has " + XPR(NT "warning: input window %"Q"..%"Q" has " "no source copies\n", stream.current_window * winsize, (stream.current_window+1) * winsize); @@ -3303,7 +3303,7 @@ main_input (xd3_cmd cmd, stream.srcwin_decided_early && stream.i_slots_used > stream.iopt_size) { - XPR(NT "warning: input position %"Q"u overflowed " + XPR(NT "warning: input position %"Q" overflowed " "instruction buffer, needed %u (vs. %u), " "consider changing -I\n", stream.current_window * winsize, @@ -3327,7 +3327,7 @@ main_input (xd3_cmd cmd, if (option_verbose > 1) { - XPR(NT "%"Q"u: in %s (%s): out %s (%s): " + XPR(NT "%"Q": in %s (%s): out %s (%s): " "total in %s: out %s: %s: srcpos %s\n", stream.current_window, main_format_bcnt (this_read, &rdb), @@ -3341,7 +3341,7 @@ main_input (xd3_cmd cmd, } else { - XPR(NT "%"Q"u: in %s: out %s: total in %s: " + XPR(NT "%"Q": in %s: out %s: total in %s: " "out %s: %s\n", stream.current_window, main_format_bcnt (this_read, &rdb), @@ -3437,12 +3437,12 @@ done: if (option_verbose > 2 && cmd == CMD_ENCODE) { - XPR(NT "source copies: %"Q"u (%"Q"u bytes)\n", + XPR(NT "source copies: %"Q" (%"Q" bytes)\n", stream.n_scpy, stream.l_scpy); - XPR(NT "target copies: %"Q"u (%"Q"u bytes)\n", + XPR(NT "target copies: %"Q" (%"Q" bytes)\n", stream.n_tcpy, stream.l_tcpy); - XPR(NT "adds: %"Q"u (%"Q"u bytes)\n", stream.n_add, stream.l_add); - XPR(NT "runs: %"Q"u (%"Q"u bytes)\n", stream.n_run, stream.l_run); + XPR(NT "adds: %"Q" (%"Q" bytes)\n", stream.n_add, stream.l_add); + XPR(NT "runs: %"Q" (%"Q" bytes)\n", stream.n_run, stream.l_run); } #endif @@ -3454,7 +3454,7 @@ done: long end_time = get_millisecs_now (); xoff_t nwrite = ofile != NULL ? ofile->nwrite : 0; - XPR(NT "finished in %s; input %"Q"u output %"Q"u bytes (%0.2f%%)\n", + XPR(NT "finished in %s; input %"Q" output %"Q" bytes (%0.2f%%)\n", main_format_millis (end_time - start_time, &tm), ifile->nread, nwrite, 100.0 * nwrite / ifile->nread); } debian/patches/unversioned_shebangs.diff0000664000000000000000000000177612155060124015673 0ustar #Author: Luca Falavigna #Bug-Debian: http://bugs.debian.org/625751 #Last-Update: 2011-05-08 Index: xdelta3-0y.dfsg/testing/xdelta3-regtest.py =================================================================== --- xdelta3-0y.dfsg.orig/testing/xdelta3-regtest.py 2011-05-08 13:22:23.412081704 +0200 +++ xdelta3-0y.dfsg/testing/xdelta3-regtest.py 2011-05-08 13:21:53.188081717 +0200 @@ -1,4 +1,4 @@ -#!/usr/bin/python2.6 +#!/usr/bin/python # xdelta 3 - delta compression tools and library # Copyright (C) 2003, 2006, 2007, 2008. Joshua P. MacDonald # Index: xdelta3-0y.dfsg/testing/xdelta3-test.py =================================================================== --- xdelta3-0y.dfsg.orig/testing/xdelta3-test.py 2011-05-08 13:22:23.344081704 +0200 +++ xdelta3-0y.dfsg/testing/xdelta3-test.py 2011-05-08 13:21:57.100081790 +0200 @@ -1,4 +1,4 @@ -#!/usr/bin/python2.7 +#!/usr/bin/python # xdelta 3 - delta compression tools and library # Copyright (C) 2003, 2006, 2007. Joshua P. MacDonald # debian/patches/manpage_lzma0000664000000000000000000000152712157344571013213 0ustar --- a/xdelta3-main.h +++ b/xdelta3-main.h @@ -4031,7 +4031,8 @@ main_help (void) XPR(NTR "compression options:\n"); XPR(NTR " -s source source file to copy from (if any)\n"); - XPR(NTR " -S [djw|fgk] enable/disable secondary compression\n"); + XPR(NTR " -S [djw|fgk|lzma|none] \n"); + XPR(NTR " enable/disable secondary compression\n"); XPR(NTR " -N disable small string-matching compression\n"); XPR(NTR " -D disable external decompression (encode/decode)\n"); XPR(NTR " -R disable external recompression (decode)\n"); --- a/xdelta3.1 +++ b/xdelta3.1 @@ -92,7 +92,7 @@ compression options: .RI source source file to copy from (if any) .TP -.BI "\-S " [djw|fgk] +.BI "\-S " [djw|fgk|lzma|none] enable/disable secondary compression .TP .BI \-N debian/patches/CVE-2014-9765.patch0000664000000000000000000002160712656405412013264 0ustar From 969e65d3a5d70442f5bafd726bcef47a0b48edd8 Mon Sep 17 00:00:00 2001 From: "josh.macdonald" Date: Sun, 12 Oct 2014 05:24:22 +0000 Subject: [PATCH] Add appheader tests; fix buffer overflow in main_get_appheader --- xdelta3/xdelta3-main.h | 5 +- xdelta3/xdelta3-test.h | 131 +++++++++++++++++++++++++++++++++++++++---------- 2 files changed, 108 insertions(+), 28 deletions(-) Index: xdelta3-3.0.7-dfsg/xdelta3-main.h =================================================================== --- xdelta3-3.0.7-dfsg.orig/xdelta3-main.h 2016-02-09 10:51:34.706689729 -0500 +++ xdelta3-3.0.7-dfsg/xdelta3-main.h 2016-02-09 10:51:34.702689688 -0500 @@ -2812,14 +2812,15 @@ if (appheadsz > 0) { + const int kMaxArgs = 4; char *start = (char*)apphead; char *slash; int place = 0; - char *parsed[4]; + char *parsed[kMaxArgs]; memset (parsed, 0, sizeof (parsed)); - while ((slash = strchr (start, '/')) != NULL) + while ((slash = strchr (start, '/')) != NULL && place < (kMaxArgs-1)) { *slash = 0; parsed[place++] = start; Index: xdelta3-3.0.7-dfsg/xdelta3-test.h =================================================================== --- xdelta3-3.0.7-dfsg.orig/xdelta3-test.h 2016-02-09 10:51:34.706689729 -0500 +++ xdelta3-3.0.7-dfsg/xdelta3-test.h 2016-02-09 10:51:34.702689688 -0500 @@ -1,5 +1,5 @@ /* xdelta 3 - delta compression tools and library Copyright (C) 2001, - * 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012. + * 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012. * Joshua P. MacDonald * * This program is free software; you can redistribute it and/or modify @@ -54,7 +54,7 @@ /* only MSBs of the array mt[]. */ /* 2002/01/09 modified by Makoto Matsumoto */ mt->mt_buffer_[i] = - (1812433253UL * (mt->mt_buffer_[i-1] ^ + (1812433253UL * (mt->mt_buffer_[i-1] ^ (mt->mt_buffer_[i-1] >> 30)) + i); } } @@ -69,20 +69,20 @@ int kk; for (kk = 0; kk < MT_LEN - MT_IA; kk++) { - y = (mt->mt_buffer_[kk] & UPPER_MASK) | + y = (mt->mt_buffer_[kk] & UPPER_MASK) | (mt->mt_buffer_[kk + 1] & LOWER_MASK); - mt->mt_buffer_[kk] = mt->mt_buffer_[kk + MT_IA] ^ + mt->mt_buffer_[kk] = mt->mt_buffer_[kk + MT_IA] ^ (y >> 1) ^ mag01[y & 0x1UL]; } for (;kk < MT_LEN - 1; kk++) { - y = (mt->mt_buffer_[kk] & UPPER_MASK) | + y = (mt->mt_buffer_[kk] & UPPER_MASK) | (mt->mt_buffer_[kk + 1] & LOWER_MASK); - mt->mt_buffer_[kk] = mt->mt_buffer_[kk + (MT_IA - MT_LEN)] ^ + mt->mt_buffer_[kk] = mt->mt_buffer_[kk + (MT_IA - MT_LEN)] ^ (y >> 1) ^ mag01[y & 0x1UL]; } - y = (mt->mt_buffer_[MT_LEN - 1] & UPPER_MASK) | + y = (mt->mt_buffer_[MT_LEN - 1] & UPPER_MASK) | (mt->mt_buffer_[0] & LOWER_MASK); - mt->mt_buffer_[MT_LEN - 1] = mt->mt_buffer_[MT_IA - 1] ^ + mt->mt_buffer_[MT_LEN - 1] = mt->mt_buffer_[MT_IA - 1] ^ (y >> 1) ^ mag01[y & 0x1UL]; mt->mt_index_ = 0; } @@ -166,7 +166,7 @@ { stream->msg = "abnormal command termination"; } - return XD3_INTERNAL; + return ret; } return 0; } @@ -257,8 +257,10 @@ static int test_make_inputs (xd3_stream *stream, xoff_t *ss_out, xoff_t *ts_out) { - usize_t ts = (mt_random (&static_mtrand) % TEST_FILE_MEAN) + TEST_FILE_MEAN / 2; - usize_t ss = (mt_random (&static_mtrand) % TEST_FILE_MEAN) + TEST_FILE_MEAN / 2; + usize_t ts = (mt_random (&static_mtrand) % TEST_FILE_MEAN) + + TEST_FILE_MEAN / 2; + usize_t ss = (mt_random (&static_mtrand) % TEST_FILE_MEAN) + + TEST_FILE_MEAN / 2; uint8_t *buf = (uint8_t*) malloc (ts + ss), *sbuf = buf, *tbuf = buf + ss; usize_t sadd = 0, sadd_max = (usize_t)(ss * TEST_ADD_RATIO); FILE *tf = NULL, *sf = NULL; @@ -421,7 +423,7 @@ fclose (orig); fclose (recons); - if (diffs != 0) + if (diffs != 0) { return XD3_INTERNAL; } @@ -429,12 +431,12 @@ } static int -test_save_copy (const char *origname) +test_copy_to (const char *from, const char *to) { char buf[TESTBUFSIZE]; int ret; - snprintf_func (buf, TESTBUFSIZE, "cp -f %s %s", origname, TEST_COPY_FILE); + snprintf_func (buf, TESTBUFSIZE, "cp -f %s %s", from, to); if ((ret = system (buf)) != 0) { @@ -445,6 +447,12 @@ } static int +test_save_copy (const char *origname) +{ + return test_copy_to(origname, TEST_COPY_FILE); +} + +static int test_file_size (const char* file, xoff_t *size) { struct stat sbuf; @@ -499,7 +507,7 @@ inp = buf->base; max = buf->base + buf->next - trunto; - if ((ret = xd3_read_uint32_t (stream, & inp, max, & rval)) != + if ((ret = xd3_read_uint32_t (stream, & inp, max, & rval)) != XD3_INVALID_INPUT || !MSG_IS (msg)) { @@ -1654,11 +1662,11 @@ if ((buf = (uint8_t*) malloc (TWO_MEGS_AND_DELTA)) == NULL) { return ENOMEM; } memset (buf, 0, TWO_MEGS_AND_DELTA); - for (i = 0; i < (2 << 20); i += 256) + for (i = 0; i < (2 << 20); i += 256) { int j; int off = mt_random(& static_mtrand) % 10; - for (j = 0; j < 256; j++) + for (j = 0; j < 256; j++) { buf[i + j] = j + off; } @@ -1683,11 +1691,11 @@ } /* Test transfer of exactly 32bits worth of data. */ - if ((ret = test_streaming (stream, - buf, - buf + (1 << 20), - buf + (2 << 20), - 1 << 12))) + if ((ret = test_streaming (stream, + buf, + buf + (1 << 20), + buf + (2 << 20), + 1 << 12))) { goto fail; } @@ -1889,7 +1897,7 @@ } /* First encode */ - snprintf_func (ecmd, TESTBUFSIZE, "%s %s -f %s %s %s %s %s %s %s", + snprintf_func (ecmd, TESTBUFSIZE, "%s %s -f %s %s %s %s %s %s %s", program_name, test_softcfg_str, has_adler32 ? "" : "-n ", has_apphead ? "-A=encode_apphead " : "-A= ", @@ -1910,7 +1918,7 @@ snprintf_func (recmd, TESTBUFSIZE, "%s recode %s -f %s %s %s %s %s", program_name, test_softcfg_str, recoded_adler32 ? "" : "-n ", - !change_apphead ? "" : + !change_apphead ? "" : (recoded_apphead ? "-A=recode_apphead " : "-A= "), recoded_secondary ? "-S djw " : "-S none ", TEST_DELTA_FILE, @@ -2361,6 +2369,76 @@ return 0; } +/* This tests that the default appheader works */ +static int +test_appheader (xd3_stream *stream, int ignore) +{ + int i; + int ret; + char buf[TESTBUFSIZE]; + char bogus[TESTBUFSIZE]; + xoff_t ssize, tsize; + test_setup (); + + if ((ret = test_make_inputs (stream, &ssize, &tsize))) { return ret; } + + snprintf_func (buf, TESTBUFSIZE, "%s -q -f -e -s %s %s %s", program_name, + TEST_SOURCE_FILE, TEST_TARGET_FILE, TEST_DELTA_FILE); + if ((ret = do_cmd (stream, buf))) { return ret; } + + if ((ret = test_copy_to (program_name, TEST_RECON2_FILE))) { return ret; } + + snprintf_func (buf, TESTBUFSIZE, "chmod 0700 %s", TEST_RECON2_FILE); + if ((ret = do_cmd (stream, buf))) { return ret; } + + if ((ret = test_save_copy (TEST_TARGET_FILE))) { return ret; } + if ((ret = test_copy_to (TEST_SOURCE_FILE, TEST_TARGET_FILE))) { return ret; } + + if ((ret = test_compare_files (TEST_TARGET_FILE, TEST_COPY_FILE)) == 0) + { + return XD3_INVALID; // I.e., files are different! + } + + // Test that the target file is restored. + snprintf_func (buf, TESTBUFSIZE, "(cd /tmp && %s -q -f -d %s)", + TEST_RECON2_FILE, + TEST_DELTA_FILE); + if ((ret = do_cmd (stream, buf))) { return ret; } + + if ((ret = test_compare_files (TEST_TARGET_FILE, TEST_COPY_FILE)) != 0) + { + return ret; + } + + // Test a malicious string w/ entries > 4 in the appheader by having + // the encoder write it: + for (i = 0; i < TESTBUFSIZE / 4; ++i) + { + bogus[2*i] = 'G'; + bogus[2*i+1] = '/'; + } + bogus[TESTBUFSIZE/2-1] = 0; + + snprintf_func (buf, TESTBUFSIZE, + "%s -q -f -A=%s -e -s %s %s %s", program_name, bogus, + TEST_SOURCE_FILE, TEST_TARGET_FILE, TEST_DELTA_FILE); + if ((ret = do_cmd (stream, buf))) { return ret; } + // Then read it: + snprintf_func (buf, TESTBUFSIZE, "(cd /tmp && %s -q -f -d %s)", + TEST_RECON2_FILE, + TEST_DELTA_FILE); + if ((ret = do_cmd (stream, buf)) == 0) + { + return XD3_INVALID; // Impossible + } + if (!WIFEXITED(ret)) + { + return XD3_INVALID; // Must have crashed! + } + + return 0; +} + /*********************************************************************** Source identical optimization ***********************************************************************/ @@ -2603,7 +2681,7 @@ default: CHECK(0); } - snprintf_func (rptr, rbuf+TESTBUFSIZE-rptr, "%d/%d", + snprintf_func (rptr, rbuf+TESTBUFSIZE-rptr, "%d/%d", inst->pos, inst->size); rptr += strlen (rptr); @@ -2848,6 +2926,7 @@ DO_TEST (force_behavior, 0, 0); DO_TEST (stdout_behavior, 0, 0); DO_TEST (no_output, 0, 0); + DO_TEST (appheader, 0, 0); DO_TEST (command_line_arguments, 0, 0); #if EXTERNAL_COMPRESSION debian/patches/fix_lzma_test.patch0000664000000000000000000000142412661070640014513 0ustar Description: fix lzma test so we can run the builtin tests Origin: backport, https://github.com/jmacd/xdelta-devel/commit/806836136cec2eeb763b4b539d4d41bc2c5bbf5c Index: xdelta3-3.0.7-dfsg/xdelta3-test.h =================================================================== --- xdelta3-3.0.7-dfsg.orig/xdelta3-test.h 2016-02-09 10:51:34.702689688 -0500 +++ xdelta3-3.0.7-dfsg/xdelta3-test.h 2016-02-17 08:10:32.719624404 -0500 @@ -1061,9 +1061,9 @@ } /* Check expected non-failures */ - if (non_failures != expected_non_failures) + if (non_failures > expected_non_failures) { - XPR(NT "non-failures %u; expected %u", + XPR(NT "non-failures %u > expected %u", non_failures, expected_non_failures); stream->msg = "incorrect"; return XD3_INTERNAL; debian/patches/series0000664000000000000000000000016412661070473012043 0ustar unversioned_shebangs.diff regtest_size_t printf_uint64 Q_not_u manpage_lzma CVE-2014-9765.patch fix_lzma_test.patch debian/patches/printf_uint640000664000000000000000000000155212157531733013267 0ustar --- a/xdelta3.h +++ b/xdelta3.h @@ -152,25 +152,25 @@ typedef uint32_t usize_t; #ifndef _FILE_OFFSET_BITS #define _FILE_OFFSET_BITS 64 #endif - typedef uint64_t xoff_t; #define SIZEOF_XOFF_T 8 #define SIZEOF_USIZE_T 4 #ifndef WIN32 -#if SIZEOF_SIZE_T == 8 -#define Q "z" -#else -#define Q "ll" -#endif -#else -#define Q "I64" -#endif -#else +#define __STDC_FORMAT_MACROS +#include +#define Q PRIu64 /* to print uint64_t that is xoff_t */ +#define Qd PRId64 /* to print int64_t */ +#else /*WIN32*/ +#define Q "I64u" +#define Qd "I64d" +#endif /*WIN32*/ +#else /* XD3_USE_LARGEFILE64 */ typedef uint32_t xoff_t; #define SIZEOF_XOFF_T 4 #define SIZEOF_USIZE_T 4 -#define Q -#endif +#define Q "u" +#define Qd "d" +#endif /* XD3_USE_LARGEFILE64 */ #define USE_UINT32 (SIZEOF_USIZE_T == 4 || \ SIZEOF_XOFF_T == 4 || REGRESSION_TEST) debian/patches/regtest_size_t0000664000000000000000000000046112157266740013607 0ustar --- a/testing/regtest.cc +++ b/testing/regtest.cc @@ -12,7 +12,7 @@ public: Options() : encode_srcwin_maxsz(1<<20), block_size(Constants::BLOCK_SIZE), size_known(false) { } - size_t encode_srcwin_maxsz; + uint64_t encode_srcwin_maxsz; size_t block_size; bool size_known; }; debian/python-xdelta3.examples0000664000000000000000000000003012155060050013576 0ustar testing/xdelta3-test.py debian/copyright0000664000000000000000000000164211570527434011136 0ustar Xdelta3 was designed and implemented by Joshua MacDonald. Xdelta3 was downloaded from http://xdelta.org/ but the file draft-korn-vcdiff.txt was deleted from the original tar file, since it is not freely redistributable. Copyright: XDelta3 - A binary delta generator. Copyright (C) 2001, 2003-2010. Joshua P. MacDonald This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. On Debian systems, a copy of the GNU General Public License is available in /usr/share/common-licenses/GPL. debian/README0000664000000000000000000003027011570527402010055 0ustar (the following info come from the file xdelta3.c) Xdelta 3 The goal of this library is to to implement both the (stand-alone) data-compression and delta-compression aspects of VCDIFF encoding, and to support a programming interface that works like Zlib (http://www.gzip.org/zlib.html). See RFC3284: The VCDIFF Generic Differencing and Compression Data Format. VCDIFF is a unified encoding that combines data-compression and delta-encoding ("differencing"). VCDIFF has a detailed byte-code instruction set with many features. The instruction format supports an immediate size operand for small COPYs and ADDs (e.g., under 18 bytes). There are also instruction "modes", which are used to compress COPY addresses by using two address caches. An instruction mode refers to slots in the NEAR and SAME caches for recent addresses. NEAR remembers the previous 4 (by default) COPY addresses, and SAME catches frequent re-uses of the same address using a 3-way (by default) 256-entry associative cache of [ADDR mod 256], the encoded byte. A hit in the NEAR/SAME cache requires 0/1 ADDR bytes. VCDIFF has a default instruction table, but an alternate instruction tables may themselves be be delta-compressed and included in the encoding header. This allows even more freedom. There are 9 instruction modes in the default code table, 4 near, 3 same, VCD_SELF (absolute encoding) and VCD_HERE (relative to the current position). ---------------------------------------------------------------------- Algorithms Aside from the details of encoding and decoding, there are a bunch of algorithms needed. 1. STRING-MATCH. A two-level fingerprinting approach is used. A single loop computes the two checksums -- small and large -- at successive offsets in the TARGET file. The large checksum is more accurate and is used to discover SOURCE matches, which are potentially very long. The small checksum is used to discover copies within the TARGET. Small matching, which is more expensive, usually dominates the large STRING-MATCH costs in this code - the more exhaustive the search, the better the results. Either of the two string-matching mechanisms may be disabled. 2. INSTRUCTION SELECTION. The IOPT buffer here represents a queue used to store overlapping copy instructions. There are two possible optimizations that go beyond a greedy search. Both of these fall into the category of "non-greedy matching" optimizations. The first optimization stems from backward SOURCE-COPY matching. When a new SOURCE-COPY instruction covers a previous instruction in the target completely, it is erased from the queue. Randal Burns originally analyzed these algorithms and did a lot of related work (\cite the 1.5-pass algorithm). The second optimization comes by the encoding of common very-small COPY and ADD instructions, for which there are special DOUBLE-code instructions, which code two instructions in a single byte. The cost of bad instruction-selection overhead is relatively high for data-compression, relative to delta-compression, so this second optimization is fairly important. With "lazy" matching (the name used in Zlib for a similar optimization), the string-match algorithm searches after a match for potential overlapping copy instructions. In Xdelta and by default, VCDIFF, the minimum match size is 4 bytes, whereas Zlib searches with a 3-byte minimum. This feature, combined with double instructions, provides a nice challenge. Search in this file for "black magic", a heuristic. 3. STREAM ALIGNMENT. Stream alignment is needed to compress large inputs in constant space. See xd3_srcwin_move_point(). 4. WINDOW SELECTION. When the IOPT buffer flushes, in the first call to xd3_iopt_finish_encoding containing any kind of copy instruction, the parameters of the source window must be decided: the offset into the source and the length of the window. Since the IOPT buffer is finite, the program may be forced to fix these values before knowing the best offset/length. 5. SECONDARY COMPRESSION. VCDIFF supports a secondary encoding to be applied to the individual sections of the data format, which are ADDRess, INSTruction, and DATA. Several secondary compressor variations are implemented here, although none is standardized yet. One is an adaptive huffman algorithm -- the FGK algorithm (Faller, Gallager, and Knuth, 1985). This compressor is extremely slow. The other is a simple static Huffman routine, which is the base case of a semi-adaptive scheme published by D.J. Wheeler and first widely used in bzip2 (by Julian Seward). This is a very interesting algorithm, originally published in nearly cryptic form by D.J. Wheeler. !!!NOTE!!! Because these are not standardized, secondary compression remains off by default. ftp://ftp.cl.cam.ac.uk/users/djw3/bred3.{c,ps} -------------------------------------------------------------------- Other Features 1. USER CONVENIENCE For user convenience, it is essential to recognize Gzip-compressed files and automatically Gzip-decompress them prior to delta-compression (or else no delta-compression will be achieved unless the user manually decompresses the inputs). The compressed represention competes with Xdelta, and this must be hidden from the command-line user interface. The Xdelta-1.x encoding was simple, not compressed itself, so Xdelta-1.x uses Zlib internally to compress the representation. This implementation supports external compression, which implements the necessary fork() and pipe() mechanics. There is a tricky step involved to support automatic detection of a compressed input in a non-seekable input. First you read a bit of the input to detect magic headers. When a compressed format is recognized, exec() the external compression program and create a second child process to copy the original input stream. [Footnote: There is a difficulty related to using Gzip externally. It is not possible to decompress and recompress a Gzip file transparently. If FILE.GZ had a cryptographic signature, then, after: (1) Gzip-decompression, (2) Xdelta-encoding, (3) Gzip-compression the signature could be broken. The only way to solve this problem is to guess at Gzip's compression level or control it by other means. I recommend that specific implementations of any compression scheme store information needed to exactly re-compress the input, that way external compression is transparent - however, this won't happen here until it has stabilized.] 2. APPLICATION-HEADER This feature was introduced in RFC3284. It allows any application to include a header within the VCDIFF file format. This allows general inter-application data exchange with support for application-specific extensions to communicate metadata. 3. VCDIFF CHECKSUM An optional checksum value is included with each window, which can be used to validate the final result. This verifies the correct source file was used for decompression as well as the obvious advantage: checking the implementation (and underlying) correctness. 4. LIGHT WEIGHT The code makes efforts to avoid copying data more than necessary. The code delays many initialization tasks until the first use, it optimizes for identical (perfectly matching) inputs. It does not compute any checksums until the first lookup misses. Memory usage is reduced. String-matching is templatized (by slightly gross use of CPP) to hard-code alternative compile-time defaults. The code has few outside dependencies. ---------------------------------------------------------------------- The default rfc3284 instruction table: (see RFC for the explanation) TYPE SIZE MODE TYPE SIZE MODE INDEX -------------------------------------------------------------------- 1. Run 0 0 Noop 0 0 0 2. Add 0, [1,17] 0 Noop 0 0 [1,18] 3. Copy 0, [4,18] 0 Noop 0 0 [19,34] 4. Copy 0, [4,18] 1 Noop 0 0 [35,50] 5. Copy 0, [4,18] 2 Noop 0 0 [51,66] 6. Copy 0, [4,18] 3 Noop 0 0 [67,82] 7. Copy 0, [4,18] 4 Noop 0 0 [83,98] 8. Copy 0, [4,18] 5 Noop 0 0 [99,114] 9. Copy 0, [4,18] 6 Noop 0 0 [115,130] 10. Copy 0, [4,18] 7 Noop 0 0 [131,146] 11. Copy 0, [4,18] 8 Noop 0 0 [147,162] 12. Add [1,4] 0 Copy [4,6] 0 [163,174] 13. Add [1,4] 0 Copy [4,6] 1 [175,186] 14. Add [1,4] 0 Copy [4,6] 2 [187,198] 15. Add [1,4] 0 Copy [4,6] 3 [199,210] 16. Add [1,4] 0 Copy [4,6] 4 [211,222] 17. Add [1,4] 0 Copy [4,6] 5 [223,234] 18. Add [1,4] 0 Copy 4 6 [235,238] 19. Add [1,4] 0 Copy 4 7 [239,242] 20. Add [1,4] 0 Copy 4 8 [243,246] 21. Copy 4 [0,8] Add 1 0 [247,255] -------------------------------------------------------------------- Reading the source: Overview This file includes itself in several passes to macro-expand certain sections with variable forms. Just read ahead, there's only a little confusion. I know this sounds ugly, but hard-coding some of the string-matching parameters results in a 10-15% increase in string-match performance. The only time this hurts is when you have unbalanced #if/endifs. A single compilation unit tames the Makefile. In short, this is to allow the above-described hack without an explodingMakefile. The single compilation unit includes the core library features, configurable string-match templates, optional main() command-line tool, misc optional features, and a regression test. Features are controled with CPP #defines, see Makefile.am. The initial __XDELTA3_C_HEADER_PASS__ starts first, the INLINE and TEMPLATE sections follow. Easy stuff first, hard stuff last. Optional features include: xdelta3-main.h The command-line interface, external compression support, POSIX-specific, info & VCDIFF-debug tools. xdelta3-second.h The common secondary compression routines. xdelta3-decoder.h All decoding routines. xdelta3-djw.h The semi-adaptive huffman secondary encoder. xdelta3-fgk.h The adaptive huffman secondary encoder. xdelta3-test.h The unit test covers major algorithms, encoding and decoding. There are single-bit error decoding tests. There are 32/64-bit file size boundary tests. There are command-line tests. There are compression tests. There are external compression tests. There are string-matching tests. There should be more tests... Additional headers include: xdelta3.h The public header file. xdelta3-cfgs.h The default settings for default, built-in encoders. These are hard-coded at compile-time. There is also a single soft-coded string matcher for experimenting with arbitrary values. xdelta3-list.h A cyclic list template Misc little debug utilities: badcopy.c Randomly modifies an input file based on two parameters: (1) the probability that a byte in the file is replaced with a pseudo-random value, and (2) the mean change size. Changes are generated using an expoential distribution which approximates the expected error_prob distribution. debian/rules0000775000000000000000000000015712155057621010257 0ustar #!/usr/bin/make -f include /usr/share/cdbs/1/rules/debhelper.mk include /usr/share/cdbs/1/class/autotools.mk