pax_global_header00006660000000000000000000000064125762150040014514gustar00rootroot0000000000000052 comment=d780c37ab0c0804903c6b9a872d0a2349e667f31 zbackup-1.4.4/000077500000000000000000000000001257621500400131615ustar00rootroot00000000000000zbackup-1.4.4/.gitignore000066400000000000000000000001421257621500400151460ustar00rootroot00000000000000*.o CMakeFiles/ CMakeCache.txt Makefile cmake_install.cmake /zbackup.pb.cc /zbackup.pb.h /zbackup zbackup-1.4.4/.travis.yml000066400000000000000000000004031257621500400152670ustar00rootroot00000000000000sudo: false language: cpp cache: ccache addons: apt: packages: - cmake - libssl-dev - libprotobuf-dev - protobuf-compiler - liblzma-dev - zlib1g-dev - liblzo2-dev script: - mkdir objdir - cd objdir - cmake ../ - make zbackup-1.4.4/CMakeLists.txt000066400000000000000000000032641257621500400157260ustar00rootroot00000000000000# Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS # Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE cmake_minimum_required( VERSION 2.8.2 ) project( zbackup ) set( CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake" ) set( CMAKE_BUILD_TYPE Release ) find_package( ZLIB REQUIRED ) include_directories( ${ZLIB_INCLUDE_DIRS} ) find_package( OpenSSL REQUIRED ) include_directories( ${OPENSSL_INCLUDE_DIR} ) find_package( Protobuf REQUIRED ) include_directories( ${PROTOBUF_INCLUDE_DIRS} ) include_directories( ${CMAKE_CURRENT_BINARY_DIR} ) find_program(PROTOBUF_PROTOC_CHECK NAMES protoc DOC "Protobuf compiler binary") IF(${PROTOBUF_PROTOC_CHECK} STREQUAL "PROTOBUF_PROTOC_CHECK-NOTFOUND") MESSAGE(FATAL_ERROR "Could not find protobuf compiler. Make sure protobuf-compiler package is installed.") ENDIF(${PROTOBUF_PROTOC_CHECK} STREQUAL "PROTOBUF_PROTOC_CHECK-NOTFOUND") PROTOBUF_GENERATE_CPP( protoSrcs protoHdrs zbackup.proto ) find_package( Threads REQUIRED ) find_package( LibLZMA REQUIRED ) include_directories( ${LIBLZMA_INCLUDE_DIRS} ) find_package( LibLZO COMPONENTS LIBLZO_HAS_LZO1X_DECOMPRESS_SAFE LIBLZO_HAS_LZO1X_1_COMPRESS ) if (LIBLZO_FOUND) ADD_DEFINITIONS(-DHAVE_LIBLZO) include_directories( ${LIBLZO_INCLUDE_DIRS} ) else (LIBLZO_FOUND) set(LIBLZO_LIBRARIES) endif (LIBLZO_FOUND) file( GLOB sourceFiles "*.cc" ) add_executable( zbackup ${sourceFiles} ${protoSrcs} ${protoHdrs} ) target_link_libraries( zbackup ${PROTOBUF_LIBRARIES} ${OPENSSL_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${ZLIB_LIBRARIES} ${LIBLZMA_LIBRARIES} ${LIBLZO_LIBRARIES} ) install( TARGETS zbackup DESTINATION bin ) zbackup-1.4.4/CONTRIBUTORS000066400000000000000000000007351257621500400150460ustar00rootroot00000000000000This file contains a list of people who have made contributions to the ZBackup. Original design and implementation: Konstantin Isakov Code contributions: Benjamin Koch Vladimir Stackov Gleb Golubitsky ikatson Eugene Agafonov Antonia Stevens Feel free to add yourself to this list in your pull-request. Please modify this file instead of source headers. zbackup-1.4.4/LICENSE000066400000000000000000000046001257621500400141660ustar00rootroot00000000000000ZBackup, a versatile deduplicating backup tool Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License (GPL) as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. The full text of versions 2 and 3 of the GPL can be found respectively in the files LICENSE-GPLV2 and LICENSE-GPLV3. EXCEPTION: This distribution of ZBackup may be linked against OpenSSL according to the terms of the section below entitled "OpenSSL Exception." This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. _OpenSSL Exception_ 0. Definitions "ZBackup" means ZBackup software licensed under version 2 or any later version of the GNU General Public License (collectively, "GPL"), or a work based on such software and licensed under the GPL. "OpenSSL" means OpenSSL toolkit software distributed by the OpenSSL Project and licensed under the OpenSSL Licenses, or a work based on such software and licensed under the OpenSSL Licenses. "OpenSSL Licenses" means the OpenSSL License and Original SSLeay License under which the OpenSSL Project distributes the OpenSSL toolkit software, as those licenses appear in the file LICENSE-OPENSSL. 1. Exception You have permission to copy, modify, propagate, and distribute a work formed by combining OpenSSL with ZBackup, or a work derivative of such a combination, even if such copying, modification, propagation, or distribution would otherwise violate the terms of the GPL. You must comply with the GPL in all respects for all of the code used other than OpenSSL. You may include this OpenSSL Exception and its grant of permissions when you distribute ZBackup. Inclusion of this notice with such a distribution constitutes a grant of such permission. If you do not wish to grant these permissions, remove this section entitled "OpenSSL Exception" from your distribution. zbackup-1.4.4/LICENSE-GPLV2000066400000000000000000000432541257621500400150260ustar00rootroot00000000000000 GNU GENERAL PUBLIC LICENSE Version 2, June 1991 Copyright (C) 1989, 1991 Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed. Preamble The licenses for most software are designed to take away your freedom to share and change it. By contrast, the GNU General Public License is intended to guarantee your freedom to share and change free software--to make sure the software is free for all its users. This General Public License applies to most of the Free Software Foundation's software and to any other program whose authors commit to using it. (Some other Free Software Foundation software is covered by the GNU Lesser General Public License instead.) You can apply it to your programs, too. When we speak of free software, we are referring to freedom, not price. Our General Public Licenses are designed to make sure that you have the freedom to distribute copies of free software (and charge for this service if you wish), that you receive source code or can get it if you want it, that you can change the software or use pieces of it in new free programs; and that you know you can do these things. To protect your rights, we need to make restrictions that forbid anyone to deny you these rights or to ask you to surrender the rights. These restrictions translate to certain responsibilities for you if you distribute copies of the software, or if you modify it. For example, if you distribute copies of such a program, whether gratis or for a fee, you must give the recipients all the rights that you have. You must make sure that they, too, receive or can get the source code. And you must show them these terms so they know their rights. We protect your rights with two steps: (1) copyright the software, and (2) offer you this license which gives you legal permission to copy, distribute and/or modify the software. Also, for each author's protection and ours, we want to make certain that everyone understands that there is no warranty for this free software. If the software is modified by someone else and passed on, we want its recipients to know that what they have is not the original, so that any problems introduced by others will not reflect on the original authors' reputations. Finally, any free program is threatened constantly by software patents. We wish to avoid the danger that redistributors of a free program will individually obtain patent licenses, in effect making the program proprietary. To prevent this, we have made it clear that any patent must be licensed for everyone's free use or not licensed at all. The precise terms and conditions for copying, distribution and modification follow. GNU GENERAL PUBLIC LICENSE TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION 0. This License applies to any program or other work which contains a notice placed by the copyright holder saying it may be distributed under the terms of this General Public License. The "Program", below, refers to any such program or work, and a "work based on the Program" means either the Program or any derivative work under copyright law: that is to say, a work containing the Program or a portion of it, either verbatim or with modifications and/or translated into another language. (Hereinafter, translation is included without limitation in the term "modification".) Each licensee is addressed as "you". Activities other than copying, distribution and modification are not covered by this License; they are outside its scope. The act of running the Program is not restricted, and the output from the Program is covered only if its contents constitute a work based on the Program (independent of having been made by running the Program). Whether that is true depends on what the Program does. 1. You may copy and distribute verbatim copies of the Program's source code as you receive it, in any medium, provided that you conspicuously and appropriately publish on each copy an appropriate copyright notice and disclaimer of warranty; keep intact all the notices that refer to this License and to the absence of any warranty; and give any other recipients of the Program a copy of this License along with the Program. You may charge a fee for the physical act of transferring a copy, and you may at your option offer warranty protection in exchange for a fee. 2. You may modify your copy or copies of the Program or any portion of it, thus forming a work based on the Program, and copy and distribute such modifications or work under the terms of Section 1 above, provided that you also meet all of these conditions: a) You must cause the modified files to carry prominent notices stating that you changed the files and the date of any change. b) You must cause any work that you distribute or publish, that in whole or in part contains or is derived from the Program or any part thereof, to be licensed as a whole at no charge to all third parties under the terms of this License. c) If the modified program normally reads commands interactively when run, you must cause it, when started running for such interactive use in the most ordinary way, to print or display an announcement including an appropriate copyright notice and a notice that there is no warranty (or else, saying that you provide a warranty) and that users may redistribute the program under these conditions, and telling the user how to view a copy of this License. (Exception: if the Program itself is interactive but does not normally print such an announcement, your work based on the Program is not required to print an announcement.) These requirements apply to the modified work as a whole. If identifiable sections of that work are not derived from the Program, and can be reasonably considered independent and separate works in themselves, then this License, and its terms, do not apply to those sections when you distribute them as separate works. But when you distribute the same sections as part of a whole which is a work based on the Program, the distribution of the whole must be on the terms of this License, whose permissions for other licensees extend to the entire whole, and thus to each and every part regardless of who wrote it. Thus, it is not the intent of this section to claim rights or contest your rights to work written entirely by you; rather, the intent is to exercise the right to control the distribution of derivative or collective works based on the Program. In addition, mere aggregation of another work not based on the Program with the Program (or with a work based on the Program) on a volume of a storage or distribution medium does not bring the other work under the scope of this License. 3. You may copy and distribute the Program (or a work based on it, under Section 2) in object code or executable form under the terms of Sections 1 and 2 above provided that you also do one of the following: a) Accompany it with the complete corresponding machine-readable source code, which must be distributed under the terms of Sections 1 and 2 above on a medium customarily used for software interchange; or, b) Accompany it with a written offer, valid for at least three years, to give any third party, for a charge no more than your cost of physically performing source distribution, a complete machine-readable copy of the corresponding source code, to be distributed under the terms of Sections 1 and 2 above on a medium customarily used for software interchange; or, c) Accompany it with the information you received as to the offer to distribute corresponding source code. (This alternative is allowed only for noncommercial distribution and only if you received the program in object code or executable form with such an offer, in accord with Subsection b above.) The source code for a work means the preferred form of the work for making modifications to it. For an executable work, complete source code means all the source code for all modules it contains, plus any associated interface definition files, plus the scripts used to control compilation and installation of the executable. However, as a special exception, the source code distributed need not include anything that is normally distributed (in either source or binary form) with the major components (compiler, kernel, and so on) of the operating system on which the executable runs, unless that component itself accompanies the executable. If distribution of executable or object code is made by offering access to copy from a designated place, then offering equivalent access to copy the source code from the same place counts as distribution of the source code, even though third parties are not compelled to copy the source along with the object code. 4. You may not copy, modify, sublicense, or distribute the Program except as expressly provided under this License. Any attempt otherwise to copy, modify, sublicense or distribute the Program is void, and will automatically terminate your rights under this License. However, parties who have received copies, or rights, from you under this License will not have their licenses terminated so long as such parties remain in full compliance. 5. You are not required to accept this License, since you have not signed it. However, nothing else grants you permission to modify or distribute the Program or its derivative works. These actions are prohibited by law if you do not accept this License. Therefore, by modifying or distributing the Program (or any work based on the Program), you indicate your acceptance of this License to do so, and all its terms and conditions for copying, distributing or modifying the Program or works based on it. 6. Each time you redistribute the Program (or any work based on the Program), the recipient automatically receives a license from the original licensor to copy, distribute or modify the Program subject to these terms and conditions. You may not impose any further restrictions on the recipients' exercise of the rights granted herein. You are not responsible for enforcing compliance by third parties to this License. 7. If, as a consequence of a court judgment or allegation of patent infringement or for any other reason (not limited to patent issues), conditions are imposed on you (whether by court order, agreement or otherwise) that contradict the conditions of this License, they do not excuse you from the conditions of this License. If you cannot distribute so as to satisfy simultaneously your obligations under this License and any other pertinent obligations, then as a consequence you may not distribute the Program at all. For example, if a patent license would not permit royalty-free redistribution of the Program by all those who receive copies directly or indirectly through you, then the only way you could satisfy both it and this License would be to refrain entirely from distribution of the Program. If any portion of this section is held invalid or unenforceable under any particular circumstance, the balance of the section is intended to apply and the section as a whole is intended to apply in other circumstances. It is not the purpose of this section to induce you to infringe any patents or other property right claims or to contest validity of any such claims; this section has the sole purpose of protecting the integrity of the free software distribution system, which is implemented by public license practices. Many people have made generous contributions to the wide range of software distributed through that system in reliance on consistent application of that system; it is up to the author/donor to decide if he or she is willing to distribute software through any other system and a licensee cannot impose that choice. This section is intended to make thoroughly clear what is believed to be a consequence of the rest of this License. 8. If the distribution and/or use of the Program is restricted in certain countries either by patents or by copyrighted interfaces, the original copyright holder who places the Program under this License may add an explicit geographical distribution limitation excluding those countries, so that distribution is permitted only in or among countries not thus excluded. In such case, this License incorporates the limitation as if written in the body of this License. 9. The Free Software Foundation may publish revised and/or new versions of the General Public License from time to time. Such new versions will be similar in spirit to the present version, but may differ in detail to address new problems or concerns. Each version is given a distinguishing version number. If the Program specifies a version number of this License which applies to it and "any later version", you have the option of following the terms and conditions either of that version or of any later version published by the Free Software Foundation. If the Program does not specify a version number of this License, you may choose any version ever published by the Free Software Foundation. 10. If you wish to incorporate parts of the Program into other free programs whose distribution conditions are different, write to the author to ask for permission. For software which is copyrighted by the Free Software Foundation, write to the Free Software Foundation; we sometimes make exceptions for this. Our decision will be guided by the two goals of preserving the free status of all derivatives of our free software and of promoting the sharing and reuse of software generally. NO WARRANTY 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. END OF TERMS AND CONDITIONS How to Apply These Terms to Your New Programs If you develop a new program, and you want it to be of the greatest possible use to the public, the best way to achieve this is to make it free software which everyone can redistribute and change under these terms. To do so, attach the following notices to the program. It is safest to attach them to the start of each source file to most effectively convey the exclusion of warranty; and each file should have at least the "copyright" line and a pointer to where the full notice is found. Copyright (C) This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. Also add information on how to contact you by electronic and paper mail. If the program is interactive, make it output a short notice like this when it starts in an interactive mode: Gnomovision version 69, Copyright (C) year name of author Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. This is free software, and you are welcome to redistribute it under certain conditions; type `show c' for details. The hypothetical commands `show w' and `show c' should show the appropriate parts of the General Public License. Of course, the commands you use may be called something other than `show w' and `show c'; they could even be mouse-clicks or menu items--whatever suits your program. You should also get your employer (if you work as a programmer) or your school, if any, to sign a "copyright disclaimer" for the program, if necessary. Here is a sample; alter the names: Yoyodyne, Inc., hereby disclaims all copyright interest in the program `Gnomovision' (which makes passes at compilers) written by James Hacker. , 1 April 1989 Ty Coon, President of Vice This General Public License does not permit incorporating your program into proprietary programs. If your program is a subroutine library, you may consider it more useful to permit linking proprietary applications with the library. If this is what you want to do, use the GNU Lesser General Public License instead of this License. zbackup-1.4.4/LICENSE-GPLV3000066400000000000000000001045131257621500400150230ustar00rootroot00000000000000 GNU GENERAL PUBLIC LICENSE Version 3, 29 June 2007 Copyright (C) 2007 Free Software Foundation, Inc. Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed. Preamble The GNU General Public License is a free, copyleft license for software and other kinds of works. The licenses for most software and other practical works are designed to take away your freedom to share and change the works. By contrast, the GNU General Public License is intended to guarantee your freedom to share and change all versions of a program--to make sure it remains free software for all its users. We, the Free Software Foundation, use the GNU General Public License for most of our software; it applies also to any other work released this way by its authors. You can apply it to your programs, too. When we speak of free software, we are referring to freedom, not price. Our General Public Licenses are designed to make sure that you have the freedom to distribute copies of free software (and charge for them if you wish), that you receive source code or can get it if you want it, that you can change the software or use pieces of it in new free programs, and that you know you can do these things. To protect your rights, we need to prevent others from denying you these rights or asking you to surrender the rights. Therefore, you have certain responsibilities if you distribute copies of the software, or if you modify it: responsibilities to respect the freedom of others. For example, if you distribute copies of such a program, whether gratis or for a fee, you must pass on to the recipients the same freedoms that you received. You must make sure that they, too, receive or can get the source code. And you must show them these terms so they know their rights. Developers that use the GNU GPL protect your rights with two steps: (1) assert copyright on the software, and (2) offer you this License giving you legal permission to copy, distribute and/or modify it. For the developers' and authors' protection, the GPL clearly explains that there is no warranty for this free software. For both users' and authors' sake, the GPL requires that modified versions be marked as changed, so that their problems will not be attributed erroneously to authors of previous versions. Some devices are designed to deny users access to install or run modified versions of the software inside them, although the manufacturer can do so. This is fundamentally incompatible with the aim of protecting users' freedom to change the software. The systematic pattern of such abuse occurs in the area of products for individuals to use, which is precisely where it is most unacceptable. Therefore, we have designed this version of the GPL to prohibit the practice for those products. If such problems arise substantially in other domains, we stand ready to extend this provision to those domains in future versions of the GPL, as needed to protect the freedom of users. Finally, every program is threatened constantly by software patents. States should not allow patents to restrict development and use of software on general-purpose computers, but in those that do, we wish to avoid the special danger that patents applied to a free program could make it effectively proprietary. To prevent this, the GPL assures that patents cannot be used to render the program non-free. The precise terms and conditions for copying, distribution and modification follow. TERMS AND CONDITIONS 0. Definitions. "This License" refers to version 3 of the GNU General Public License. "Copyright" also means copyright-like laws that apply to other kinds of works, such as semiconductor masks. "The Program" refers to any copyrightable work licensed under this License. Each licensee is addressed as "you". "Licensees" and "recipients" may be individuals or organizations. To "modify" a work means to copy from or adapt all or part of the work in a fashion requiring copyright permission, other than the making of an exact copy. The resulting work is called a "modified version" of the earlier work or a work "based on" the earlier work. A "covered work" means either the unmodified Program or a work based on the Program. To "propagate" a work means to do anything with it that, without permission, would make you directly or secondarily liable for infringement under applicable copyright law, except executing it on a computer or modifying a private copy. Propagation includes copying, distribution (with or without modification), making available to the public, and in some countries other activities as well. To "convey" a work means any kind of propagation that enables other parties to make or receive copies. Mere interaction with a user through a computer network, with no transfer of a copy, is not conveying. An interactive user interface displays "Appropriate Legal Notices" to the extent that it includes a convenient and prominently visible feature that (1) displays an appropriate copyright notice, and (2) tells the user that there is no warranty for the work (except to the extent that warranties are provided), that licensees may convey the work under this License, and how to view a copy of this License. If the interface presents a list of user commands or options, such as a menu, a prominent item in the list meets this criterion. 1. Source Code. The "source code" for a work means the preferred form of the work for making modifications to it. "Object code" means any non-source form of a work. A "Standard Interface" means an interface that either is an official standard defined by a recognized standards body, or, in the case of interfaces specified for a particular programming language, one that is widely used among developers working in that language. The "System Libraries" of an executable work include anything, other than the work as a whole, that (a) is included in the normal form of packaging a Major Component, but which is not part of that Major Component, and (b) serves only to enable use of the work with that Major Component, or to implement a Standard Interface for which an implementation is available to the public in source code form. A "Major Component", in this context, means a major essential component (kernel, window system, and so on) of the specific operating system (if any) on which the executable work runs, or a compiler used to produce the work, or an object code interpreter used to run it. The "Corresponding Source" for a work in object code form means all the source code needed to generate, install, and (for an executable work) run the object code and to modify the work, including scripts to control those activities. However, it does not include the work's System Libraries, or general-purpose tools or generally available free programs which are used unmodified in performing those activities but which are not part of the work. For example, Corresponding Source includes interface definition files associated with source files for the work, and the source code for shared libraries and dynamically linked subprograms that the work is specifically designed to require, such as by intimate data communication or control flow between those subprograms and other parts of the work. The Corresponding Source need not include anything that users can regenerate automatically from other parts of the Corresponding Source. The Corresponding Source for a work in source code form is that same work. 2. Basic Permissions. All rights granted under this License are granted for the term of copyright on the Program, and are irrevocable provided the stated conditions are met. This License explicitly affirms your unlimited permission to run the unmodified Program. The output from running a covered work is covered by this License only if the output, given its content, constitutes a covered work. This License acknowledges your rights of fair use or other equivalent, as provided by copyright law. You may make, run and propagate covered works that you do not convey, without conditions so long as your license otherwise remains in force. You may convey covered works to others for the sole purpose of having them make modifications exclusively for you, or provide you with facilities for running those works, provided that you comply with the terms of this License in conveying all material for which you do not control copyright. Those thus making or running the covered works for you must do so exclusively on your behalf, under your direction and control, on terms that prohibit them from making any copies of your copyrighted material outside their relationship with you. Conveying under any other circumstances is permitted solely under the conditions stated below. Sublicensing is not allowed; section 10 makes it unnecessary. 3. Protecting Users' Legal Rights From Anti-Circumvention Law. No covered work shall be deemed part of an effective technological measure under any applicable law fulfilling obligations under article 11 of the WIPO copyright treaty adopted on 20 December 1996, or similar laws prohibiting or restricting circumvention of such measures. When you convey a covered work, you waive any legal power to forbid circumvention of technological measures to the extent such circumvention is effected by exercising rights under this License with respect to the covered work, and you disclaim any intention to limit operation or modification of the work as a means of enforcing, against the work's users, your or third parties' legal rights to forbid circumvention of technological measures. 4. Conveying Verbatim Copies. You may convey verbatim copies of the Program's source code as you receive it, in any medium, provided that you conspicuously and appropriately publish on each copy an appropriate copyright notice; keep intact all notices stating that this License and any non-permissive terms added in accord with section 7 apply to the code; keep intact all notices of the absence of any warranty; and give all recipients a copy of this License along with the Program. You may charge any price or no price for each copy that you convey, and you may offer support or warranty protection for a fee. 5. Conveying Modified Source Versions. You may convey a work based on the Program, or the modifications to produce it from the Program, in the form of source code under the terms of section 4, provided that you also meet all of these conditions: a) The work must carry prominent notices stating that you modified it, and giving a relevant date. b) The work must carry prominent notices stating that it is released under this License and any conditions added under section 7. This requirement modifies the requirement in section 4 to "keep intact all notices". c) You must license the entire work, as a whole, under this License to anyone who comes into possession of a copy. This License will therefore apply, along with any applicable section 7 additional terms, to the whole of the work, and all its parts, regardless of how they are packaged. This License gives no permission to license the work in any other way, but it does not invalidate such permission if you have separately received it. d) If the work has interactive user interfaces, each must display Appropriate Legal Notices; however, if the Program has interactive interfaces that do not display Appropriate Legal Notices, your work need not make them do so. A compilation of a covered work with other separate and independent works, which are not by their nature extensions of the covered work, and which are not combined with it such as to form a larger program, in or on a volume of a storage or distribution medium, is called an "aggregate" if the compilation and its resulting copyright are not used to limit the access or legal rights of the compilation's users beyond what the individual works permit. Inclusion of a covered work in an aggregate does not cause this License to apply to the other parts of the aggregate. 6. Conveying Non-Source Forms. You may convey a covered work in object code form under the terms of sections 4 and 5, provided that you also convey the machine-readable Corresponding Source under the terms of this License, in one of these ways: a) Convey the object code in, or embodied in, a physical product (including a physical distribution medium), accompanied by the Corresponding Source fixed on a durable physical medium customarily used for software interchange. b) Convey the object code in, or embodied in, a physical product (including a physical distribution medium), accompanied by a written offer, valid for at least three years and valid for as long as you offer spare parts or customer support for that product model, to give anyone who possesses the object code either (1) a copy of the Corresponding Source for all the software in the product that is covered by this License, on a durable physical medium customarily used for software interchange, for a price no more than your reasonable cost of physically performing this conveying of source, or (2) access to copy the Corresponding Source from a network server at no charge. c) Convey individual copies of the object code with a copy of the written offer to provide the Corresponding Source. This alternative is allowed only occasionally and noncommercially, and only if you received the object code with such an offer, in accord with subsection 6b. d) Convey the object code by offering access from a designated place (gratis or for a charge), and offer equivalent access to the Corresponding Source in the same way through the same place at no further charge. You need not require recipients to copy the Corresponding Source along with the object code. If the place to copy the object code is a network server, the Corresponding Source may be on a different server (operated by you or a third party) that supports equivalent copying facilities, provided you maintain clear directions next to the object code saying where to find the Corresponding Source. Regardless of what server hosts the Corresponding Source, you remain obligated to ensure that it is available for as long as needed to satisfy these requirements. e) Convey the object code using peer-to-peer transmission, provided you inform other peers where the object code and Corresponding Source of the work are being offered to the general public at no charge under subsection 6d. A separable portion of the object code, whose source code is excluded from the Corresponding Source as a System Library, need not be included in conveying the object code work. A "User Product" is either (1) a "consumer product", which means any tangible personal property which is normally used for personal, family, or household purposes, or (2) anything designed or sold for incorporation into a dwelling. In determining whether a product is a consumer product, doubtful cases shall be resolved in favor of coverage. For a particular product received by a particular user, "normally used" refers to a typical or common use of that class of product, regardless of the status of the particular user or of the way in which the particular user actually uses, or expects or is expected to use, the product. A product is a consumer product regardless of whether the product has substantial commercial, industrial or non-consumer uses, unless such uses represent the only significant mode of use of the product. "Installation Information" for a User Product means any methods, procedures, authorization keys, or other information required to install and execute modified versions of a covered work in that User Product from a modified version of its Corresponding Source. The information must suffice to ensure that the continued functioning of the modified object code is in no case prevented or interfered with solely because modification has been made. If you convey an object code work under this section in, or with, or specifically for use in, a User Product, and the conveying occurs as part of a transaction in which the right of possession and use of the User Product is transferred to the recipient in perpetuity or for a fixed term (regardless of how the transaction is characterized), the Corresponding Source conveyed under this section must be accompanied by the Installation Information. But this requirement does not apply if neither you nor any third party retains the ability to install modified object code on the User Product (for example, the work has been installed in ROM). The requirement to provide Installation Information does not include a requirement to continue to provide support service, warranty, or updates for a work that has been modified or installed by the recipient, or for the User Product in which it has been modified or installed. Access to a network may be denied when the modification itself materially and adversely affects the operation of the network or violates the rules and protocols for communication across the network. Corresponding Source conveyed, and Installation Information provided, in accord with this section must be in a format that is publicly documented (and with an implementation available to the public in source code form), and must require no special password or key for unpacking, reading or copying. 7. Additional Terms. "Additional permissions" are terms that supplement the terms of this License by making exceptions from one or more of its conditions. Additional permissions that are applicable to the entire Program shall be treated as though they were included in this License, to the extent that they are valid under applicable law. If additional permissions apply only to part of the Program, that part may be used separately under those permissions, but the entire Program remains governed by this License without regard to the additional permissions. When you convey a copy of a covered work, you may at your option remove any additional permissions from that copy, or from any part of it. (Additional permissions may be written to require their own removal in certain cases when you modify the work.) You may place additional permissions on material, added by you to a covered work, for which you have or can give appropriate copyright permission. Notwithstanding any other provision of this License, for material you add to a covered work, you may (if authorized by the copyright holders of that material) supplement the terms of this License with terms: a) Disclaiming warranty or limiting liability differently from the terms of sections 15 and 16 of this License; or b) Requiring preservation of specified reasonable legal notices or author attributions in that material or in the Appropriate Legal Notices displayed by works containing it; or c) Prohibiting misrepresentation of the origin of that material, or requiring that modified versions of such material be marked in reasonable ways as different from the original version; or d) Limiting the use for publicity purposes of names of licensors or authors of the material; or e) Declining to grant rights under trademark law for use of some trade names, trademarks, or service marks; or f) Requiring indemnification of licensors and authors of that material by anyone who conveys the material (or modified versions of it) with contractual assumptions of liability to the recipient, for any liability that these contractual assumptions directly impose on those licensors and authors. All other non-permissive additional terms are considered "further restrictions" within the meaning of section 10. If the Program as you received it, or any part of it, contains a notice stating that it is governed by this License along with a term that is a further restriction, you may remove that term. If a license document contains a further restriction but permits relicensing or conveying under this License, you may add to a covered work material governed by the terms of that license document, provided that the further restriction does not survive such relicensing or conveying. If you add terms to a covered work in accord with this section, you must place, in the relevant source files, a statement of the additional terms that apply to those files, or a notice indicating where to find the applicable terms. Additional terms, permissive or non-permissive, may be stated in the form of a separately written license, or stated as exceptions; the above requirements apply either way. 8. Termination. You may not propagate or modify a covered work except as expressly provided under this License. Any attempt otherwise to propagate or modify it is void, and will automatically terminate your rights under this License (including any patent licenses granted under the third paragraph of section 11). However, if you cease all violation of this License, then your license from a particular copyright holder is reinstated (a) provisionally, unless and until the copyright holder explicitly and finally terminates your license, and (b) permanently, if the copyright holder fails to notify you of the violation by some reasonable means prior to 60 days after the cessation. Moreover, your license from a particular copyright holder is reinstated permanently if the copyright holder notifies you of the violation by some reasonable means, this is the first time you have received notice of violation of this License (for any work) from that copyright holder, and you cure the violation prior to 30 days after your receipt of the notice. Termination of your rights under this section does not terminate the licenses of parties who have received copies or rights from you under this License. If your rights have been terminated and not permanently reinstated, you do not qualify to receive new licenses for the same material under section 10. 9. Acceptance Not Required for Having Copies. You are not required to accept this License in order to receive or run a copy of the Program. Ancillary propagation of a covered work occurring solely as a consequence of using peer-to-peer transmission to receive a copy likewise does not require acceptance. However, nothing other than this License grants you permission to propagate or modify any covered work. These actions infringe copyright if you do not accept this License. Therefore, by modifying or propagating a covered work, you indicate your acceptance of this License to do so. 10. Automatic Licensing of Downstream Recipients. Each time you convey a covered work, the recipient automatically receives a license from the original licensors, to run, modify and propagate that work, subject to this License. You are not responsible for enforcing compliance by third parties with this License. An "entity transaction" is a transaction transferring control of an organization, or substantially all assets of one, or subdividing an organization, or merging organizations. If propagation of a covered work results from an entity transaction, each party to that transaction who receives a copy of the work also receives whatever licenses to the work the party's predecessor in interest had or could give under the previous paragraph, plus a right to possession of the Corresponding Source of the work from the predecessor in interest, if the predecessor has it or can get it with reasonable efforts. You may not impose any further restrictions on the exercise of the rights granted or affirmed under this License. For example, you may not impose a license fee, royalty, or other charge for exercise of rights granted under this License, and you may not initiate litigation (including a cross-claim or counterclaim in a lawsuit) alleging that any patent claim is infringed by making, using, selling, offering for sale, or importing the Program or any portion of it. 11. Patents. A "contributor" is a copyright holder who authorizes use under this License of the Program or a work on which the Program is based. The work thus licensed is called the contributor's "contributor version". A contributor's "essential patent claims" are all patent claims owned or controlled by the contributor, whether already acquired or hereafter acquired, that would be infringed by some manner, permitted by this License, of making, using, or selling its contributor version, but do not include claims that would be infringed only as a consequence of further modification of the contributor version. For purposes of this definition, "control" includes the right to grant patent sublicenses in a manner consistent with the requirements of this License. Each contributor grants you a non-exclusive, worldwide, royalty-free patent license under the contributor's essential patent claims, to make, use, sell, offer for sale, import and otherwise run, modify and propagate the contents of its contributor version. In the following three paragraphs, a "patent license" is any express agreement or commitment, however denominated, not to enforce a patent (such as an express permission to practice a patent or covenant not to sue for patent infringement). To "grant" such a patent license to a party means to make such an agreement or commitment not to enforce a patent against the party. If you convey a covered work, knowingly relying on a patent license, and the Corresponding Source of the work is not available for anyone to copy, free of charge and under the terms of this License, through a publicly available network server or other readily accessible means, then you must either (1) cause the Corresponding Source to be so available, or (2) arrange to deprive yourself of the benefit of the patent license for this particular work, or (3) arrange, in a manner consistent with the requirements of this License, to extend the patent license to downstream recipients. "Knowingly relying" means you have actual knowledge that, but for the patent license, your conveying the covered work in a country, or your recipient's use of the covered work in a country, would infringe one or more identifiable patents in that country that you have reason to believe are valid. If, pursuant to or in connection with a single transaction or arrangement, you convey, or propagate by procuring conveyance of, a covered work, and grant a patent license to some of the parties receiving the covered work authorizing them to use, propagate, modify or convey a specific copy of the covered work, then the patent license you grant is automatically extended to all recipients of the covered work and works based on it. A patent license is "discriminatory" if it does not include within the scope of its coverage, prohibits the exercise of, or is conditioned on the non-exercise of one or more of the rights that are specifically granted under this License. You may not convey a covered work if you are a party to an arrangement with a third party that is in the business of distributing software, under which you make payment to the third party based on the extent of your activity of conveying the work, and under which the third party grants, to any of the parties who would receive the covered work from you, a discriminatory patent license (a) in connection with copies of the covered work conveyed by you (or copies made from those copies), or (b) primarily for and in connection with specific products or compilations that contain the covered work, unless you entered into that arrangement, or that patent license was granted, prior to 28 March 2007. Nothing in this License shall be construed as excluding or limiting any implied license or other defenses to infringement that may otherwise be available to you under applicable patent law. 12. No Surrender of Others' Freedom. If conditions are imposed on you (whether by court order, agreement or otherwise) that contradict the conditions of this License, they do not excuse you from the conditions of this License. If you cannot convey a covered work so as to satisfy simultaneously your obligations under this License and any other pertinent obligations, then as a consequence you may not convey it at all. For example, if you agree to terms that obligate you to collect a royalty for further conveying from those to whom you convey the Program, the only way you could satisfy both those terms and this License would be to refrain entirely from conveying the Program. 13. Use with the GNU Affero General Public License. Notwithstanding any other provision of this License, you have permission to link or combine any covered work with a work licensed under version 3 of the GNU Affero General Public License into a single combined work, and to convey the resulting work. The terms of this License will continue to apply to the part which is the covered work, but the special requirements of the GNU Affero General Public License, section 13, concerning interaction through a network will apply to the combination as such. 14. Revised Versions of this License. The Free Software Foundation may publish revised and/or new versions of the GNU General Public License from time to time. Such new versions will be similar in spirit to the present version, but may differ in detail to address new problems or concerns. Each version is given a distinguishing version number. If the Program specifies that a certain numbered version of the GNU General Public License "or any later version" applies to it, you have the option of following the terms and conditions either of that numbered version or of any later version published by the Free Software Foundation. If the Program does not specify a version number of the GNU General Public License, you may choose any version ever published by the Free Software Foundation. If the Program specifies that a proxy can decide which future versions of the GNU General Public License can be used, that proxy's public statement of acceptance of a version permanently authorizes you to choose that version for the Program. Later license versions may give you additional or different permissions. However, no additional obligations are imposed on any author or copyright holder as a result of your choosing to follow a later version. 15. Disclaimer of Warranty. THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 16. Limitation of Liability. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. 17. Interpretation of Sections 15 and 16. If the disclaimer of warranty and limitation of liability provided above cannot be given local legal effect according to their terms, reviewing courts shall apply local law that most closely approximates an absolute waiver of all civil liability in connection with the Program, unless a warranty or assumption of liability accompanies a copy of the Program in return for a fee. END OF TERMS AND CONDITIONS How to Apply These Terms to Your New Programs If you develop a new program, and you want it to be of the greatest possible use to the public, the best way to achieve this is to make it free software which everyone can redistribute and change under these terms. To do so, attach the following notices to the program. It is safest to attach them to the start of each source file to most effectively state the exclusion of warranty; and each file should have at least the "copyright" line and a pointer to where the full notice is found. Copyright (C) This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program. If not, see . Also add information on how to contact you by electronic and paper mail. If the program does terminal interaction, make it output a short notice like this when it starts in an interactive mode: Copyright (C) This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. This is free software, and you are welcome to redistribute it under certain conditions; type `show c' for details. The hypothetical commands `show w' and `show c' should show the appropriate parts of the General Public License. Of course, your program's commands might be different; for a GUI interface, you would use an "about box". You should also get your employer (if you work as a programmer) or school, if any, to sign a "copyright disclaimer" for the program, if necessary. For more information on this, and how to apply and follow the GNU GPL, see . The GNU General Public License does not permit incorporating your program into proprietary programs. If your program is a subroutine library, you may consider it more useful to permit linking proprietary applications with the library. If this is what you want to do, use the GNU Lesser General Public License instead of this License. But first, please read . zbackup-1.4.4/LICENSE-OPENSSL000066400000000000000000000142071257621500400153130ustar00rootroot00000000000000 LICENSE ISSUES ============== The OpenSSL toolkit stays under a dual license, i.e. both the conditions of the OpenSSL License and the original SSLeay license apply to the toolkit. See below for the actual license texts. Actually both licenses are BSD-style Open Source licenses. In case of any license issues related to OpenSSL please contact openssl-core@openssl.org. OpenSSL License --------------- /* ==================================================================== * Copyright (c) 1998-2011 The OpenSSL Project. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. * * 3. All advertising materials mentioning features or use of this * software must display the following acknowledgment: * "This product includes software developed by the OpenSSL Project * for use in the OpenSSL Toolkit. (http://www.openssl.org/)" * * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to * endorse or promote products derived from this software without * prior written permission. For written permission, please contact * openssl-core@openssl.org. * * 5. Products derived from this software may not be called "OpenSSL" * nor may "OpenSSL" appear in their names without prior written * permission of the OpenSSL Project. * * 6. Redistributions of any form whatsoever must retain the following * acknowledgment: * "This product includes software developed by the OpenSSL Project * for use in the OpenSSL Toolkit (http://www.openssl.org/)" * * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE OpenSSL PROJECT OR * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED * OF THE POSSIBILITY OF SUCH DAMAGE. * ==================================================================== * * This product includes cryptographic software written by Eric Young * (eay@cryptsoft.com). This product includes software written by Tim * Hudson (tjh@cryptsoft.com). * */ Original SSLeay License ----------------------- /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com) * All rights reserved. * * This package is an SSL implementation written * by Eric Young (eay@cryptsoft.com). * The implementation was written so as to conform with Netscapes SSL. * * This library is free for commercial and non-commercial use as long as * the following conditions are aheared to. The following conditions * apply to all code found in this distribution, be it the RC4, RSA, * lhash, DES, etc., code; not just the SSL code. The SSL documentation * included with this distribution is covered by the same copyright terms * except that the holder is Tim Hudson (tjh@cryptsoft.com). * * Copyright remains Eric Young's, and as such any Copyright notices in * the code are not to be removed. * If this package is used in a product, Eric Young should be given attribution * as the author of the parts of the library used. * This can be in the form of a textual message at program startup or * in documentation (online or textual) provided with the package. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * "This product includes cryptographic software written by * Eric Young (eay@cryptsoft.com)" * The word 'cryptographic' can be left out if the rouines from the library * being used are not cryptographic related :-). * 4. If you include any Windows specific code (or a derivative thereof) from * the apps directory (application code) you must include an acknowledgement: * "This product includes software written by Tim Hudson (tjh@cryptsoft.com)" * * THIS SOFTWARE IS PROVIDED BY ERIC YOUNG ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * The licence and distribution terms for any publically available version or * derivative of this code cannot be changed. i.e. this code cannot simply be * copied and put under another distribution licence * [including the GNU Public Licence.] */ zbackup-1.4.4/README.md000066400000000000000000000463541257621500400144540ustar00rootroot00000000000000[![Build Status](https://travis-ci.org/zbackup/zbackup.svg)](https://travis-ci.org/zbackup/zbackup) # Introduction **zbackup** is a globally-deduplicating backup tool, based on the ideas found in [rsync](http://rsync.samba.org/). Feed a large `.tar` into it, and it will store duplicate regions of it only once, then compress and optionally encrypt the result. Feed another `.tar` file, and it will also re-use any data found in any previous backups. This way only new changes are stored, and as long as the files are not very different, the amount of storage required is very low. Any of the backup files stored previously can be read back in full at any time. The program is format-agnostic, so you can feed virtually any files to it (any types of archives, proprietary formats, even raw disk images -- but see [Caveats](#caveats)). This is achieved by sliding a window with a rolling hash over the input at a byte granularity and checking whether the block in focus was ever met already. If a rolling hash matches, an additional full cryptographic hash is calculated to ensure the block is indeed the same. The deduplication happens then. # Features The program has the following features: * Parallel LZMA or LZO compression of the stored data * Built-in AES encryption of the stored data * Possibility to delete old backup data * Use of a 64-bit rolling hash, keeping the amount of soft collisions to zero * Repository consists of immutable files. No existing files are ever modified * Written in C++ only with only modest library dependencies * Safe to use in production (see [below](#safety)) * Possibility to exchange data between repos without recompression # Build dependencies * `cmake` >= 2.8.2 (though it should not be too hard to compile the sources by hand if needed) * `libssl-dev` for all encryption, hashing and random numbers * `libprotobuf-dev` and `protobuf-compiler` for data serialization * `liblzma-dev` for compression * `liblzo2-dev` for compression (optional) * `zlib1g-dev` for adler32 calculation # Quickstart To build and install: ```bash cd zbackup cmake . make sudo make install # or just run as ./zbackup ``` Zbackup is also part of the [Debian](https://packages.debian.org/search?keywords=zbackup), [Ubuntu](http://packages.ubuntu.com/search?keywords=zbackup) and [Arch Linux](https://aur.archlinux.org/packages/zbackup/) distributions of GNU/Linux. To use: ```bash zbackup init --non-encrypted /my/backup/repo tar c /my/precious/data | zbackup backup /my/backup/repo/backups/backup-`date '+%Y-%m-%d'` zbackup restore /my/backup/repo/backups/backup-`date '+%Y-%m-%d'` > /my/precious/backup-restored.tar ``` If you have a lot of RAM to spare, you can use it to speed-up the restore process -- to use 512 MB more, pass `--cache-size 512mb` when restoring. If encryption is wanted, create a file with your password: ``` bash # more secure to use an editor echo mypassword > ~/.my_backup_password chmod 600 ~/.my_backup_password ``` Then init the repo the following way: ```bash zbackup init --password-file ~/.my_backup_password /my/backup/repo ``` And always pass the same argument afterwards: ```bash tar c /my/precious/data | zbackup --password-file ~/.my_backup_password backup /my/backup/repo/backups/backup-`date '+%Y-%m-%d'` zbackup --password-file ~/.my_backup_password restore /my/backup/repo/backups/backup-`date '+%Y-%m-%d'` > /my/precious/backup-restored.tar ``` If you have a 32-bit system and a lot of cores, consider lowering the number of compression threads by passing `--threads 4` or `--threads 2` if the program runs out of address space when backing up (see why [below](#caveats), item 2). There should be no problem on a 64-bit system. # Caveats * While you can pipe any data into the program, the data should be uncompressed and unencrypted -- otherwise no deduplication could be performed on it. `zbackup` would compress and encrypt the data itself, so there's no need to do that yourself. So just run `tar c` and pipe it into `zbackup` directly. If backing up disk images employing encryption, pipe the unencrypted version (the one you normally mount). If you create `.zip` or `.rar` files, use no compression (`-0` or `-m0`) and no encryption. * Parallel LZMA compression uses a lot of RAM (several hundreds of megabytes, depending on the number of threads used), and ten times more virtual address space. The latter is only relevant on 32-bit architectures where it's limited to 2 or 3 GB. If you hit the ceiling, lower the number of threads with `--threads`. * Since the data is deduplicated, there's naturally no redundancy in it. A loss of a single file can lead to a loss of virtually all data. Make sure you store it on a redundant storage (RAID1, a cloud provider etc). * The encryption key, if used, is stored in the `info` file in the root of the repo. It is encrypted with your password. Technically thus you can change your password without re-encrypting any data, and as long as no one possesses the old `info` file and knows your old password, you would be safe (even though the actual option to change password is not implemented yet -- someone who needs this is welcome to create a pull request -- the possibility is all there). Also note that it is crucial you don't lose your `info` file, as otherwise the whole backup would be lost. # Limitations * Right now the only modes supported are reading from standard input and writing to standard output. FUSE mounts and NBD servers may be added later if someone contributes the code. * The program keeps all known blocks in an in-RAM hash table, which may create scalability problems for very large repos (see [below](#scalability)). * The only encryption mode currently implemented is `AES-128` in `CBC` mode with `PKCS#7` padding. If you believe that this is not secure enough, patches are welcome. Before you jump to conclusions however, read [this article](http://www.schneier.com/blog/archives/2009/07/another_new_aes.html). * The only compression mode supported is LZMA, which suits backups very nicely. * It's only possible to fully restore the backup in order to get to a required file, without any option to quickly pick it out. `tar` would not allow to do it anyway, but e.g. for `zip` files it could have been possible. This is possible to implement though, e.g. by exposing the data over a FUSE filesystem. * There's no option to specify block and bundle sizes other than the default ones (currently `64k` and `2MB` respectively), though it's trivial to add command-line switches for those. Most of those limitations can be lifted by implementing the respective features. # Safety Is it safe to use `zbackup` for production data? Being free software, the program comes with no warranty of any kind. That said, it's perfectly safe for production, and here's why. When performing a backup, the program never modifies or deletes any existing files -- only new ones are created. It specifically checks for that, and the code paths involved are short and easy to inspect. Furthermore, each backup is protected by its `SHA256` sum, which is calculated before piping the data into the deduplication logic. The code path doing that is also short and easy to inspect. When a backup is being restored, its `SHA256` is calculated again and compared against the stored one. The program would fail on a mismatch. Therefore, to ensure safety it is enough to restore each backup to `/dev/null` immediately after creating it. If it restores fine, it will restore fine ever after. To add some statistics, the author of the program has been using an older version of `zbackup` internally for over a year. The `SHA256` check never ever failed. Again, even if it does, you would know immediately, so no work would be lost. Therefore you are welcome to try the program in production, and if you like it, stick with it. # Usage notes The repository has the following directory structure: ``` /repo backups/ bundles/ 00/ 01/ 02/ ... index/ info ``` * The `backups` directory contain your backups. Those are very small files which are needed for restoration. They are encrypted if encryption is enabled. The names can be arbitrary. It is possible to arrange files in subdirectories, too. Free renaming is also allowed. * The `bundles` directory contains the bulk of data. Each bundle internally contains multiple small chunks, compressed together and encrypted. Together all those chunks account for all deduplicated data stored. * The `index` directory contains the full index of all chunks in the repository, together with their bundle names. A separate index file is created for each backup session. Technically those files are redundant, all information is contained in the bundles themselves. However, having a separate `index` is nice for two reasons: 1) it's faster to read as it incurs less seeks, and 2) it allows making backups while storing bundles elsewhere. Bundles are only needed when restoring -- otherwise it's sufficient to only have `index`. One could then move all newly created bundles into another machine after each backup. * `info` is a very important file which contains all global repository metadata, such as chunk and bundle sizes, and an encryption key encrypted with the user password. It is paramount not to lose it, so backing it up separately somewhere might be a good idea. On the other hand, if you absolutely don't trust your remote storage provider, you might consider not storing it with the rest of the data. It would then be impossible to decrypt it at all, even if your password gets known later. The program does not have any facilities for sending your backup over the network. You can `rsync` the repo to another computer or use any kind of cloud storage capable of storing files. Since `zbackup` never modifies any existing files, the latter is especially easy -- just tell the upload tool you use not to upload any files which already exist on the remote side (e.g. with `gsutil` it's `gsutil cp -R -n /my/backup gs:/mybackup/`). To aid with creating backups, there's an utility called `tartool` included with `zbackup`. The idea is the following: one sprinkles empty files called `.backup` and `.no-backup` across the entire filesystem. Directories where `.backup` files are placed are marked for backing up. Similarly, directories with `.no-backup` files are marked not to be backed up. Additionally, it is possible to place `.backup-XYZ` in the same directory where `XYZ` is to mark `XYZ` for backing up, or place `.no-backup-XYZ` to mark it not to be backed up. Then `tartool` can be run with three arguments -- the root directory to start from (can be `/`), the output `includes` file, and the output `excludes` file. The tool traverses over the given directory noting the `.backup*` and `.no-backup*` files and creating include and exclude lists for the `tar` utility. The `tar` utility could then be run as `tar c --files-from includes --exclude-from excludes` to store all chosen data. # Scalability This section tries do address the question on the maximum amount of data which can be held in a backup repository. What is meant here is the deduplicated data. The number of bytes in all source files ever fed into the repository doesn't matter, but the total size of the resulting repository does. Internally all input data is split into small blocks called chunks (up to `64k` each by default). Chunks are collected into bundles (up to `2MB` each by default), and those bundles are then compressed and encrypted. There are then two problems with the total number of chunks in the repository: * Hashes of all existing chunks are needed to be kept in RAM while the backup is ongoing. Since the sliding window performs checking with a single-byte granularity, lookups would otherwise be too slow. The amount of data needed to be stored is technically only 24 bytes for each chunk, where the size of the chunk is up to `64k`. In an example real-life `18GB` repo, only `18MB` are taken by in its hash index. Multiply this roughly by two to have an estimate of RAM needed to store this index as an in-RAM hash table. However, as this size is proportional to the total size of the repo, for `2TB` repo you could already require `2GB` of RAM. Most repos are much smaller though, and as long as the deduplication works properly, in many cases you can store terabytes of highly-redundant backup files in a `20GB` repo easily. * We use a 64-bit rolling hash, which allows to have an `O(1)` lookup cost at each byte we process. Due to [birthday paradox](https://en.wikipedia.org/wiki/Birthday_paradox), we would start having collisions when we approach `2^32` hashes. If each chunk we have is `32k` on average, we would get there when our repo grows to `128TB`. We would still be able to continue, but as the number of collisions would grow, we would have to resort to calculating the full hash of a block at each byte more and more often, which would result in a considerable slowdown. All in all, as long as the amount of RAM permits, one can go up to several terabytes in deduplicated data, and start having some slowdown after having hundreds of terabytes, RAM-permitting. # Design choices * We use a 64-bit modified Rabin-Karp rolling hash (see `rolling_hash.hh` for details), while most other programs use a 32-bit one. As noted previously, one problem with the hash size is its birthday bound, which with the 32-bit hash is met after having only `2^16` hashes. The choice of a 64-bit hash allows us to scale much better while having virtually the same calculation cost on a typical 64-bit machine. * `rsync` uses `MD5` as its strong hash. While `MD5` is known to be fast, it is also known to be broken, allowing a malicious user to craft colliding inputs. `zbackup` uses `SHA1` instead. The cost of `SHA1` calculations on modern machines is actually less than that of `MD5` (run `openssl speed md5 sha1` on yours), so it's a win-win situation. We only keep the first 128 bits of the `SHA1` output, and therefore together with the rolling hash we have a 192-bit hash for each chunk. It's a multiple of 8 bytes which is a nice properly on 64-bit machines, and it is long enough not to worry about possible collisions. * `AES-128` in `CBC` mode with `PKCS#7` padding is used for encryption. This seems to be a reasonbly safe classic solution. Each encrypted file has a random IV as its first 16 bytes. * We use Google's [protocol buffers](https://developers.google.com/protocol-buffers/) to represent data structures in binary form. They are very efficient and relatively simple to use. # Compression zbackup uses LZMA to compress stored data. It compresses very well, but it will slow down your backup (unless you have a very fast CPU). LZO is much faster, but the files will be bigger. If you don't want your backup process to be cpu-bound, you should consider using LZO. However, there are some caveats: 1. LZO is so fast that other parts of zbackup consume significant portions of the CPU. In fact, it is only using one core on my machine because compression is the only thing that can run in parallel. 2. I've hacked the LZO support in a day. You shouldn't trust it. Please make sure that restore works before you assume that your data is safe. That may still be faster than a backup with LZMA ;-) 3. LZMA is still the default, so make sure that you use the `--compression lzo` argument when you init the repo or whenever you do a backup. You can mix LZMA and LZO in a repository. Each bundle file has a field that says how it was compressed, so zbackup will use the right method to decompress it. You could use an old zbackup respository with only LZMA bundles and start using LZO. However, please think twice before you do that because old versions of zbackup won't be able to read those bundles. # Improvements There's a lot to be improved in the program. It was released with the minimum amount of functionality to be useful. It is also stable. This should hopefully stimulate people to join the development and add all those other fancy features. Here's a list of ideas: * Additional options, such as configurable chunk and bundle sizes etc. * A command to change password. * Improved garbage collection. The program should support ability to specify maximum index file size / maximum index file count (for better compatibility with cloud storages as well) or something like retention policy. * A command to fsck the repo by doing something close to what garbage collection does, but also checking all hashes and so on. * Parallel decompression. Right now decompression is single-threaded, but it is possible to look ahead in the stream and perform prefetching. * Support for mounting the repo over FUSE. Random access to data would then be possible. * Support for exposing a backed up file over a userspace NBD server. It would then be possible to mount raw disk images without extracting them. * Support for other encryption types (preferably for everything `openssl` supports with its `evp`). * Support for other compression methods. * You name it! # Communication * The program's website is at . * Development happens at . * Discussion forum is at . Please ask for help there! The author is reachable over email at . Please be constructive and don't ask for help using the program, though. In most cases it's best to stick to the forum, unless you have something to discuss with the author in private. # Similar projects `zbackup` is certainly not the first project to embrace the idea of using a rolling hash for deduplication. Here's a list of other projects the author found on the web: * [bup](https://github.com/bup/bup), based on storing data in `git` packs. No possibility of removing old data. This program was the initial inspiration for `zbackup`. * [ddar](http://www.synctus.com/ddar/), seems to be a little bit outdated. Contains a nice list of alternatives with comparisons. * [rdiff-backup](http://www.nongnu.org/rdiff-backup/), based on the original `rsync` algorithm. Does not do global deduplication, only working over the files with the same file name. * [duplicity](http://duplicity.nongnu.org/), which looks similar to `rdiff-backup` with regards to mode of operation. * Some filesystems (most notably [ZFS](http://en.wikipedia.org/wiki/ZFS) and [Btrfs](http://en.wikipedia.org/wiki/Btrfs)) provide deduplication features. They do so only at block level though, without a sliding window, so they can not accomodate to arbitrary byte insertion/deletion in the middle of data. * [Attic](https://attic-backup.org/), which looks very similar to zbackup. # Credits Copyright (c) 2012-2014 Konstantin Isakov () and ZBackup contributors, see CONTRIBUTORS. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. zbackup-1.4.4/adler32.hh000066400000000000000000000015551257621500400147440ustar00rootroot00000000000000// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS // Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE #ifndef ADLER32_HH_INCLUDED__ #define ADLER32_HH_INCLUDED__ #include #include #include /// A simple wrapper to calculate adler32 class Adler32 { public: typedef uint32_t Value; Adler32(): value( ( Value ) adler32( 0, 0, 0 ) ) {} void add( void const * data, size_t size ) { // When size is 0, we assume a no-op was requested and 'data' should be // ignored. However, adler32() has a special semantic for NULL 'data'. // Therefore we check the size before calling it if ( size ) value = ( Value ) adler32( value, ( Bytef const * ) data, size ); } Value result() const { return value; } private: Value value; }; #endif zbackup-1.4.4/appendallocator.cc000066400000000000000000000046371257621500400166520ustar00rootroot00000000000000// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS // Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE #include #include #include "appendallocator.hh" AppendAllocator::AppendAllocator( unsigned blockSize_, unsigned granularity ): alignMask( granularity - 1 ), // We may decide to enlarge the block to make sure it is a multiple of // granularity. An improperly sized block would just waste the leftover // bytes blockSize( ( blockSize_ + alignMask ) & ~alignMask ), leftInBlock( -1 ) { } char * AppendAllocator::allocateBytes( unsigned size ) { // For zero-sized allocations, we always return a non-zero pointer. To do // that, we need to make sure we have it if ( !size && !blocks.empty() ) return nextAvailable; if ( leftInBlock < (int) size ) { unsigned toAllocate = ( size <= blockSize ? blockSize : size ); // Need a new block char * p = (char *) malloc( toAllocate ); if ( !p ) throw std::bad_alloc(); blocks.push_back( Record( p, nextAvailable, leftInBlock ) ); leftInBlock = (int) toAllocate; nextAvailable = p; } // We may need to allocate more than was asked to preserve granularity int toTake = (int) ( ( size + alignMask ) & ~alignMask ); char * result = nextAvailable; nextAvailable += toTake; leftInBlock -= toTake; // leftInBlock can become negative here, as toTake can // actually be larger than the space left due to an added alignment return result; } void AppendAllocator::returnBytes( unsigned size ) { if ( !size ) return; // If we are pointing to the start of the block, we need to free it and go // back to the previous one if ( nextAvailable == blocks.back().data ) { if ( blocks.size() == 1 ) throw std::bad_alloc(); free( blocks.back().data ); leftInBlock = blocks.back().prevLeftInBlock; nextAvailable = blocks.back().prevNextAvailable; blocks.pop_back(); } unsigned toTake = ( size + alignMask ) & ~alignMask; // There must be enough used bytes in the block if ( nextAvailable - blocks.back().data < (int) toTake ) throw std::bad_alloc(); nextAvailable -= toTake; leftInBlock += toTake; } void AppendAllocator::clear() { for ( unsigned x = blocks.size(); x--; ) free( blocks[ x ].data ); blocks.clear(); leftInBlock = -1; } AppendAllocator::~AppendAllocator() { clear(); } zbackup-1.4.4/appendallocator.hh000066400000000000000000000041161257621500400166540ustar00rootroot00000000000000// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS // Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE #ifndef APPENDALLOCATOR_HH_INCLUDED__ #define APPENDALLOCATOR_HH_INCLUDED__ #include #include #include #include /// A simple "add-only" memory allocation mechanism. class AppendAllocator { unsigned alignMask; unsigned blockSize; struct Record { char * data; char * prevNextAvailable; int prevLeftInBlock; Record( char * data_, char * prevNextAvailable_, int prevLeftInBlock_ ): data( data_ ), prevNextAvailable( prevNextAvailable_ ), prevLeftInBlock( prevLeftInBlock_ ) {} }; std::vector< Record > blocks; char * nextAvailable; int leftInBlock; // Can become < 0 due to added alignment public: /// blockSize is the amount of bytes allocated for each of the underlying /// storage blocks. granularity makes sure you allocate objects with /// the proper alignment. It must be a power of 2 AppendAllocator( unsigned blockSize, unsigned granularity ); ~AppendAllocator(); /// Removes all data from the append allocator. void clear(); /// Allocates a size-sized memory block. The only way to free it is to /// destroy the whole AppendAllocator. Can throw bad_alloc in an out-of- /// memory situation char * allocateBytes( unsigned size ); /// Returns the allocated bytes back. The size must match the size passed /// to allocateBytes() on the last invocation. Calls to allocateBytes()/ /// returnBytes() must follow the stack order - returnBytes() should undo /// the previous allocateBytes() void returnBytes( unsigned size ); /// Allocates memory to hold 'count' objects of T. In essense, it just does /// multiplication and type casting template< typename T > T * allocateObjects( unsigned count ) { return (T *) allocateBytes( count * sizeof( T ) ); } /// Returns the allocated objects back template< typename T > void returnObjects( unsigned count ) { returnBytes( count * sizeof( T ) ); } }; #endif zbackup-1.4.4/backup_creator.cc000066400000000000000000000164611257621500400164640ustar00rootroot00000000000000// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS // Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE #include #include #include "backup_creator.hh" #include "check.hh" #include "debug.hh" #include "message.hh" #include "page_size.hh" #include "static_assert.hh" namespace { unsigned const MinChunkSize = 256; } BackupCreator::BackupCreator( StorageInfo const & info, ChunkIndex & chunkIndex, ChunkStorage::Writer & chunkStorageWriter ): chunkMaxSize( info.chunk_max_size() ), chunkIndex( chunkIndex ), chunkStorageWriter( chunkStorageWriter ), ringBufferFill( 0 ), chunkToSaveFill( 0 ), backupDataStream( new google::protobuf::io::StringOutputStream( &backupData ) ), chunkIdGenerated( false ) { // In our ring buffer we have enough space to store one chunk plus an extra // page for buffering the input ringBuffer.resize( chunkMaxSize + getPageSize() ); begin = ringBuffer.data(); end = &ringBuffer.back() + 1; head = begin; tail = head; chunkToSave.resize( chunkMaxSize ); } void * BackupCreator::getInputBuffer() { return head; } size_t BackupCreator::getInputBufferSize() { if ( tail > head ) return tail - head; else if ( tail == head && ringBufferFill ) return 0; else return end - head; } void BackupCreator::handleMoreData( unsigned added ) { // Note: head is never supposed to wrap around in the middle of the operation, // as getInputBufferSize() never returns a value which could result in a // wrap-around while( added ) { // If we don't have a full chunk, we need to consume data until we have // one if ( ringBufferFill < chunkMaxSize ) { unsigned left = chunkMaxSize - ringBufferFill; bool canFullyFill = added >= left; unsigned toFill = canFullyFill ? left : added; added -= toFill; ringBufferFill += toFill; while ( toFill-- ) rollingHash.rollIn( *head++ ); if ( head == end ) head = begin; // If we've managed to fill in the complete chunk, attempt matching it if ( canFullyFill ) addChunkIfMatched(); } else { // At this point we have a full chunk in the ring buffer, so we can rotate // over a byte chunkToSave[ chunkToSaveFill++ ] = *tail; if ( chunkToSaveFill == chunkMaxSize ) // Got the full chunk - save it saveChunkToSave(); rollingHash.rotate( *head++, *tail++ ); if ( head == end ) head = begin; if ( tail == end ) tail = begin; addChunkIfMatched(); --added; // A byte was consumed } } } void BackupCreator::saveChunkToSave() { CHECK( chunkToSaveFill > 0, "chunk to save is empty" ); if ( chunkToSaveFill < 128 ) // TODO: make this value configurable { // The amount of data is too small - emit without creating a new chunk BackupInstruction instr; instr.set_bytes_to_emit( chunkToSave.data(), chunkToSaveFill ); outputInstruction( instr ); } else { // Output as a chunk ChunkId id; id.rollingHash = RollingHash::digest( chunkToSave.data(), chunkToSaveFill ); unsigned char sha1Value[ SHA_DIGEST_LENGTH ]; SHA1( (unsigned char const *) chunkToSave.data(), chunkToSaveFill, sha1Value ); STATIC_ASSERT( sizeof( id.cryptoHash ) <= sizeof( sha1Value ) ); memcpy( id.cryptoHash, sha1Value, sizeof( id.cryptoHash ) ); // Save it to the store if it's not there already chunkStorageWriter.add( id, chunkToSave.data(), chunkToSaveFill ); BackupInstruction instr; instr.set_chunk_to_emit( id.toBlob() ); outputInstruction( instr ); } chunkToSaveFill = 0; } void BackupCreator::finish() { dPrintf( "At finish: %u, %u\n", chunkToSaveFill, ringBufferFill ); // At this point we may have some bytes in chunkToSave, and some in the ring // buffer. We need to save both if ( chunkToSaveFill + ringBufferFill > chunkMaxSize ) { // We have more than a full chunk in chunkToSave and ringBuffer together, so // save the first part as a full chunk first // Move data from ring buffer to have full chunk in chunkToSave. moveFromRingBufferToChunkToSave( chunkMaxSize - chunkToSaveFill ); saveChunkToSave(); } // Concatenate the rest of data and save it too CHECK( chunkToSaveFill + ringBufferFill <= chunkMaxSize, "had more than two " "full chunks at backup finish" ); moveFromRingBufferToChunkToSave( ringBufferFill ); if ( chunkToSaveFill ) saveChunkToSave(); } void BackupCreator::moveFromRingBufferToChunkToSave( unsigned toMove ) { // If tail is before head, all data in the ring buffer is in one contiguous // piece. If not, it's in two pieces if ( tail < head ) { memcpy( chunkToSave.data() + chunkToSaveFill, tail, toMove ); tail += toMove; } else { unsigned toEnd = end - tail; unsigned firstPart = toEnd < toMove ? toEnd : toMove; memcpy( chunkToSave.data() + chunkToSaveFill, tail, firstPart ); tail += firstPart; if ( toMove > firstPart ) { unsigned secondPart = toMove - firstPart; memcpy( chunkToSave.data() + chunkToSaveFill + firstPart, begin, secondPart ); tail = begin + secondPart; } } if ( tail == end ) tail = begin; chunkToSaveFill += toMove; ringBufferFill -= toMove; } ChunkId const & BackupCreator::getChunkId() { if ( !chunkIdGenerated ) { // Calculate SHA1 SHA_CTX ctx; SHA1_Init( &ctx ); if ( tail < head ) { // Tail is before head - all the block is in one contiguous piece SHA1_Update( &ctx, tail, head - tail ); } else { // Tail is after head - the block consists of two pieces SHA1_Update( &ctx, tail, end - tail ); SHA1_Update( &ctx, begin, head - begin ); } unsigned char sha1Value[ SHA_DIGEST_LENGTH ]; SHA1_Final( sha1Value, &ctx ); generatedChunkId.rollingHash = rollingHash.digest(); memcpy( generatedChunkId.cryptoHash, sha1Value, sizeof( generatedChunkId.cryptoHash ) ); chunkIdGenerated = true; } return generatedChunkId; } void BackupCreator::addChunkIfMatched() { chunkIdGenerated = false; if ( chunkIndex.findChunk( rollingHash.digest(), *this ) ) { // verbosePrintf( "Reuse of chunk %lu\n", rollingHash.digest() ); // Before emitting the matched chunk, we need to make sure any bytes // which came before it are saved first if ( chunkToSaveFill ) saveChunkToSave(); // Add the record BackupInstruction instr; instr.set_chunk_to_emit( getChunkId().toBlob() ); outputInstruction( instr ); // The block was consumed from the ring buffer - remove the block from it tail = head; ringBufferFill = 0; rollingHash.reset(); } } void BackupCreator::outputInstruction( BackupInstruction const & instr ) { // TODO: once backupData becomes large enough, spawn another BackupCreator and // feed data to it. This way we wouldn't have to store the entire backupData // in RAM Message::serialize( instr, *backupDataStream ); } void BackupCreator::getBackupData( string & str ) { CHECK( backupDataStream.get(), "getBackupData() called twice" ); backupDataStream.reset(); str.swap( backupData ); } zbackup-1.4.4/backup_creator.hh000066400000000000000000000056421257621500400164750ustar00rootroot00000000000000// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS // Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE #ifndef BACKUP_CREATOR_HH_INCLUDED__ #define BACKUP_CREATOR_HH_INCLUDED__ #include #include #include #include #include "chunk_id.hh" #include "chunk_index.hh" #include "chunk_storage.hh" #include "file.hh" #include "nocopy.hh" #include "rolling_hash.hh" #include "sptr.hh" #include "zbackup.pb.h" using std::vector; using std::string; /// Creates a backup by processing input data and matching/writing chunks class BackupCreator: ChunkIndex::ChunkInfoInterface, NoCopy { unsigned chunkMaxSize; ChunkIndex & chunkIndex; ChunkStorage::Writer & chunkStorageWriter; vector< char > ringBuffer; // Ring buffer vars char * begin; char * end; char * head; char * tail; unsigned ringBufferFill; /// In this buffer we assemble the next chunk to be eventually stored. We /// copy the bytes from the ring buffer. While the copying may be avoided in /// some cases, the plan is to move to multi-threaded chunk storage in the /// future, where it would be necessary in any case vector< char > chunkToSave; unsigned chunkToSaveFill; /// Number of bytes accumulated in chunkToSave /// When we have data in chunkToSave, this points to the record in backupData /// which should store it unsigned recordIndexToSaveDataInto; RollingHash rollingHash; string backupData; sptr< google::protobuf::io::StringOutputStream > backupDataStream; /// Sees if the current block in the ring buffer exists in the chunk store. /// If it does, the reference is emitted and the ring buffer is cleared void addChunkIfMatched(); /// Outputs data contained in chunkToSave as a new chunk void saveChunkToSave(); /// Move the given amount of bytes from the ring buffer to the chunk to save. /// Ring buffer must have at least that many bytes void moveFromRingBufferToChunkToSave( unsigned bytes ); /// Outputs the given instruction to the backup stream void outputInstruction( BackupInstruction const & ); bool chunkIdGenerated; ChunkId generatedChunkId; virtual ChunkId const & getChunkId(); public: BackupCreator( StorageInfo const &, ChunkIndex &, ChunkStorage::Writer & ); /// The data is fed the following way: the user fills getInputBuffer() with /// up to getInputBufferSize() bytes, then calls handleMoreData() with the /// number of bytes written void * getInputBuffer(); size_t getInputBufferSize(); void handleMoreData( unsigned ); /// Flushes any remaining data and finishes the process. No additional data /// may be added after this call is made void finish(); /// Returns the result of the backup creation. Can only be called once the /// finish() was called and the backup is complete void getBackupData( string & ); }; #endif zbackup-1.4.4/backup_exchanger.cc000066400000000000000000000030751257621500400167660ustar00rootroot00000000000000// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS // Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE #include "backup_exchanger.hh" #include "dir.hh" #include "debug.hh" namespace BackupExchanger { vector< string > findOrRebuild( string const & src, string const & dst, string const & relativePath ) { vector< string > files; Dir::Listing lst ( Dir::addPath( src, relativePath ) ); Dir::Entry entry; while ( lst.getNext( entry ) ) { string currentRelativePath ( relativePath ); if ( currentRelativePath.empty() ) currentRelativePath.assign( entry.getFileName() ); else currentRelativePath.assign( Dir::addPath( relativePath, entry.getFileName() ) ); if ( entry.isDir() ) { verbosePrintf( "Found directory %s...\n", currentRelativePath.c_str() ); string srcFullPath ( Dir::addPath( src, currentRelativePath ) ); string dstFullPath ( Dir::addPath( dst, currentRelativePath ) ); if ( !dst.empty() && !Dir::exists( dstFullPath.c_str() ) ) { verbosePrintf( "Directory %s not found in destination, creating...\n", currentRelativePath.c_str() ); Dir::create( dstFullPath.c_str() ); } vector< string > subFiles ( findOrRebuild( src, dst, currentRelativePath ) ); files.insert( files.end(), subFiles.begin(), subFiles.end() ); } else { verbosePrintf( "Found file %s...\n", currentRelativePath.c_str() ); files.push_back( currentRelativePath ); } } return files; } } zbackup-1.4.4/backup_exchanger.hh000066400000000000000000000013611257621500400167740ustar00rootroot00000000000000// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS // Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE #ifndef BACKUP_EXCHANGER_HH_INCLUDED__ #define BACKUP_EXCHANGER_HH_INCLUDED__ #include #include #include "sptr.hh" #include "tmp_mgr.hh" namespace BackupExchanger { using std::string; using std::vector; using std::pair; enum { backups, bundles, index, Flags }; /// Recreate source directory structure in destination vector< string > findOrRebuild( string const & src, string const & dst = std::string(), string const & relativePath = std::string() ); typedef pair< sptr< TemporaryFile >, string > PendingExchangeRename; } #endif zbackup-1.4.4/backup_file.cc000066400000000000000000000023171257621500400157370ustar00rootroot00000000000000// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS // Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE #include "backup_file.hh" #include "encrypted_file.hh" #include "encryption.hh" #include "message.hh" namespace BackupFile { enum { FileFormatVersion = 1 }; void save( string const & fileName, EncryptionKey const & encryptionKey, BackupInfo const & backupInfo ) { EncryptedFile::OutputStream os( fileName.c_str(), encryptionKey, Encryption::ZeroIv ); os.writeRandomIv(); FileHeader header; header.set_version( FileFormatVersion ); Message::serialize( header, os ); Message::serialize( backupInfo, os ); os.writeAdler32(); } void load( string const & fileName, EncryptionKey const & encryptionKey, BackupInfo & backupInfo ) { EncryptedFile::InputStream is( fileName.c_str(), encryptionKey, Encryption::ZeroIv ); is.consumeRandomIv(); FileHeader header; Message::parse( header, is ); if ( header.version() != FileFormatVersion ) throw exUnsupportedVersion(); Message::parse( backupInfo, is ); is.checkAdler32(); } } zbackup-1.4.4/backup_file.hh000066400000000000000000000014721257621500400157520ustar00rootroot00000000000000// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS // Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE #ifndef BACKUP_FILE_HH_INCLUDED__ #define BACKUP_FILE_HH_INCLUDED__ #include #include #include "encryption_key.hh" #include "ex.hh" #include "zbackup.pb.h" namespace BackupFile { using std::string; DEF_EX( Ex, "Backup file exception", std::exception ) DEF_EX( exUnsupportedVersion, "Unsupported version of the backup file format", Ex ) /// Saves the given BackupInfo data into the given file void save( string const & fileName, EncryptionKey const &, BackupInfo const & ); /// Loads the given BackupInfo data from the given file void load( string const & fileName, EncryptionKey const &, BackupInfo & ); } #endif zbackup-1.4.4/backup_restorer.cc000066400000000000000000000054431257621500400166700ustar00rootroot00000000000000// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS // Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE #include #include #include #include "backup_restorer.hh" #include "chunk_id.hh" #include "message.hh" #include "zbackup.pb.h" namespace BackupRestorer { using std::vector; using google::protobuf::io::CodedInputStream; void restore( ChunkStorage::Reader & chunkStorageReader, std::string const & backupData, DataSink * output, ChunkSet * chunkSet ) { google::protobuf::io::ArrayInputStream is( backupData.data(), backupData.size() ); CodedInputStream cis( &is ); CodedInputStream::Limit limit = cis.PushLimit( backupData.size() ); // The following line prevents it from barfing on large backupData. // TODO: this disables size checks for each separate message. Figure a better // way to do this while keeping them enabled. It seems we need to create an // instance of CodedInputStream for each message, but it might be expensive cis.SetTotalBytesLimit( backupData.size(), -1 ); // Used when emitting chunks string chunk; BackupInstruction instr; while ( cis.BytesUntilLimit() > 0 ) { Message::parse( instr, cis ); if ( instr.has_chunk_to_emit() ) { ChunkId id( instr.chunk_to_emit() ); if ( output ) { // Need to emit a chunk, reading it from the store size_t chunkSize; chunkStorageReader.get( id, chunk, chunkSize ); output->saveData( chunk.data(), chunkSize ); } if ( chunkSet ) { chunkSet->insert( id ); } } if ( output && instr.has_bytes_to_emit() ) { // Need to emit the bytes directly string const & bytes = instr.bytes_to_emit(); output->saveData( bytes.data(), bytes.size() ); } } cis.PopLimit( limit ); } void restoreIterations( ChunkStorage::Reader & chunkStorageReader, BackupInfo & backupInfo, std::string & backupData, ChunkSet * chunkSet ) { // Perform the iterations needed to get to the actual user backup data for ( ; ; ) { backupData.swap( *backupInfo.mutable_backup_data() ); if ( backupInfo.iterations() ) { struct StringWriter: public DataSink { string result; virtual void saveData( void const * data, size_t size ) { result.append( ( char const * ) data, size ); } } stringWriter; restore( chunkStorageReader, backupData, &stringWriter, chunkSet ); backupInfo.mutable_backup_data()->swap( stringWriter.result ); backupInfo.set_iterations( backupInfo.iterations() - 1 ); } else break; } } } zbackup-1.4.4/backup_restorer.hh000066400000000000000000000020421257621500400166720ustar00rootroot00000000000000// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS // Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE #ifndef BACKUP_RESTORER_HH_INCLUDED__ #define BACKUP_RESTORER_HH_INCLUDED__ #include #include #include #include #include "chunk_storage.hh" #include "ex.hh" /// Generic interface to stream data out class DataSink { public: virtual void saveData( void const * data, size_t size )=0; virtual ~DataSink() {} }; /// Restores the backup namespace BackupRestorer { DEF_EX( Ex, "Backup restorer exception", std::exception ) DEF_EX( exTooManyBytesToEmit, "A backup record asks to emit too many bytes", Ex ) typedef std::set< ChunkId > ChunkSet; /// Restores the given backup void restore( ChunkStorage::Reader &, std::string const & backupData, DataSink *, ChunkSet * ); /// Performs restore iterations on backupData void restoreIterations( ChunkStorage::Reader &, BackupInfo &, std::string &, ChunkSet * ); } #endif zbackup-1.4.4/bundle.cc000066400000000000000000000145171257621500400147510ustar00rootroot00000000000000// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS // Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE #include #include "bundle.hh" #include "check.hh" #include "dir.hh" #include "encryption.hh" #include "hex.hh" #include "message.hh" #include "adler32.hh" #include "compression.hh" namespace Bundle { enum { FileFormatVersion = 1, // This means, we don't use LZMA in this file. FileFormatVersionNotLZMA, // <- add more versions here // This is the first version, we do not support. FileFormatVersionFirstUnsupported }; void Creator::addChunk( string const & id, void const * data, size_t size ) { BundleInfo_ChunkRecord * record = info.add_chunk_record(); record->set_id( id ); record->set_size( size ); payload.append( ( char const * ) data, size ); } void Creator::write( std::string const & fileName, EncryptionKey const & key, Reader & reader ) { EncryptedFile::OutputStream os( fileName.c_str(), key, Encryption::ZeroIv ); os.writeRandomIv(); Message::serialize( reader.getBundleHeader(), os ); Message::serialize( reader.getBundleInfo(), os ); os.writeAdler32(); void * bufPrev = NULL; const void * bufCurr = NULL; int sizePrev = 0, sizeCurr = 0; bool readPrev = false, readCurr = false; for ( ; ; ) { bool readCurr = reader.is->Next( &bufCurr, &sizeCurr ); if ( readCurr ) { if ( readPrev ) { os.write( bufPrev, sizePrev ); readPrev = readCurr; free( bufPrev ); bufPrev = malloc( sizeCurr ); memcpy( bufPrev, bufCurr, sizeCurr ); sizePrev = sizeCurr; } else { readPrev = readCurr; bufPrev = malloc( sizeCurr ); memcpy( bufPrev, bufCurr, sizeCurr ); sizePrev = sizeCurr; } } else { if ( readPrev ) { sizePrev -= sizeof( Adler32::Value ); os.write( bufPrev, sizePrev ); os.writeAdler32(); free ( bufPrev ); break; } } } if ( reader.is.get() ) reader.is.reset(); } void Creator::write( std::string const & fileName, EncryptionKey const & key ) { EncryptedFile::OutputStream os( fileName.c_str(), key, Encryption::ZeroIv ); os.writeRandomIv(); BundleFileHeader header; const_sptr compression = Compression::CompressionMethod::defaultCompression; header.set_compression_method( compression->getName() ); // The old code only support lzma, so we will bump up the version, if we're // using lzma. This will make it fail cleanly. if ( compression->getName() == "lzma" ) header.set_version( FileFormatVersion ); else header.set_version( FileFormatVersionNotLZMA ); Message::serialize( header, os ); Message::serialize( info, os ); os.writeAdler32(); // Compress sptr encoder = compression->createEncoder(); encoder->setInput( payload.data(), payload.size() ); for ( ; ; ) { { void * data; int size; if ( !os.Next( &data, &size ) ) { encoder.reset(); throw exBundleWriteFailed(); } if ( !size ) continue; encoder->setOutput( data, size ); } // Perform the compression if ( encoder->process(true) ) { if ( encoder->getAvailableOutput() ) os.BackUp( encoder->getAvailableOutput() ); break; } } encoder.reset(); os.writeAdler32(); } Reader::Reader( string const & fileName, EncryptionKey const & key, bool keepStream ) { is = new EncryptedFile::InputStream( fileName.c_str(), key, Encryption::ZeroIv ); is->consumeRandomIv(); Message::parse( header, *is ); if ( header.version() >= FileFormatVersionFirstUnsupported ) throw exUnsupportedVersion(); Message::parse( info, *is ); is->checkAdler32(); size_t payloadSize = 0; for ( int x = info.chunk_record_size(); x--; ) payloadSize += info.chunk_record( x ).size(); payload.resize( payloadSize ); if ( keepStream ) return; sptr decoder = Compression::CompressionMethod::findCompression( header.compression_method() )->createDecoder(); decoder->setOutput( &payload[ 0 ], payload.size() ); for ( ; ; ) { { void const * data; int size; if ( !is->Next( &data, &size ) ) { decoder.reset(); throw exBundleReadFailed(); } if ( !size ) continue; decoder->setInput( data, size ); } if ( decoder->process(false) ) { if ( decoder->getAvailableInput() ) is->BackUp( decoder->getAvailableInput() ); break; } if ( !decoder->getAvailableOutput() && decoder->getAvailableInput() ) { // Apparently we have more data than we were expecting decoder.reset(); throw exTooMuchData(); } } decoder.reset(); is->checkAdler32(); if ( is.get() ) is.reset(); // Populate the map char const * next = payload.data(); for ( int x = 0, count = info.chunk_record_size(); x < count; ++x ) { BundleInfo_ChunkRecord const & record = info.chunk_record( x ); pair< Chunks::iterator, bool > res = chunks.insert( Chunks::value_type( record.id(), Chunks::mapped_type( next, record.size() ) ) ); if ( !res.second ) throw exDuplicateChunks(); // Duplicate key encountered next += record.size(); } } bool Reader::get( string const & chunkId, string & chunkData, size_t & chunkDataSize ) { Chunks::iterator i = chunks.find( chunkId ); if ( i != chunks.end() ) { size_t sz = i->second.second; if ( chunkData.size() < sz ) chunkData.resize( sz ); memcpy( &chunkData[ 0 ], i->second.first, sz ); chunkDataSize = sz; return true; } else return false; } string generateFileName( Id const & id, string const & bundlesDir, bool createDirs ) { string hex( toHex( ( unsigned char * ) &id, sizeof( id ) ) ); // TODO: make this scheme more flexible and allow it to scale, or at least // be configurable string level1( Dir::addPath( bundlesDir, hex.substr( 0, 2 ) ) ); if ( createDirs && !Dir::exists( level1 ) ) Dir::create( level1 ); return string( Dir::addPath( level1, hex ) ); } } zbackup-1.4.4/bundle.hh000066400000000000000000000073201257621500400147550ustar00rootroot00000000000000// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS // Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE #ifndef BUNDLE_HH_INCLUDED__ #define BUNDLE_HH_INCLUDED__ #include #include #include #include #include #include #include "encryption_key.hh" #include "ex.hh" #include "nocopy.hh" #include "static_assert.hh" #include "zbackup.pb.h" #include "encrypted_file.hh" #include "sptr.hh" namespace Bundle { using std::string; using std::pair; using std::map; enum { /// The number of bytes the bundle id has. We chose 192-bit just to be on /// the safer side. It is also a multiple of 8 bytes, which is good for /// alignment IdSize = 24 }; /// Id of the bundle is IdSize bytes. Can and should be used as a POD type struct Id { char blob[ IdSize ]; bool operator == ( Id const & other ) const { return memcmp( blob, other.blob, sizeof( blob ) ) == 0; } bool operator != ( Id const & other ) const { return ! operator == ( other ); } bool operator < ( Id const & other ) const { return memcmp( blob, other.blob, sizeof( blob ) ) < 0; } }; STATIC_ASSERT( sizeof( Id ) == IdSize ); /// Reads the bundle and allows accessing chunks class Reader: NoCopy { BundleInfo info; BundleFileHeader header; /// Unpacked payload string payload; /// Maps chunk id blob to its contents and size typedef map< string, pair< char const *, size_t > > Chunks; Chunks chunks; public: DEF_EX( Ex, "Bundle reader exception", std::exception ) DEF_EX( exBundleReadFailed, "Bundle read failed", Ex ) DEF_EX( exUnsupportedVersion, "Unsupported version of the index file format", Ex ) DEF_EX( exTooMuchData, "More data than expected in a bundle", Ex ) DEF_EX( exDuplicateChunks, "Chunks with the same id found in a bundle", Ex ) Reader( string const & fileName, EncryptionKey const & key, bool keepStream = false ); /// Reads the chunk into chunkData and returns true, or returns false if there /// was no such chunk in the bundle. chunkData may be enlarged but won't /// be shrunk. The size of the actual chunk would be stored in chunkDataSize bool get( string const & chunkId, string & chunkData, size_t & chunkDataSize ); BundleInfo getBundleInfo() { return info; } BundleFileHeader getBundleHeader() { return header; } string getPayload() { return payload; } sptr< EncryptedFile::InputStream > is; }; /// Creates a bundle by adding chunks to it until it's full, then compressing /// it and writing out to disk class Creator { BundleInfo info; string payload; public: DEF_EX( Ex, "Bundle creator exception", std::exception ) DEF_EX( exBundleWriteFailed, "Bundle write failed", Ex ) /// Adds a chunk with the given id void addChunk( string const & chunkId, void const * data, size_t size ); /// Returns the number of bytes comprising all chunk bodies so far size_t getPayloadSize() const { return payload.size(); } /// Compresses and writes the bundle to the given file. The operation is /// time-consuming - calling this function from a worker thread could be /// warranted void write( string const & fileName, EncryptionKey const & ); void write( string const & fileName, EncryptionKey const &, Bundle::Reader & reader ); /// Returns the current BundleInfo record - this is used for index files BundleInfo const & getCurrentBundleInfo() const { return info; } }; /// Generates a full file name for a bundle with the given id. If createDirs /// is true, any intermediate directories will be created if they don't exist /// already string generateFileName( Id const &, string const & bundlesDir, bool createDirs ); } #endif zbackup-1.4.4/check.hh000066400000000000000000000017421257621500400145630ustar00rootroot00000000000000// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS // Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE #ifndef CHECK_HH_INCLUDED__ #define CHECK_HH_INCLUDED__ #include #include #include // Run-time assertion macro // Usage: CHECK( value == 16, "Value is not 16: %d", value ); // This will abort() if the value is not 16 with the message stating so. // TODO: show the backtrace here, without using __FILE__ __LINE__ #define CHECK( condition, message, ... ) ({if (!(condition)) \ { \ fprintf( stderr, "Check failed: " ); \ fprintf( stderr, message, ##__VA_ARGS__ ); \ fprintf( stderr, "\nAt %s:%d\n", __FILE__, __LINE__ ); \ abort(); \ }}) #define FAIL( ... ) CHECK( false, __VA_ARGS__ ) // Debug-only versions. Only instantiated in debug builds #ifndef NDEBUG #define DCHECK CHECK #define DFAIL FAIL #else #define DCHECK( ... ) #define DFAIL( ... ) #endif #endif zbackup-1.4.4/chunk_id.cc000066400000000000000000000024411257621500400152550ustar00rootroot00000000000000// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS // Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE #include "chunk_id.hh" #include #include "endian.hh" #include "check.hh" string ChunkId::toBlob() const { string out( BlobSize, 0 ); toBlob( &out[ 0 ] ); return out; } void ChunkId::toBlob( void * outPtr ) const { char * out = ( char * ) outPtr; RollingHash::Digest v = toLittleEndian( rollingHash ); memcpy( out, cryptoHash, sizeof( cryptoHash ) ); memcpy( out + sizeof( cryptoHash ), &v, sizeof( v ) ); } void ChunkId::setFromBlob( void const * data ) { char const * blob = ( char const * ) data; RollingHash::Digest v; memcpy( cryptoHash, blob, sizeof( cryptoHash ) ); memcpy( &v, blob + sizeof( cryptoHash ), sizeof( v ) ); rollingHash = fromLittleEndian( v ); } bool operator <( const ChunkId &lhs, const ChunkId &rhs ) { int r = memcmp( &lhs.cryptoHash, &rhs.cryptoHash, sizeof( lhs.cryptoHash ) ); if ( r != 0 ) return r < 0; return memcmp( &lhs.rollingHash, &rhs.rollingHash, sizeof( lhs.rollingHash ) ) < 0; } ChunkId::ChunkId( string const & blob ) { CHECK( blob.size() == BlobSize, "incorrect blob size: %zu", blob.size() ); setFromBlob( blob.data() ); } zbackup-1.4.4/chunk_id.hh000066400000000000000000000017331257621500400152720ustar00rootroot00000000000000// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS // Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE #ifndef CHUNK_ID_HH_INCLUDED__ #define CHUNK_ID_HH_INCLUDED__ #include #include "rolling_hash.hh" using std::string; /// Chunk is identified by its crypto hash concatenated with its rolling hash struct ChunkId { typedef char CryptoHashPart[ 16 ]; CryptoHashPart cryptoHash; typedef RollingHash::Digest RollingHashPart; RollingHashPart rollingHash; enum { BlobSize = sizeof( CryptoHashPart ) + sizeof( RollingHashPart ) }; string toBlob() const; /// Faster version - should point to a buffer with at least BlobSize bytes void toBlob( void * ) const; /// Set the chunk id data reading from the given blob void setFromBlob( void const * ); ChunkId() {} ChunkId( string const & blob ); }; bool operator <( const ChunkId &lhs, const ChunkId &rhs ); #endif zbackup-1.4.4/chunk_index.cc000066400000000000000000000111121257621500400157630ustar00rootroot00000000000000// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS // Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE #include #include #include #include #include "chunk_index.hh" #include "debug.hh" #include "dir.hh" #include "index_file.hh" #include "zbackup.pb.h" ChunkIndex::Chain::Chain( ChunkId const & id, Bundle::Id const * bundleId ): next( 0 ), bundleId( bundleId ) { memcpy( cryptoHash, id.cryptoHash, sizeof( cryptoHash ) ); } bool ChunkIndex::Chain::equalsTo( ChunkId const & id ) { return memcmp( cryptoHash, id.cryptoHash, sizeof( cryptoHash ) ) == 0; } void ChunkIndex::loadIndex( IndexProcessor & ip ) { Dir::Listing lst( indexPath ); Dir::Entry entry; verbosePrintf( "Loading index...\n" ); while( lst.getNext( entry ) ) { verbosePrintf( "Loading index file %s...\n", entry.getFileName().c_str() ); try { string indexFn = Dir::addPath( indexPath, entry.getFileName() ); IndexFile::Reader reader( key, indexFn ); ip.startIndex( indexFn ); BundleInfo info; Bundle::Id bundleId; while( reader.readNextRecord( info, bundleId ) ) { Bundle::Id * savedId = storage.allocateObjects< Bundle::Id >( 1 ); memcpy( savedId, &bundleId, sizeof( bundleId ) ); ChunkId id; ip.startBundle( *savedId ); for ( int x = info.chunk_record_size(); x--; ) { BundleInfo_ChunkRecord const & record = info.chunk_record( x ); if ( record.id().size() != ChunkId::BlobSize ) throw exIncorrectChunkIdSize(); id.setFromBlob( record.id().data() ); ip.processChunk( id ); } ip.finishBundle( *savedId, info ); } ip.finishIndex( indexFn ); } catch( std::exception & e ) { fprintf( stderr, "error: %s\n", e.what() ); continue; } } verbosePrintf( "Index loaded.\n" ); } void ChunkIndex::startIndex( string const & ) { } void ChunkIndex::startBundle( Bundle::Id const & bundleId ) { lastBundleId = &bundleId; } void ChunkIndex::processChunk( ChunkId const & chunkId ) { registerNewChunkId( chunkId, lastBundleId ); } void ChunkIndex::finishBundle( Bundle::Id const &, BundleInfo const & ) { } void ChunkIndex::finishIndex( string const & ) { } ChunkIndex::ChunkIndex( EncryptionKey const & key, TmpMgr & tmpMgr, string const & indexPath, bool prohibitChunkIndexLoading ): key( key ), tmpMgr( tmpMgr ), indexPath( indexPath ), storage( 65536, 1 ), lastBundleId( NULL ) { if ( !prohibitChunkIndexLoading ) loadIndex( *this ); } Bundle::Id const * ChunkIndex::findChunk( ChunkId::RollingHashPart rollingHash, ChunkInfoInterface & chunkInfo ) { HashTable::iterator i = hashTable.find( rollingHash ); ChunkId const * id = 0; if ( i != hashTable.end() ) { if ( !id ) id = &chunkInfo.getChunkId(); // Check the chains for ( Chain * chain = i->second; chain; chain = chain->next ) if ( chain->equalsTo( *id ) ) return chain->bundleId; } return NULL; } namespace { struct ChunkInfoImmediate: public ChunkIndex::ChunkInfoInterface { ChunkId const & id; ChunkInfoImmediate( ChunkId const & id ): id( id ) {} virtual ChunkId const & getChunkId() { return id; } }; } Bundle::Id const * ChunkIndex::findChunk( ChunkId const & chunkId ) { ChunkInfoImmediate chunkInfo( chunkId ); return findChunk( chunkId.rollingHash, chunkInfo ); } ChunkIndex::Chain * ChunkIndex::registerNewChunkId( ChunkId const & id, Bundle::Id const * bundleId ) { HashTable::iterator i = hashTable.insert( std::make_pair( id.rollingHash, ( Chain *) 0 ) ).first; Chain ** chain = &i->second; // Check the chains for ( ; *chain; chain = &( ( *chain )->next ) ) if ( ( *chain )->equalsTo( id ) ) { return NULL; // The entry existed already } // Create a new chain *chain = new ( storage.allocateObjects< Chain >( 1 ) ) Chain( id, bundleId ); return *chain; } bool ChunkIndex::addChunk( ChunkId const & id, Bundle::Id const & bundleId ) { if ( Chain * chain = registerNewChunkId( id, NULL ) ) { // Allocate or re-use bundle id if ( !lastBundleId || *lastBundleId != bundleId ) { Bundle::Id * allocatedId = storage.allocateObjects< Bundle::Id >( 1 ); memcpy( allocatedId, &bundleId, Bundle::IdSize ); lastBundleId = allocatedId; } chain->bundleId = lastBundleId; return true; } else return false; } zbackup-1.4.4/chunk_index.hh000066400000000000000000000074561257621500400160150ustar00rootroot00000000000000// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS // Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE #ifndef CHUNK_INDEX_HH_INCLUDED__ #define CHUNK_INDEX_HH_INCLUDED__ // is obsolete, but requires C++11. Make up your // mind, GNU people! #undef __DEPRECATED #include #include #include #include #include #include #include "appendallocator.hh" #include "bundle.hh" #include "chunk_id.hh" #include "dir.hh" #include "encryption_key.hh" #include "endian.hh" #include "ex.hh" #include "index_file.hh" #include "nocopy.hh" #include "rolling_hash.hh" #include "tmp_mgr.hh" using std::vector; /// __gnu_cxx::hash is not defined for unsigned long long. As uint64_t is /// typedefed as unsigned long long on all 32-bit architectures and on some /// 64-bit ones, we need to define this. Our keys should have more or less /// uniform bit distribution, so on 32-bit systems returning the lower 32 bits /// should be fine namespace __gnu_cxx { template<> struct hash< unsigned long long > { size_t operator()( unsigned long long v ) const { return v; } }; } class IndexProcessor { public: virtual void startIndex( string const & ) = 0; virtual void startBundle( Bundle::Id const & ) = 0; virtual void processChunk( ChunkId const & ) = 0; virtual void finishBundle( Bundle::Id const &, BundleInfo const & ) = 0; virtual void finishIndex( string const & ) = 0; }; /// Maintains an in-memory hash table allowing to check whether we have a /// specific chunk or not, and if we do, get the bundle id it's in class ChunkIndex: NoCopy, IndexProcessor { struct Chain { ChunkId::CryptoHashPart cryptoHash; Chain * next; Bundle::Id const * bundleId; Chain( ChunkId const &, Bundle::Id const * bundleId ); bool equalsTo( ChunkId const & id ); }; /// This hash map stores all known chunk ids /// TODO: implement a custom hash table for better performance typedef __gnu_cxx::hash_map< RollingHash::Digest, Chain * > HashTable; EncryptionKey const & key; TmpMgr & tmpMgr; string indexPath; AppendAllocator storage; HashTable hashTable; /// Stores the last used bundle id, which can be re-used Bundle::Id const * lastBundleId; public: DEF_EX( Ex, "Chunk index exception", std::exception ) DEF_EX( exIncorrectChunkIdSize, "Incorrect chunk id size encountered", Ex ) ChunkIndex( EncryptionKey const &, TmpMgr &, string const & indexPath, bool ); struct ChunkInfoInterface { /// Returns the full id of the chunk. This function is only called if that /// full id is actually needed, as its generation requires the expensive /// calculation of the full hash virtual ChunkId const & getChunkId()=0; virtual ~ChunkInfoInterface() {} }; /// If the given chunk exists, its bundle id is returned, otherwise NULL Bundle::Id const * findChunk( ChunkId::RollingHashPart, ChunkInfoInterface & ); /// If the given chunk exists, its bundle id is returned, otherwise NULL Bundle::Id const * findChunk( ChunkId const & ); /// Adds a new chunk to the index if it did not exist already. Returns true /// if added, false if existed already bool addChunk( ChunkId const &, Bundle::Id const & ); void startIndex( string const & ); void startBundle( Bundle::Id const & ); void processChunk( ChunkId const & ); void finishBundle( Bundle::Id const &, BundleInfo const & ); void finishIndex( string const & ); void loadIndex( IndexProcessor & ); private: /// Inserts new chunk id into the in-memory hash table. Returns the created /// Chain if it was inserted, NULL if it existed before Chain * registerNewChunkId( ChunkId const & id, Bundle::Id const * ); }; #endif zbackup-1.4.4/chunk_storage.cc000066400000000000000000000146531257621500400163350ustar00rootroot00000000000000// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS // Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE #include "check.hh" #include "chunk_storage.hh" #include "debug.hh" #include "dir.hh" #include "hex.hh" #include "random.hh" namespace ChunkStorage { Writer::Writer( StorageInfo const & storageInfo, EncryptionKey const & encryptionKey, TmpMgr & tmpMgr, ChunkIndex & index, string const & bundlesDir, string const & indexDir, size_t maxCompressorsToRun ): storageInfo( storageInfo ), encryptionKey( encryptionKey ), tmpMgr( tmpMgr ), index( index ), bundlesDir( bundlesDir ), indexDir( indexDir ), hasCurrentBundleId( false ), maxCompressorsToRun( maxCompressorsToRun ), runningCompressors( 0 ) { verbosePrintf( "Using up to %zu thread(s) for compression\n", maxCompressorsToRun ); } Writer::~Writer() { waitForAllCompressorsToFinish(); } bool Writer::add( ChunkId const & id, void const * data, size_t size ) { if ( index.addChunk( id, getCurrentBundleId() ) ) { // Added to the index? Emit to the bundle then if ( getCurrentBundle().getPayloadSize() + size > storageInfo.bundle_max_payload_size() ) finishCurrentBundle(); getCurrentBundle().addChunk( id.toBlob(), data, size ); return true; } else return false; } void Writer::addBundle( BundleInfo const & bundleInfo, Bundle::Id const & bundleId ) { if ( !indexFile.get() ) { // Create a new index file indexTempFile = tmpMgr.makeTemporaryFile(); indexFile = new IndexFile::Writer( encryptionKey, indexTempFile->getFileName() ); } indexFile->add( bundleInfo, bundleId ); } void Writer::commit() { finishCurrentBundle(); waitForAllCompressorsToFinish(); // Move all bundles for ( size_t x = pendingBundleRenames.size(); x--; ) { PendingBundleRename & r = pendingBundleRenames[ x ]; r.first->moveOverTo( Bundle::generateFileName( r.second, bundlesDir, true ) ); } pendingBundleRenames.clear(); // Move the index file if ( indexFile.get() ) { indexFile.reset(); // Generate a random filename unsigned char buf[ 24 ]; // Same comments as for Bundle::IdSize Random::genaratePseudo( buf, sizeof( buf ) ); indexTempFile->moveOverTo( Dir::addPath( indexDir, toHex( buf, sizeof( buf ) ) ) ); indexTempFile.reset(); } } void Writer::reset() { finishCurrentBundle(); waitForAllCompressorsToFinish(); pendingBundleRenames.clear(); if ( indexFile.get() ) { indexFile.reset(); } } Bundle::Creator & Writer::getCurrentBundle() { if ( !currentBundle.get() ) currentBundle = new Bundle::Creator; return *currentBundle; } void Writer::finishCurrentBundle() { if ( !currentBundle.get() ) return; Bundle::Id const & bundleId = getCurrentBundleId(); addBundle( currentBundle->getCurrentBundleInfo(), bundleId ); sptr< TemporaryFile > file = tmpMgr.makeTemporaryFile(); pendingBundleRenames.push_back( PendingBundleRename( file, bundleId ) ); // Create a new compressor // Wait for some compressors to finish if there are too many of them Lock _( runningCompressorsMutex ); while ( runningCompressors >= maxCompressorsToRun ) runningCompressorsCondition.wait( runningCompressorsMutex ); Compressor * compressor = new Compressor( *this, currentBundle, file->getFileName() ); currentBundle.reset(); hasCurrentBundleId = false; compressor->start(); ++runningCompressors; } void Writer::waitForAllCompressorsToFinish() { Lock _( runningCompressorsMutex ); while ( runningCompressors ) runningCompressorsCondition.wait( runningCompressorsMutex ); } Bundle::Id const & Writer::getCurrentBundleId() { if ( !hasCurrentBundleId ) { // Generate a new one Random::genaratePseudo( ¤tBundleId, sizeof( currentBundleId ) ); hasCurrentBundleId = true; } return currentBundleId; } Writer::Compressor::Compressor( Writer & writer, sptr< Bundle::Creator > const & bundleCreator, string const & fileName ): writer( writer ), bundleCreator( bundleCreator ), fileName( fileName ) { } void * Writer::Compressor::Compressor::threadFunction() throw() { try { bundleCreator->write( fileName, writer.encryptionKey ); } catch( std::exception & e ) { FAIL( "Bunding writing failed: %s", e.what() ); } { Lock _( writer.runningCompressorsMutex ); CHECK( writer.runningCompressors, "no running compressors" ); --writer.runningCompressors; writer.runningCompressorsCondition.signal(); } detach(); // We're in detached thread, so no further cleanup is necessary delete this; return NULL; } Reader::Reader( StorageInfo const & storageInfo, EncryptionKey const & encryptionKey, ChunkIndex & index, string const & bundlesDir, size_t maxCacheSizeBytes ): storageInfo( storageInfo ), encryptionKey( encryptionKey ), index( index ), bundlesDir( bundlesDir ), // We need to have at least one cached reader, otherwise we would have to // unpack a bundle each time a chunk is read, even for consecutive chunks // in the same bundle cachedReaders( maxCacheSizeBytes < storageInfo.bundle_max_payload_size() ? 1 : maxCacheSizeBytes / storageInfo.bundle_max_payload_size() ) { verbosePrintf( "Using up to %zu MB of RAM as cache\n", maxCacheSizeBytes / 1048576 ); } void Reader::get( ChunkId const & chunkId, string & data, size_t & size ) { if ( Bundle::Id const * bundleId = index.findChunk( chunkId ) ) { Bundle::Reader & reader = getReaderFor( *bundleId ); reader.get( chunkId.toBlob(), data, size ); } else { string blob = chunkId.toBlob(); throw exNoSuchChunk( toHex( ( unsigned char const * ) blob.data(), blob.size() ) ); } } Bundle::Reader & Reader::getReaderFor( Bundle::Id const & id ) { sptr< Bundle::Reader > & reader = cachedReaders.entry< Bundle::Reader >( string( ( char const * ) &id, sizeof( id ) ) ); if ( !reader.get() ) { // Load the bundle reader = new Bundle::Reader( Bundle::generateFileName( id, bundlesDir, false ), encryptionKey ); } return *reader; } } zbackup-1.4.4/chunk_storage.hh000066400000000000000000000103251257621500400163370ustar00rootroot00000000000000// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS // Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE #ifndef CHUNK_STORAGE_HH_INCLUDED__ #define CHUNK_STORAGE_HH_INCLUDED__ #include #include #include #include #include #include "bundle.hh" #include "chunk_id.hh" #include "chunk_index.hh" #include "encryption_key.hh" #include "ex.hh" #include "file.hh" #include "index_file.hh" #include "mt.hh" #include "nocopy.hh" #include "objectcache.hh" #include "sptr.hh" #include "tmp_mgr.hh" #include "zbackup.pb.h" namespace ChunkStorage { using std::string; using std::vector; using std::pair; DEF_EX( Ex, "Chunk storage exception", std::exception ) /// Allows adding new chunks to the storage by filling up new bundles with them /// and writing new index files class Writer: NoCopy { public: /// All new bundles and index files are created as temp files. Call commit() /// to move them to their permanent locations. commit() is never called /// automatically! Writer( StorageInfo const &, EncryptionKey const &, TmpMgr &, ChunkIndex & index, string const & bundlesDir, string const & indexDir, size_t maxCompressorsToRun ); /// Adds the given chunk to the store. If such a chunk has already existed /// in the index, does nothing and returns false bool add( ChunkId const &, void const * data, size_t size ); /// Adds an existing bundle to the index void addBundle( BundleInfo const &, Bundle::Id const & bundleId ); /// Commits all newly created bundles. Must be called before destroying the /// object -- otherwise all work will be removed from the temp dir and lost void commit(); /// Throw away all current changes. void reset(); ~Writer(); private: /// Performs the compression in a separate thread. Destroys itself once done class Compressor: public Thread { Writer & writer; sptr< Bundle::Creator > bundleCreator; string fileName; public: Compressor( Writer &, sptr< Bundle::Creator > const &, string const & fileName ); protected: virtual void * threadFunction() throw(); }; friend class Compressor; /// Returns the id of the currently written bundle. If there's none, generates /// one. If a bundle hasn't yet started, still generates it - once the bundle /// is started, it will be used then Bundle::Id const & getCurrentBundleId(); /// Returns *currentBundle or creates a new one Bundle::Creator & getCurrentBundle(); /// Writes the current bundle and deallocates it void finishCurrentBundle(); /// Wait for all compressors to finish void waitForAllCompressorsToFinish(); StorageInfo const & storageInfo; EncryptionKey const & encryptionKey; TmpMgr & tmpMgr; ChunkIndex & index; string bundlesDir, indexDir; sptr< TemporaryFile > indexTempFile; sptr< IndexFile::Writer > indexFile; sptr< Bundle::Creator > currentBundle; Bundle::Id currentBundleId; bool hasCurrentBundleId; size_t maxCompressorsToRun; Mutex runningCompressorsMutex; Condition runningCompressorsCondition; size_t runningCompressors; /// Maps temp file of the bundle to its id blob typedef pair< sptr< TemporaryFile >, Bundle::Id > PendingBundleRename; vector< PendingBundleRename > pendingBundleRenames; }; /// Allows retrieving existing chunks by extracting them from the bundles with /// the help of an Index object class Reader: NoCopy { public: DEF_EX_STR( exNoSuchChunk, "no such chunk found:", Ex ) Reader( StorageInfo const &, EncryptionKey const &, ChunkIndex & index, string const & bundlesDir, size_t maxCacheSizeBytes ); /// Loads the given chunk from the store into the given buffer. May throw file /// and decompression exceptions. 'data' may be enlarged but won't be shrunk. /// The size of the actual chunk would be stored in 'size' void get( ChunkId const &, string & data, size_t & size ); /// Retrieves the reader for the given bundle id. May employ caching Bundle::Reader & getReaderFor( Bundle::Id const & ); private: StorageInfo const & storageInfo; EncryptionKey const & encryptionKey; ChunkIndex & index; string bundlesDir; ObjectCache cachedReaders; }; } #endif zbackup-1.4.4/cmake/000077500000000000000000000000001257621500400142415ustar00rootroot00000000000000zbackup-1.4.4/cmake/FindLibLZMA.cmake000066400000000000000000000133251257621500400172420ustar00rootroot00000000000000# - Find LibLZMA # Find LibLZMA headers and library # # LIBLZMA_FOUND - True if liblzma is found. # LIBLZMA_INCLUDE_DIRS - Directory where liblzma headers are located. # LIBLZMA_LIBRARIES - Lzma libraries to link against. # LIBLZMA_HAS_AUTO_DECODER - True if lzma_auto_decoder() is found (required). # LIBLZMA_HAS_EASY_ENCODER - True if lzma_easy_encoder() is found (required). # LIBLZMA_HAS_LZMA_PRESET - True if lzma_lzma_preset() is found (required). # LIBLZMA_VERSION_MAJOR - The major version of lzma # LIBLZMA_VERSION_MINOR - The minor version of lzma # LIBLZMA_VERSION_PATCH - The patch version of lzma # LIBLZMA_VERSION_STRING - version number as a string (ex: "5.0.3") #============================================================================= # Copyright 2008 Per Øyvind Karlsen # Copyright 2009 Alexander Neundorf # Copyright 2009 Helio Chissini de Castro # Copyright 2012 Mario Bensi # # Distributed under the OSI-approved BSD License (the "License"): # # CMake - Cross Platform Makefile Generator # Copyright 2000-2011 Kitware, Inc., Insight Software Consortium # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # * Neither the names of Kitware, Inc., the Insight Software Consortium, # nor the names of their contributors may be used to endorse or promote # products derived from this software without specific prior written # permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT # HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # # ------------------------------------------------------------------------------ # # The above copyright and license notice applies to distributions of # CMake in source and binary form. Some source files contain additional # notices of original copyright by their contributors; see each source # for details. Third-party software packages supplied with CMake under # compatible licenses provide their own copyright notices documented in # corresponding subdirectories. # # ------------------------------------------------------------------------------ # # CMake was initially developed by Kitware with the following sponsorship: # # * National Library of Medicine at the National Institutes of Health # as part of the Insight Segmentation and Registration Toolkit (ITK). # # * US National Labs (Los Alamos, Livermore, Sandia) ASC Parallel # Visualization Initiative. # # * National Alliance for Medical Image Computing (NAMIC) is funded by the # National Institutes of Health through the NIH Roadmap for Medical Research, # Grant U54 EB005149. # # * Kitware, Inc. #============================================================================= find_path(LIBLZMA_INCLUDE_DIR lzma.h ) find_library(LIBLZMA_LIBRARY lzma) if(LIBLZMA_INCLUDE_DIR AND EXISTS "${LIBLZMA_INCLUDE_DIR}/lzma/version.h") file(STRINGS "${LIBLZMA_INCLUDE_DIR}/lzma/version.h" LIBLZMA_HEADER_CONTENTS REGEX "#define LZMA_VERSION_[A-Z]+ [0-9]+") string(REGEX REPLACE ".*#define LZMA_VERSION_MAJOR ([0-9]+).*" "\\1" LIBLZMA_VERSION_MAJOR "${LIBLZMA_HEADER_CONTENTS}") string(REGEX REPLACE ".*#define LZMA_VERSION_MINOR ([0-9]+).*" "\\1" LIBLZMA_VERSION_MINOR "${LIBLZMA_HEADER_CONTENTS}") string(REGEX REPLACE ".*#define LZMA_VERSION_PATCH ([0-9]+).*" "\\1" LIBLZMA_VERSION_PATCH "${LIBLZMA_HEADER_CONTENTS}") set(LIBLZMA_VERSION_STRING "${LIBLZMA_VERSION_MAJOR}.${LIBLZMA_VERSION_MINOR}.${LIBLZMA_VERSION_PATCH}") unset(LIBLZMA_HEADER_CONTENTS) endif() # We're using new code known now as XZ, even library still been called LZMA # it can be found in http://tukaani.org/xz/ # Avoid using old codebase if (LIBLZMA_LIBRARY) include(CheckLibraryExists) CHECK_LIBRARY_EXISTS(${LIBLZMA_LIBRARY} lzma_auto_decoder "" LIBLZMA_HAS_AUTO_DECODER) CHECK_LIBRARY_EXISTS(${LIBLZMA_LIBRARY} lzma_easy_encoder "" LIBLZMA_HAS_EASY_ENCODER) CHECK_LIBRARY_EXISTS(${LIBLZMA_LIBRARY} lzma_lzma_preset "" LIBLZMA_HAS_LZMA_PRESET) endif () include(FindPackageHandleStandardArgs) FIND_PACKAGE_HANDLE_STANDARD_ARGS(LibLZMA DEFAULT_MSG LIBLZMA_INCLUDE_DIR LIBLZMA_LIBRARY LIBLZMA_HAS_AUTO_DECODER LIBLZMA_HAS_EASY_ENCODER LIBLZMA_HAS_LZMA_PRESET ) if (LIBLZMA_FOUND) set(LIBLZMA_LIBRARIES ${LIBLZMA_LIBRARY}) set(LIBLZMA_INCLUDE_DIRS ${LIBLZMA_INCLUDE_DIR}) endif () mark_as_advanced( LIBLZMA_INCLUDE_DIR LIBLZMA_LIBRARY ) zbackup-1.4.4/cmake/FindLibLZO.cmake000066400000000000000000000122671257621500400171470ustar00rootroot00000000000000# - Find LibLZO # Find LibLZO headers and library # # LIBLZO_FOUND - True if liblzo is found. # LIBLZO_INCLUDE_DIRS - Directory where liblzo headers are located. # LIBLZO_LIBRARIES - Lzma libraries to link against. # LIBLZO_HAS_AUTO_DECODER - True if lzo_auto_decoder() is found (required). # LIBLZO_HAS_EASY_ENCODER - True if lzo_easy_encoder() is found (required). # LIBLZO_HAS_LZO_PRESET - True if lzo_lzo_preset() is found (required). # LIBLZO_VERSION_MAJOR - The major version of lzo # LIBLZO_VERSION_MINOR - The minor version of lzo # LIBLZO_VERSION_PATCH - The patch version of lzo # LIBLZO_VERSION_STRING - version number as a string (ex: "5.0.3") #============================================================================= # Copyright 2008 Per Øyvind Karlsen # Copyright 2009 Alexander Neundorf # Copyright 2009 Helio Chissini de Castro # Copyright 2012 Mario Bensi # Copyright 2012-2014 Konstantin Isakov and ZBackup # contributors, see CONTRIBUTORS # # Distributed under the OSI-approved BSD License (the "License"): # # CMake - Cross Platform Makefile Generator # Copyright 2000-2011 Kitware, Inc., Insight Software Consortium # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # * Neither the names of Kitware, Inc., the Insight Software Consortium, # nor the names of their contributors may be used to endorse or promote # products derived from this software without specific prior written # permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT # HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # # ------------------------------------------------------------------------------ # # The above copyright and license notice applies to distributions of # CMake in source and binary form. Some source files contain additional # notices of original copyright by their contributors; see each source # for details. Third-party software packages supplied with CMake under # compatible licenses provide their own copyright notices documented in # corresponding subdirectories. # # ------------------------------------------------------------------------------ # # CMake was initially developed by Kitware with the following sponsorship: # # * National Library of Medicine at the National Institutes of Health # as part of the Insight Segmentation and Registration Toolkit (ITK). # # * US National Labs (Los Alamos, Livermore, Sandia) ASC Parallel # Visualization Initiative. # # * National Alliance for Medical Image Computing (NAMIC) is funded by the # National Institutes of Health through the NIH Roadmap for Medical Research, # Grant U54 EB005149. # # * Kitware, Inc. #============================================================================= find_path(LIBLZO_INCLUDE_DIR lzo/lzo1x.h ) find_library(LIBLZO_LIBRARY lzo2) if(LIBLZO_INCLUDE_DIR AND EXISTS "${LIBLZO_INCLUDE_DIR}/lzo/lzoconf.h") file(STRINGS "${LIBLZO_INCLUDE_DIR}/lzo/lzoconf.h" LIBLZO_HEADER_CONTENTS REGEX "#define LZO_VERSION_STRING.+\"[^\"]+\"") string(REGEX REPLACE ".*#define LZO_VERSION_STRING.+\"([^\"]+)\".*" "\\1" LIBLZO_VERSION_STRING "${LIBLZO_HEADER_CONTENTS}") unset(LIBLZO_HEADER_CONTENTS) endif() # We're just using two functions. if (LIBLZO_LIBRARY) include(CheckLibraryExists) CHECK_LIBRARY_EXISTS(${LIBLZO_LIBRARY} lzo1x_decompress_safe "" LIBLZO_HAS_LZO1X_DECOMPRESS_SAFE) CHECK_LIBRARY_EXISTS(${LIBLZO_LIBRARY} lzo1x_1_compress "" LIBLZO_HAS_LZO1X_1_COMPRESS) endif () include(FindPackageHandleStandardArgs) FIND_PACKAGE_HANDLE_STANDARD_ARGS(LibLZO DEFAULT_MSG LIBLZO_INCLUDE_DIR LIBLZO_LIBRARY LIBLZO_HAS_LZO1X_DECOMPRESS_SAFE LIBLZO_HAS_LZO1X_1_COMPRESS ) if (LIBLZO_FOUND) set(LIBLZO_LIBRARIES ${LIBLZO_LIBRARY}) set(LIBLZO_INCLUDE_DIRS ${LIBLZO_INCLUDE_DIR}) endif () mark_as_advanced( LIBLZO_INCLUDE_DIR LIBLZO_LIBRARY ) zbackup-1.4.4/compression.cc000066400000000000000000000452121257621500400160350ustar00rootroot00000000000000// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS // Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE #include #include "compression.hh" #include "check.hh" namespace Compression { EnDecoder::EnDecoder() { } EnDecoder::~EnDecoder() { } CompressionMethod::~CompressionMethod() { } // LZMA #include class LZMAEnDecoder : public EnDecoder { protected: static lzma_stream initValue; lzma_stream strm; public: LZMAEnDecoder() { strm = initValue; } void setInput( const void* data, size_t size ) { strm.next_in = (const uint8_t *) data; strm.avail_in = size; } void setOutput( void* data, size_t size ) { strm.next_out = (uint8_t *) data; strm.avail_out = size; } size_t getAvailableInput() { return strm.avail_in; } size_t getAvailableOutput() { return strm.avail_out; } bool process( bool finish ) { lzma_ret ret = lzma_code( &strm, ( finish ? LZMA_FINISH : LZMA_RUN ) ); CHECK( ret == LZMA_OK || ret == LZMA_STREAM_END, "lzma_code error: %d", (int) ret ); return ( ret == LZMA_STREAM_END ); } ~LZMAEnDecoder() { lzma_end( &strm ); } }; lzma_stream LZMAEnDecoder::initValue = LZMA_STREAM_INIT; class LZMAEncoder : public LZMAEnDecoder { public: LZMAEncoder() { uint32_t preset = 6; // TODO: make this customizable, although 6 seems to be // the best option lzma_ret ret = lzma_easy_encoder( &strm, preset, LZMA_CHECK_CRC64 ); CHECK( ret == LZMA_OK, "lzma_easy_encoder error: %d", (int) ret ); } }; class LZMADecoder : public LZMAEnDecoder { public: LZMADecoder() { lzma_ret ret = lzma_stream_decoder( &strm, UINT64_MAX, 0 ); CHECK( ret == LZMA_OK,"lzma_stream_decoder error: %d", (int) ret ); } }; class LZMACompression : public CompressionMethod { public: sptr createEncoder() const { return new LZMAEncoder(); } sptr createDecoder() const { return new LZMADecoder(); } std::string getName() const { return "lzma"; } }; // LZO // liblzo implements a lot of algorithms "for unlimited backward compatibility" // The web site says: // "My experiments have shown that LZO1B is good with a large blocksize // or with very redundant data, LZO1F is good with a small blocksize or // with binary data and that LZO1X is often the best choice of all. // LZO1Y and LZO1Z are almost identical to LZO1X - they can achieve a // better compression ratio on some files. // Beware, your mileage may vary." // => I'm using LZO1X, as suggested #include // Unfortunately, liblzo always works with the whole data, so it doesn't support // the streaming approach that most other libraries use. This means that we have // to use a big buffer for the data. The class NoStreamEnDecoder implements this // so we can use it, if there is another library like liblzo. // Collect all data and process it in one pass class NoStreamEnDecoder : public EnDecoder { std::string accDataIn, accDataOut; const char* dataIn; char* dataOut; size_t availIn, availOut; bool processed; size_t posInAccDataOut; protected: // you must implement these: // Should we try with the existing output buffer which has availOut // bytes of free space? If you know that this will fail, return false. // You may peek into dataIn which contains the complete compressed data. virtual bool shouldTryWith( const char* dataIn, size_t availIn, size_t availOut ) =0; // We will allocate a buffer for the output data. How big should it be? // You may peek into dataIn which contains the complete compressed data. virtual size_t suggestOutputSize( const char* dataIn, size_t availIn ) =0; // Is this input complete? // An encoder should return false. virtual bool isCompleteInput( const char* dataIn, size_t availIn ) =0; // Process the data in dataIn and put the result into dataOut. You musn't // write more than availOut bytes! If the output buffer is big enough, // process the data and store the output size in outputSize. If the output // buffer is too small, return false and we will give you a bigger one. If // any other error occurrs, abort the program. We don't have any better // error handling. Sorry. Do NOT return false for errors that won't be // remedied by a bigger buffer! virtual bool doProcess( const char* dataIn, size_t availIn, char* dataOut, size_t availOut, size_t& outputSize ) =0; void setUnusedInput( size_t unused ) { this->dataIn += availIn - unused; this->availIn = unused; } public: NoStreamEnDecoder() { dataIn = dataOut = NULL; availIn = availOut = posInAccDataOut = 0; processed = false; } void setInput( const void* data, size_t size ) { dataIn = (const char *) data; availIn = size; } void setOutput( void* data, size_t size ) { dataOut = (char *) data; availOut = size; } size_t getAvailableInput() { return availIn; } size_t getAvailableOutput() { return availOut; } bool process( bool finish ) { // try to process the input, if we haven't done it, yet if ( !processed ) { // data has not been encoded if ( accDataIn.empty() ) { // this is the first piece of data if ( finish || isCompleteInput( dataIn, availIn ) ) { // special case: all the data has been passed at once // -> process it without using accDataIn processFinish( dataIn, availIn ); } } // if we didn't process the data, put it into accumulator if ( !processed ) { // accumulate data in accDataIn accDataIn.append( dataIn, availIn ); // If this was the last bit of data, we process it, now. if ( finish || isCompleteInput( accDataIn.data(), accDataIn.size() ) ) { processFinish( accDataIn.data(), accDataIn.size() ); } } } // If the input has been processed, try to copy some of it to the output buffer. if ( processed ) { // data has been encoded or decoded, remaining output is in accDataOut // -> copy to output if (availOut > 0 && accDataOut.size() - posInAccDataOut > 0) { size_t sz = availOut; if ( sz > accDataOut.size() - posInAccDataOut ) sz = accDataOut.size() - posInAccDataOut; memcpy( dataOut, accDataOut.data() + posInAccDataOut, sz ); dataOut += sz; availOut -= sz; posInAccDataOut += sz; } // no more data left? -> return true return ( accDataOut.size() - posInAccDataOut == 0 ); } else { // not yet processed, so we cannot be done return false; } } private: void processFinish( const char* dataIn, size_t availIn ) { // should we try with the existing output buffer? if ( shouldTryWith( dataIn, availIn, availOut ) ) { size_t outputSize; if ( doProcess( dataIn, availIn, dataOut, availOut, outputSize ) ) { // it worked :-) processed = true; availOut -= outputSize; return ; } } // we use our own buffer size_t bufferSize = suggestOutputSize( dataIn, availIn ); do { accDataOut.resize(bufferSize); size_t outputSize; //TODO doc says we mustn't modify the pointer returned by data()... if ( doProcess( dataIn, availIn, (char*) accDataOut.data(), bufferSize, outputSize ) ) { // buffer is big enough accDataOut.resize( outputSize ); processed = true; return ; } // try a bigger one bufferSize *= 2; } while (true); } }; #include // like NoStreamEnDecoder, but also adds the uncompressed size before the stream //NOTE You should make sure that the compression function doesn't overwrite any // memory, if this information is corrupted! This could be exploited by a // malicious person and there is nothing I can do about it. I could check for // an overflow, but when control gets back to this class, it is already too // late, as one 'ret' instruction is enough to do harm. class NoStreamAndUnknownSizeDecoder : public NoStreamEnDecoder { protected: // You implement this one: // If you don't know the real decoded size, don't change outputSize. virtual bool doProcessNoSize( const char* dataIn, size_t availIn, char* dataOut, size_t availOut, size_t& outputSize ) =0; bool shouldTryWith( const char* dataIn, size_t availIn, size_t availOut ) { return suggestOutputSize( dataIn, availIn ) <= availOut; } // Is this input complete? bool isCompleteInput( const char* dataIn, size_t availIn ) { if ( availIn < 2*sizeof(uint64_t) ) return false; dataIn += sizeof(uint64_t); size_t inputSize = le32toh( *(uint32_t*) dataIn ); return ( availIn >= inputSize + 2*sizeof(uint64_t) ); } size_t suggestOutputSize( const char* dataIn, size_t availIn ) { CHECK( availIn >= sizeof(uint64_t), "not enough input data" ); // We're not using size_t because we need a type that has the same size on all // architectures. A 32-bit host won't be able to open files with more than // 4GB (actually much less), so 4 byte are enough. Even a 64-bit host would // have some trouble with allocating 8GB of RAM just for our buffers ;-) //NOTE If your compiler doesn't accept this cast, your size_t is smaller than // uint32_t. In that case, you are in trouble... size_t outputSize = le32toh( *(uint32_t*) dataIn ); return outputSize; } bool doProcess( const char* dataIn, size_t availIn, char* dataOut, size_t availOut, size_t& outputSize ) { if ( availIn < 2*sizeof( uint64_t ) ) return false; //NOTE We skip 8 bytes. If we later decide to drop compatibility with 32-bit // hosts, we can save a 64-bit size. Well, that will be much later, when // we can easily hold two copies of a 4GB file in main memory :-D size_t neededOutputSize = le32toh( *(uint32_t*) dataIn ); dataIn += sizeof(uint64_t); size_t inputSize = le32toh( *(uint32_t*) dataIn ); dataIn += sizeof(uint64_t); /*if ( outputSize < neededOutputSize ) return false;*/ outputSize = neededOutputSize; availIn -= 2*sizeof( uint64_t ); // We might not need all of our input data. setUnusedInput( availIn - inputSize ); availIn = inputSize; size_t reportedOutputSize = neededOutputSize; if ( !doProcessNoSize( dataIn, availIn, dataOut, availOut, reportedOutputSize ) ) return false; CHECK( reportedOutputSize == neededOutputSize, "Size of decoded data is different than expected" ); return true; } }; // encoder for NoStreamAndUnknownSizeDecoder class NoStreamAndUnknownSizeEncoder : public NoStreamEnDecoder { protected: // You implement this one: virtual bool doProcessNoSize( const char* dataIn, size_t availIn, char* dataOut, size_t availOut, size_t& outputSize ) =0; bool shouldTryWith( const char*, size_t, size_t availOut ) { // If the compression doesn't use any spaces... return availOut > sizeof( uint64_t ); } bool isCompleteInput( const char* dataIn, size_t availIn ) { // We cannot know whether the user wants to send more data. // -> return false; user must use finish=true to signal end of data return false; } size_t getOverhead() { return 2*sizeof( uint64_t ); } size_t suggestOutputSize( const char*, size_t availIn ) { // We assume that the compression won't make the data any bigger. return availIn + getOverhead(); } bool doProcess( const char* dataIn, size_t availIn, char* dataOut, size_t availOut, size_t& outputSize ) { CHECK( availIn <= UINT32_MAX, "You want to compress more than 4GB of data?! Sorry, we don't support that, yet." ); memcpy(dataOut, "ABCDEFGHIJKLMNOP", 16); // store size *(uint32_t*)dataOut = htole32( availIn ); uint32_t* compressedSize = (uint32_t*) ( dataOut + sizeof( uint64_t ) ); // compressed data goes after the size // We skip more than we actually use; see NoStreamAndUnknownSizeDecoder::doProcess(...). dataOut += getOverhead(); availOut -= getOverhead(); if ( !doProcessNoSize( dataIn, availIn, dataOut, availOut, outputSize ) ) return false; CHECK( outputSize <= UINT32_MAX, "The compressed data is more than 4GB?! Sorry, we don't support that, yet." ); *compressedSize = htole32( (uint32_t) outputSize ); outputSize += getOverhead(); return true; } }; #ifdef HAVE_LIBLZO #include // finally, we can implement lzo class LZO1X_1_Decoder : public NoStreamAndUnknownSizeDecoder { protected: bool doProcessNoSize( const char* dataIn, size_t availIn, char* dataOut, size_t availOut, size_t& outputSize ) { // same argument is used for available output size and size of decompressed data outputSize = availOut; int ret = lzo1x_decompress_safe( (const lzo_bytep) dataIn, availIn, (lzo_bytep) dataOut, (lzo_uintp) &outputSize, NULL ); if ( ret == LZO_E_OUTPUT_OVERRUN ) return false; CHECK( ret >= LZO_E_OK, "lzo1x_decompress_safe failed (code %d)", ret ); return true; } }; class LZO1X_1_Compression; class LZO1X_1_Encoder : public NoStreamAndUnknownSizeEncoder { const LZO1X_1_Compression* compression; static size_t calcMaxCompressedSize(size_t availIn); public: LZO1X_1_Encoder(const LZO1X_1_Compression* compression) { this->compression = compression; } protected: bool doProcessNoSize( const char* dataIn, size_t availIn, char* dataOut, size_t availOut, size_t& outputSize ); bool shouldTryWith( const char*, size_t, size_t availOut ); size_t suggestOutputSize( const char*, size_t availIn ); }; class LZO1X_1_Compression : public CompressionMethod { static bool initialized; static void init() { //TODO This is not thread-safe. Does it have to be? if (!initialized) { int ret = lzo_init(); CHECK( ret == LZO_E_OK, "lzo_init failed (%d)", ret ); initialized = true; } } public: sptr createEncoder() const { init(); return new LZO1X_1_Encoder(this); } sptr createDecoder() const { init(); return new LZO1X_1_Decoder(); } std::string getName() const { return "lzo1x_1"; } lzo_voidp getWorkmem( size_t size ) const { return new char[size]; } void giveBackWorkmem( lzo_voidp wrkmem ) const { //TODO I think we should keep the memory around and reuse it. After all // it is only a few kilobytes and we will need it a lot. However, I // won't risk anything here because I don't know whether this will be // called by more than one thread. delete[] (char*)wrkmem; } }; bool LZO1X_1_Compression::initialized = false; size_t LZO1X_1_Encoder::calcMaxCompressedSize( size_t availIn ) { // It seems that lzo1x_1_compress does NOT check whether the buffer is big enough. // The documentation refers to example/simple.c which says: // "Because the input block may be incompressible, we must provide a little more // output space in case that compression is not possible." // -> We use the same formula. return (availIn + availIn / 16 + 64 + 3); } bool LZO1X_1_Encoder::shouldTryWith( const char* dataIn, size_t availIn, size_t availOut ) { return availOut >= suggestOutputSize( dataIn, availIn ); } size_t LZO1X_1_Encoder::suggestOutputSize( const char*, size_t availIn ) { // It seems that lzo1x_1_compress does NOT check whether the buffer is big enough. // The documentation refers to example/simple.c which says: // "Because the input block may be incompressible, we must provide a little more // output space in case that compression is not possible." // -> We use the same formula. return calcMaxCompressedSize( availIn ) + getOverhead(); } bool LZO1X_1_Encoder::doProcessNoSize( const char* dataIn, size_t availIn, char* dataOut, size_t availOut, size_t& outputSize ) { // It seems that lzo1x_1_compress does NOT check whether the buffer is big enough. // Therefore, we won't try it unless we are sure that the buffer is big enough. if ( availOut < calcMaxCompressedSize( availIn ) ) return false; // same argument is used for available output size (haha, see above) // and size of decompressed data outputSize = availOut; lzo_voidp wrkmem = compression->getWorkmem(LZO1X_1_MEM_COMPRESS); int ret = lzo1x_1_compress( (const lzo_bytep) dataIn, availIn, (lzo_bytep) dataOut, (lzo_uintp) &outputSize, wrkmem ); compression->giveBackWorkmem(wrkmem); if ( ret == LZO_E_OUTPUT_OVERRUN ) return false; CHECK( ret >= LZO_E_OK, "lzo1x_1_compress failed (code %d)", ret ); return true; } #endif // HAVE_LIBLZO // register them static const_sptr const compressions[] = { new LZMACompression(), # ifdef HAVE_LIBLZO new LZO1X_1_Compression(), # endif // NULL entry marks end of list. Don't remove it! NULL }; const_sptr CompressionMethod::defaultCompression = compressions[0]; const_sptr CompressionMethod::findCompression( const std::string& name, bool optional ) { for ( const const_sptr* c = compressions+0; *c; ++c ) { if ( (*c)->getName() == name ) { return (*c); } } if ( !optional ) { throw exUnsupportedCompressionMethod( name ); } return NULL; } // iterator over compressions CompressionMethod::iterator::iterator( const const_sptr* ptr ) : ptr( ptr) { } CompressionMethod::iterator::iterator( const iterator& it ) : ptr(it.ptr) { } CompressionMethod::iterator& CompressionMethod::iterator::operator =( const iterator& it ) { this->ptr = it.ptr; return *this; } bool CompressionMethod::iterator::operator ==( const iterator& other ) const { // special case: one has ptr==NULL (end iterator returned by end()) and the // other has *ptr==NULL (end iterator obtained by calling ++) if ( !ptr && ( !other.ptr || !*other.ptr ) ) return true; else if ( !other.ptr && ( !ptr || !*ptr ) ) return true; else return (ptr == other.ptr); } bool CompressionMethod::iterator::operator !=( const iterator& other ) const { return !( *this == other ); } bool CompressionMethod::iterator::atEnd() const { return !ptr || !*ptr; } CompressionMethod::iterator& CompressionMethod::iterator::operator ++() { CHECK( ptr && *ptr, "Cannot increment the end iterator" ); ++ptr; return *this; } const_sptr CompressionMethod::iterator::operator *() { CHECK( ptr && *ptr, "Cannot dereference the end iterator" ); return *ptr; } CompressionMethod::iterator CompressionMethod::begin() { return iterator(compressions); } CompressionMethod::iterator CompressionMethod::end() { return iterator(NULL); } } zbackup-1.4.4/compression.hh000066400000000000000000000050061257621500400160440ustar00rootroot00000000000000// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS // Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE #ifndef COMPRESSION_HH_INCLUDED__ #define COMPRESSION_HH_INCLUDED__ #include "sptr.hh" #include "ex.hh" #include "nocopy.hh" namespace Compression { DEF_EX( Ex, "Compression exception", std::exception ) DEF_EX_STR( exUnsupportedCompressionMethod, "Unsupported compression method: ", Ex ) // used for encoding or decoding class EnDecoder: NoCopy { protected: EnDecoder(); public: virtual ~EnDecoder(); // encoder can read up to size bytes from data virtual void setInput ( const void* data, size_t size ) =0; // how many bytes of the last input haven't been used, yet? virtual size_t getAvailableInput() =0; // encoder can write up to size bytes to output virtual void setOutput( void* data, size_t size ) =0; // how many bytes of free space are remaining in the output buffer virtual size_t getAvailableOutput() =0; // process some bytes // finish: will you pass more data to the encoder via setOutput? // NOTE You must eventually set finish to true. // returns, whether all output bytes have been written virtual bool process( bool finish ) =0; }; // compression method class CompressionMethod { public: virtual ~CompressionMethod(); // returns name of compression method // This name is saved in the file header of the compressed file. virtual std::string getName() const =0; virtual sptr createEncoder() const =0; virtual sptr createDecoder() const =0; // find a compression by name // If optional is false, it will either return a valid CompressionMethod // object or abort the program. If optional is true, it will return // NULL, if it cannot find the a compression with that name. static const_sptr findCompression( const std::string& name, bool optional = false ); static const_sptr defaultCompression; class iterator { friend class CompressionMethod; const const_sptr* ptr; iterator( const const_sptr* ptr ); public: iterator( const iterator& it ); iterator& operator =( const iterator& it ); bool operator ==( const iterator& other ) const; bool operator !=( const iterator& other ) const; bool atEnd() const; iterator& operator ++(); const_sptr operator *(); }; static iterator begin(); static iterator end(); }; } #endif zbackup-1.4.4/debug.cc000066400000000000000000000003211257621500400145520ustar00rootroot00000000000000// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS // Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE bool verboseMode = true; zbackup-1.4.4/debug.hh000066400000000000000000000011031257621500400145630ustar00rootroot00000000000000// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS // Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE #ifndef DEBUG_HH_INCLUDED__ #define DEBUG_HH_INCLUDED__ #include // Macros we use to output debugging information #ifndef NDEBUG #define dPrintf( ... ) (fprintf( stderr, __VA_ARGS__ )) #else #define dPrintf( ... ) #endif extern bool verboseMode; #define verbosePrintf( ... ) ({ if ( verboseMode ) \ fprintf( stderr, __VA_ARGS__ ); }) #endif zbackup-1.4.4/dir.cc000066400000000000000000000047631257621500400142600ustar00rootroot00000000000000// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS // Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE #include #include #include #include #include #include #include "dir.hh" DIR * dir; namespace Dir { bool exists( string const & name ) { struct stat buf; return stat( name.c_str(), &buf ) == 0 && S_ISDIR( buf.st_mode ); } void create( string const & name ) { if ( mkdir( name.c_str(), 0777 ) != 0 ) throw exCantCreate( name ); } void remove( string const & name ) { if ( rmdir( name.c_str() ) != 0 ) throw exCantRemove( name ); } string addPath( string const & first, string const & second ) { if ( first.empty() ) return second; if ( second.empty() ) return first; if ( first[ first.size() - 1 ] == separator() ) return first + second; else return first + separator() + second; } string getRealPath( string const & path ) { if ( char * r = realpath( path.c_str(), NULL ) ) { string result( r ); free( r ); return result; } else throw exCantGetRealPath( path ); } string getDirName( string const & path ) { char const * c = path.c_str(); std::vector< char > copy( c, c + path.size() + 1 ); return dirname( copy.data() ); } bool isDirEmpty( string const & path ) { Listing lst(path); Entry tmp; return !lst.getNext(tmp); } Listing::Listing( string const & dirName ): dirName( dirName ) { dir = opendir( dirName.c_str() ); if ( !dir ) throw exCantList( dirName ); } Listing::~Listing() { closedir( dir ); } bool Listing::getNext( Entry & result ) { dirent entry; dirent * entryPtr; struct stat entryStats; for ( ; ; ) { if ( readdir_r( dir, &entry, &entryPtr ) != 0 ) throw exCantList( dirName ); if ( !entryPtr ) return false; #ifndef __APPLE__ if ( fstatat( dirfd( dir ), entry.d_name, &entryStats, AT_SYMLINK_NOFOLLOW ) != 0 ) #else if ( lstat( addPath( dirName, entry.d_name ).c_str(), &entryStats ) != 0) #endif throw exCantList( dirName ); bool isDir = S_ISDIR( entryStats.st_mode ); bool isSymLink = S_ISLNK( entryStats.st_mode ); if ( isDir && ( entry.d_name[ 0 ] == '.' && ( !entry.d_name[ 1 ] || entry.d_name[ 1 ] == '.' ) ) ) { // Skip the . or .. entries continue; } result = Entry( entry.d_name, isDir, isSymLink ); return true; } } } zbackup-1.4.4/dir.hh000066400000000000000000000040551257621500400142640ustar00rootroot00000000000000// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS // Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE #ifndef DIR_HH_INCLUDED__ #define DIR_HH_INCLUDED__ #include #include #include #include #include "ex.hh" #include "nocopy.hh" using std::string; /// Directory-related operations namespace Dir { DEF_EX( Ex, "Directory exception", std::exception ) DEF_EX_STR( exCantCreate, "Can't create directory", Ex ) DEF_EX_STR( exCantRemove, "Can't remove directory", Ex ) DEF_EX_STR( exCantList, "Can't list directory", Ex ) DEF_EX_STR( exCantGetRealPath, "Can't real path of", Ex ) /// Checks whether the given dir exists or not bool exists( string const & ); /// Creates the given directory void create( string const & ); /// Removes the given directory. It must be empty to be removed void remove( string const & ); /// Adds one path to another, e.g. for /hello/world and baz/bar, returns /// /hello/world/baz/bar string addPath( string const & first, string const & second ); /// Returns the canonicalized absolute pathname with symlinks resolved string getRealPath( string const & ); /// Returns the directory part of the given path string getDirName( string const & ); /// Checkes whether directory is empty bool isDirEmpty( string const & ); /// A separator used to separate names in the path. inline char separator() { return '/'; } class Entry { string fileName; bool dir; bool symlink; public: Entry() {} Entry( string const & fileName, bool dir, bool symlink ): fileName( fileName ), dir( dir ), symlink( symlink ) {} string const & getFileName() const { return fileName; } bool isDir() const { return dir; } bool isSymLink() const { return symlink; } }; /// Allows listing the directory class Listing: NoCopy { string dirName; DIR * dir; public: Listing( string const & dirName ); ~Listing(); /// Return true if entry was filled, false if end of dir was encountered bool getNext( Entry & ); }; } #endif zbackup-1.4.4/encrypted_file.cc000066400000000000000000000227401257621500400164710ustar00rootroot00000000000000// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS // Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE #include #include #include "check.hh" #include "encrypted_file.hh" #include "endian.hh" #include "page_size.hh" #include "random.hh" namespace EncryptedFile { using Encryption::BlockSize; InputStream::InputStream( char const * fileName, EncryptionKey const & key, void const * iv_ ): file( fileName, UnbufferedFile::ReadOnly ), filePos( 0 ), key( key ), // Our buffer must be larger than BlockSize, as otherwise we won't be able // to handle PKCS#7 padding properly buffer( std::max( getPageSize(), ( unsigned ) BlockSize * 2 ) ), fill( 0 ), remainder( 0 ), backedUp( false ) { if ( key.hasKey() ) { memcpy( iv, iv_, sizeof( iv ) ); // Since we use padding, file size should be evenly dividable by the cipher // block size, and we should have at least one block UnbufferedFile::Offset size = file.size(); if ( !size || size % BlockSize ) throw exIncorrectFileSize(); } } bool InputStream::Next( void const ** data, int * size ) { // If we backed up, return the unconsumed data if ( backedUp ) backedUp = false; else { try { // Update adler32 for the previous block adler32.add( start, fill ); // Read more data if ( filePos && !remainder ) { // Once we're read a full block, we always have a remainder. If not, // this means we've hit the end of file already fill = 0; return false; } // If we have a remainder, move it to the beginning of buffer and make // it start the next block memmove( buffer.data(), start + fill, remainder ); start = buffer.data(); fill = file.read( start + remainder, buffer.size() - remainder ) + remainder; // remainder should techically be 0 now, but decrypt() will update it // anyway // remainder = 0; decrypt(); } catch( UnbufferedFile::exReadError & ) { fill = 0; // To make sure state is remaining consistent return false; } } *data = start; *size = fill; filePos += fill; return *size; } void InputStream::BackUp( int count ) { CHECK( count >= 0, "count is negative" ); if ( !backedUp ) { CHECK( (size_t) count <= fill, "Backing up too much" ); size_t consumed = fill - count; adler32.add( start, consumed ); start += consumed; fill = count; filePos -= count; backedUp = fill; // Don't make the next Next() return 0 bytes } else { CHECK( count == 0, "backing up after being backed up already" ); } } bool InputStream::Skip( int count ) { CHECK( count >= 0, "count is negative" ); // We always need to read and decrypt data, as otherwise both the state of // CBC and adler32 would be incorrect void const * data; int size; while( count ) { if ( !Next( &data, &size ) ) return false; else if ( size > count ) { BackUp( size - count ); break; } else count -= size; } return true; } int64_t InputStream::ByteCount() const { return filePos; } Adler32::Value InputStream::getAdler32() { // This makes all data consumed, if not already BackUp( 0 ); return adler32.result(); } void InputStream::read( void * buf, size_t size ) { void const * data; int avail; char * n = ( char * ) buf; while( size ) { if ( !Next( &data, &avail ) ) throw exReadFailed(); else if ( avail > ( ssize_t ) size ) { memcpy( n, data, size ); BackUp( avail - size ); break; } else { memcpy( n, data, avail ); n += avail; size -= avail; } } } void InputStream::checkAdler32() { Adler32::Value ours = getAdler32(); Adler32::Value r; read( &r, sizeof( r ) ); if ( ours != fromLittleEndian( r ) ) throw exAdlerMismatch(); } void InputStream::consumeRandomIv() { if ( key.hasKey() ) { char iv[ Encryption::IvSize ]; read( iv, sizeof( iv ) ); // read() can throw exceptions, Skip() can't } } void InputStream::decrypt() { if ( fill == buffer.size() ) { // When we have the full buffer, we set the last block of it aside and // treat the rest as the normal CBC sequence. The last block in the buffer // may be the last block of file, in which case we would need to handle // padding. That may happen the next time the function is called remainder = BlockSize; fill -= BlockSize; doDecrypt(); } else { // This is an end of file. Decrypt data treating the last block being // padded // Since we always have padding in the file and the last block is always // set apart when reading full buffers, we must have at least one block // to decrypt here doDecrypt(); // Unpad the last block if ( key.hasKey() ) fill -= BlockSize - Encryption::unpad( start + fill - BlockSize ); // We have not left any remainder this time remainder = 0; } } void InputStream::doDecrypt() { if ( !key.hasKey() ) return; // Since we use padding, file size should be evenly dividable by the cipher's // block size, and we should always have at least one block. When we get here, // we would always get the proper fill value unless those characteristics are // not met. We check for the same condition on construction, but the file // size can change while we are reading it // We don't throw an exception here as the interface we implement doesn't // support them CHECK( fill > 0 && !( fill % BlockSize ), "incorrect size of the encrypted " "file - must be non-zero and in multiples of %u", ( unsigned ) BlockSize ); // Copy the next iv prior to decrypting the data in place, as it will // not be available afterwards char newIv[ Encryption::IvSize ]; memcpy( newIv, Encryption::getNextDecryptionIv( start, fill ), sizeof( newIv ) ); // Decrypt the data Encryption::decrypt( iv, key.getKey(), start, start, fill ); // Copy the new iv memcpy( iv, newIv, sizeof( iv ) ); } OutputStream::OutputStream( char const * fileName, EncryptionKey const & key, void const * iv_ ): file( fileName, UnbufferedFile::WriteOnly ), filePos( 0 ), key( key ), buffer( getPageSize() ), start( buffer.data() ), avail( 0 ), backedUp( false ) { if ( key.hasKey() ) memcpy( iv, iv_, sizeof( iv ) ); } bool OutputStream::Next( void ** data, int * size ) { // If we backed up, return the unconsumed data if ( backedUp ) backedUp = false; else { try { // Update adler32 for the previous block adler32.add( start, avail ); // Encrypt and write the buffer if it had data if ( filePos ) encryptAndWrite( buffer.size() ); start = buffer.data(); avail = buffer.size(); } catch( UnbufferedFile::exWriteError & ) { avail = 0; // To make sure state is remaining consistent return false; } } *data = start; *size = avail; filePos += avail; return *size; } void OutputStream::BackUp( int count ) { CHECK( count >= 0, "count is negative" ); if ( !backedUp ) { CHECK( (size_t) count <= avail, "Backing up too much" ); size_t consumed = avail - count; adler32.add( start, consumed ); start += consumed; avail = count; filePos -= count; backedUp = avail; // Don't make the next Next() return 0 bytes } else { CHECK( count == 0, "backing up after being backed up already" ); } } int64_t OutputStream::ByteCount() const { return filePos; } Adler32::Value OutputStream::getAdler32() { // This makes all data consumed, if not already BackUp( 0 ); return adler32.result(); } void OutputStream::write( void const * buf, size_t size ) { void * data; int avail; char const * n = ( char const * ) buf; while( size ) { if ( !Next( &data, &avail ) ) throw exReadFailed(); else if ( avail > ( ssize_t ) size ) { memcpy( data, n, size ); BackUp( avail - size ); break; } else { memcpy( data, n, avail ); n += avail; size -= avail; } } } void OutputStream::writeAdler32() { Adler32::Value v = toLittleEndian( getAdler32() ); write( &v, sizeof( v ) ); } void OutputStream::writeRandomIv() { if ( key.hasKey() ) { char iv[ Encryption::IvSize ]; Random::genaratePseudo( iv, sizeof( iv ) ); write( iv, sizeof( iv ) ); } } void OutputStream::encryptAndWrite( size_t bytes ) { if ( key.hasKey() ) { CHECK( bytes > 0 && !( bytes % BlockSize ), "incorrect number of bytes to " "encrypt and write - must be non-zero and in multiples of %u", ( unsigned ) BlockSize ); void const * nextIv = Encryption::encrypt( iv, key.getKey(), buffer.data(), buffer.data(), bytes ); memcpy( iv, nextIv, sizeof( iv ) ); } file.write( buffer.data(), bytes ); } OutputStream::~OutputStream() { // This makes all data consumed, if not already BackUp( 0 ); // If we have the full buffer, write it first if ( start == buffer.data() + buffer.size() ) { encryptAndWrite( buffer.size() ); start = buffer.data(); } size_t bytesToWrite = start - buffer.data(); if ( key.hasKey() ) { // Perform padding size_t remainderSize = bytesToWrite % BlockSize; Encryption::pad( start - remainderSize, remainderSize ); bytesToWrite += BlockSize - remainderSize; } encryptAndWrite( bytesToWrite ); } } zbackup-1.4.4/encrypted_file.hh000066400000000000000000000121641257621500400165020ustar00rootroot00000000000000// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS // Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE #ifndef ENCRYPTED_FILE_HH_INCLUDED__ #define ENCRYPTED_FILE_HH_INCLUDED__ #include #include #include #include #include #include #include "adler32.hh" #include "encryption.hh" #include "encryption_key.hh" #include "ex.hh" #include "unbuffered_file.hh" /// Google's ZeroCopyStream implementations which read and write files encrypted /// with our encryption mechanism. They also calculate adler32 of all file /// content and write/check it at the end. /// Encryption-wise we implement AES-128 in CBC mode with PKCS#7 padding. We /// don't use EVP for this currently - everyone is welcome to change this, and /// to add support for arbitrary ciphers, key lengths and modes of operations as /// well. When no encryption key is set, no encryption or padding is done, but /// everything else works the same way otherwise namespace EncryptedFile { DEF_EX( Ex, "Encrypted file exception", std::exception ) DEF_EX( exFileCorrupted, "encrypted file data is currupted", Ex ) DEF_EX( exIncorrectFileSize, "size of the encrypted file is incorrect", exFileCorrupted ) DEF_EX( exReadFailed, "read failed", Ex ) // Only thrown by InputStream::read() DEF_EX( exAdlerMismatch, "adler32 mismatch", Ex ) class InputStream: public google::protobuf::io::ZeroCopyInputStream { public: /// Opens the input file. If EncryptionKey contains no key, the input won't be /// decrypted and iv would be ignored InputStream( char const * fileName, EncryptionKey const &, void const * iv ); virtual bool Next( void const ** data, int * size ); virtual void BackUp( int count ); virtual bool Skip( int count ); virtual int64_t ByteCount() const; /// Returns adler32 of all data read so far. Calling this makes backing up /// for the previous Next() call impossible - the data has to be consumed Adler32::Value getAdler32(); /// Performs a traditional read, for convenience purposes void read( void * buf, size_t size ); /// Reads an adler32 value from the stream and compares with checkAdler32(). /// Throws an exception on mismatch void checkAdler32(); /// Reads and discards the number of bytes equivalent to an IV size. This is /// used when no IV is initially provided. /// If there's no encryption key set, does nothing void consumeRandomIv(); /// Closes the file ~InputStream() {} private: UnbufferedFile file; UnbufferedFile::Offset filePos; EncryptionKey const & key; char iv[ Encryption::IvSize ]; std::vector< char > buffer; char * start; /// Points to the start of the data currently held in buffer size_t fill; /// Number of bytes held in buffer size_t remainder; /// Number of bytes held in buffer just after the main /// 'fill'-bytes portion. We have to keep those to implement /// PKCS#7 padding bool backedUp; /// True if the BackUp operation was performed, and the buffer /// contents are therefore unconsumed Adler32 adler32; /// Decrypts 'fill' bytes at 'start', adjusting 'fill' and setting 'remainder' void decrypt(); /// Only used by decrypt() void doDecrypt(); }; class OutputStream: public google::protobuf::io::ZeroCopyOutputStream { public: /// Creates the output file. If EncryptionKey contains no key, the output /// won't be encrypted and iv would be ignored OutputStream( char const * fileName, EncryptionKey const &, void const * iv ); virtual bool Next( void ** data, int * size ); virtual void BackUp( int count ); virtual int64_t ByteCount() const; /// Returns adler32 of all data written so far. Calling this makes backing up /// for the previous Next() call impossible - the data has to be consumed Adler32::Value getAdler32(); /// Performs a traditional write, for convenience purposes void write( void const * buf, size_t size ); /// Writes the current adler32 value returned by getAdler32() to the stream void writeAdler32(); /// Writes the number of random bytes equivalent to an IV size. This is used /// when no IV is initially provided, and provides an equivalent of having /// a random IV when used just after the stream has been opened. /// If there's no encryption key set, does nothing void writeRandomIv(); /// Finishes writing and closes the file ~OutputStream(); private: UnbufferedFile file; UnbufferedFile::Offset filePos; EncryptionKey const & key; char iv[ Encryption::IvSize ]; std::vector< char > buffer; char * start; /// Points to the start of the area currently available for /// writing to in buffer size_t avail; /// Number of bytes available for writing to in buffer bool backedUp; /// True if the BackUp operation was performed, and the buffer /// contents are therefore unconsumed Adler32 adler32; /// Encrypts and writes 'bytes' bytes from the beginning of the buffer. /// 'bytes' must be non-zero and in multiples of BlockSize void encryptAndWrite( size_t bytes ); }; } #endif zbackup-1.4.4/encryption.cc000066400000000000000000000064331257621500400156700ustar00rootroot00000000000000// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS // Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE #include #include "check.hh" #include "encryption.hh" #include "static_assert.hh" namespace Encryption { char const ZeroIv[ IvSize ] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; void const * encrypt( void const * iv, void const * keyData, void const * inData, void * outData, size_t size ) { unsigned char block[ BlockSize ]; CHECK( !( size % BlockSize ), "size of data to encrypt is not a multiple of " "block size" ); AES_KEY key; AES_set_encrypt_key( ( unsigned char const * ) keyData, KeySize * 8, &key ); void const * prev = iv; // We do the operation in block size multiples. We do XOR in size_t // multiples. The operation is endian-neutral // Make sure that BlockSize is a multiple of the size of size_t STATIC_ASSERT( !( BlockSize % sizeof( size_t ) ) ); size_t const * inS = ( size_t const * ) inData; unsigned char * out = ( unsigned char * ) outData; for ( size_t count = size / BlockSize; count--; ) { size_t const * prevS = ( size_t const * ) prev; size_t * blockS = ( size_t * ) block; for ( size_t x = BlockSize / sizeof( size_t ); x--; ) *blockS++ = *inS++ ^ *prevS++; AES_encrypt( block, out, &key ); prev = out; out += BlockSize; } return prev; } void const * getNextDecryptionIv( void const * in, size_t size ) { CHECK( !( size % BlockSize ), "size of data to decrypt is not a multiple of " "block size" ); return ( char const * ) in + size - BlockSize; } void decrypt( void const * iv, void const * keyData, void const * inData, void * outData, size_t size ) { CHECK( !( size % BlockSize ), "size of data to decrypt is not a multiple of " "block size" ); AES_KEY key; AES_set_decrypt_key( ( unsigned char const * ) keyData, KeySize * 8, &key ); // We decrypt from the end to the beginning unsigned char const * in = ( unsigned char const * ) inData + size; unsigned char * out = ( unsigned char * ) outData + size; size_t count = size / BlockSize; size_t const * prevS = ( size_t const * )( in - BlockSize ); size_t * outS = ( size_t * ) out; while( count-- ) { if ( prevS == inData ) prevS = ( size_t const * )( ( unsigned char const * ) iv + BlockSize ); in -= BlockSize; AES_decrypt( in, ( unsigned char * ) outS - BlockSize, &key ); for ( size_t x = BlockSize / sizeof( size_t ); x--; ) *--outS ^= *--prevS; } } void pad( void * data, size_t size ) { CHECK( size < BlockSize, "size to pad is too large: %zu bytes", size ); unsigned char * p = ( unsigned char * ) data + size; unsigned char v = BlockSize - size; for ( size_t count = v; count--; ) *p++ = v; } size_t unpad( void const * data ) { unsigned char const * p = ( unsigned char const * ) data + BlockSize - 1; unsigned char v = *p; if ( !v || v > BlockSize ) throw exBadPadding(); // Check the rest of the padding for ( size_t count = v - 1; count--; ) if ( *--p != v ) throw exBadPadding(); return BlockSize - v; } } zbackup-1.4.4/encryption.hh000066400000000000000000000041551257621500400157010ustar00rootroot00000000000000// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS // Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE #ifndef ENCRYPTION_HH_INCLUDED__ #define ENCRYPTION_HH_INCLUDED__ #include #include #include "ex.hh" /// What we implement right now is AES-128 in CBC mode with PKCS#7 padding namespace Encryption { enum { KeySize = 16, /// Size of the key in bytes IvSize = 16, /// Size of the IV data in bytes BlockSize = 16 /// Cipher block size in bytes }; DEF_EX( exBadPadding, "Bad padding encountered", std::exception ) /// Encrypts 'size' bytes of the data pointed to by 'in', outputting 'size' /// bytes to 'out'. 'key' points to KeySize bytes of the key data. 'iv' points /// to IvSize bytes used as an initialization vector. 'in' and 'out' can be the /// same. 'size' must be a multiple of BlockSize. Returns a pointer to the /// IV which should be used to continue encrypting, which in CBC is the last /// encrypted block void const * encrypt( void const * iv, void const * key, void const * in, void * out, size_t size ); /// Returns a pointer to the IV which should be used to decrypt the block next /// to the given one, which in CBC is the last encrypted block. Note that if an /// in-place decryption is performed, this IV should be saved first, as it will /// be overwritten with the decrypted data. For size == 0, the returned pointer /// is invalid void const * getNextDecryptionIv( void const * in, size_t size ); /// The reverse of encrypt() void decrypt( void const * iv, void const * key, void const * in, void * out, size_t size ); /// Pads the last block to be encrypted, pointed to by 'data', 'size' bytes, /// which should be less than BlockSize, to occupy BlockSize bytes void pad( void * data, size_t size ); /// Returns the size of the padded data. The data itself is unchanged - use the /// first bytes of 'data'. Can throw exBadPadding size_t unpad( void const * data ); /// The IV consisting of zero bytes. Use it when there is no IV extern char const ZeroIv[ IvSize ]; } #endif zbackup-1.4.4/encryption_key.cc000066400000000000000000000064531257621500400165420ustar00rootroot00000000000000// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS // Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE #include #include #include #include #include "check.hh" #include "encryption_key.hh" #include "random.hh" namespace { /// Derives an encryption key from a password and key info void deriveKey( string const & password, EncryptionKeyInfo const & info, void * key, unsigned keySize ) { CHECK( PKCS5_PBKDF2_HMAC_SHA1( password.data(), password.size(), (unsigned char const *) info.salt().data(), info.salt().size(), info.rounds(), keySize, (unsigned char *) key ) == 1, "encryption key derivation failed" ); } string calculateKeyHmac( void const * key, unsigned keySize, string const & input ) { char result[ EVP_MAX_MD_SIZE ]; unsigned resultSize; CHECK( HMAC( EVP_sha1(), (unsigned char const *) key, keySize, (unsigned char const *) input.data(), input.size(), (unsigned char *) result, &resultSize ), "encryption key HMAC calcuation failed" ); return string( result, result + resultSize ); } } EncryptionKey::EncryptionKey( string const & password, EncryptionKeyInfo const * info ) { if ( !info ) isSet = false; else { isSet = true; char derivedKey[ KeySize ]; deriveKey( password, *info, derivedKey, sizeof( derivedKey ) ); AES_KEY aesKey; AES_set_decrypt_key( ( unsigned char const * ) derivedKey, 128, &aesKey ); AES_decrypt( ( unsigned char const * ) info->encrypted_key().data(), ( unsigned char * ) key, &aesKey ); if ( calculateKeyHmac( key, sizeof( key ), info->key_check_input() ) != info->key_check_hmac() ) throw exInvalidPassword(); } } EncryptionKey::~EncryptionKey() { // Clear the key from memory memset( key, 0, sizeof( key ) ); } void EncryptionKey::generate( string const & password, EncryptionKeyInfo & info ) { // Use this buf for salts char buf[ 16 ]; Random::genaratePseudo( buf, sizeof( buf ) ); info.set_salt( buf, sizeof( buf ) ); info.set_rounds( 10000 ); // TODO: make this configurable char derivedKey[ KeySize ]; deriveKey( password, info, derivedKey, sizeof( derivedKey ) ); char key[ KeySize ]; Random::genarateTrue( key, sizeof( key ) ); // Fill in the HMAC verification part Random::genaratePseudo( buf, sizeof( buf ) ); info.set_key_check_input( buf, sizeof( buf ) ); info.set_key_check_hmac( calculateKeyHmac( key, sizeof( key ), info.key_check_input() ) ); // Encrypt the key AES_KEY aesKey; AES_set_encrypt_key( ( unsigned char const * ) derivedKey, 128, &aesKey ); char encryptedKey[ sizeof( key ) ]; AES_encrypt( ( unsigned char const * ) key, ( unsigned char * ) encryptedKey, &aesKey ); info.set_encrypted_key( encryptedKey, sizeof( encryptedKey ) ); // Clear the key from memory memset( key, 0, sizeof( key ) ); } EncryptionKey const & EncryptionKey::noKey() { static EncryptionKey key( string(), NULL ); return key; } zbackup-1.4.4/encryption_key.hh000066400000000000000000000026441257621500400165520ustar00rootroot00000000000000// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS // Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE #ifndef ENCRYPTION_KEY_HH_INCLUDED__ #define ENCRYPTION_KEY_HH_INCLUDED__ #include #include #include "ex.hh" #include "zbackup.pb.h" using std::string; class EncryptionKey { bool isSet; unsigned const static KeySize = 16; // TODO: make this configurable char key[ KeySize ]; public: DEF_EX( exInvalidPassword, "Invalid password specified", std::exception ) /// Decodes the encryption key from the given info and password. If info is /// passed as NULL, the password is ignored and no key is set EncryptionKey( string const & password, EncryptionKeyInfo const * ); ~EncryptionKey(); /// Returns true if key was set, false otherwise. bool hasKey() const { return isSet; } /// Returns the key. Check if there is one with hasKey() first. Note: the key /// should not be copied, as it may be allocated in a locked page in the /// future void const * getKey() const { return key; } /// Returns key size, in bytes unsigned getKeySize() const { return sizeof( key ); } /// Generates new key info using the given password static void generate( string const & password, EncryptionKeyInfo & ); /// Returns a static instance without any key set static EncryptionKey const & noKey(); }; #endif zbackup-1.4.4/endian.hh000066400000000000000000000031541257621500400147430ustar00rootroot00000000000000// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS // Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE #ifndef ENDIAN_HH_INCLUDED__ #define ENDIAN_HH_INCLUDED__ #include #include #ifdef __APPLE__ #include #else #include #endif #if __BYTE_ORDER == __LITTLE_ENDIAN /// Converts the given host-order value to big-endian value inline uint32_t toBigEndian( uint32_t v ) { return htonl( v ); } /// Converts the given host-order value to little-endian value inline uint32_t toLittleEndian( uint32_t v ) { return v; } inline uint64_t toLittleEndian( uint64_t v ) { return v; } /// Converts the given little-endian value to host-order value inline uint32_t fromLittleEndian( uint32_t v ) { return v; } inline uint64_t fromLittleEndian( uint64_t v ) { return v; } #elif __BYTE_ORDER == __BIG_ENDIAN // Note: the functions used are non-standard. Add more ifdefs if needed /// Converts the given host-order value to big-endian value inline uint32_t toBigEndian( uint32_t v ) { return v; } /// Converts the given host-order value to little-endian value inline uint32_t toLittleEndian( uint32_t v ) { return htole32( v ); } inline uint64_t toLittleEndian( uint64_t v ) { return htole64( v ); } /// Converts the given little-endian value to host-order value inline uint32_t fromLittleEndian( uint32_t v ) { return le32toh( v ); } inline uint64_t fromLittleEndian( uint64_t v ) { return le64toh( v ); } #else #error Please add support for architectures different from little-endian and\ big-endian. #endif #endif zbackup-1.4.4/ex.hh000066400000000000000000000032531257621500400141210ustar00rootroot00000000000000// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS // Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE #ifndef EX_HH_INCLUDED__ #define EX_HH_INCLUDED__ #include #include #include /// A way to declare an exception class fast /// Do like this: /// DEF_EX( exErrorInFoo, "An error in foo encountered", std::exception ) /// DEF_EX( exFooNotFound, "Foo was not found", exErrorInFoo ) #define DEF_EX( exName, exDescription, exParent ) \ class exName: public exParent { \ public: \ virtual const char * what() const throw() { return (exDescription); } \ virtual ~exName() throw() {} }; /// Same as DEF_EX, but takes a runtime string argument, which gets concatenated /// with the description. /// /// DEF_EX_STR( exCantOpen, "can't open file", std::exception ) /// ... /// throw exCantOpen( "example.txt" ); /// /// what() would return "can't open file example.txt" #define DEF_EX_STR( exName, exDescription, exParent ) \ class exName: public exParent { \ std::string value; \ public: \ exName( std::string const & value_ ): value( std::string( exDescription ) + " " + value_ ) {} \ exName( char const * value_, unsigned size ): value( std::string( exDescription ) + " " + std::string( value_, size ) ) {} \ virtual const char * what() const throw() { return value.c_str(); } \ virtual ~exName() throw() {} }; /// An exception class to wrap leave code into an std::exception class exLeaveWrapped: public std::exception { char buf[ 32 ]; public: exLeaveWrapped( int error ) { sprintf( buf, "%d", error ); } char const * what() const throw() { return buf; } }; #endif zbackup-1.4.4/file.cc000066400000000000000000000174711257621500400144210ustar00rootroot00000000000000// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS // Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE #include #include #include #include #include #ifdef __APPLE__ #include #else #include #endif #include #include #include "file.hh" enum { // We employ a writing buffer to considerably speed up file operations when // they consists of many small writes. The default size for the buffer is 64k WriteBufferSize = 65536 }; bool File::exists( char const * filename ) throw() { #ifdef __WIN32 struct _stat buf; return _stat( filename, &buf ) == 0; #else struct stat buf; // EOVERFLOW rationale: if the file is too large, it still does exist return stat( filename, &buf ) == 0 || errno == EOVERFLOW; #endif } void File::erase( std::string const & filename ) throw( exCantErase ) { if ( remove( filename.c_str() ) != 0 ) throw exCantErase( filename ); } void File::rename( std::string const & from, std::string const & to ) throw( exCantRename ) { int res = 0; res = ::rename( from.c_str(), to.c_str() ); if ( 0 != res ) { if ( EXDEV == errno ) { int read_fd; int write_fd; struct stat stat_buf; off_t offset = 0; /* Open the input file. */ read_fd = ::open( from.c_str(), O_RDONLY ); /* Stat the input file to obtain its size. */ fstat( read_fd, &stat_buf ); /* Open the output file for writing, with the same permissions as the source file. */ write_fd = ::open( to.c_str(), O_WRONLY | O_CREAT, stat_buf.st_mode ); /* Blast the bytes from one file to the other. */ #ifdef __APPLE__ if ( -1 == sendfile(write_fd, read_fd, offset, &stat_buf.st_size, NULL, 0) ) throw exCantRename( from + " to " + to ); #else if ( -1 == sendfile(write_fd, read_fd, &offset, stat_buf.st_size) ) throw exCantRename( from + " to " + to ); #endif /* Close up. */ ::close( read_fd ); ::close( write_fd ); File::erase ( from ); } else throw exCantRename( from + " to " + to ); } } void File::open( char const * filename, OpenMode mode ) throw( exCantOpen ) { char const * m; switch( mode ) { case Update: m = "r+b"; break; case WriteOnly: m = "wb"; break; default: m = "rb"; } f = fopen( filename, m ); if ( !f ) throw exCantOpen( std::string( filename ) + ": " + strerror( errno ) ); } File::File( char const * filename, OpenMode mode ) throw( exCantOpen ): writeBuffer( 0 ) { open( filename, mode ); } File::File( std::string const & filename, OpenMode mode ) throw( exCantOpen ): writeBuffer( 0 ) { open( filename.c_str(), mode ); } void File::read( void * buf, size_t size ) throw( exReadError, exWriteError ) { if ( !size ) return; if ( writeBuffer ) flushWriteBuffer(); size_t result = fread( buf, size, 1, f ); if ( result != 1 ) { if ( !ferror( f ) ) throw exShortRead(); else throw exReadErrorDetailed( f ); } } size_t File::readRecords( void * buf, size_t size, size_t count ) throw( exWriteError ) { if ( writeBuffer ) flushWriteBuffer(); return fread( buf, size, count, f ); } void File::write( void const * buf, size_t size ) throw( exWriteError ) { if ( !size ) return; if ( size >= WriteBufferSize ) { // If the write is large, there's not much point in buffering flushWriteBuffer(); size_t result = fwrite( buf, size, 1, f ); if ( result != 1 ) throw exWriteError(); return; } if ( !writeBuffer ) { // Allocate the writing buffer since we don't have any yet writeBuffer = new char[ WriteBufferSize ]; writeBufferLeft = WriteBufferSize; } size_t toAdd = size < writeBufferLeft ? size : writeBufferLeft; memcpy( writeBuffer + ( WriteBufferSize - writeBufferLeft ), buf, toAdd ); size -= toAdd; writeBufferLeft -= toAdd; if ( !writeBufferLeft ) // Out of buffer? Flush it { flushWriteBuffer(); if ( size ) // Something's still left? Add to buffer { memcpy( writeBuffer, (char const *)buf + toAdd, size ); writeBufferLeft -= size; } } } size_t File::writeRecords( void const * buf, size_t size, size_t count ) throw( exWriteError ) { flushWriteBuffer(); return fwrite( buf, size, count, f ); } char * File::gets( char * s, int size, bool stripNl ) throw( exWriteError ) { if ( writeBuffer ) flushWriteBuffer(); char * result = fgets( s, size, f ); if ( result && stripNl ) { size_t len = strlen( result ); char * last = result + len; while( len-- ) { --last; if ( *last == '\n' || *last == '\r' ) *last = 0; else break; } } return result; } std::string File::gets( bool stripNl ) throw( exReadError, exWriteError ) { char buf[ 1024 ]; if ( !gets( buf, sizeof( buf ), stripNl ) ) { if ( !ferror( f ) ) throw exShortRead(); else throw exReadErrorDetailed( f ); } return std::string( buf ); } void File::seek( long offset ) throw( exSeekError, exWriteError ) { if ( writeBuffer ) flushWriteBuffer(); if ( fseek( f, offset, SEEK_SET ) != 0 ) throw exSeekError(); } void File::seekCur( long offset ) throw( exSeekError, exWriteError ) { if ( writeBuffer ) flushWriteBuffer(); if ( fseek( f, offset, SEEK_CUR ) != 0 ) throw exSeekError(); } void File::seekEnd( long offset ) throw( exSeekError, exWriteError ) { if ( writeBuffer ) flushWriteBuffer(); if ( fseek( f, offset, SEEK_END ) != 0 ) throw exSeekError(); } void File::rewind() throw( exSeekError, exWriteError ) { seek( 0 ); } size_t File::tell() throw( exSeekError ) { long result = ftell( f ); if ( result == -1 ) throw exSeekError(); if ( writeBuffer ) result += ( WriteBufferSize - writeBufferLeft ); return ( size_t ) result; } size_t File::size() throw( exSeekError, exWriteError ) { size_t cur = tell(); seekEnd( 0 ); size_t result = tell(); seek( cur ); return result; } bool File::eof() throw( exWriteError ) { if ( writeBuffer ) flushWriteBuffer(); return feof( f ); } FILE * File::file() throw( exWriteError ) { flushWriteBuffer(); return f; } FILE * File::release() throw( exWriteError ) { releaseWriteBuffer(); FILE * c = f; f = 0; return c; } void File::close() throw( exWriteError ) { fclose( release() ); } File::~File() throw() { if ( f ) { try { releaseWriteBuffer(); } catch( exWriteError & ) { } fclose( f ); } } void File::flushWriteBuffer() throw( exWriteError ) { if ( writeBuffer && writeBufferLeft != WriteBufferSize ) { size_t result = fwrite( writeBuffer, WriteBufferSize - writeBufferLeft, 1, f ); if ( result != 1 ) throw exWriteError(); writeBufferLeft = WriteBufferSize; } } void File::releaseWriteBuffer() throw( exWriteError ) { flushWriteBuffer(); if ( writeBuffer ) { delete [] writeBuffer; writeBuffer = 0; } } File::exReadErrorDetailed::exReadErrorDetailed( int fd ) { buildDescription( fd ); } File::exReadErrorDetailed::exReadErrorDetailed( FILE * f ) { buildDescription( fileno( f ) ); } void File::exReadErrorDetailed::buildDescription( int fd ) { description = "Error reading from file "; char path[ PATH_MAX ]; char procFdLink[ 48 ]; sprintf( procFdLink, "/proc/self/fd/%d", fd ); int pathChars = readlink( procFdLink, path, sizeof( path ) ); if ( pathChars < 0 ) description += "(unknown)"; else description.append( path, pathChars ); } const char * File::exReadErrorDetailed::what() const throw() { return description.c_str(); } File::exReadErrorDetailed::~exReadErrorDetailed() throw () { } zbackup-1.4.4/file.hh000066400000000000000000000124611257621500400144250ustar00rootroot00000000000000// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS // Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE #ifndef FILE_HH_INCLUDED__ #define FILE_HH_INCLUDED__ #include #include #include #include #include "ex.hh" using std::string; /// A simple wrapper over FILE * operations with added write-buffering class File { FILE * f; char * writeBuffer; size_t writeBufferLeft; public: DEF_EX( Ex, "File exception", std::exception ) DEF_EX_STR( exCantOpen, "Can't open", Ex ) DEF_EX( exReadError, "Error reading from file", Ex ) DEF_EX( exShortRead, "Short read from the file", exReadError ) DEF_EX( exWriteError, "Error writing to the file", Ex ) DEF_EX( exSeekError, "File seek error", Ex ) DEF_EX_STR( exCantErase, "Can't erase file", Ex ) DEF_EX_STR( exCantRename, "Can't rename file", Ex ) enum OpenMode { ReadOnly, WriteOnly, Update }; typedef long Offset; File( char const * filename, OpenMode ) throw( exCantOpen ); File( std::string const & filename, OpenMode ) throw( exCantOpen ); /// Reads the number of bytes to the buffer, throws an error if it /// failed to fill the whole buffer (short read, i/o error etc) void read( void * buf, size_t size ) throw( exReadError, exWriteError ); template< typename T > void read( T & value ) throw( exReadError, exWriteError ) { read( &value, sizeof( value ) ); } template< typename T > T read() throw( exReadError, exWriteError ) { T value; read( value ); return value; } /// Attempts reading at most 'count' records sized 'size'. Returns /// the number of records it managed to read, up to 'count' size_t readRecords( void * buf, size_t size, size_t count ) throw( exWriteError ); /// Writes the number of bytes from the buffer, throws an error if it /// failed to write the whole buffer (short write, i/o error etc). /// This function employs write buffering, and as such, writes may not /// end up on disk immediately, or a short write may occur later /// than it really did. If you don't want write buffering, use /// writeRecords() function instead void write( void const * buf, size_t size ) throw( exWriteError ); template< typename T > void write( T const & value ) throw( exWriteError ) { write( &value, sizeof( value ) ); } /// Attempts writing at most 'count' records sized 'size'. Returns /// the number of records it managed to write, up to 'count'. /// This function does not employ buffering, but flushes the buffer if it /// was used before size_t writeRecords( void const * buf, size_t size, size_t count ) throw( exWriteError ); /// Reads a string from the file. Unlike the normal fgets(), this one /// can strip the trailing newline character, if this was requested. /// Returns either s or 0 if no characters were read char * gets( char * s, int size, bool stripNl = false ) throw( exWriteError ); /// Like the above, but uses its own local internal buffer (1024 bytes /// currently), and strips newlines by default std::string gets( bool stripNl = true ) throw( exReadError, exWriteError ); /// Seeks in the file, relative to its beginning void seek( long offset ) throw( exSeekError, exWriteError ); /// Seeks in the file, relative to the current position void seekCur( long offset ) throw( exSeekError, exWriteError ); /// Seeks in the file, relative to the end of file void seekEnd( long offset = 0 ) throw( exSeekError, exWriteError ); /// Seeks to the beginning of file void rewind() throw( exSeekError, exWriteError ); /// Tells the current position within the file, relative to its beginning size_t tell() throw( exSeekError ); /// Returns file size size_t size() throw( exSeekError, exWriteError ); /// Returns true if end-of-file condition is set bool eof() throw( exWriteError ); /// Returns the underlying FILE * record, so other operations can be /// performed on it FILE * file() throw( exWriteError ); /// Releases the file handle out of the control of the class. No further /// operations are valid. The file will not be closed on destruction FILE * release() throw( exWriteError ); /// Closes the file. No further operations are valid void close() throw( exWriteError ); /// Checks if the file exists or not static bool exists( char const * filename ) throw(); static bool exists( std::string const & filename ) throw() { return exists( filename.c_str() ); } ~File() throw(); /// Erases the given file static void erase( std::string const & ) throw( exCantErase ); /// Renames the given file static void rename( std::string const & from, std::string const & to ) throw( exCantRename ); /// Throwing this class instead of exReadError will make the description /// include the file name class exReadErrorDetailed: public exReadError { string description; public: exReadErrorDetailed( int fd ); exReadErrorDetailed( FILE * f ); virtual const char * what() const throw(); virtual ~exReadErrorDetailed() throw (); private: void buildDescription( int fd ); }; private: void open( char const * filename, OpenMode ) throw( exCantOpen ); void flushWriteBuffer() throw( exWriteError ); void releaseWriteBuffer() throw( exWriteError ); }; #endif zbackup-1.4.4/hex.cc000066400000000000000000000014621257621500400142570ustar00rootroot00000000000000// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS // Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE #include "hex.hh" using std::string; namespace { /// Converts 'size' bytes pointed to by 'in' into a hex string pointed to by /// 'out'. It should have at least size * 2 bytes. No trailing zero is added void hexify( unsigned char const * in, unsigned size, char * out ) { while( size-- ) { unsigned char v = *in++; *out++ = ( v >> 4 < 10 ) ? '0' + ( v >> 4 ) : 'a' + ( v >> 4 ) - 10; *out++ = ( ( v & 0xF ) < 10 ) ? '0' + ( v & 0xF ) : 'a' + ( v & 0xF ) - 10; } } } string toHex( unsigned char const * in, unsigned size ) { string result( size * 2, 0 ); hexify( in, size, &result[ 0 ] ); return result; } zbackup-1.4.4/hex.hh000066400000000000000000000006051257621500400142670ustar00rootroot00000000000000// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS // Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE #ifndef HEX_HH_INCLUDED__ #define HEX_HH_INCLUDED__ #include /// Converts 'size' bytes pointed to by 'in' into a hex string std::string toHex( unsigned char const * in, unsigned size ); #endif zbackup-1.4.4/index_file.cc000066400000000000000000000033421257621500400156000ustar00rootroot00000000000000// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS // Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE #include #include "bundle.hh" #include "encryption.hh" #include "index_file.hh" #include "message.hh" namespace IndexFile { enum { FileFormatVersion = 1 }; Writer::Writer( EncryptionKey const & key, string const & fileName ): stream( fileName.c_str(), key, Encryption::ZeroIv ) { stream.writeRandomIv(); FileHeader header; header.set_version( FileFormatVersion ); Message::serialize( header, stream ); } void Writer::add( BundleInfo const & info, Bundle::Id const & bundleId ) { IndexBundleHeader header; header.set_id( &bundleId, sizeof( bundleId ) ); Message::serialize( header, stream ); Message::serialize( info, stream ); } Writer::~Writer() { // Final record which does not have a bundle id IndexBundleHeader header; Message::serialize( header, stream ); stream.writeAdler32(); } Reader::Reader( EncryptionKey const & key, string const & fileName ): stream( fileName.c_str(), key, Encryption::ZeroIv ) { stream.consumeRandomIv(); FileHeader header; Message::parse( header, stream ); if ( header.version() != FileFormatVersion ) throw exUnsupportedVersion(); } bool Reader::readNextRecord( BundleInfo & info, Bundle::Id & bundleId ) { IndexBundleHeader header; Message::parse( header, stream ); if ( header.has_id() ) { if ( header.id().size() != sizeof( bundleId ) ) throw exIncorrectBundleIdSize(); memcpy( &bundleId, header.id().data(), sizeof( bundleId ) ); Message::parse( info, stream ); return true; } else { stream.checkAdler32(); return false; } } } zbackup-1.4.4/index_file.hh000066400000000000000000000031661257621500400156160ustar00rootroot00000000000000// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS // Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE #ifndef INDEX_FILE_HH_INCLUDED__ #define INDEX_FILE_HH_INCLUDED__ #include #include #include "adler32.hh" #include "bundle.hh" #include "encrypted_file.hh" #include "encryption_key.hh" #include "ex.hh" #include "file.hh" #include "nocopy.hh" #include "zbackup.pb.h" /// Index files store all existing chunk ids and their bundle ids. This /// information can also be retrieved by scanning all bundle files, but that /// would incur a lot of disk seeks which we want to minimize here namespace IndexFile { using std::string; /// Creates index files class Writer: NoCopy { EncryptedFile::OutputStream stream; public: /// Creates a new chunk log. Initially it is stored in a temporary file Writer( EncryptionKey const &, string const & fileName ); /// Adds a bundle info to the log void add( BundleInfo const &, Bundle::Id const & bundleId ); /// Finalizes the file ~Writer(); }; /// Reads index files class Reader: NoCopy { EncryptedFile::InputStream stream; public: DEF_EX( Ex, "Index file reader exception", std::exception ) DEF_EX( exUnsupportedVersion, "Unsupported version of the index file format", Ex ) DEF_EX( exIncorrectBundleIdSize, "Incorrect bundle id size encountered", Ex ) Reader( EncryptionKey const &, string const & fileName ); /// Reads the next record from the file. Returns false if no more records can /// be found bool readNextRecord( BundleInfo &, Bundle::Id & bundleId ); }; } #endif zbackup-1.4.4/message.cc000066400000000000000000000021201257621500400151070ustar00rootroot00000000000000// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS // Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE #include "message.hh" #include namespace Message { void serialize( MessageLite const & message, ZeroCopyOutputStream & stream ) { CodedOutputStream cos( &stream ); serialize( message, cos ); } void serialize( MessageLite const & message, CodedOutputStream & cos ) { cos.WriteVarint32( message.ByteSize() ); message.SerializeWithCachedSizes( &cos ); if ( cos.HadError() ) throw exCantSerialize( message.GetTypeName() ); } void parse( MessageLite & message, ZeroCopyInputStream & stream ) { CodedInputStream cis( &stream ); parse( message, cis ); } void parse( MessageLite & message, CodedInputStream & cis ) { uint32_t v; if ( !cis.ReadVarint32( &v ) ) throw exCantParse( message.GetTypeName() ); CodedInputStream::Limit limit = cis.PushLimit( v ); if( !message.ParseFromCodedStream( &cis ) ) throw exCantParse( message.GetTypeName() ); cis.PopLimit( limit ); } } zbackup-1.4.4/message.hh000066400000000000000000000025421257621500400151310ustar00rootroot00000000000000// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS // Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE #ifndef MESSAGE_HH_INCLUDED__ #define MESSAGE_HH_INCLUDED__ #include #include #include #include #include "ex.hh" /// Some utilities for protobuffer messages namespace Message { DEF_EX( Ex, "Message exception", std::exception ) DEF_EX_STR( exCantParse, "Can't parse message", Ex ) DEF_EX_STR( exCantSerialize, "Can't serialize message", Ex ) using google::protobuf::io::ZeroCopyOutputStream; using google::protobuf::io::ZeroCopyInputStream; using google::protobuf::io::CodedInputStream; using google::protobuf::io::CodedOutputStream; using google::protobuf::MessageLite; /// Serializes the given message to the given zero-copy stream void serialize( MessageLite const &, ZeroCopyOutputStream & ); /// Serializes the given message to the given coded stream void serialize( MessageLite const &, CodedOutputStream & ); /// Reads and parses the given message from the given zero-copy stream void parse( MessageLite &, ZeroCopyInputStream & ); /// Reads and parses the given message from the given coded stream void parse( MessageLite &, CodedInputStream & ); } #endif zbackup-1.4.4/mt.cc000066400000000000000000000025751257621500400141210ustar00rootroot00000000000000// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS // Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE #include "mt.hh" #include #include "check.hh" Mutex::Mutex() { pthread_mutex_init( &mutex, 0 ); } void Mutex::lock() { pthread_mutex_lock( &mutex ); } void Mutex::unlock() { pthread_mutex_unlock( &mutex ); } Mutex::~Mutex() { pthread_mutex_destroy( &mutex ); } Condition::Condition() { pthread_cond_init( &cond, 0 ); } void Condition::signal() { pthread_cond_signal( &cond ); } void Condition::broadcast() { pthread_cond_broadcast( &cond ); } void Condition::wait( Mutex & m ) { pthread_cond_wait( &cond, &m.mutex ); } Condition::~Condition() { pthread_cond_destroy( &cond ); } void * Thread::__thread_routine( void * param ) { return ( (Thread *)param ) -> threadFunction(); } void Thread::start() { CHECK( pthread_create( &thread, 0, &__thread_routine, this ) == 0, "pthread_create() failed" ); } void Thread::detach() { CHECK( pthread_detach( thread ) == 0, "pthread_detach() failed" ); } void * Thread::join() { void * ret; pthread_join( thread, &ret ); return ret; } size_t getNumberOfCpus() { long result = sysconf( _SC_NPROCESSORS_ONLN ); // Handle -1 and also sanitize the 0 value which wouldn't make sense return result < 1 ? 1 : result; } zbackup-1.4.4/mt.hh000066400000000000000000000025311257621500400141230ustar00rootroot00000000000000// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS // Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE #ifndef MT_HH_INCLUDED__ #define MT_HH_INCLUDED__ #include #include #include "nocopy.hh" /// Multithreading class Condition; class Mutex { friend class Condition; pthread_mutex_t mutex; public: Mutex(); /// Please consider using the Lock class instead void lock(); void unlock(); ~Mutex(); }; class Lock: NoCopy { Mutex * m; public: Lock( Mutex & mutex ): m( &mutex ) { m->lock(); } ~Lock() { m->unlock(); } }; /// Condition variable. Atomically unlocks the given mutex before it suspends /// waiting for event, and upon the awakening reacquires it class Condition { pthread_cond_t cond; public: Condition(); void signal(); void broadcast(); /// Mutex must be locked on entrance void wait( Mutex & m ); ~Condition(); }; class Thread { public: void start(); void detach(); void * join(); virtual ~Thread() {} protected: /// This is the function that is meant to work in a separate thread virtual void * threadFunction() throw()=0; private: pthread_t thread; static void * __thread_routine( void * ); }; /// Returns the number of CPUs this system has size_t getNumberOfCpus(); #endif zbackup-1.4.4/nocopy.hh000066400000000000000000000007331257621500400150140ustar00rootroot00000000000000// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS // Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE #ifndef NOCOPY_HH_INCLUDED__ #define NOCOPY_HH_INCLUDED__ /// A simple class to disallow copying of the class objects. Inherit from it to /// use it class NoCopy { public: NoCopy() {} private: NoCopy( NoCopy const & ); NoCopy & operator = ( NoCopy const & ); }; #endif // NOCOPY_HH zbackup-1.4.4/objectcache.cc000066400000000000000000000021371257621500400157250ustar00rootroot00000000000000// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS // Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE #include "objectcache.hh" ObjectCache::ObjectCache( unsigned maxSize_ ): maxObjects( maxSize_ ), totalObjects( 0 ) { } bool ObjectCache::remove( ObjectId const & id ) { Objects tmp; tmp.push_back( Object() ); tmp.back().id = id; ObjectMap::iterator i = objectMap.find( tmp.begin() ); if ( i == objectMap.end() ) return false; // Make sure that in case a destructor raises an exception, the cache // is left in a consistent state. Reference * ref = (*i)->reference; objects.erase( *i ); objectMap.erase( i ); --totalObjects; delete ref; return true; } void ObjectCache::clear() { for ( Objects::iterator i = objects.begin(); i != objects.end(); ) { // Make sure that in case a destructor raises an exception, the cache // is left in a consistent state. Reference * ref = i->reference; objectMap.erase( i ); objects.erase( i++ ); --totalObjects; delete ref; } } zbackup-1.4.4/objectcache.hh000066400000000000000000000062041257621500400157360ustar00rootroot00000000000000// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS // Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE #ifndef OBJECTCACHE_HH_INCLUDED__ #define OBJECTCACHE_HH_INCLUDED__ #include #include #include #include #include "sptr.hh" #include "nocopy.hh" /// ObjectCache allows caching dynamically-allocated objects of any type. The /// size of the cache is upper-bound and is specified at construction-time. /// Newly added or recently found objects are placed to the top of the internal /// stack. When there's no space in the cache, object become removed from the /// bottom of it class ObjectCache: NoCopy { public: ObjectCache( unsigned maxObjects ); /// Id of the object being stored in the cache typedef std::string ObjectId; /// Returns a reference to the stored object with the given id, or creates /// one if none existed. The caller must know the expected type of the object /// and specify it explicitly template< class T > sptr< T > & entry( ObjectId const & ); /// Removes a stored object with the given id. Returns true if the object /// was removed, false if it didn't exist in the cache bool remove( ObjectId const & ); /// Deletes all the objects from cache void clear(); ~ObjectCache() { clear(); } private: /// Base class for a reference to an object being stored struct Reference: NoCopy { virtual ~Reference() {} }; /// Having this class allows to delete T via virtual destructor accessible /// from the base Reference class template< class T > struct ReferenceTo: public Reference { sptr< T > ref; }; struct Object { ObjectId id; Reference * reference; }; typedef std::list< Object > Objects; struct ObjectsIteratorComp { bool operator () ( Objects::iterator const & x, Objects::iterator const & y ) { return x->id < y->id; } }; typedef std::set< Objects::iterator, ObjectsIteratorComp > ObjectMap; unsigned maxObjects; Objects objects; unsigned totalObjects; ObjectMap objectMap; }; template< class T > sptr< T > & ObjectCache::entry( ObjectId const & id ) { Objects tmp; tmp.push_back( Object() ); tmp.back().id = id; std::pair< ObjectMap::iterator, bool > r = objectMap.insert( tmp.begin() ); if ( r.second ) { // The object was created // Init the reference ReferenceTo< T > * refTo = new ReferenceTo< T >(); tmp.back().reference = refTo; // Add the object to top of our objects objects.splice( objects.begin(), tmp ); ++totalObjects; // evict an entry at the bottom, if needed if ( totalObjects > maxObjects ) { Objects::iterator i = --objects.end(); objectMap.erase( i ); Reference * ref = i->reference; objects.pop_back(); --totalObjects; delete ref; // We expect that it may throw } return refTo->ref; } else { // The object was existent // Move it to the top objects.splice( objects.begin(), objects, *r.first ); return dynamic_cast< ReferenceTo< T > & >( *objects.front().reference ).ref; } } #endif // OBJECTCACHE_HH zbackup-1.4.4/page_size.cc000066400000000000000000000005451257621500400154420ustar00rootroot00000000000000// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS // Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE #include "page_size.hh" #include unsigned getPageSize() { static unsigned value = 0; if ( !value ) value = sysconf( _SC_PAGESIZE ); return value; } zbackup-1.4.4/page_size.hh000066400000000000000000000005071257621500400154520ustar00rootroot00000000000000// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS // Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE #ifndef PAGE_SIZE_HH_INCLUDED__ #define PAGE_SIZE_HH_INCLUDED__ /// Returns the page size used by this system unsigned getPageSize(); #endif zbackup-1.4.4/random.cc000066400000000000000000000010241257621500400147450ustar00rootroot00000000000000// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS // Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE #include "random.hh" #include namespace Random { void genarateTrue( void * buf, unsigned size ) { if ( RAND_bytes( (unsigned char *) buf, size ) != 1 ) throw exCantGenerate(); } void genaratePseudo( void * buf, unsigned size ) { if ( RAND_pseudo_bytes( (unsigned char *) buf, size ) < 0 ) throw exCantGenerate(); } } zbackup-1.4.4/random.hh000066400000000000000000000012241257621500400147610ustar00rootroot00000000000000// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS // Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE #ifndef RANDOM_HH_INCLUDED__ #define RANDOM_HH_INCLUDED__ #include #include "ex.hh" namespace Random { DEF_EX( exCantGenerate, "Error generating random sequence, try later", std::exception ) /// This one fills the buffer with true randomness, suitable for a key void genarateTrue( void * buf, unsigned size ); /// This one fills the buffer with pseudo randomness, suitable for salts but not /// keys void genaratePseudo( void * buf, unsigned size ); } #endif zbackup-1.4.4/rolling_hash.cc000066400000000000000000000012111257621500400161340ustar00rootroot00000000000000// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS // Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE #include "rolling_hash.hh" RollingHash::RollingHash() { reset(); } void RollingHash::reset() { count = 0; factor = 0; nextFactor = 1; value = 0; } RollingHash::Digest RollingHash::digest( void const * buf, unsigned size ) { // TODO: this can be optimized, as in this case there's no need to calculate // factor values. RollingHash hash; for ( char const * p = ( char const * )buf; size--; ) hash.rollIn( *p++ ); return hash.digest(); } zbackup-1.4.4/rolling_hash.hh000066400000000000000000000044741257621500400161640ustar00rootroot00000000000000// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS // Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE #ifndef ROLLING_HASH_HH_INCLUDED__ #define ROLLING_HASH_HH_INCLUDED__ #include #include // Modified Rabin-Karp rolling hash with the base of 257 and the modulo of 2^64. // The canonical RK hash calculates the following value (e.g. for 4 bytes): // hash = ( v1*b^3 + v2*b^2 + v3*b + v4 ) % m // where v1, v2, v3 and v4 are the sequence of bytes, b is the base and m // is the modulo. // We add b^4 in the mix: // hash = ( b^4 + v1*b^3 + v2*b^2 + v3*b + v4 ) % m // This fixes collisions where sequences only differ in the amount of zero // bytes in the beginning (those amount to zero in the canonical RK), since the // power of b in the first member depends on the total number of bytes in the // sequence. // The choice of base: 257 is easy to multiply by (only two bits are set), and // is the first prime larger than the value of any byte. It's easy to create // collisions with the smaller primes: two-byte sequences '1, 0' and '0, base' // would collide, for example. // The choice of modulo: 32-bit is impractical due to birthday paradox -- you // get a collision with the 50% probability having only 77000 hashes. With // 64-bit, the number of hashes to have the same probability would be 5.1 // billion. With the block size of 64k, that would amount to 303 terabytes of // data stored, which should be enough for our purposes. // Note: ( a = ( a << 8 ) + a ) is equivalent to ( a *= 257 ) class RollingHash { uint64_t factor; uint64_t nextFactor; uint64_t value; size_t count; public: typedef uint64_t Digest; RollingHash(); void reset(); void rollIn( char c ) { factor = nextFactor; nextFactor = ( nextFactor << 8 ) + nextFactor; // nextFactor *= 257 value = ( value << 8 ) + value; value += ( unsigned char ) c; ++count; } void rotate( char in, char out ) { value -= uint64_t( ( unsigned char ) out ) * factor; value = ( value << 8 ) + value; // value *= 257 value += ( unsigned char ) in; } Digest digest() const { return value + nextFactor; } size_t size() const { return count; } static Digest digest( void const * buf, unsigned size ); }; #endif zbackup-1.4.4/sha256.cc000066400000000000000000000010451257621500400145000ustar00rootroot00000000000000// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS // Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE #include "sha256.hh" Sha256::Sha256() { SHA256_Init( &ctx ); } void Sha256::add( void const * data, size_t size ) { SHA256_Update( &ctx, data, size ); } void Sha256::finish( void * result ) { SHA256_Final( ( unsigned char * ) result, &ctx ); } string Sha256::finish() { char buf[ Size ]; finish( buf ); return string( buf, buf + sizeof( buf ) ); } zbackup-1.4.4/sha256.hh000066400000000000000000000013171257621500400145140ustar00rootroot00000000000000// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS // Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE #ifndef SHA256_HH_INCLUDED__ #define SHA256_HH_INCLUDED__ #include #include #include using std::string; /// A simple wrapper over openssl class Sha256 { SHA256_CTX ctx; public: enum { // Number of bytes a digest has Size = SHA256_DIGEST_LENGTH }; Sha256(); /// Adds more data void add( void const * data, size_t size ); /// Result should point at at least Size bytes void finish( void * result ); /// Returns result as a string blob string finish(); }; #endif zbackup-1.4.4/sptr.hh000066400000000000000000000057611257621500400145030ustar00rootroot00000000000000// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS // Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE #ifndef SPTR_HH_INCLUDED__ #define SPTR_HH_INCLUDED__ /// A generic non-intrusive smart-pointer template. We could use boost::, tr1:: /// or whatever, but since there's no standard solution yet, it isn't worth /// the dependency given the simplicity of the template template< class T > class sptr_base { template< class TT > friend class sptr_base; T * p; unsigned * count; void increment() { if ( count ) ++*count; } public: sptr_base(): p( 0 ), count( 0 ) {} sptr_base( T * p_ ): p( p_ ), count( p ? new unsigned( 1 ) : 0 ) { } sptr_base( sptr_base< T > const & other ): p( other.p ), count( other.count ) { increment(); } // TT is meant to be a derivative of T template< class TT > sptr_base( sptr_base< TT > const & other ): p( ( T * ) other.p ), count( other.count ) { increment(); } void reset() { if ( count ) { if ( ! -- *count ) { delete count; count = 0; if ( p ) { T * p_ = p; p = 0; delete p_; } } else { p = 0; count = 0; } } } unsigned use_count() const { return count; } sptr_base & operator = ( sptr_base const & other ) { if ( &other != this ) { reset(); p = other.p; count = other.count; increment(); } return * this; } operator bool( void ) const { return !!p; } bool operator ! ( void ) const { return !p; } bool operator == ( sptr_base const & other ) const { return p == other.p; } bool operator != ( sptr_base const & other ) const { return p != other.p; } ~sptr_base() { reset(); } protected: T * get_base( void ) const { return p; } }; template< class T > class sptr: public sptr_base< T > { public: sptr() {} sptr( T * p ): sptr_base< T >( p ) {} // TT is meant to be a derivative of T template< class TT > sptr( sptr< TT > const & other ): sptr_base< T >( other ) {} // Retrieval T * get( void ) const { return sptr_base< T > :: get_base(); } T * operator -> ( void ) const { return get(); } T & operator * ( void ) const { return * get(); } // Check operator bool( void ) const { return get(); } bool operator ! ( void ) const { return !get(); } }; template< class T > class const_sptr: public sptr_base< T > { public: const_sptr() {} const_sptr( T * p_ ): sptr_base< T >( p_ ) {} const_sptr( sptr< T > const & other ): sptr_base< T >( other ) {} // TT is meant to be a derivative of T template< class TT > const_sptr( sptr_base< TT > const & other ): sptr_base< T >( other ) {} // Retrieval T const * get( void ) const { return sptr_base< T > :: get_base(); } T const * operator -> ( void ) const { return get(); } T const & operator * ( void ) const { return * get(); } }; #endif zbackup-1.4.4/static_assert.hh000066400000000000000000000012621257621500400163530ustar00rootroot00000000000000// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS // Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE #ifndef STATIC_ASSERT_HH_INCLUDED__ #define STATIC_ASSERT_HH_INCLUDED__ // Based on the one from the Boost library. It wouldn't make sense to depend on // boost just for that namespace StaticAssert { template < bool > struct AssertionFailure; template <> struct AssertionFailure< true > {}; template< int > struct Test {}; } #define STATIC_ASSERT( B ) \ typedef ::StaticAssert::Test< \ sizeof( ::StaticAssert::AssertionFailure< bool( B ) > ) >\ static_assert_typedef_ ## __LINE__ #endif zbackup-1.4.4/storage_info_file.cc000066400000000000000000000021011257621500400171400ustar00rootroot00000000000000// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS // Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE #include #include "encrypted_file.hh" #include "message.hh" #include "storage_info_file.hh" namespace StorageInfoFile { enum { FileFormatVersion = 1 }; void save( string const & fileName, StorageInfo const & storageInfo ) { EncryptedFile::OutputStream os( fileName.c_str(), EncryptionKey::noKey(), NULL ); FileHeader header; header.set_version( FileFormatVersion ); Message::serialize( header, os ); Message::serialize( storageInfo, os ); os.writeAdler32(); } void load( string const & fileName, StorageInfo & storageInfo ) { EncryptedFile::InputStream is( fileName.c_str(), EncryptionKey::noKey(), NULL ); FileHeader header; Message::parse( header, is ); if ( header.version() != FileFormatVersion ) throw exUnsupportedVersion(); Message::parse( storageInfo, is ); is.checkAdler32(); } } zbackup-1.4.4/storage_info_file.hh000066400000000000000000000014551257621500400171650ustar00rootroot00000000000000// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS // Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE #ifndef STORAGE_INFO_FILE_HH_INCLUDED__ #define STORAGE_INFO_FILE_HH_INCLUDED__ #include #include #include "encryption_key.hh" #include "ex.hh" #include "zbackup.pb.h" namespace StorageInfoFile { using std::string; DEF_EX( Ex, "Storage info file exception", std::exception ) DEF_EX( exUnsupportedVersion, "Unsupported version of the storage info file format", Ex ) /// Saves the given StorageInfo data into the given file void save( string const & fileName, StorageInfo const & ); /// Loads the given StorageInfo data from the given file void load( string const & fileName, StorageInfo & ); } #endif zbackup-1.4.4/tartool/000077500000000000000000000000001257621500400146455ustar00rootroot00000000000000zbackup-1.4.4/tartool/CMakeLists.txt000066400000000000000000000005711257621500400174100ustar00rootroot00000000000000# Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS # Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE cmake_minimum_required( VERSION 2.6.0 ) project( tartool ) set( CMAKE_BUILD_TYPE Release ) add_executable( tartool tartool.cc ../file.cc ../dir.cc ) install( TARGETS tartool DESTINATION bin ) zbackup-1.4.4/tartool/tartool.cc000066400000000000000000000120741257621500400166440ustar00rootroot00000000000000// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS // Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE #include #include #include #include #include #include "../dir.hh" #include "../file.hh" using std::string; using std::vector; using std::map; void mention( File & file, string const & path ) { file.write( path.data(), path.size() ); file.write( '\n' ); } bool startsWith( string const & s, char const * prefix ) { for ( char const * sPtr = s.c_str(), * pPtr = prefix; *pPtr; ++sPtr, ++pPtr ) if ( *sPtr != *pPtr ) return false; return true; } void scanDirIgnoringErrors( string const & path, File & includes, File & excludes, bool currentlyIncluded ); void scanDir( string const & path, File & includes, File & excludes, bool currentlyIncluded ) { Dir::Entry entry; vector< string > subdirs; vector< string > namedIncludes, namedExcludes; typedef map< string, bool > FileList; FileList fileList; bool doBackup = false; bool dontBackup = false; for ( Dir::Listing dir( path ); dir.getNext( entry ); ) { string const & fileName = entry.getFileName(); if ( entry.isDir() ) { if ( !entry.isSymLink() ) subdirs.push_back( fileName ); } else if ( fileName == ".backup" ) doBackup = true; if ( fileName == ".no-backup" ) dontBackup = true; else if ( startsWith( fileName, ".backup-" ) ) namedIncludes.push_back( fileName.substr( 8 ) ); else if ( startsWith( fileName, ".no-backup-" ) ) namedExcludes.push_back( fileName.substr( 11 ) ); } // If both are mentioned, backup if ( doBackup ) dontBackup = false; if ( doBackup && !currentlyIncluded ) { mention( includes, path ); currentlyIncluded = true; } if ( dontBackup && currentlyIncluded ) { mention( excludes, path ); currentlyIncluded = false; } // If we have any effective named lists, build the fileList map and process // them. if ( ( !currentlyIncluded && !namedIncludes.empty() ) || ( currentlyIncluded && !namedExcludes.empty() ) ) { for ( Dir::Listing dir( path ); dir.getNext( entry ); ) fileList[ entry.getFileName() ] = entry.isDir() && !entry.isSymLink(); if ( !currentlyIncluded ) { for ( vector< string > :: const_iterator i = namedIncludes.begin(); i != namedIncludes.end(); ++i ) { FileList::iterator entry = fileList.find( *i ); if ( entry != fileList.end() ) { mention( includes, Dir::addPath( path, *i ) ); if ( entry->second ) // Is it a dir? Scan it then. scanDir( Dir::addPath( path, entry->first ), includes, excludes, true ); // Make sure we don't process it twice. fileList.erase( entry ); } else fprintf( stderr, "Warning: named include %s does not exist in %s\n", i->c_str(), path.c_str() ); } } else { for ( vector< string > :: const_iterator i = namedExcludes.begin(); i != namedExcludes.end(); ++i ) { FileList::iterator entry = fileList.find( *i ); if ( entry != fileList.end() ) { mention( excludes, Dir::addPath( path, *i ) ); if ( entry->second ) // Is it a dir? Scan it then. scanDir( Dir::addPath( path, entry->first ), includes, excludes, false ); // Make sure we don't process it twice. fileList.erase( entry ); } else fprintf( stderr, "Warning: named exclude %s does not exist in %s\n", i->c_str(), path.c_str() ); } } // Scan the rest of dirs for ( FileList::const_iterator i = fileList.begin(); i != fileList.end(); ++i ) if ( i->second ) scanDirIgnoringErrors( Dir::addPath( path, i->first ), includes, excludes, currentlyIncluded ); } else { // No named lists -- just process all the dirs for ( size_t x = 0; x < subdirs.size(); ++x ) scanDirIgnoringErrors( Dir::addPath( path, subdirs[ x ] ), includes, excludes, currentlyIncluded ); } } void scanDirIgnoringErrors( string const & path, File & includes, File & excludes, bool currentlyIncluded ) { try { scanDir( path, includes, excludes, currentlyIncluded ); } catch( Dir::exCantList & e ) { fprintf( stderr, "Warning: %s\n", e.what() ); } } int main( int argc, char *argv[] ) { if ( argc != 4 ) { fprintf( stderr, "Usage: %s \n", *argv ); return EXIT_FAILURE; } try { File includes( argv[ 2 ], File::WriteOnly ); File excludes( argv[ 3 ], File::WriteOnly ); scanDir( argv[ 1 ], includes, excludes, false ); return EXIT_SUCCESS; } catch( std::exception & e ) { fprintf( stderr, "Error: %s\n", e.what() ); return EXIT_FAILURE; } } zbackup-1.4.4/tests/000077500000000000000000000000001257621500400143235ustar00rootroot00000000000000zbackup-1.4.4/tests/TODO.txt000066400000000000000000000000751257621500400156330ustar00rootroot00000000000000Convert those to cmake -- they still use qmake at the moment zbackup-1.4.4/tests/bundle/000077500000000000000000000000001257621500400155745ustar00rootroot00000000000000zbackup-1.4.4/tests/bundle/bundle.pro000066400000000000000000000022521257621500400175700ustar00rootroot00000000000000###################################################################### # Automatically generated by qmake (2.01a) Sun Jul 14 20:54:52 2013 ###################################################################### TEMPLATE = app TARGET = DEPENDPATH += . INCLUDEPATH += . CONFIG = debug LIBS += -lcrypto -lprotobuf -lz -lprotobuf -llzma -llzo2 DEFINES += __STDC_FORMAT_MACROS DEFINES += HAVE_LIBLZO # Input SOURCES += test_bundle.cc \ ../../unbuffered_file.cc \ ../../tmp_mgr.cc \ ../../page_size.cc \ ../../random.cc \ ../../encryption_key.cc \ ../../encryption.cc \ ../../encrypted_file.cc \ ../../file.cc \ ../../dir.cc \ ../../bundle.cc \ ../../message.cc \ ../../hex.cc \ ../../compression.cc \ ../../zbackup.pb.cc HEADERS += \ ../../unbuffered_file.hh \ ../../tmp_mgr.hh \ ../../adler32.hh \ ../../page_size.hh \ ../../random.hh \ ../../encryption_key.hh \ ../../encrypted_file.hh \ ../../encryption.hh \ ../../ex.hh \ ../../file.hh \ ../../dir.hh \ ../../bundle.hh \ ../../message.hh \ ../../hex.hh \ ../../compression.hh \ ../../message.hh \ ../../zbackup.pb.h zbackup-1.4.4/tests/bundle/test_bundle.cc000066400000000000000000000122631257621500400204170ustar00rootroot00000000000000// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS // Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE #include #include #include #include "../../encrypted_file.hh" #include "../../encryption_key.hh" #include "../../random.hh" #include "../../tmp_mgr.hh" #include "../../check.hh" #include "../../adler32.hh" #include "../../bundle.hh" #include "../../compression.hh" #include "../../message.hh" using namespace Compression; char tmpbuf[100]; void testCompatibility() { // The LZO code uses a different file header than the previous code // because it adds the compression_method field. Nevertheless, it // must be compatible with previous code. TmpMgr tmpMgr( "/dev/shm" ); sptr< TemporaryFile > tempFile = tmpMgr.makeTemporaryFile(); std::string fileName = tempFile->getFileName(); EncryptionKey noKey( std::string(), NULL ); // Write old header, read as new header { { EncryptedFile::OutputStream os( fileName.c_str(), noKey, Encryption::ZeroIv ); FileHeader header; header.set_version( 42 ); Message::serialize( header, os ); } { EncryptedFile::InputStream is( fileName.c_str(), noKey, Encryption::ZeroIv ); BundleFileHeader header; Message::parse( header, is ); CHECK( header.version() == 42, "version is wrong when reading old header with new program" ); CHECK( header.compression_method() == "lzma", "compression_method is wrong when reading old header with new program" ); } } // Write new header, read as old header //NOTE In the real code, this will only work, if the file uses LZMA. If it doesn't, the version // field is increased and the old code will refuse to read the file. { { EncryptedFile::OutputStream os( fileName.c_str(), noKey, Encryption::ZeroIv ); BundleFileHeader header; header.set_version( 42 ); Message::serialize( header, os ); } { EncryptedFile::InputStream is( fileName.c_str(), noKey, Encryption::ZeroIv ); FileHeader header; Message::parse( header, is ); CHECK( header.version() == 42, "version is wrong when reading new header with old program" ); // cannot check compression_method because the field doesn't exist } } printf("compatibility test successful.\n"); } void readAndWrite( EncryptionKey const & key, const_sptr compression1, const_sptr compression2 ) { // temporary file for the bundle TmpMgr tmpMgr( "/dev/shm" ); sptr< TemporaryFile > tempFile = tmpMgr.makeTemporaryFile(); // some chunk data int chunkCount = rand() % 30; size_t chunkSize = rand() % 20 ? 64*1024 : 10; char** chunks = new char*[chunkCount]; string* chunkIds = new string[chunkCount]; CompressionMethod::defaultCompression = compression1; // write bundle { Bundle::Creator bundle; for (int i=0;igetFileName().c_str(), key ); } CompressionMethod::defaultCompression = compression2; // read it and compare { Bundle::Reader bundle( tempFile->getFileName().c_str(), key ); for (int i=0;i > compressions; for ( CompressionMethod::iterator it = CompressionMethod::begin(); it!=CompressionMethod::end(); ++it ) { printf( "supported compression: %s\n", (*it)->getName().c_str() ); compressions.push_back( *it ); } for ( size_t iteration = 100; iteration--; ) { // default compression while writing the file const_sptr compression1 = compressions[ rand() % compressions.size() ]; // default compression while reading the file // The reader should ignore it and always use the compression that was used for the file. const_sptr compression2 = compressions[ rand() % compressions.size() ]; readAndWrite( ( rand() & 1 ) ? key : noKey, compression1, compression2 ); } printf("\n"); return 0; } zbackup-1.4.4/tests/encrypted_file/000077500000000000000000000000001257621500400173175ustar00rootroot00000000000000zbackup-1.4.4/tests/encrypted_file/encrypted_file.pro000066400000000000000000000016451257621500400230430ustar00rootroot00000000000000###################################################################### # Automatically generated by qmake (2.01a) Sun Jul 14 20:54:52 2013 ###################################################################### TEMPLATE = app TARGET = DEPENDPATH += . INCLUDEPATH += . LIBS += -lcrypto -lprotobuf -lz DEFINES += __STDC_FORMAT_MACROS # Input SOURCES += test_encrypted_file.cc \ ../../unbuffered_file.cc \ ../../tmp_mgr.cc \ ../../page_size.cc \ ../../random.cc \ ../../encryption_key.cc \ ../../encryption.cc \ ../../encrypted_file.cc \ ../../file.cc \ ../../dir.cc \ ../../zbackup.pb.cc HEADERS += \ ../../unbuffered_file.hh \ ../../tmp_mgr.hh \ ../../adler32.hh \ ../../page_size.hh \ ../../random.hh \ ../../encryption_key.hh \ ../../encrypted_file.hh \ ../../encryption.hh \ ../../ex.hh \ ../../file.hh \ ../../dir.hh \ ../../zbackup.pb.h zbackup-1.4.4/tests/encrypted_file/test_encrypted_file.cc000066400000000000000000000112251257621500400236620ustar00rootroot00000000000000// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS // Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE #include #include "../../encrypted_file.hh" #include "../../encryption_key.hh" #include "../../random.hh" #include "../../tmp_mgr.hh" #include "../../check.hh" #include "../../adler32.hh" char rnd[ 16384 ]; Adler32::Value adler( int sz ) { Adler32 a; a.add( rnd, sz ); return a.result(); } void readAndWrite( EncryptionKey const & key, bool writeBackups, bool readBackups, bool readSkips ) { TmpMgr tmpMgr( "/dev/shm" ); sptr< TemporaryFile > tempFile = tmpMgr.makeTemporaryFile(); int fileSize = rand() % ( sizeof( rnd ) + 1 ); fprintf( stderr, "Run with %d bytes, %s%s%s%sfile %s...\n", fileSize, key.hasKey() ? "" : "no encryption, ", writeBackups ? "write backups, " : "", readBackups ? "read backups, " : "", readSkips ? "read skips, " : "", tempFile->getFileName().c_str() ); char iv[ Encryption::IvSize ]; Random::genaratePseudo( iv, sizeof( iv ) ); // Write { EncryptedFile::OutputStream out( tempFile->getFileName().c_str(), key, iv ); char const * next = rnd; int avail = 0; for ( int left = fileSize; left; ) { CHECK( out.ByteCount() == fileSize - left, "Incorrect bytecount in the " "middle of writing" ); void * data; CHECK( out.Next( &data, &avail ), "out.Next() returned false" ); CHECK( avail > 0, "out.Next() returned zero size" ); bool doBackup = writeBackups && ( rand() & 1 ); int backup; if ( doBackup ) { backup = rand() % ( avail + 1 ); // Make sure we don't back up and then need to back up again to finish // the write if ( avail > left ) backup = avail - left; avail -= backup; } int toWrite = avail > left ? left : avail; memcpy( data, next, toWrite ); if ( doBackup ) out.BackUp( backup ); next += toWrite; left -= toWrite; avail -= toWrite; if ( !avail && ( rand() & 1 ) ) { CHECK( adler( next - rnd ) == out.getAdler32(), "bad adler32 in the middle of writing" ); } } if ( avail || ( rand() & 1 ) ) out.BackUp( avail ); CHECK( out.ByteCount() == fileSize, "Incorrect bytecount after writing" ); if ( rand() & 1 ) { CHECK( adler( fileSize ) == out.getAdler32(), "bad adler32 of the written file" ); } } // Read back { EncryptedFile::InputStream in( tempFile->getFileName().c_str(), key, iv ); char const * next = rnd; void const * data; int avail = 0; for ( int left = fileSize; left; ) { if ( readSkips && ( rand() & 1 ) ) { int toSkip = rand() % ( left + 1 ); in.Skip( toSkip ); next += toSkip; left -= toSkip; avail = 0; continue; } CHECK( in.ByteCount() == fileSize - left, "Incorrect bytecount in the " "middle of reading" ); CHECK( in.Next( &data, &avail ), "file ended while %d were still left", left ); CHECK( avail > 0, "in.Next() returned zero size" ); bool doBackup = readBackups && ( rand() & 1 ); int backup; if ( doBackup ) { backup = rand() % ( avail + 1 ); avail -= backup; } int toRead = avail > left ? left : avail; CHECK( memcmp( next, data, toRead ) == 0, "Different bytes read than " "expected at offset %d", int( next - rnd ) ); if ( doBackup ) in.BackUp( backup ); next += toRead; left -= toRead; avail -= toRead; if ( !avail && ( rand() & 1 ) ) { CHECK( adler( next - rnd ) == in.getAdler32(), "bad adler32 in the middle of the reading" ); } } CHECK( in.ByteCount() == fileSize, "Incorrect bytecount after reading" ); CHECK( !avail, "at least %d bytes still available", avail ); CHECK( !in.Next( &data, &avail ), "file should have ended but resulted in " "%d more bytes", avail ); if ( rand() & 1 ) { CHECK( adler( fileSize ) == in.getAdler32(), "bad adler32 of the read file" ); } } } int main() { Random::genaratePseudo( rnd, sizeof( rnd ) ); EncryptionKeyInfo keyInfo; EncryptionKey::generate( "blah", keyInfo ); EncryptionKey key( "blah", &keyInfo ); EncryptionKey noKey( std::string(), NULL ); for ( size_t iteration = 100000; iteration--; ) readAndWrite( ( rand() & 1 ) ? key : noKey, rand() & 1, rand() & 1, rand() & 1 ); } zbackup-1.4.4/tests/rolling_hash/000077500000000000000000000000001257621500400167745ustar00rootroot00000000000000zbackup-1.4.4/tests/rolling_hash/rolling_hash.pro000066400000000000000000000006241257621500400221710ustar00rootroot00000000000000###################################################################### # Automatically generated by qmake (2.01a) ?? ???. 8 14:05:16 2012 ###################################################################### TEMPLATE = app TARGET = DEPENDPATH += . INCLUDEPATH += . LIBS += -lcrypto # Input SOURCES += test_rolling_hash.cc ../../rolling_hash.cc \ ../../random.cc HEADERS += \ ../../random.hh zbackup-1.4.4/tests/rolling_hash/test_rolling_hash.cc000066400000000000000000000065511257621500400230220ustar00rootroot00000000000000// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS // Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE #include #include #include #include #include #include #include "../../rolling_hash.hh" #include "../../random.hh" using std::vector; using std::map; using std::set; using std::pair; using std::make_pair; int main() { // Generate a buffer with random data, then pick slices there and try // different strategies of rolling to them vector< char > data( 65536 ); Random::genaratePseudo( data.data(), data.size() ); for ( unsigned iteration = 0; iteration < 5000; ++iteration ) { unsigned sliceBegin = rand() % data.size(); unsigned sliceSize = 1 + ( rand() % ( data.size() - sliceBegin ) ); // Calculate the hash by roll-ins only uint64_t rollIns; { RollingHash hash; for ( unsigned x = 0; x < sliceSize; ++x ) hash.rollIn( data[ sliceBegin + x ] ); rollIns = hash.digest(); } // Calculate the hash by rolling-in from the beginning of data to sliceSize, // then rotating to sliceBegin uint64_t rotates; { RollingHash hash; for ( unsigned x = 0; x < sliceSize; ++x ) hash.rollIn( data[ x ] ); for ( unsigned x = 0; x < sliceBegin; ++x ) hash.rotate( data[ sliceSize + x ], data[ x ] ); rotates = hash.digest(); } if ( rollIns != rotates ) { fprintf( stderr, "Error in iteration %u: %016lx vs %016lx\n", iteration, rollIns, rotates ); return EXIT_FAILURE; } printf( "Iteration %u: %016lx\n", iteration, rollIns ); } fprintf( stderr, "Rolling hash test produced equal results\n" ); // Test collisions // Maps the hash to the ranges. Ideally each hash should be mapped to a // single range map< uint64_t, set< pair< unsigned, unsigned > > > collisions; size_t collisionValuesCount = 0; for ( unsigned iteration = 0; iteration < 500000; ++iteration ) { unsigned sliceBegin = rand() % ( data.size() - 7 ); // A minimum of 16 should be enough to ensure every unique slice corresponds // to a unique random sequence with a very high probability unsigned sliceSize = 16 + ( rand() % ( data.size() - sliceBegin ) ); // Calculate the hash by roll-ins (fastest) uint64_t rollIns; { RollingHash hash; for ( unsigned x = 0; x < sliceSize; ++x ) hash.rollIn( data[ sliceBegin + x ] ); rollIns = hash.digest(); } if ( collisions[ rollIns ].insert( make_pair( sliceBegin, sliceSize ) ).second ) ++collisionValuesCount; if ( ! ( ( iteration + 1 ) % 1000 ) ) printf( "Iteration %u: %016lx\n", iteration, rollIns ); } size_t collisionsFound = collisionValuesCount - collisions.size(); double collisionsPercentage = double( collisionsFound ) * 100 / collisionValuesCount; fprintf( stderr, "Collisions: %.04f%% (%zu in %zu)\n", collisionsPercentage, collisionsFound, collisionValuesCount ); if ( collisionsFound ) { // The probability of a collision in 500000 hashes is one to ~6 billions fprintf( stderr, "Found a collision, which should be highly unlikely\n" ); return EXIT_FAILURE; } fprintf( stderr, "Rolling hash test succeeded\n" ); return EXIT_SUCCESS; } zbackup-1.4.4/tmp_mgr.cc000066400000000000000000000024341257621500400151400ustar00rootroot00000000000000// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS // Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE #include "tmp_mgr.hh" #include #include #include "dir.hh" #include "file.hh" TemporaryFile::TemporaryFile( string const & fileName ): fileName( fileName ) { } void TemporaryFile::moveOverTo( string const & destinationFileName, bool mayOverwrite ) { if ( !mayOverwrite && File::exists( destinationFileName ) ) throw TmpMgr::exWontOverwrite( destinationFileName ); File::rename( fileName, destinationFileName ); fileName.clear(); } TemporaryFile::~TemporaryFile() { if ( !fileName.empty() ) File::erase( fileName ); } string const & TemporaryFile::getFileName() const { return fileName; } TmpMgr::TmpMgr( string const & path ): path( path ) { if ( !Dir::exists( path ) ) Dir::create( path ); } sptr< TemporaryFile > TmpMgr::makeTemporaryFile() { string name( Dir::addPath( path, "XXXXXX") ); int fd = mkstemp( &name[ 0 ] ); if ( fd == -1 || close( fd ) != 0 ) throw exCantCreate( path ); return new TemporaryFile( name ); } TmpMgr::~TmpMgr() { try { Dir::remove( path ); } catch( Dir::exCantRemove & ) { } } zbackup-1.4.4/tmp_mgr.hh000066400000000000000000000035711257621500400151550ustar00rootroot00000000000000// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS // Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE #ifndef TMP_MGR_HH_INCLUDED__ #define TMP_MGR_HH_INCLUDED__ #include #include #include "dir.hh" #include "ex.hh" #include "file.hh" #include "nocopy.hh" #include "sptr.hh" /// A temporary file class TemporaryFile: NoCopy { public: /// Returns the temporary file's file name. The file may already be existent - /// it is supposed to be overwritten then string const & getFileName() const; /// Renames this temporary file over the given file name. If the destination /// file exists already, it gets replaced if mayOverwrite is true, or throws /// an exception otherwise void moveOverTo( string const & destinationFileName, bool mayOverwrite = false ); /// Removes the file from the disk, unless moveOverTo() was called previously ~TemporaryFile(); private: /// Use TmpMgr::makeTemporaryFile() instead of this constructor TemporaryFile( string const & fileName ); string fileName; friend class TmpMgr; }; /// Allows creating temporary files and later either removing them or moving /// them over to the target ones class TmpMgr: NoCopy { string path; public: DEF_EX( Ex, "Temporary file manager exception", std::exception ) DEF_EX_STR( exCantCreate, "Can't create a temporary file in dir", Ex ) DEF_EX_STR( exWontOverwrite, "Won't overwrite existing file", Ex ) /// Creates the given directory if it doesn't exist already and uses it to /// store temporary files. TmpMgr( string const & path ); /// Creates an new empty temporary file and returns its full file name, /// including the path. The file is then supposed to be overwritten sptr< TemporaryFile > makeTemporaryFile(); /// Removes the temporary directory, if possible ~TmpMgr(); }; #endif zbackup-1.4.4/unbuffered_file.cc000066400000000000000000000042501257621500400166150ustar00rootroot00000000000000// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS // Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE #define _LARGEFILE64_SOURCE #include #include #include #include #include #include "check.hh" #include "unbuffered_file.hh" #ifdef __APPLE__ #define lseek64 lseek #endif UnbufferedFile::UnbufferedFile( char const * fileName, Mode mode ) throw( exCantOpen ) { int flags = ( mode == WriteOnly ? ( O_WRONLY | O_CREAT | O_TRUNC ) : O_RDONLY ); #ifndef __APPLE__ flags |= O_LARGEFILE; #endif fd = open( fileName, flags, 0666 ); if ( fd < 0 ) throw exCantOpen( fileName ); } size_t UnbufferedFile::read( void * buf, size_t size ) throw( exReadError ) { char * next = ( char * ) buf; size_t left = size; while( left ) { ssize_t rd = ::read( fd, next, left ); if ( rd < 0 ) { if ( errno != EINTR ) throw exReadError(); } else if ( rd > 0 ) { CHECK( ( size_t ) rd <= left, "read too many bytes from a file" ); next += rd; left -= rd; } else break; } return size - left; } void UnbufferedFile::write( void const * buf, size_t size ) throw( exWriteError ) { char const * next = ( char const * ) buf; size_t left = size; while( left ) { ssize_t written = ::write( fd, next, left ); if ( written < 0 ) { if ( errno != EINTR ) throw exWriteError(); } else { CHECK( ( size_t ) written <= left, "wrote too many bytes to a file" ); next += written; left -= written; } } } UnbufferedFile::Offset UnbufferedFile::size() throw( exSeekError ) { Offset cur = lseek64( fd, 0, SEEK_CUR ); if ( cur < 0 ) throw exSeekError(); Offset result = lseek64( fd, 0, SEEK_END ); if ( result < 0 || lseek64( fd, cur, SEEK_SET ) < 0 ) throw exSeekError(); return result; } void UnbufferedFile::seekCur( Offset offset ) throw( exSeekError ) { if ( lseek64( fd, offset, SEEK_CUR ) < 0 ) throw exSeekError(); } UnbufferedFile::~UnbufferedFile() throw() { close( fd ); } zbackup-1.4.4/unbuffered_file.hh000066400000000000000000000034451257621500400166340ustar00rootroot00000000000000// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS // Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE #ifndef UNBUFFERED_FILE_HH_INCLUDED__ #define UNBUFFERED_FILE_HH_INCLUDED__ #include #include #include #include #include "ex.hh" #include "nocopy.hh" /// A file which does not employ its own buffering. /// TODO: add support for memory-mapped I/O, with the interface which would look /// like that of a zero-copy stream. However, since we can do encryption in- /// place, both interfaces should be available - when there's no memory-mapped /// I/O available, the user should still provide its own buffer (and then do /// in-place encryption in it). class UnbufferedFile: NoCopy { public: DEF_EX( Ex, "Unbuffered file exception", std::exception ) DEF_EX_STR( exCantOpen, "Can't open file", Ex ) DEF_EX( exReadError, "File read error", Ex ) DEF_EX( exWriteError, "File write error", Ex ) DEF_EX( exSeekError, "File seek error", Ex ) enum Mode { ReadOnly, WriteOnly }; typedef int64_t Offset; /// Opens the given file UnbufferedFile( char const * fileName, Mode ) throw( exCantOpen ); /// Reads up to 'size' bytes into the buffer. Returns the number of bytes /// read. If the value returned is less than the 'size' provided, the end of /// file was reached size_t read( void * buf, size_t size ) throw( exReadError ); /// Writes 'size' bytes void write( void const * buf, size_t size ) throw( exWriteError ); /// Returns file size Offset size() throw( exSeekError ); /// Seeks to the given offset, relative to the current file offset void seekCur( Offset ) throw( exSeekError ); ~UnbufferedFile() throw(); private: int fd; }; #endif zbackup-1.4.4/zbackup.cc000066400000000000000000000524211257621500400151330ustar00rootroot00000000000000// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS // Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE #include #include #include #include #include #include #include #include #include #include "backup_creator.hh" #include "backup_file.hh" #include "backup_restorer.hh" #include "compression.hh" #include "debug.hh" #include "dir.hh" #include "encryption_key.hh" #include "ex.hh" #include "file.hh" #include "mt.hh" #include "sha256.hh" #include "sptr.hh" #include "storage_info_file.hh" #include "zbackup.hh" #include "index_file.hh" #include "bundle.hh" #include "zcollector.hh" using std::vector; using std::bitset; using std::iterator; ZBackup::ZBackup( string const & storageDir, string const & password, size_t threads ): ZBackupBase( storageDir, password ), chunkStorageWriter( storageInfo, encryptionkey, tmpMgr, chunkIndex, getBundlesPath(), getIndexPath(), threads ) { } void ZBackup::backupFromStdin( string const & outputFileName ) { if ( isatty( fileno( stdin ) ) ) throw exWontReadFromTerminal(); if ( File::exists( outputFileName ) ) throw exWontOverwrite( outputFileName ); Sha256 sha256; BackupCreator backupCreator( storageInfo, chunkIndex, chunkStorageWriter ); time_t startTime = time( 0 ); uint64_t totalDataSize = 0; for ( ; ; ) { size_t toRead = backupCreator.getInputBufferSize(); // dPrintf( "Reading up to %u bytes from stdin\n", toRead ); void * inputBuffer = backupCreator.getInputBuffer(); size_t rd = fread( inputBuffer, 1, toRead, stdin ); if ( !rd ) { if ( feof( stdin ) ) { dPrintf( "No more input on stdin\n" ); break; } else throw exStdinError(); } sha256.add( inputBuffer, rd ); backupCreator.handleMoreData( rd ); totalDataSize += rd; } // Finish up with the creator backupCreator.finish(); string serialized; backupCreator.getBackupData( serialized ); BackupInfo info; info.set_sha256( sha256.finish() ); info.set_size( totalDataSize ); // Shrink the serialized data iteratively until it wouldn't shrink anymore for ( ; ; ) { BackupCreator backupCreator( storageInfo, chunkIndex, chunkStorageWriter ); char const * ptr = serialized.data(); size_t left = serialized.size(); while( left ) { size_t bufferSize = backupCreator.getInputBufferSize(); size_t toCopy = bufferSize > left ? left : bufferSize; memcpy( backupCreator.getInputBuffer(), ptr, toCopy ); backupCreator.handleMoreData( toCopy ); ptr += toCopy; left -= toCopy; } backupCreator.finish(); string newGen; backupCreator.getBackupData( newGen ); if ( newGen.size() < serialized.size() ) { serialized.swap( newGen ); info.set_iterations( info.iterations() + 1 ); } else break; } dPrintf( "Iterations: %u\n", info.iterations() ); info.mutable_backup_data()->swap( serialized ); info.set_time( time( 0 ) - startTime ); // Commit the bundles to the disk before creating the final output file chunkStorageWriter.commit(); // Now save the resulting BackupInfo sptr< TemporaryFile > tmpFile = tmpMgr.makeTemporaryFile(); BackupFile::save( tmpFile->getFileName(), encryptionkey, info ); tmpFile->moveOverTo( outputFileName ); } ZRestore::ZRestore( string const & storageDir, string const & password, size_t cacheSize ): ZBackupBase( storageDir, password ), chunkStorageReader( storageInfo, encryptionkey, chunkIndex, getBundlesPath(), cacheSize ) { } void ZRestore::restoreToStdin( string const & inputFileName ) { if ( isatty( fileno( stdout ) ) ) throw exWontWriteToTerminal(); BackupInfo backupInfo; BackupFile::load( inputFileName, encryptionkey, backupInfo ); string backupData; // Perform the iterations needed to get to the actual user backup data BackupRestorer::restoreIterations( chunkStorageReader, backupInfo, backupData, NULL ); struct StdoutWriter: public DataSink { Sha256 sha256; virtual void saveData( void const * data, size_t size ) { sha256.add( data, size ); if ( fwrite( data, size, 1, stdout ) != 1 ) throw exStdoutError(); } } stdoutWriter; BackupRestorer::restore( chunkStorageReader, backupData, &stdoutWriter, NULL ); if ( stdoutWriter.sha256.finish() != backupInfo.sha256() ) throw exChecksumError(); } ZExchange::ZExchange( string const & srcStorageDir, string const & srcPassword, string const & dstStorageDir, string const & dstPassword, bool prohibitChunkIndexLoading ): srcZBackupBase( srcStorageDir, srcPassword, prohibitChunkIndexLoading ), dstZBackupBase( dstStorageDir, dstPassword, prohibitChunkIndexLoading ) { } void ZExchange::exchange( string const & srcPath, string const & dstPath, bitset< BackupExchanger::Flags > const & exchange ) { vector< BackupExchanger::PendingExchangeRename > pendingExchangeRenames; if ( exchange.test( BackupExchanger::bundles ) ) { verbosePrintf( "Searching for bundles...\n" ); vector< string > bundles = BackupExchanger::findOrRebuild( srcZBackupBase.getBundlesPath(), dstZBackupBase.getBundlesPath() ); for ( std::vector< string >::iterator it = bundles.begin(); it != bundles.end(); ++it ) { verbosePrintf( "Processing bundle file %s... ", it->c_str() ); string outputFileName ( Dir::addPath( dstZBackupBase.getBundlesPath(), *it ) ); if ( !File::exists( outputFileName ) ) { sptr< Bundle::Reader > reader = new Bundle::Reader( Dir::addPath ( srcZBackupBase.getBundlesPath(), *it ), srcZBackupBase.encryptionkey, true ); sptr< Bundle::Creator > creator = new Bundle::Creator; sptr< TemporaryFile > bundleTempFile = dstZBackupBase.tmpMgr.makeTemporaryFile(); creator->write( bundleTempFile->getFileName(), dstZBackupBase.encryptionkey, *reader ); if ( creator.get() && reader.get() ) { creator.reset(); reader.reset(); pendingExchangeRenames.push_back( BackupExchanger::PendingExchangeRename( bundleTempFile, outputFileName ) ); verbosePrintf( "done.\n" ); } } else { verbosePrintf( "file exists - skipped.\n" ); } } verbosePrintf( "Bundle exchange completed.\n" ); } if ( exchange.test( BackupExchanger::index ) ) { verbosePrintf( "Searching for indicies...\n" ); vector< string > indicies = BackupExchanger::findOrRebuild( srcZBackupBase.getIndexPath(), dstZBackupBase.getIndexPath() ); for ( std::vector< string >::iterator it = indicies.begin(); it != indicies.end(); ++it ) { verbosePrintf( "Processing index file %s... ", it->c_str() ); string outputFileName ( Dir::addPath( dstZBackupBase.getIndexPath(), *it ) ); if ( !File::exists( outputFileName ) ) { sptr< IndexFile::Reader > reader = new IndexFile::Reader( srcZBackupBase.encryptionkey, Dir::addPath( srcZBackupBase.getIndexPath(), *it ) ); sptr< TemporaryFile > indexTempFile = dstZBackupBase.tmpMgr.makeTemporaryFile(); sptr< IndexFile::Writer > writer = new IndexFile::Writer( dstZBackupBase.encryptionkey, indexTempFile->getFileName() ); BundleInfo bundleInfo; Bundle::Id bundleId; while( reader->readNextRecord( bundleInfo, bundleId ) ) { writer->add( bundleInfo, bundleId ); } if ( writer.get() && reader.get() ) { writer.reset(); reader.reset(); pendingExchangeRenames.push_back( BackupExchanger::PendingExchangeRename( indexTempFile, outputFileName ) ); verbosePrintf( "done.\n" ); } } else { verbosePrintf( "file exists - skipped.\n" ); } } verbosePrintf( "Index exchange completed.\n" ); } if ( exchange.test( BackupExchanger::backups ) ) { BackupInfo backupInfo; verbosePrintf( "Searching for backups...\n" ); vector< string > backups = BackupExchanger::findOrRebuild( srcZBackupBase.getBackupsPath(), dstZBackupBase.getBackupsPath() ); for ( std::vector< string >::iterator it = backups.begin(); it != backups.end(); ++it ) { verbosePrintf( "Processing backup file %s... ", it->c_str() ); string outputFileName ( Dir::addPath( dstZBackupBase.getBackupsPath(), *it ) ); if ( !File::exists( outputFileName ) ) { BackupFile::load( Dir::addPath( srcZBackupBase.getBackupsPath(), *it ), srcZBackupBase.encryptionkey, backupInfo ); sptr< TemporaryFile > tmpFile = dstZBackupBase.tmpMgr.makeTemporaryFile(); BackupFile::save( tmpFile->getFileName(), dstZBackupBase.encryptionkey, backupInfo ); pendingExchangeRenames.push_back( BackupExchanger::PendingExchangeRename( tmpFile, outputFileName ) ); verbosePrintf( "done.\n" ); } else { verbosePrintf( "file exists - skipped.\n" ); } } verbosePrintf( "Backup exchange completed.\n" ); } if ( pendingExchangeRenames.size() > 0 ) { verbosePrintf( "Moving files from temp directory to appropriate places... " ); for ( size_t x = pendingExchangeRenames.size(); x--; ) { BackupExchanger::PendingExchangeRename & r = pendingExchangeRenames[ x ]; r.first->moveOverTo( r.second ); if ( r.first.get() ) { r.first.reset(); } } pendingExchangeRenames.clear(); verbosePrintf( "done.\n" ); } } DEF_EX( exExchangeWithLessThanTwoKeys, "Specify password flag (--non-encrypted or --password-file)" " for import/export operation twice (first for source and second for destination)", std::exception ) DEF_EX( exNonEncryptedWithKey, "--non-encrypted and --password-file are incompatible", std::exception ) DEF_EX( exSpecifyEncryptionOptions, "Specify either --password-file or --non-encrypted", std::exception ) DEF_EX_STR( exInvalidThreadsValue, "Invalid threads value specified:", std::exception ) int main( int argc, char *argv[] ) { try { size_t const defaultThreads = getNumberOfCpus(); size_t threads = defaultThreads; size_t const defaultCacheSizeMb = 40; size_t cacheSizeMb = defaultCacheSizeMb; bool printHelp = false; bool forcedCompressionMethod = false; vector< char const * > args; vector< string > passwords; bitset< BackupExchanger::Flags > exchange; for( int x = 1; x < argc; ++x ) { if ( strcmp( argv[ x ], "--password-file" ) == 0 && x + 1 < argc ) { // Read the password char const * passwordFile = argv[ x + 1 ]; string passwordData; if ( passwordFile ) { File f( passwordFile, File::ReadOnly ); passwordData.resize( f.size() ); f.read( &passwordData[ 0 ], passwordData.size() ); // If the password ends with \n, remove that last \n. Many editors will // add \n there even if a user doesn't want them to if ( !passwordData.empty() && passwordData[ passwordData.size() - 1 ] == '\n' ) passwordData.resize( passwordData.size() - 1 ); passwords.push_back( passwordData ); } ++x; } else if ( strcmp( argv[ x ], "--exchange" ) == 0 && x + 1 < argc ) { char const * exchangeValue = argv[ x + 1 ]; if ( strcmp( exchangeValue, "backups" ) == 0 ) exchange.set( BackupExchanger::backups ); else if ( strcmp( exchangeValue, "bundles" ) == 0 ) exchange.set( BackupExchanger::bundles ); else if ( strcmp( exchangeValue, "index" ) == 0 ) exchange.set( BackupExchanger::index ); else { fprintf( stderr, "Invalid exchange value specified: %s\n" "Must be one of the following: backups, bundles, index\n", exchangeValue ); return EXIT_FAILURE; } ++x; } else if ( strcmp( argv[ x ], "--non-encrypted" ) == 0 ) { passwords.push_back( "" ); } else if ( strcmp( argv[ x ], "--silent" ) == 0 ) verboseMode = false; else if ( strcmp( argv[ x ], "--threads" ) == 0 && x + 1 < argc ) { int n; if ( sscanf( argv[ x + 1 ], "%zu %n", &threads, &n ) != 1 || argv[ x + 1 ][ n ] || threads < 1 ) throw exInvalidThreadsValue( argv[ x + 1 ] ); ++x; } else if ( strcmp( argv[ x ], "--cache-size" ) == 0 && x + 1 < argc ) { char suffix[ 16 ]; int n; if ( sscanf( argv[ x + 1 ], "%zu %15s %n", &cacheSizeMb, suffix, &n ) == 2 && !argv[ x + 1 ][ n ] ) { // Check the suffix for ( char * c = suffix; *c; ++c ) *c = tolower( *c ); if ( strcmp( suffix, "mb" ) != 0 ) { fprintf( stderr, "Invalid suffix specified in cache size: %s. " "The only supported suffix is 'mb' for megabytes\n", argv[ x + 1 ] ); return EXIT_FAILURE; } ++x; } else { fprintf( stderr, "Invalid cache size value specified: %s. " "Must be a number with the 'mb' suffix, e.g. '100mb'\n", argv[ x + 1 ] ); return EXIT_FAILURE; } } else if ( strcmp( argv[ x ], "--compression" ) == 0 && x + 1 < argc ) { forcedCompressionMethod = true; // next argument names the compression method ++x; if ( strcmp( argv[ x ], "lzma" ) == 0 ) { const_sptr lzma = Compression::CompressionMethod::findCompression( "lzma" ); if ( !lzma ) { fprintf( stderr, "zbackup is compiled without LZMA support, but the code " "would support it. If you install liblzma (including development files) " "and recompile zbackup, you can use LZMA.\n" ); return EXIT_FAILURE; } Compression::CompressionMethod::defaultCompression = lzma; } else if ( strcmp( argv[ x ], "lzo" ) == 0 ) { const_sptr lzo = Compression::CompressionMethod::findCompression( "lzo1x_1" ); if ( !lzo ) { fprintf( stderr, "zbackup is compiled without LZO support, but the code " "would support it. If you install liblzo2 (including development files) " "and recompile zbackup, you can use LZO.\n" ); return EXIT_FAILURE; } Compression::CompressionMethod::defaultCompression = lzo; } else { fprintf( stderr, "zbackup doesn't support compression method '%s'. You may need a newer version.\n", argv[ x ] ); return EXIT_FAILURE; } } else if ( strcmp( argv[ x ], "--help" ) == 0 || strcmp( argv[ x ], "-h" ) == 0 ) { printHelp = true; } else args.push_back( argv[ x ] ); } if ( args.size() < 1 || printHelp ) { fprintf( stderr, "ZBackup, a versatile deduplicating backup tool, version 1.4.3\n" "Copyright (c) 2012-2014 Konstantin Isakov and\n" "ZBackup contributors\n" "Comes with no warranty. Licensed under GNU GPLv2 or later + OpenSSL.\n" "Visit the project's home page at http://zbackup.org/\n\n" "Usage: %s [flags] [command args]\n" " Flags: --non-encrypted|--password-file \n" " password flag should be specified twice if import/export\n" " command specified\n" " --silent (default is verbose)\n" " --threads (default is %zu on your system)\n" " --cache-size MB (default is %zu)\n" " --exchange [backups|bundles|index] (can be\n" " specified multiple times)\n" " --compression (default is lzma)\n" " --help|-h show this message\n" " Commands:\n" " init - initializes new storage;\n" " backup - performs a backup from stdin;\n" " restore - restores a backup to stdout;\n" " export -\n" " performs export from source to destination storage;\n" " import -\n" " performs import from source to destination storage;\n" " gc [fast|deep] - performs garbage collection.\n" " Default is fast.\n" " For export/import storage path must be valid (initialized) storage.\n" "", *argv, defaultThreads, defaultCacheSizeMb ); return EXIT_FAILURE; } if ( passwords.size() > 1 && ( ( passwords[ 0 ].empty() && !passwords[ 1 ].empty() ) || ( !passwords[ 0 ].empty() && passwords[ 1 ].empty() ) ) && ( strcmp( args[ 0 ], "export" ) != 0 && strcmp( args[ 0 ], "import" ) != 0 ) ) throw exNonEncryptedWithKey(); else if ( passwords.size() < 2 && ( strcmp( args[ 0 ], "export" ) == 0 || strcmp( args[ 0 ], "import" ) == 0 ) ) throw exExchangeWithLessThanTwoKeys(); else if ( passwords.size() < 1 ) throw exSpecifyEncryptionOptions(); if ( strcmp( args[ 0 ], "init" ) == 0 ) { // Perform the init if ( args.size() != 2 ) { fprintf( stderr, "Usage: %s init \n", *argv ); return EXIT_FAILURE; } ZBackup::initStorage( args[ 1 ], passwords[ 0 ], !passwords[ 0 ].empty() ); } else if ( strcmp( args[ 0 ], "backup" ) == 0 ) { // Perform the backup if ( args.size() != 2 ) { fprintf( stderr, "Usage: %s backup \n", *argv ); return EXIT_FAILURE; } ZBackup zb( ZBackup::deriveStorageDirFromBackupsFile( args[ 1 ] ), passwords[ 0 ], threads ); if ( !forcedCompressionMethod ) zb.useDefaultCompressionMethod(); zb.backupFromStdin( args[ 1 ] ); } else if ( strcmp( args[ 0 ], "restore" ) == 0 ) { // Perform the restore if ( args.size() != 2 ) { fprintf( stderr, "Usage: %s restore \n", *argv ); return EXIT_FAILURE; } ZRestore zr( ZRestore::deriveStorageDirFromBackupsFile( args[ 1 ] ), passwords[ 0 ], cacheSizeMb * 1048576 ); if ( !forcedCompressionMethod ) zr.useDefaultCompressionMethod(); zr.restoreToStdin( args[ 1 ] ); } else if ( strcmp( args[ 0 ], "export" ) == 0 || strcmp( args[ 0 ], "import" ) == 0 ) { if ( args.size() != 3 ) { fprintf( stderr, "Usage: %s %s \n", *argv, args[ 0 ] ); return EXIT_FAILURE; } if ( exchange.none() ) { fprintf( stderr, "Specify any --exchange flag\n" ); return EXIT_FAILURE; } int src, dst; if ( strcmp( args[ 0 ], "export" ) == 0 ) { src = 1; dst = 2; } else if ( strcmp( args[ 0 ], "import" ) == 0 ) { src = 2; dst = 1; } dPrintf( "%s src: %s\n", args[ 0 ], args[ src ] ); dPrintf( "%s dst: %s\n", args[ 0 ], args[ dst ] ); ZExchange ze( ZBackupBase::deriveStorageDirFromBackupsFile( args[ src ], true ), passwords[ src - 1 ], ZBackupBase::deriveStorageDirFromBackupsFile( args[ dst ], true ), passwords[ dst - 1 ], true ); ze.exchange( args[ src ], args[ dst ], exchange ); } else if ( strcmp( args[ 0 ], "gc" ) == 0 ) { // Perform the garbage collection if ( args.size() < 2 || args.size() > 3 ) { fprintf( stderr, "Usage: %s %s [fast|deep] \n", *argv, args[ 0 ] ); return EXIT_FAILURE; } int fieldStorage = 1; int fieldAction = 2; if ( args.size() == 3 ) { fieldStorage = 2; fieldAction = 1; } if ( args.size() > 2 && strcmp( args[ fieldAction ], "fast" ) == 0 ) { ZCollector zc( ZBackupBase::deriveStorageDirFromBackupsFile( args[ fieldStorage ], true ), passwords[ 0 ], threads, cacheSizeMb * 1048576 ); zc.gc( false ); } else if ( args.size() > 2 && strcmp( args[ fieldAction ], "deep" ) == 0 ) { ZCollector zc( ZBackupBase::deriveStorageDirFromBackupsFile( args[ fieldStorage ], true ), passwords[ 0 ], threads, cacheSizeMb * 1048576 ); zc.gc( true ); } else { ZCollector zc( ZBackupBase::deriveStorageDirFromBackupsFile( args[ fieldStorage ], true ), passwords[ 0 ], threads, cacheSizeMb * 1048576 ); zc.gc( false ); } } else { fprintf( stderr, "Error: unknown command line option: %s\n", args[ 0 ] ); return EXIT_FAILURE; } } catch( std::exception & e ) { fprintf( stderr, "%s\n", e.what() ); return EXIT_FAILURE; } return EXIT_SUCCESS; } zbackup-1.4.4/zbackup.hh000066400000000000000000000035271257621500400151500ustar00rootroot00000000000000// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS // Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE #ifndef ZBACKUP_HH_INCLUDED__ #define ZBACKUP_HH_INCLUDED__ #include #include #include #include #include #include "chunk_id.hh" #include "chunk_index.hh" #include "chunk_storage.hh" #include "encryption_key.hh" #include "ex.hh" #include "tmp_mgr.hh" #include "zbackup.pb.h" #include "zbackup_base.hh" #include "backup_exchanger.hh" using std::string; using std::vector; using std::bitset; class ZBackup: public ZBackupBase { ChunkStorage::Writer chunkStorageWriter; public: ZBackup( string const & storageDir, string const & password, size_t threads ); /// Backs up the data from stdin void backupFromStdin( string const & outputFileName ); }; class ZRestore: public ZBackupBase { ChunkStorage::Reader chunkStorageReader; public: ZRestore( string const & storageDir, string const & password, size_t cacheSize ); /// Restores the data to stdin void restoreToStdin( string const & inputFileName ); }; class ZCollect: public ZBackupBase { ChunkStorage::Reader chunkStorageReader; size_t threads; public: ZCollect( string const & storageDir, string const & password, size_t threads, size_t cacheSize ); void gc(); }; class ZExchange { ZBackupBase srcZBackupBase; ZBackupBase dstZBackupBase; public: ZExchange( string const & srcStorageDir, string const & srcPassword, string const & dstStorageDir, string const & dstPassword, bool prohibitChunkIndexLoading ); /// Exchanges the data between storages void exchange( string const & srcFileName, string const & dstFileName, bitset< BackupExchanger::Flags > const & exchange ); }; #endif zbackup-1.4.4/zbackup.proto000066400000000000000000000103051257621500400157040ustar00rootroot00000000000000// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS // Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE // Protobuffers used in zbackup // This stores the key used for the encryption of all the blocks. The key itself // is stored in the encrypted form. A user supplies a password - it is used // together with salt and rounds to generate a decryption key for the actual // key used for block encryption. This way we can change the password without // re-encrypting all the blocks message EncryptionKeyInfo { // The decryption key is derived from the password, salt and rounds using // PKCS5_PBKDF2_HMAC_SHA1 // Salt to use together with the user password required bytes salt = 1; // Rounds of hashing to apply when generating the key used to decrypt the // block key required uint32 rounds = 2; // Stores the block encryption key, in an encrypted form itself required bytes encrypted_key = 3; // Used to check that the key was decrypted correctly - see the next field required bytes key_check_input = 4; // HMAC of key_check_input using the decrypted key. Used to check that the // key was indeed decrypted correctly required bytes key_check_hmac = 5; } message StorageInfo { // Maximum chunk size used when storing chunks required uint32 chunk_max_size = 1; // Maximum number of bytes a bundle can hold. Only real chunk bytes are // counted, not metadata. Any bundle should be able to contain at least // one arbitrary single chunk, so this should not be smaller than // chunk_max_size required uint32 bundle_max_payload_size = 2; // If present, used for encryption/decryption of all data optional EncryptionKeyInfo encryption_key = 3; // Default compression for new bundles optional string default_compression_method = 4 [default = "lzma"]; } message BundleInfo { // Info about a single chunk stored message ChunkRecord { // Id of the chunk required bytes id = 1; // Size of the chunk required uint32 size = 2; } // A sequence of chunk records repeated ChunkRecord chunk_record = 1; } message FileHeader { // File format version required uint32 version = 1; } message BundleFileHeader { // File format version required uint32 version = 1; // Compression method that is used for this file // If the program doesn't support that field, it will try LZMA. If it is // LZMA, that will work. If it isn't, it will have aborted before because // the version in FileHeader is higher than it can support. optional string compression_method = 2 [default = "lzma"]; } message IndexBundleHeader { // Id of the bundle following in the stream. If not present, indicates the // end of log file optional bytes id = 1; } // A single instruction. Backups are made of a sequence of those instructions, // which are executed one after another message BackupInstruction { // Both fields can present simultaneously. They are evaluated in the same // order they are listed here // If present, the chunk with that id should be emitted to the data flow optional bytes chunk_to_emit = 1; // If present, the bytes contained in the field should be emitted to the // data flow optional bytes bytes_to_emit = 2; } message BackupInfo { // The backup data. Since usually the field is quite large for real life /// backups, we process its serialized data with the same backup algorithm // iteratively until it doesn't shrink. The content of this field represents // the last iteration of that process. If iterations = 0, it directly // represents the user's backup data. If iterations = 1, it represents the // backed up BackupData which would represent the user's backed up data once // it is restored, and so on. // The type is 'bytes' as the result is serialized required bytes backup_data = 1; // Number of times backup_data should be restored with the 'restore' algorithm // before we get what we need to restore for the end user optional uint32 iterations = 2 [default = 0]; // Number of bytes in the backup data required uint64 size = 3; // SHA-256 of the original data required bytes sha256 = 4; // Time spent creating the backup, in seconds optional int64 time = 5; } zbackup-1.4.4/zbackup_base.cc000066400000000000000000000076371257621500400161360ustar00rootroot00000000000000// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS // Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE #include "zbackup_base.hh" #include "storage_info_file.hh" #include "compression.hh" using std::string; Paths::Paths( string const & storageDir ): storageDir( storageDir ) { } string Paths::getTmpPath() { return string( Dir::addPath( storageDir, "tmp" ) ); } string Paths::getBundlesPath() { return string( Dir::addPath( storageDir, "bundles" ) ); } string Paths::getStorageInfoPath() { return string( Dir::addPath( storageDir, "info" ) ); } string Paths::getIndexPath() { return string( Dir::addPath( storageDir, "index" ) ); } string Paths::getBackupsPath() { return string( Dir::addPath( storageDir, "backups" ) ); } ZBackupBase::ZBackupBase( string const & storageDir, string const & password ): Paths( storageDir ), storageInfo( loadStorageInfo() ), encryptionkey( password, storageInfo.has_encryption_key() ? &storageInfo.encryption_key() : 0 ), tmpMgr( getTmpPath() ), chunkIndex( encryptionkey, tmpMgr, getIndexPath(), false ) { } ZBackupBase::ZBackupBase( string const & storageDir, string const & password, bool prohibitChunkIndexLoading ): Paths( storageDir ), storageInfo( loadStorageInfo() ), encryptionkey( password, storageInfo.has_encryption_key() ? &storageInfo.encryption_key() : 0 ), tmpMgr( getTmpPath() ), chunkIndex( encryptionkey, tmpMgr, getIndexPath(), prohibitChunkIndexLoading ) { } StorageInfo ZBackupBase::loadStorageInfo() { StorageInfo storageInfo; StorageInfoFile::load( getStorageInfoPath(), storageInfo ); return storageInfo; } void ZBackupBase::initStorage( string const & storageDir, string const & password, bool isEncrypted ) { StorageInfo storageInfo; // TODO: make the following configurable storageInfo.set_chunk_max_size( 65536 ); storageInfo.set_bundle_max_payload_size( 0x200000 ); if ( isEncrypted ) EncryptionKey::generate( password, *storageInfo.mutable_encryption_key() ); storageInfo.set_default_compression_method( Compression::CompressionMethod::defaultCompression->getName() ); Paths paths( storageDir ); if ( !Dir::exists( storageDir ) ) Dir::create( storageDir ); if ( !Dir::exists( paths.getBundlesPath() ) ) Dir::create( paths.getBundlesPath() ); if ( !Dir::exists( paths.getBackupsPath() ) ) Dir::create( paths.getBackupsPath() ); if ( !Dir::exists( paths.getIndexPath() ) ) Dir::create( paths.getIndexPath() ); string storageInfoPath( paths.getStorageInfoPath() ); if ( File::exists( storageInfoPath ) ) throw exWontOverwrite( storageInfoPath ); StorageInfoFile::save( storageInfoPath, storageInfo ); } string ZBackupBase::deriveStorageDirFromBackupsFile( string const & backupsFile, bool allowOutside ) { // TODO: handle cases when there's a backup/ folder within the backup/ folder // correctly if ( allowOutside ) return Dir::getRealPath( backupsFile ); string realPath = Dir::getRealPath( Dir::getDirName( backupsFile ) ); size_t pos; if ( realPath.size() >= 8 && strcmp( realPath.c_str() + realPath.size() - 8, "/backups") == 0 ) pos = realPath.size() - 8; else pos = realPath.rfind( "/backups/" ); if ( pos == string::npos ) throw exCantDeriveStorageDir( backupsFile ); else return realPath.substr( 0, pos ); } void ZBackupBase::useDefaultCompressionMethod() { std::string compression_method_name = storageInfo.default_compression_method(); const_sptr compression = Compression::CompressionMethod::findCompression( compression_method_name ); Compression::CompressionMethod::defaultCompression = compression; } zbackup-1.4.4/zbackup_base.hh000066400000000000000000000041721257621500400161370ustar00rootroot00000000000000// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS // Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE #ifndef ZBACKUP_BASE_HH_INCLUDED__ #define ZBACKUP_BASE_HH_INCLUDED__ #include #include #include "ex.hh" #include "chunk_index.hh" struct Paths { std::string storageDir; Paths( std::string const & storageDir ); std::string getTmpPath(); std::string getRestorePath(); std::string getCreatePath(); std::string getBundlesPath(); std::string getStorageInfoPath(); std::string getIndexPath(); std::string getBackupsPath(); }; class ZBackupBase: public Paths { public: DEF_EX( Ex, "ZBackup exception", std::exception ) DEF_EX_STR( exWontOverwrite, "Won't overwrite existing file", Ex ) DEF_EX( exStdinError, "Error reading from standard input", Ex ) DEF_EX( exWontReadFromTerminal, "Won't read data from a terminal", exStdinError ) DEF_EX( exStdoutError, "Error writing to standard output", Ex ) DEF_EX( exWontWriteToTerminal, "Won't write data to a terminal", exStdoutError ) DEF_EX( exSerializeError, "Failed to serialize data", Ex ) DEF_EX( exParseError, "Failed to parse data", Ex ) DEF_EX( exChecksumError, "Checksum error", Ex ) DEF_EX_STR( exCantDeriveStorageDir, "The path must be within the backups/ dir:", Ex ) /// Opens the storage ZBackupBase( std::string const & storageDir, std::string const & password ); ZBackupBase( std::string const & storageDir, std::string const & password, bool prohibitChunkIndexLoading ); /// Creates new storage static void initStorage( std::string const & storageDir, std::string const & password, bool isEncrypted ); /// For a given file within the backups/ dir in the storage, returns its /// storage dir or throws an exception static std::string deriveStorageDirFromBackupsFile( std::string const & backupsFile, bool allowOutside = false ); void useDefaultCompressionMethod(); StorageInfo storageInfo; EncryptionKey encryptionkey; TmpMgr tmpMgr; ChunkIndex chunkIndex; private: StorageInfo loadStorageInfo(); }; #endif zbackup-1.4.4/zcollector.cc000066400000000000000000000154251257621500400156570ustar00rootroot00000000000000// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS // Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE #include "zcollector.hh" #include "backup_restorer.hh" #include "backup_file.hh" #include "backup_exchanger.hh" #include "debug.hh" using std::string; using std::iterator; namespace { class BundleCollector: public IndexProcessor { private: Bundle::Id savedId; int totalChunks, usedChunks, indexTotalChunks, indexUsedChunks; int indexModifiedBundles, indexKeptBundles, indexRemovedBundles; bool indexModified, indexNecessary; vector< string > filesToUnlink; BackupRestorer::ChunkSet overallChunkSet; std::set< Bundle::Id > overallBundleSet; void copyUsedChunks( BundleInfo const & info ) { // Copy used chunks to the new index string chunk; size_t chunkSize; for ( int x = info.chunk_record_size(); x--; ) { BundleInfo_ChunkRecord const & record = info.chunk_record( x ); ChunkId id( record.id() ); if ( usedChunkSet.find( id ) != usedChunkSet.end() ) { chunkStorageReader->get( id, chunk, chunkSize ); chunkStorageWriter->add( id, chunk.data(), chunkSize ); } } } public: string bundlesPath; ChunkStorage::Reader *chunkStorageReader; ChunkStorage::Writer *chunkStorageWriter; BackupRestorer::ChunkSet usedChunkSet; bool gcRepack, gcDeep; void startIndex( string const & indexFn ) { indexModified = indexNecessary = false; indexTotalChunks = indexUsedChunks = 0; indexModifiedBundles = indexKeptBundles = indexRemovedBundles = 0; } void finishIndex( string const & indexFn ) { verbosePrintf( "Chunks used: %d/%d, bundles: %d kept, %d modified, %d removed\n", indexUsedChunks, indexTotalChunks, indexKeptBundles, indexModifiedBundles, indexRemovedBundles); if ( indexModified ) { filesToUnlink.push_back( indexFn ); commit(); } else { chunkStorageWriter->reset(); if ( gcDeep && !indexNecessary ) // this index was a complete copy so we don't need it filesToUnlink.push_back( indexFn ); } } void startBundle( Bundle::Id const & bundleId ) { savedId = bundleId; totalChunks = 0; usedChunks = 0; } void processChunk( ChunkId const & chunkId ) { if ( gcDeep ) { if ( overallChunkSet.find ( chunkId ) == overallChunkSet.end() ) overallChunkSet.insert( chunkId ); else return; } totalChunks++; if ( usedChunkSet.find( chunkId ) != usedChunkSet.end() ) { usedChunks++; indexNecessary = true; } } void finishBundle( Bundle::Id const & bundleId, BundleInfo const & info ) { string i = Bundle::generateFileName( savedId, "", false ); indexTotalChunks += totalChunks; indexUsedChunks += usedChunks; if ( 0 == usedChunks && 0 != totalChunks ) { dPrintf( "Deleting %s bundle\n", i.c_str() ); filesToUnlink.push_back( Dir::addPath( bundlesPath, i ) ); indexModified = true; indexRemovedBundles++; } else if ( usedChunks < totalChunks ) { dPrintf( "%s: used %d/%d chunks\n", i.c_str(), usedChunks, totalChunks ); filesToUnlink.push_back( Dir::addPath( bundlesPath, i ) ); indexModified = true; copyUsedChunks( info ); indexModifiedBundles++; } else { if ( gcRepack ) { filesToUnlink.push_back( Dir::addPath( bundlesPath, i ) ); indexModified = true; copyUsedChunks( info ); indexModifiedBundles++; } else { if ( gcDeep && 0 == totalChunks ) { if ( overallBundleSet.find ( bundleId ) == overallBundleSet.end() ) { overallBundleSet.insert( bundleId ); dPrintf( "Deleting %s bundle\n", i.c_str() ); filesToUnlink.push_back( Dir::addPath( bundlesPath, i ) ); indexModified = true; indexRemovedBundles++; } else { // trigger index update indexModified = true; } } else { if ( gcDeep && overallBundleSet.find ( bundleId ) == overallBundleSet.end() ) overallBundleSet.insert( bundleId ); chunkStorageWriter->addBundle( info, savedId ); dPrintf( "Keeping %s bundle\n", i.c_str() ); indexKeptBundles++; } } } } void commit() { for ( int i = filesToUnlink.size(); i--; ) { dPrintf( "Unlinking %s\n", filesToUnlink[i].c_str() ); unlink( filesToUnlink[i].c_str() ); } filesToUnlink.clear(); chunkStorageWriter->commit(); } }; } ZCollector::ZCollector( string const & storageDir, string const & password, size_t threads, size_t cacheSize ): ZBackupBase( storageDir, password ), chunkStorageReader( storageInfo, encryptionkey, chunkIndex, getBundlesPath(), cacheSize ) { this->threads = threads; } void ZCollector::gc( bool gcDeep ) { ChunkIndex chunkReindex( encryptionkey, tmpMgr, getIndexPath(), true ); ChunkStorage::Writer chunkStorageWriter( storageInfo, encryptionkey, tmpMgr, chunkReindex, getBundlesPath(), getIndexPath(), threads ); string fileName; Dir::Entry entry; string backupsPath = getBackupsPath(); BundleCollector collector; collector.bundlesPath = getBundlesPath(); collector.chunkStorageReader = &this->chunkStorageReader; collector.chunkStorageWriter = &chunkStorageWriter; collector.gcRepack = false; collector.gcDeep = gcDeep; verbosePrintf( "Performing garbage collection...\n" ); verbosePrintf( "Searching for backups...\n" ); vector< string > backups = BackupExchanger::findOrRebuild( getBackupsPath() ); for ( std::vector< string >::iterator it = backups.begin(); it != backups.end(); ++it ) { string backup( Dir::addPath( getBackupsPath(), *it ) ); verbosePrintf( "Checking backup %s...\n", backup.c_str() ); BackupInfo backupInfo; BackupFile::load( backup , encryptionkey, backupInfo ); string backupData; BackupRestorer::restoreIterations( chunkStorageReader, backupInfo, backupData, &collector.usedChunkSet ); BackupRestorer::restore( chunkStorageReader, backupData, NULL, &collector.usedChunkSet ); } verbosePrintf( "Checking bundles...\n" ); chunkIndex.loadIndex( collector ); collector.commit(); verbosePrintf( "Cleaning up...\n" ); string bundlesPath = getBundlesPath(); Dir::Listing bundleLst( bundlesPath ); while( bundleLst.getNext( entry ) ) { const string dirPath = Dir::addPath( bundlesPath, entry.getFileName()); if ( entry.isDir() && Dir::isDirEmpty( dirPath ) ) { Dir::remove(dirPath); } } verbosePrintf( "Garbage collection complete\n" ); } zbackup-1.4.4/zcollector.hh000066400000000000000000000011001257621500400156520ustar00rootroot00000000000000// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS // Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE #ifndef Z_COLLECTOR_HH_INCLUDED__ #define Z_COLLECTOR_HH_INCLUDED__ #include "zbackup_base.hh" #include "chunk_storage.hh" class ZCollector : public ZBackupBase { ChunkStorage::Reader chunkStorageReader; size_t threads; public: ZCollector( std::string const & storageDir, std::string const & password, size_t threads, size_t cacheSize ); void gc( bool ); }; #endif