pax_global_header00006660000000000000000000000064122661167710014523gustar00rootroot0000000000000052 comment=f193efe2bc370ee1d98ec2e8e8cb9af9edda4b5b yrmcds-1.0.3/000077500000000000000000000000001226611677100130255ustar00rootroot00000000000000yrmcds-1.0.3/.gitignore000066400000000000000000000001671226611677100150210ustar00rootroot00000000000000# C/C++ *.[oa] # Editors *~ .*.swp # Directories html lz4 # binaries yrmcdsd yrmcdsd.map *.exe # misc. COPYING.hpp yrmcds-1.0.3/COPYING000066400000000000000000000024121226611677100140570ustar00rootroot00000000000000Copyright (c) 2013, Cybozu. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. yrmcds-1.0.3/ChangeLog000066400000000000000000000033411226611677100146000ustar00rootroot00000000000000version 1.0.3 * fix a compilation problem on non-SSE Intel 32bit CPU. * remove test/async as it is not related to any part of yrmcds. version 1.0.2 * fix siphash implementation to avoid unaligned memory access. * improve Makefile for better platform portability. * fix test/reactor to use pipes instead of stdin. version 1.0.1 * fix an infinite loop when sending 0 byte object via binary protocol. * improve binary protocol compatibility with memcached. Specifically, Delete/Append/Prepend/Increment/Decrement with CAS are supported now. * improve flush logic of temporary files. * fix some potential bugs: #23, #24 version 1.0.0 * [security] MurmurHash was replaced with SipHash. version 0.9.7 * add a new statistics option "stats ops" to report the total count of command executions for each command. version 0.9.6 * improve scalability by reducing the reactor thread's load. * add "max_connections" config option. * fixed some memory corruption bugs. version 0.9.5 * change UID if "user" is defined in the configuration file. * change GID if "group" is defined in the configuration file. * fixed a memory synchronization bug. version 0.9.4 * add proper memory fences. version 0.9.3 * enhances STATS to include the number of connections. * replication starts quickly by eliminating wasteful wait. * moves recv(2) from the reactor to workers, reducing the reactor load. * fixed bugs: #10 version 0.9.2 * fixed bugs: #4, #6 version 0.9.1 * implements the server-side locking mechanism. * new command line option "-v" shows the version and copyrights. * fixed bugs: #2 version 0.9.0 * The first public release. * implements memcached ASCII and binary protocols. * implements master-slave replication. yrmcds-1.0.3/Makefile000066400000000000000000000046331226611677100144730ustar00rootroot00000000000000# Makefile for yrmcds # Prerequisites: gcc 4.8+ or clang 3.3+ PREFIX = /usr/local DEFAULT_CONFIG = $(PREFIX)/etc/yrmcds.conf CACHELINE_SIZE := $(shell getconf LEVEL1_DCACHE_LINESIZE) ifeq ($(CACHELINE_SIZE), 0) CACHELINE_SIZE = 32 endif CC = gcc CXX = g++ CPPFLAGS = -I. -DCACHELINE_SIZE=$(CACHELINE_SIZE) CPPFLAGS += -DDEFAULT_CONFIG=$(DEFAULT_CONFIG) -DUSE_TCMALLOC OPTFLAGS = -O2 #-flto DEBUGFLAGS = -gdwarf-3 #-fsanitize=address WARNFLAGS = -Wall -Wnon-virtual-dtor -Woverloaded-virtual CPUFLAGS = #-march=core2 -mtune=corei7 CXXFLAGS = -std=gnu++11 $(OPTFLAGS) $(DEBUGFLAGS) $(shell getconf LFS_CFLAGS) $(WARNFLAGS) $(CPUFLAGS) LDFLAGS = -L. $(shell getconf LFS_LDFLAGS) LIBTCMALLOC = -ltcmalloc_minimal LDLIBS = $(shell getconf LFS_LIBS) -lyrmcds $(LIBTCMALLOC) -lpthread CLDOC = LD_LIBRARY_PATH=/usr/local/clang/lib cldoc HEADERS = $(wildcard src/*.hpp cybozu/*.hpp) SOURCES = $(wildcard src/*.cpp cybozu/*.cpp) OBJECTS = $(patsubst %.cpp,%.o,$(SOURCES)) EXE = yrmcdsd TESTS = $(patsubst %.cpp,%,$(wildcard test/*.cpp)) LIB = libyrmcds.a LIB_OBJECTS = $(filter-out src/main.o,$(OBJECTS)) PACKAGES = build-essential libgoogle-perftools-dev python-pip all: $(EXE) lib: $(LIB) tests: $(TESTS) strip: $(EXE) nm -n $(EXE) | grep -v '\( [aNUw] \)\|\(__crc_\)\|\( \$$[adt]\)' > $(EXE).map strip $(EXE) install: $(EXE) cp etc/upstart /etc/init/yrmcds.conf cp etc/logrotate /etc/logrotate.d/yrmcds cp etc/yrmcds.conf $(DEFAULT_CONFIG) cp $(EXE) $(PREFIX)/sbin/yrmcds install -o nobody -g nogroup -m 644 /dev/null /var/log/yrmcds.log COPYING.hpp: COPYING echo -n 'static char COPYING[] = R"(' > $@ cat $< >>$@ echo ')";' >>$@ src/main.o: COPYING.hpp $(OBJECTS): $(HEADERS) $(LIB): $(LIB_OBJECTS) $(AR) rcus $@ $^ $(EXE): src/main.o $(LIB) $(CXX) $(CXXFLAGS) $(LDFLAGS) -o $@ $< $(LDLIBS) test/%.exe: test/%.o $(LIB) $(CXX) $(CXXFLAGS) $(LDFLAGS) -o $@ $< $(LDLIBS) $(TESTS): $(LIB) @$(MAKE) -s $@.exe ./$@.exe @echo html: @clang++ -v 2>/dev/null || (echo "No clang++ in PATH." && false) rm -rf html $(CLDOC) generate $(CPPFLAGS) $(CXXFLAGS) -w -D__STRICT_ANSI__ -- --output html --merge docs $(HEADERS) serve: html @cd html; python -m SimpleHTTPServer 8888 || true clean: rm -f src/*.o cybozu/*.o test/*.o test/*.exe COPYING.hpp $(EXE) $(EXE).map $(LIB) setup: sudo apt-get install -y --install-recommends $(PACKAGES) sudo pip install cldoc --upgrade .PHONY: all strip lib tests install html serve clean setup $(TESTS) yrmcds-1.0.3/README.md000066400000000000000000000064721226611677100143150ustar00rootroot00000000000000yrmcds ====== yrmcds is a memory object caching system with master/slave replication. Since its protocol is perfectly compatible with that of [memcached][], yrmcds can be used as a drop-in replacement for [memcached][]. In fact, yrmcds is [faster][bench] than memcached! The biggest benefit of yrmcds is its amazingly low cost replication system. The master server is elected dynamically from a group of servers, which eliminates static master/slave configurations. By adopting virtual-IP based replication, no modifications to applications are required. Unlike [repcached][], yrmcds is not a patch for memcached. No code is shared between yrmcds and memcached. yrmcds is developed for [cybozu.com][cybozu], a B2B cloud service widely adopted by companies in Japan. License ------- yrmcds is licensed under [the BSD 2-clause license][bsd2]. The source code contains a [SipHash][] implementation borrowed from [csiphash][] which is licensed under [the MIT license][mit]. Features -------- * Memcached text and binary protocols. * [Server-side locking](docs/locking.md). * Large objects can be stored in temporary files, not in memory. * Global LRU eviction / no slab distribution problem. * Unlike memcached, yrmcds is not involved with slabs problems. ([1][slab1], [2][slab2]) * Virtual-IP based master-slave replication. * Automatic fail-over. * Automatic recovery of redundancy. A companion client library [libyrmcds][] and a [PHP extension][php-yrmcds] are also available. See also [usage guide](docs/usage.md), [future plans](docs/future.md), [differences from memcached](docs/diffs.md), [design notes](docs/design.md) and some [benchmark results](docs/bench.md). Prerequisites ------------- * Fairly recent Linux kernel. * C++11 compiler (gcc 4.8.1+ or clang 3.3+). * [TCMalloc][tcmalloc] from Google. * GNU make. The following may help Ubuntu users to compile gcc 4.8.1: ```shell sudo apt-get install libgmp-dev libmpfr-dev libmpc-dev build-essential tar xjf gcc-4.8.1.tar.bz2 mkdir gcc-4.8.1/build cd gcc-4.8.1/build ../configure --prefix=/usr/local/gcc --disable-shared --disable-multilib \ --enable-threads --enable-__cxa_atexit --enable-languages=c,c++ \ --disable-nls make -j 4 BOOT_CFLAGS=-O2 bootstrap sudo make install make clean export PATH=/usr/local/gcc/bin:$PATH ``` Build ----- 1. Prepare TCMalloc. On Ubuntu, run `apt-get install libgoogle-perftools-dev`. 2. Run `make`. You can build yrmcds without TCMalloc by editing Makefile. Install ------- On Ubuntu, `sudo make install` installs yrmcds under `/usr/local`. An upstart script and a logrotate configuration file are installed too. [memcached]: http://memcached.org/ [bench]: https://github.com/cybozu/yrmcds/blob/master/docs/bench.md#results [repcached]: http://repcached.lab.klab.org/ [bsd2]: http://opensource.org/licenses/BSD-2-Clause [SipHash]: https://131002.net/siphash/ [csiphash]: https://github.com/majek/csiphash [mit]: http://opensource.org/licenses/MIT [libyrmcds]: http://cybozu.github.io/libyrmcds/ [php-yrmcds]: http://cybozu.github.io/php-yrmcds/ [slab1]: http://nosql.mypopescu.com/post/13506116892/memcached-internals-memory-allocation-eviction [slab2]: https://groups.google.com/forum/#!topic/memcached/DuJNy5gbQ0o [cybozu]: https://www.cybozu.com/us/ [tcmalloc]: http://goog-perftools.sourceforge.net/doc/tcmalloc.html yrmcds-1.0.3/cybozu/000077500000000000000000000000001226611677100143405ustar00rootroot00000000000000yrmcds-1.0.3/cybozu/config_parser.cpp000066400000000000000000000022751226611677100176730ustar00rootroot00000000000000// (C) 2013 Cybozu. #include "config_parser.hpp" #include namespace { const char white_spaces[] = " \f\n\r\t\v"; inline void rtrim(std::string& s) { s.erase( s.find_last_not_of(white_spaces) + 1 ); } inline void ltrim(std::string& s) { s.erase( 0, s.find_first_not_of(white_spaces) ); } } // anonymous namespace namespace cybozu { void config_parser::load(const std::string& path) { m_config.clear(); std::ifstream is(path); if( !is ) throw std::runtime_error("failed to open " + path); unsigned int lineno = 0; for( std::string l; std::getline(is, l); ) { lineno++; ltrim(l); rtrim(l); if( l.empty() || l[0] == '#' ) continue; std::size_t n = l.find('='); if( n == std::string::npos ) throw parse_error(path, lineno); std::string key = l.substr(0, n); rtrim(key); std::string value = l.substr(n+1); ltrim(value); // unquote the value std::size_t len = value.size(); if( len >= 2 && value[0] == '"' && value[len-1] == '"' ) { value = value.substr(1, len-2); } set(key, value); } } } // namespace cybozu yrmcds-1.0.3/cybozu/config_parser.hpp000066400000000000000000000075161226611677100177030ustar00rootroot00000000000000// Read and parse configuration files. // (C) 2013 Cybozu. #ifndef CYBOZU_CONFIG_PARSER_HPP #define CYBOZU_CONFIG_PARSER_HPP #include #include #include namespace cybozu { // Parse configuration files. class config_parser { std::unordered_map m_config; public: // Create an empty config_parser object. config_parser() {} // Create a and load the file at `path`. // @path The path to the configuration file. // // Create a and load the file at `path`. // Raise exception if the file contains an invalid line. explicit config_parser(const std::string& path) { load(path); } // exception for invalid file format. struct parse_error: public std::runtime_error { parse_error(const std::string& path, unsigned int lineno): std::runtime_error("Parse error in " + path + " at line " + std::to_string(lineno)) {} }; // exception for key not found error. struct not_found: public std::runtime_error { not_found(const std::string& key): std::runtime_error("Key not found: " + key) {} }; // exception for illegal value error. struct illegal_value: public std::runtime_error { illegal_value(const std::string& key): std::runtime_error("Illegal value for " + key) {} }; // Load a configuration file. // @path The path to the configuration file. // // Load a configuration file. // Raise exception if the file contains an invalid line. // All previously loaded configurations will be cleared. void load(const std::string& path); // Set a configuration value. // @key A configuration key. // @value The associated value. void set(const std::string& key, const std::string& value) { auto it = m_config.find(key); if( it == m_config.end() ) { m_config.emplace(key, value); } else { it->second = value; } } // Get a value associated with `key`. // @key A configuration key. // // Get a value associated with `key`. // Raise exception if the key is not found. // // @return A reference to the associated with `key`. const std::string& get(const std::string& key) const { auto it = m_config.find(key); if( it == m_config.end() ) throw not_found(key); return it->second; } // Return `true` if `key` exists. // @key A configuration key. bool exists(const std::string& key) const { return m_config.find(key) != m_config.end(); } // Get an integer converted from the value associated with `key`. // @key A configuration key. // // Get an integer converted from the value associated with `key`. // Raise or . // // @return An integer converted from the associated value. int get_as_int(const std::string& key) const { try { return std::stoi(get(key)); } catch(const std::invalid_argument& e) { throw illegal_value(key); } catch(const std::out_of_range& e) { throw illegal_value(key); } } // Get a boolean converted from the value associated with `key`. // @key A configuration key. // // Get a boolean converted from the value associated with `key`. // Raise or . // //@return `true` or `false` converted from the associated value. bool get_as_bool(const std::string& key) const { const std::string& s = get(key); if( s == "true" ) return true; if( s == "false" ) return false; throw illegal_value(key); } }; } // namespace cybozu #endif // CYBOZU_CONFIG_PARSER_HPP yrmcds-1.0.3/cybozu/dynbuf.hpp000066400000000000000000000127211226611677100163430ustar00rootroot00000000000000// A fast, dynamic-sized char buffer. // (C) 2013 Cybozu. #ifndef CYBOZU_DYNBUF_HPP #define CYBOZU_DYNBUF_HPP #ifdef USE_TCMALLOC # include #else # include #endif #include #include #include #include namespace cybozu { // A fast, dynamic-sized char buffer. // // There are two motivations for this class rather than using // or . One is to provide nicer interfaces to receive data // for system calls. Another is to be efficient as much as possible. class dynbuf final { public: // Constructor. // @default_capacity The default capacity of the internal buffer. // // The constructor pre-allocates an internal buffer if `default_capacity` // is not 0. Everytime is invoked, the internal buffer will be // shrunk to `default_capacity`. explicit dynbuf(std::size_t default_capacity): m_p(default_capacity ? _malloc(default_capacity) : nullptr), m_default_capacity(default_capacity), m_capacity(default_capacity) {} ~dynbuf() { if( m_p != nullptr ) _free(m_p); } dynbuf(const dynbuf&) = delete; dynbuf(dynbuf&& rhs) noexcept: m_p(nullptr), m_default_capacity(rhs.m_default_capacity), m_capacity(rhs.m_capacity), m_used(rhs.m_used) { std::swap(m_p, rhs.m_p); } dynbuf& operator=(const dynbuf&) = delete; dynbuf& operator=(dynbuf&&) = delete; void swap(dynbuf& other) noexcept { std::swap(m_p, other.m_p); std::swap(m_capacity, other.m_capacity); std::swap(m_used, other.m_used); } // Clear the contents and reset the internal buffer. void reset() { m_used = 0; if( m_default_capacity == m_capacity ) return; char* new_p = nullptr; if( m_default_capacity != 0 ) new_p = _malloc(m_default_capacity); _free(m_p); m_p = new_p; m_capacity = m_default_capacity; } // Append contents. // @p The pointer to the contents. // @len The length of the contents. void append(const char* p, std::size_t len) { if( freebytes() < len ) enlarge(len); std::memcpy(m_p+m_used, p, len); m_used += len; } // Erase contents. // @len The length to be erased. // // Erase contents from the head of the internal buffer. // Remaining data will be moved to the head. void erase(std::size_t len) { if( m_used < len ) throw std::invalid_argument(""); if( len == 0 ) return; std::size_t remain = m_used - len; if( remain == 0 ) { reset(); return; } if( remain <= m_default_capacity && m_capacity != m_default_capacity ) { char* new_p = _malloc(m_default_capacity); std::memcpy(new_p, m_p+len, remain); _free(m_p); m_p = new_p; m_capacity = m_default_capacity; m_used = remain; return; } std::memmove(m_p, m_p+len, remain); m_used = remain; } // Prepare enough free space in the internal buffer. // @len Required free space size. // // This prepares at least `len` byte free space in the internal buffer. // The caller should call after it wrote some data to the // free space. // // @return The pointer to the free space. char* prepare(std::size_t len) { if( freebytes() < len ) enlarge(len); return m_p + m_used; } // Consume the free space. // @len Size of memory to be consumed. void consume(std::size_t len) noexcept { m_used += len; } // Return the pointer to the head of the internal buffer. const char* data() const noexcept { return m_p; } // Same as STL's `empty()`. bool empty() const noexcept { return m_used == 0; } // Same as STL's `size()`. std::size_t size() const noexcept { return m_used; } private: char* m_p; const std::size_t m_default_capacity; std::size_t m_capacity; std::size_t m_used = 0; std::size_t freebytes() const noexcept { return m_capacity - m_used; } void enlarge(std::size_t additional) { const std::size_t new_capacity = m_capacity + additional; m_p = _realloc(m_p, new_capacity); m_capacity = new_capacity; } static char* _malloc(std::size_t len) { #ifdef USE_TCMALLOC char* p = (char*)tc_malloc(len); #else char* p = (char*)std::malloc(len); #endif if( p == nullptr ) throw std::runtime_error("failed to allocate memory."); return p; } static void _free(void* p) noexcept { #ifdef USE_TCMALLOC tc_free(p); #else std::free(p); #endif } static char* _realloc(void* p, std::size_t new_len) { #ifdef USE_TCMALLOC char* np = (char*)tc_realloc(p, new_len); #else char* np = (char*)std::realloc(p, new_len); #endif if( np == nullptr ) throw std::runtime_error("failed to re-allocate memory."); return np; } friend bool operator==(const dynbuf& lhs, const dynbuf& rhs); }; inline bool operator==(const dynbuf& lhs, const dynbuf& rhs) { if( lhs.m_p == rhs.m_p ) return true; if( lhs.m_used != rhs.m_used ) return false; return std::memcmp(lhs.m_p, rhs.m_p, lhs.m_used) == 0; } } // namespace cybozu #endif // CYBOZU_DYNBUF_HPP yrmcds-1.0.3/cybozu/filesystem.cpp000066400000000000000000000011221226611677100172240ustar00rootroot00000000000000// (C) 2013 Cybozu. #include "filesystem.hpp" #include "logger.hpp" #include #include namespace cybozu { bool get_stat(const std::string& path, struct stat& st) { if( ::stat(path.c_str(), &st) == 0 ) return true; if( errno == EFAULT || errno == ENOMEM ) throw_unix_error(errno, "stat"); auto ec = std::system_category().default_error_condition(3); logger::debug() << "stat failed. path=" << path << ", error=" << ec.message() << " (" << errno << ")"; return false; } } // namespace cybozu yrmcds-1.0.3/cybozu/filesystem.hpp000066400000000000000000000014011226611677100172310ustar00rootroot00000000000000// filesystem.hpp // (C) 2013 Cybozu. #ifndef CYBOZU_FILESYSTEM_HPP #define CYBOZU_FILESYSTEM_HPP #include "util.hpp" #include #include #include #include namespace cybozu { // stat(2) wrapper. // @return `true` if a file exists at `path`, `false` otherwise. bool get_stat(const std::string& path, struct stat& st); inline bool is_dir(const std::string& path) { struct stat st; if( ! get_stat(path, st) ) return false; return S_ISDIR(st.st_mode); } inline bool is_readable(const std::string& path) { return ::access(path.c_str(), R_OK) == 0; } inline bool is_writable(const std::string& path) { return ::access(path.c_str(), W_OK) == 0; } } // namespace cybozu #endif // CYBOZU_FILESYSTEM_HPP yrmcds-1.0.3/cybozu/hash_map.cpp000066400000000000000000000011261226611677100166240ustar00rootroot00000000000000// (C) 2013 Cybozu. #include "hash_map.hpp" #include namespace cybozu { unsigned int nearest_prime(unsigned int n) noexcept { static_assert( sizeof(n) >= 4, "Too small unsigned int." ); if( n == 2 ) return 2; for( unsigned int i = n|1; ; i += 2 ) { bool prime = true; unsigned int r = (unsigned int)std::rint( std::sqrt((double)i) ); for( unsigned int j = 3; j <= r; j += 2 ) { if( (i % j) == 0 ) { prime = false; break; } } if( prime ) return i; } } } // namespace cybozu yrmcds-1.0.3/cybozu/hash_map.hpp000066400000000000000000000271401226611677100166350ustar00rootroot00000000000000// The object hash map. // (C) 2013 Cybozu. #ifndef CYBOZU_HASH_MAP_HPP #define CYBOZU_HASH_MAP_HPP #include "siphash.hpp" #include #include #include #include #include #include #include #include namespace cybozu { // Key class for . class hash_key final { public: // Construct from a statically allocated memory. // @p Pointer to a statically allocated memory. // @len Length of the key. // // Construct from a statically allocated memory. // // As long as the constructed object lives, the memory pointed by `p` // must not be freed. hash_key(const char* p, std::size_t len) noexcept : m_p(p), m_len(len) { m_hash = siphash24(m_p, m_len); } // Construct by moving a . // // Construct by moving a . Sample usage: // ``` // hash_key( std::vector(p, p+len) ) // ``` hash_key(std::vector v): m_v(std::move(v)), m_p(m_v.data()), m_len(m_v.size()) { m_hash = siphash24(m_p, m_len); } // Copy constructor. hash_key(const hash_key& rhs): m_v(rhs.m_p, rhs.m_p+rhs.m_len), m_p(m_v.data()), m_len(m_v.size()), m_hash(rhs.m_hash) {} hash_key& operator=(const hash_key& rhs) = delete; // Move contructor and assign operator. hash_key(hash_key&& rhs) noexcept = default; hash_key& operator=(hash_key&& rhs) = default; std::uint64_t hash() const noexcept { return m_hash; } const char* data() const noexcept { return m_p; } std::size_t length() const noexcept { return m_len; } std::string str() const { return std::string(m_p, m_len); } private: std::vector m_v; const char* m_p; std::size_t m_len; std::uint64_t m_hash; friend bool operator==(const hash_key&, const hash_key&) noexcept; }; inline bool operator==(const hash_key& lhs, const hash_key& rhs) noexcept { if( lhs.m_len != rhs.m_len ) return false; if( lhs.m_p == rhs.m_p ) return true; return std::memcmp(lhs.m_p, rhs.m_p, lhs.m_len) == 0; } // Return the nearest prime number. unsigned int nearest_prime(unsigned int n) noexcept; // Highly concurrent object hash map. // // Keys for this hash map are whereas objects are of type `T`. // `T` must be either move-constructible or copyable. // // Each bucket in the hash map has its unique mutex to guard itself. // This design reduces contensions between threads drastically in exchange // for some functions such as dynamic resizing of the number of buckets. template class hash_map { static_assert( std::is_move_constructible::value || std::is_copy_constructible::value, "T must be move- or copy- constructible." ); public: typedef std::function handler; typedef std::function creator; // Hash map bucket. // // Each hash value corresponds to a bucket. // Member functions whose names end with `_nolock` are not thread-safe. class bucket { struct item { const hash_key key; T object; item* next; // perfect forwarding template item(const hash_key& k, X&& o, item* n): key(k), object(std::forward(o)), next(n) {} }; public: bucket(): m_objects(nullptr) {} ~bucket() { clear_nolock(); } // Handle or insert an object. // @key The object's key. // @h A function to handle an existing object. // @c A function to create a new object. // // This function can be used to handle an existing object, or // to insert a new object when such an object does not exist. // // if `h` is `nullptr` and there is an existing object for `key`, // `false` is returned. If `c` is `nullptr` and there is no // object for `key`, `false` is returned. // // If `h` is not `nullptr` and there is an existing object for // `key`, then `h` is called and the return value of `h` is // returned. This means `h` can return `false` if it failed // to handle the object. // // If `c` is not `nullptr` and there is no object for `key`, // an object is created by calling `c` and stored, then `true` // is returned. // // @return `true` if succeeded, `false` otherwise. bool apply_nolock(const hash_key& key, const handler& h, const creator& c) { for( item* p = m_objects; p != nullptr; p = p->next ) { if( p->key == key ) { if( ! h ) return false; return h(p->key, p->object); } } if( ! c ) return false; m_objects = new item(key, c(key), m_objects); return true; } // Thread-safe . bool apply(const hash_key& key, const handler& h, const creator& c) { lock_guard g(m_lock); return apply_nolock(key, h, c); } // Remove an object for `key`. // @key The object's key. // @callback A function called when an object is removed. // // This removes an object associated with `key`. If there is no // object associated with `key`, return `false`. If `callback` // is not `nullptr`, it is called when an object is removed. // // @return `true` if successfully removed, `false` otherwise. bool remove_nolock(const hash_key& key, const std::function& callback) { for( item** p = &m_objects; *p != nullptr; p = &((*p)->next) ) { if( (*p)->key == key ) { item* to_delete = *p; *p = to_delete->next; delete to_delete; if( callback ) callback(key); return true; } } return false; } // Thread-safe . bool remove(const hash_key& key, const std::function& callback) { lock_guard g(m_lock); return remove_nolock(key, callback); } // Remove an object for `key` if `pred` returns `true`. // @key The object's key. // @pred A predicate function. // // This function removes an object assiciated with `key` if a // predicate function returns `true`. This function is thread-safe. // // @return `true` if object existed, `false` otherwise. bool remove_if(const hash_key& key, const std::function& pred) { lock_guard g(m_lock); for( item** p = &m_objects; *p != nullptr; p = &((*p)->next) ) { item* to_delete = *p; if( to_delete->key == key ) { if( pred(key, to_delete->object) ) { *p = to_delete->next; delete to_delete; } return true; } } return false; } // Collect garbage objects. // @pred Predicate function. // // This function collects garbage objects. // Objects for which `pred` returns `true` will be removed. void gc(const std::function& pred) { lock_guard g(m_lock); for( item** p = &m_objects; *p != nullptr; ) { item* to_delete = *p; if( pred(to_delete->key, to_delete->object) ) { *p = to_delete->next; delete to_delete; } else { p = &(to_delete->next); } } } // Clear objects in this bucket. void clear_nolock() { while( m_objects ) { item* next = m_objects->next; delete m_objects; m_objects = next; } } private: using lock_guard = std::lock_guard; mutable std::mutex m_lock; item* m_objects; }; hash_map(unsigned int buckets): m_size(nearest_prime(buckets)), m_buckets(m_size) {} // Handle or insert an object. // @key The object's key. // @h A function to handle an existing object. // @c A function to create a new object. // // This function can be used to handle an existing object, or // to insert a new object when such an object does not exist. // // if `h` is `nullptr` and there is an existing object for `key`, // `false` is returned. If `c` is `nullptr` and there is no // object for `key`, `false` is returned. // // If `h` is not `nullptr` and there is an existing object for // `key`, then `h` is called and the return value of `h` is // returned. This means `h` can return `false` if it failed // to handle the object. // // If `c` is not `nullptr` and there is no object for `key`, // an object is created by calling `c` and stored, then `true` // is returned. // // @return `true` if succeeded, `false` otherwise. bool apply_nolock(const hash_key& key, const handler& h, const creator& c) { return get_bucket(key).apply_nolock(key, h, c); } // Thread-safe . bool apply(const hash_key& key, const handler& h, const creator& c) { return get_bucket(key).apply(key, h, c); } // Remove an object for `key`. // @key The object's key. // @callback A function called when an object is removed. // // This removes an object associated with `key`. If there is no // object associated with `key`, return `false`. If `callback` // is not `nullptr`, it is called when an object is removed. // // @return `true` if successfully removed, `false` otherwise. bool remove_nolock(const hash_key& key, const std::function& callback) { return get_bucket(key).remove_nolock(key, callback); } // Thread-safe . bool remove(const hash_key& key, const std::function& callback) { return get_bucket(key).remove(key, callback); } // Remove an object for `key` if `pred` returns `true`. // @key The object's key. // @pred A predicate function. // // This function removes an object assiciated with `key` if a // predicate function returns `true`. This function is thread-safe. // // @return `true` if object existed, `false` otherwise. bool remove_if(const hash_key& key, const std::function& pred) { return get_bucket(key).remove_if(key, pred); } /* Bucket interfaces */ // Return the number of buckets in this hash map. std::size_t bucket_count() const noexcept { return m_size; } bucket& get_bucket(const hash_key& key) noexcept { return m_buckets[key.hash() % m_size]; } // iterator. using iterator = typename std::vector::iterator; iterator begin() { return m_buckets.begin(); } iterator end() { return m_buckets.end(); } private: const std::size_t m_size; std::vector m_buckets; }; } // namespace cybozu #endif // CYBOZU_HASH_MAP_HPP yrmcds-1.0.3/cybozu/ip_address.cpp000066400000000000000000000051501226611677100171620ustar00rootroot00000000000000// (C) 2013 Cybozu. #include "ip_address.hpp" #include "util.hpp" #include #include #include #include #include namespace { class ifaddrs_wrapper { struct ifaddrs* m_addr; public: ifaddrs_wrapper() { if( getifaddrs(&m_addr) != 0 ) cybozu::throw_unix_error(errno, "getifaddrs"); } ~ifaddrs_wrapper() { freeifaddrs(m_addr); } bool find(const cybozu::ip_address& addr) const { for(auto a = m_addr; a != nullptr; a = a->ifa_next) { if( a->ifa_addr == NULL ) continue; switch(a->ifa_addr->sa_family) { case AF_INET: case AF_INET6: if( addr == cybozu::ip_address(a->ifa_addr) ) return true; } } return false; } }; } // anonymous namespace namespace cybozu { void ip_address::parse(const std::string& s) { if( inet_pton(AF_INET, s.c_str(), &addr) == 1 ) { af = addr_family::ipv4; return; } if( inet_pton(AF_INET6, s.c_str(), &addr) == 1 ) { af = addr_family::ipv6; return; } throw bad_address("Invalid address: " + s); } ip_address::ip_address(const std::string& s) { parse(s); } ip_address::ip_address(const struct sockaddr* ifa_addr) { if( ifa_addr->sa_family == AF_INET ) { af = addr_family::ipv4; const struct sockaddr_in* p = (const struct sockaddr_in*)ifa_addr; std::memcpy(&addr, &(p->sin_addr), sizeof(struct in_addr)); return; } if( ifa_addr->sa_family == AF_INET6 ) { af = addr_family::ipv6; const struct sockaddr_in6* p = (const struct sockaddr_in6*)ifa_addr; std::memcpy(&addr, &(p->sin6_addr), sizeof(struct in6_addr)); return; } throw bad_address("Invalid address family"); } bool ip_address::operator==(const ip_address& rhs) const { if( af != rhs.af ) return false; if( af == addr_family::none ) return true; if( is_v4() ) { return std::memcmp(v4addr(), rhs.v4addr(), sizeof(struct in_addr)) == 0; } return std::memcmp(v6addr(), rhs.v6addr(), sizeof(struct in6_addr)) == 0; } std::string ip_address::str() const { if( is_v4() ) { char dst[INET_ADDRSTRLEN]; return std::string(inet_ntop(AF_INET, v4addr(), dst, INET_ADDRSTRLEN)); } if( is_v6() ) { char dst[INET6_ADDRSTRLEN]; return std::string(inet_ntop(AF_INET6, v6addr(), dst, INET6_ADDRSTRLEN)); } return ""; } bool has_ip_address(const ip_address& addr) { ifaddrs_wrapper ifw; return ifw.find(addr); } } // namespace cybozu yrmcds-1.0.3/cybozu/ip_address.hpp000066400000000000000000000026111226611677100171660ustar00rootroot00000000000000// Abstract IP address. // (C) 2013 Cybozu. #ifndef CYBOZU_IP_ADDRESS_HPP #define CYBOZU_IP_ADDRESS_HPP #include #include #include namespace cybozu { // Abstract IP address. // // Abstract IP address. // Both IPv4 and IPv6 addresses can be stored transparently. class ip_address { enum class addr_family {none, ipv4, ipv6} af = addr_family::none; union { struct in_addr ipv4_addr; struct in6_addr ipv6_addr; } addr; public: ip_address() noexcept {} explicit ip_address(const std::string& s); explicit ip_address(const struct sockaddr* ifa_addr); struct bad_address: public std::runtime_error { bad_address(const std::string& s): std::runtime_error(s) {} }; // This may throw is `s` is not a valid IP address. void parse(const std::string& s); bool is_v4() const { return af == addr_family::ipv4; } bool is_v6() const { return af == addr_family::ipv6; } const struct in_addr* v4addr() const { return &(addr.ipv4_addr); } const struct in6_addr* v6addr() const { return &(addr.ipv6_addr); } bool operator==(const ip_address& rhs) const; std::string str() const; }; // `true` if this machine has `addr`. `false` otherwise. bool has_ip_address(const ip_address& addr); } // namespace cybozu #endif // CYBOZU_IP_ADDRESS_HPP yrmcds-1.0.3/cybozu/logger.cpp000066400000000000000000000027671226611677100163370ustar00rootroot00000000000000// (C) 2013 Cybozu. #include "logger.hpp" #include "util.hpp" #include #include #include #include #include #include #include namespace cybozu { void logstream::add_prefix(const char* level) { char buf[64]; std::time_t t = std::time(nullptr); std::tm ct; // beware that std::gmtime is not thread-safe. std::strftime(buf, sizeof(buf), "%Y-%m-%d %T ", ::gmtime_r(&t, &ct)); *m_os << buf << level << " "; } void logstream::output() { if( m_os.get() == nullptr ) return; *m_os << "\n"; logger::instance().log(m_os->str()); m_os.reset(nullptr); } std::atomic logger::_threshold{ severity::info }; void logger::open_nolock(const std::string& path) { if( m_fd != -1 && m_fd != STDERR_FILENO ) throw std::logic_error(" double open."); m_fd = ::open(path.c_str(), O_WRONLY|O_APPEND|O_CREAT, 0644); if( m_fd == -1 ) { m_fd = STDERR_FILENO; throw_unix_error(errno, ("open(" + path + ")").c_str()); } } void logger::log(const std::string& msg) { lock_guard g(m_lock); std::size_t to_write = msg.size(); const char* ptr = msg.data(); while( to_write != 0 ) { ssize_t n = ::write(m_fd, ptr, to_write); if( n == -1 ) { if( errno == EAGAIN || errno == EINTR ) continue; throw_unix_error(errno, " write"); } ptr += n; to_write -= n; } } } // namespace cybozu yrmcds-1.0.3/cybozu/logger.hpp000066400000000000000000000064271226611677100163410ustar00rootroot00000000000000// Thread-safe logger. // (C) 2013 Cybozu. #ifndef CYBOZU_LOGGER_HPP #define CYBOZU_LOGGER_HPP #include #include #include #include #include #include namespace cybozu { // Severity used for . enum class severity {error, warning, info, debug}; // Output stream to the logger. // // Just like other output streams, you can output a variety of objects // by `<<` operator. The log is written in a line from within the // destructor. class logstream { const bool m_valid; std::unique_ptr m_os; void add_prefix(const char* level); public: logstream(bool valid_, const char* level): m_valid(valid_), m_os(valid_ ? new std::ostringstream() : nullptr) { if( valid_ ) add_prefix(level); } logstream(logstream&& rhs) noexcept: m_valid(rhs.m_valid), m_os(std::move(rhs.m_os)) {} logstream(const logstream&) = delete; logstream& operator=(const logstream&) = delete; ~logstream() { if( m_valid ) output(); } // `true` if this stream is valid for logging. bool valid() const { return m_valid; } // Output the log line. The stream gets invalidated. void output(); template logstream& operator<< (const T& arg) { if( m_valid ) *m_os << arg; return *this; } }; // A thread-safe logger. // // A simple thread-safe logger. By default, this logger outputs // logs to the standard error. class logger { int m_fd; std::string m_path; mutable std::mutex m_lock; typedef std::lock_guard lock_guard; static std::atomic _threshold; logger(): m_fd(STDERR_FILENO), m_path("") {} void open_nolock(const std::string& path); void close_nolock() { if( m_fd == -1 || m_fd == STDERR_FILENO ) return; fsync(m_fd); ::close(m_fd); m_fd = -1; } public: // Return the singleton. // // @return object. static logger& instance() { static logger l; return l; } logger(const logger&) = delete; logger& operator=(const logger&) = delete; // fsync and close the log file. ~logger() { close_nolock(); } static severity threshold() { return _threshold.load(std::memory_order_relaxed); } static void set_threshold(severity new_threshold) { _threshold.store(new_threshold, std::memory_order_relaxed); } void open(const std::string& path) { lock_guard g(m_lock); m_path = path; open_nolock(path); } void reopen() { lock_guard g(m_lock); if( m_fd == STDERR_FILENO ) return; close_nolock(); open_nolock(m_path); } void close() { lock_guard g(m_lock); close_nolock(); } void log(const std::string& msg); static logstream error() { return logstream(threshold() >= severity::error, "ERROR"); } static logstream warning() { return logstream(threshold() >= severity::warning, "WARN"); } static logstream info() { return logstream(threshold() >= severity::info, "INFO"); } static logstream debug() { return logstream(threshold() >= severity::debug, "DEBUG"); } }; } // namespace cybozu #endif // CYBOZU_LOGGER_HPP yrmcds-1.0.3/cybozu/reactor.cpp000066400000000000000000000100011226611677100164730ustar00rootroot00000000000000// (C) 2013 Cybozu. #include "reactor.hpp" #include "util.hpp" #include #include #include namespace { const int EPOLL_SIZE = 128; const int POLLING_TIMEOUT = 100; // milli seconds } namespace cybozu { reactor::reactor(): m_fd( epoll_create1(EPOLL_CLOEXEC) ), m_running(true) { if( m_fd == -1 ) throw_unix_error(errno, "epoll_create1"); m_resources.max_load_factor(1.0); m_resources.reserve(100000); m_readables.reserve(256); m_readables_copy.reserve(256); m_drop_req.reserve(256); m_drop_req_copy.reserve(256); } reactor::~reactor() { ::close(m_fd); } void reactor::add_resource(std::unique_ptr res, int events) { if( res->m_reactor != nullptr ) throw std::logic_error(" already added!"); res->m_reactor = this; const int fd = res->fileno(); struct epoll_event ev; ev.events = events | EPOLLET; ev.data.fd = fd; if( epoll_ctl(m_fd, EPOLL_CTL_ADD, fd, &ev) == -1 ) throw_unix_error(errno, "epoll_ctl"); m_resources.emplace(fd, std::move(res)); } void reactor::modify_events(const resource& res, int events) { const int fd = res.fileno(); struct epoll_event ev; ev.events = events | EPOLLET; ev.data.fd = fd; if( epoll_ctl(m_fd, EPOLL_CTL_MOD, fd, &ev) == -1 ) throw_unix_error(errno, "epoll_ctl"); } void reactor::run(std::function callback, int interval) { namespace crn = std::chrono; const int milli_interval = interval * 1000; auto last = crn::steady_clock::now(); m_running = true; while( m_running ) { poll(); auto now = crn::steady_clock::now(); auto diff = crn::duration_cast(now - last).count(); if( diff > milli_interval ) { callback(*this); last = now; } } } void reactor::remove_resource(int fd) { resource_map::iterator it = m_resources.find(fd); if( it == m_resources.end() ) { dump_stack(); throw std::logic_error("bug in remove_resource"); } m_garbage.emplace_back( std::move(it->second) ); m_resources.erase(it); if( epoll_ctl(m_fd, EPOLL_CTL_DEL, fd, NULL) == -1 ) throw_unix_error(errno, "epoll_ctl(EPOLL_CTL_DEL)"); m_readables.erase(std::remove(m_readables.begin(), m_readables.end(), fd), m_readables.end()); } void reactor::poll() { // process readable resources std::sort(m_readables.begin(), m_readables.end()); std::unique_copy(m_readables.begin(), m_readables.end(), std::back_inserter(m_readables_copy)); m_readables.clear(); for( int fd: m_readables_copy ) { if( ! m_resources[fd]->on_readable() ) remove_resource(fd); } m_readables_copy.clear(); // process drop requests { lock_guard g(m_lock); m_drop_req_copy.swap(m_drop_req); } for( int fd: m_drop_req_copy ) remove_resource(fd); m_drop_req_copy.clear(); struct epoll_event events[EPOLL_SIZE]; int timeout = m_readables.empty() ? POLLING_TIMEOUT : 0; int n = epoll_wait(m_fd, events, EPOLL_SIZE, timeout); if( n == -1 ) { if( errno == EINTR ) return; throw_unix_error(errno, "epoll_wait"); } for( int i = 0; i < n; ++i ) { const struct epoll_event& ev = events[i]; const int fd = ev.data.fd; resource& r = *(m_resources[fd]); if( ev.events & EPOLLERR ) { if( ! r.on_error() ) remove_resource(fd); continue; } if( ev.events & EPOLLHUP ) { if( ! r.on_hangup() ) { remove_resource(fd); continue; } } if( ev.events & EPOLLIN ) { if( ! r.on_readable() ) { remove_resource(fd); continue; } } if( ev.events & EPOLLOUT ) { if( ! r.on_writable() ) remove_resource(fd); } } } } // namespace cybozu yrmcds-1.0.3/cybozu/reactor.hpp000066400000000000000000000202061226611677100165100ustar00rootroot00000000000000// A reactor implementation using epoll(2). // (C) 2013 Cybozu. #ifndef CYBOZU_REACTOR_HPP #define CYBOZU_REACTOR_HPP #include "logger.hpp" #include "spinlock.hpp" #include "util.hpp" #include #include #include #include #include #include #include namespace cybozu { class reactor; // An abstraction of a file descriptor. // // An abstraction of a file descriptor for use with . // The file descriptor should be set non-blocking. // // All member functions except for are for the // reactor thread. Sub classes can add methods for the other threads. class resource { public: // Constructor. // @fd A UNIX file descriptor. explicit resource(int fd): m_fd(fd) {} resource(const resource&) = delete; resource& operator=(const resource&) = delete; // Close the file descriptor. virtual ~resource() { ::close(m_fd); } // Return the UNIX file descriptor for this resource. int fileno() const { return m_fd; } // `true` if this resource is still valid. bool valid() const { lock_guard g(m_lock); return m_valid; } // Invalidate this resource. // // This is for the reactor thread only. // You may call this from within or . bool invalidate() { lock_guard g(m_lock); if( ! m_valid ) return true; invalidate_( std::move(g) ); return false; } // Invalidate this resource then request the reactor to remove this. // // This method invalidates this resource then requests the reactor // thread to remove this resource from the reactor. // // **Do not use this in the reactor thread**. void invalidate_and_close(); // A template method called from within invalidate. // // A template method called from within invalidate. // Subclasses can override this to clean up something. virtual void on_invalidate() {} // Called when the reactor finds this resource is readable. // // This method is called when the reactor finds reading from this // resource will not block. // // `on_readable` is allowed to stop reading from `m_fd` before `EAGAIN` // or `EWOULDBLOCK` for fairness. If you stop reading, make sure // to call `m_reactor->add_readable(*this)` from within `on_readable`. // // If some error happened, execute `return invalidate();`. // // `invalidate()` can be called when `recv` or `read` returns 0, // but it is up to you (or your application's protocol). In general, // the client may still be waiting for the response. // // @return `true` or return value of . virtual bool on_readable() = 0; // Called when the reactor finds this resource is writable. // // This method is called when the reactor finds this resource gets // writable. Unlike , `on_writable` must try to write // all pending data until it encounters `EAGAIN` or `EWOULDBLOCK`. // // If some error happened, execute `return invalidate();`. // // @return `true` or return value of . virtual bool on_writable() = 0; // Called when the reactor finds this resource has hanged up. // // This method is called when the reactor detects unexpected hangup. virtual bool on_hangup() { return invalidate(); } // Called when the reactor finds an error on this resource. // // This method is called when the reactor detects some error. virtual bool on_error() { return invalidate(); } protected: const int m_fd; reactor* m_reactor = nullptr; friend class reactor; private: bool m_valid = true; mutable spinlock m_lock; typedef std::unique_lock lock_guard; void invalidate_(lock_guard g) { m_valid = false; g.unlock(); on_invalidate(); } }; // The reactor. // // This reactor internally uses epoll with edge-triggered mode. class reactor { public: reactor(); ~reactor(); // Events to poll. enum reactor_event { EVENT_IN = EPOLLIN, EVENT_OUT = EPOLLOUT, }; // Add a resource to the reactor. // @events `EVENT_IN`, `EVENT_OUT`, or bitwise OR of them. // // Add a resource to the reactor. The ownership of the resource // will be moved to the reactor. Only the reactor thread can use this. void add_resource(std::unique_ptr res, int events); // Modify epoll events for a resource. // @events `EVENT_IN`, `EVENT_OUT`, or bitwise OR of them. void modify_events(const resource& res, int events); // Run the reactor loop. // @callback Function called at each interval. // @interval Interval between two callbacks in seconds. void run(std::function callback, int interval = 1); // Run the reactor once. void run_once() { poll(); } // Return the number of registered and still valid resources. std::size_t size() const { return m_resources.size(); } // Quit the run loop gracefully. // // Call this to quit the run loop gracefully. // Only the reactor thread can use this. void quit() { m_running = false; } // Invalidate all registered resources. // // Invalidate all registered resources to unblock other threads. void invalidate() { for( auto& t: m_resources ) t.second->invalidate(); } // Add a resource to the readable resource list. // // Only may call this when it stops reading // from a resource before it encounters `EAGAIN` or `EWOULDBLOCK`. void add_readable(const resource& res) { m_readables.push_back(res.fileno()); } // Add a removal request for a resource. // // Resources can be shared with threads other than the reactor thread. // When such a thread successfully invalidates a resource, the thread // need to request the reactor thread to remove the resource by // calling this. void request_removal(const resource& res) { lock_guard g(m_lock); m_drop_req.push_back(res.fileno()); } bool has_garbage() const noexcept { return ! m_garbage.empty(); } // Fix the garbage resources to be destructed at the next . // // Fix the garbage resources to be destructed at the next . // Only the reactor thread can use this. // // @return `false` if there are no garbage resources, `true` otherwise. bool fix_garbage() { if( ! m_garbage_copy.empty() ) throw std::logic_error("No gc() was called after fix_garbage()!"); std::size_t n = m_garbage.size(); if( n == 0 ) return false; logger::debug() << "reactor: collecting " << n << " resources."; m_garbage_copy.swap(m_garbage); return true; } // Destruct garbage resources fixed by . // // Destruct garbage resources fixed by . // Only the reactor thread can use this. void gc() { logger::debug() << "reactor: collected garbage resources."; m_garbage_copy.clear(); } // Remove a registered resource. // This is only for the reactor thread. void remove_resource(const resource& res) { remove_resource(res.fileno()); } private: const int m_fd; bool m_running; typedef std::unordered_map> resource_map; resource_map m_resources; std::vector m_readables; std::vector m_readables_copy; // resource close request queue and its guarding lock. mutable spinlock m_lock; typedef std::lock_guard lock_guard; std::vector m_drop_req; std::vector m_drop_req_copy; // pending destruction lists std::vector> m_garbage; std::vector> m_garbage_copy; void remove_resource(int fd); void poll(); }; inline void resource::invalidate_and_close() { lock_guard g(m_lock); if( ! m_valid ) return; invalidate_( std::move(g) ); m_reactor->request_removal(*this); } } // namespace cybozu #endif // CYBOZU_REACTOR_HPP yrmcds-1.0.3/cybozu/signal.cpp000066400000000000000000000022551226611677100163250ustar00rootroot00000000000000#include "signal.hpp" #include #include #include #include namespace { const char ABORT_MESSAGE[] = "got SIGABRT.\n"; void handle_abort [[noreturn]] (int) { ::write(STDERR_FILENO, ABORT_MESSAGE, sizeof(ABORT_MESSAGE) - 1); cybozu::dump_stack(); std::abort(); } } // anonymous namespace namespace cybozu { std::unique_ptr signal_setup(std::initializer_list sigs) { sigset_t mask[1]; sigemptyset(mask); for( int i: sigs ) sigaddset(mask, i); int e = pthread_sigmask(SIG_BLOCK, mask, NULL); if( e != 0 ) throw_unix_error(e, "pthread_sigmask"); // signal disposition is a per-process attribute. struct sigaction act; std::memset(&act, 0, sizeof(act)); act.sa_handler = SIG_IGN; if( sigaction(SIGPIPE, &act, NULL) == -1 ) throw_unix_error(errno, "sigaction"); std::memset(&act, 0, sizeof(act)); act.sa_handler = handle_abort; act.sa_flags = SA_RESETHAND; if( sigaction(SIGABRT, &act, NULL) == -1 ) throw_unix_error(errno, "sigaction"); return std::unique_ptr( new signal_reader(mask) ); } } // namespace cybozu yrmcds-1.0.3/cybozu/signal.hpp000066400000000000000000000053671226611677100163410ustar00rootroot00000000000000/* Signal handling resource using signalfd(2). * * (C) 2013 Cybozu. */ #ifndef CYBOZU_SIGNAL_HPP #define CYBOZU_SIGNAL_HPP #include "reactor.hpp" #include "util.hpp" #include #include #include #include namespace cybozu { // A subclass for [`signalfd`](http://manpages.ubuntu.com/manpages/precise/en/man2/signalfd.2.html). class signal_reader: public resource { public: using callback_t = std::function; // Constructor. // @mask A set of signals to be handled by this resource. // @callback Callback function to handle received signals. // // Construct a signal reading resource. Signals set in `mask` // need to be blocked on all threads. signal_reader(const sigset_t *mask, callback_t callback): resource( signalfd(-1, mask, SFD_NONBLOCK|SFD_CLOEXEC) ), m_callback(callback) { if( m_fd == -1 ) throw_unix_error(errno, "signalfd"); } // Constructor. // @mask A set of signals to be handled by this resource. // // Construct a signal reading resource. Signals set in `mask` // need to be blocked on all threads. This constructor leaves // the callback function empty. Use `set_handler` to set one. explicit signal_reader(const sigset_t *mask): signal_reader(mask, nullptr) {} virtual ~signal_reader() {} void set_handler(const callback_t& callback) { m_callback = callback; } private: callback_t m_callback; virtual bool on_readable() override final { while( true ) { struct signalfd_siginfo si; ssize_t n = read(m_fd, &si, sizeof(si)); if( n == -1 ) { if( errno == EINTR ) continue; if( errno == EAGAIN || errno ==EWOULDBLOCK ) return true; throw_unix_error(errno, "read"); } if( n != sizeof(si) ) throw std::runtime_error(" bug?"); if( m_callback ) m_callback(si, *m_reactor); } } virtual bool on_writable() override final { return true; } }; // Block signals and create for blocked signals. // @sigs List of signals to be blocked. // // This function blocks given signals and creates a // prepared to read blocked signals. In addition, this configures // `SIGPIPE` to be ignored, and `SIGABRT` produces the stack trace. // // This function should be called from the main thread before any // other threads are created. std::unique_ptr signal_setup(std::initializer_list sigs); } // namespace cybozu #endif // CYBOZU_SIGNAL_HPP yrmcds-1.0.3/cybozu/siphash.cpp000066400000000000000000000065731226611677100165160ustar00rootroot00000000000000/* Copyright (c) 2013 Marek Majkowski Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. Original location: https://github.com/majek/csiphash/ Solution inspired by code from: Samuel Neves (supercop/crypto_auth/siphash24/little) djb (supercop/crypto_auth/siphash24/little2) Jean-Philippe Aumasson (https://131002.net/siphash/siphash24.c) */ // Cosmetic changes for C++ on Linux by @ymmt2005 #include "siphash.hpp" #include #include #define ROTATE(x, b) (std::uint64_t)( ((x) << (b)) | ( (x) >> (64 - (b))) ) #define HALF_ROUND(a,b,c,d,s,t) \ a += b; c += d; \ b = ROTATE(b, s) ^ a; \ d = ROTATE(d, t) ^ c; \ a = ROTATE(a, 32); #define DOUBLE_ROUND(v0,v1,v2,v3) \ HALF_ROUND(v0,v1,v2,v3,13,16); \ HALF_ROUND(v2,v1,v0,v3,17,21); \ HALF_ROUND(v0,v1,v2,v3,13,16); \ HALF_ROUND(v2,v1,v0,v3,17,21); using std::uint8_t; using std::uint32_t; using std::uint64_t; namespace { uint64_t static_k0; uint64_t static_k1; } namespace cybozu { void siphash24_seed(const char key[16]) { uint64_t t; std::memcpy(&t, key, sizeof(t)); static_k0 = le64toh(t); std::memcpy(&t, key + 8, sizeof(t)); static_k1 = le64toh(t); } uint64_t siphash24(const void *src, std::size_t src_sz) { uint64_t b = (uint64_t)src_sz << 56; const uint8_t *m = (const uint8_t*)src; uint64_t v0 = static_k0 ^ 0x736f6d6570736575ULL; uint64_t v1 = static_k1 ^ 0x646f72616e646f6dULL; uint64_t v2 = static_k0 ^ 0x6c7967656e657261ULL; uint64_t v3 = static_k1 ^ 0x7465646279746573ULL; while (src_sz >= 8) { uint64_t mi; std::memcpy(&mi, m, sizeof(mi)); mi = le64toh(mi); m += 8; src_sz -= 8; v3 ^= mi; DOUBLE_ROUND(v0,v1,v2,v3); v0 ^= mi; } switch (src_sz) { case 7: b |= uint64_t(m[6]) << 48; case 6: b |= uint64_t(m[5]) << 40; case 5: b |= uint64_t(m[4]) << 32; case 4: b |= uint64_t(m[3]) << 24; case 3: b |= uint64_t(m[2]) << 16; case 2: b |= uint64_t(m[1]) << 8; case 1: b |= uint64_t(m[0]); } v3 ^= b; DOUBLE_ROUND(v0,v1,v2,v3); v0 ^= b; v2 ^= 0xff; DOUBLE_ROUND(v0,v1,v2,v3); DOUBLE_ROUND(v0,v1,v2,v3); return (v0 ^ v1) ^ (v2 ^ v3); } } // namespace cybozu yrmcds-1.0.3/cybozu/siphash.hpp000066400000000000000000000013041226611677100165060ustar00rootroot00000000000000// siphash.hpp // (C) 2013 Cybozu. #ifndef CYBOZU_SIPHASH_HPP #define CYBOZU_SIPHASH_HPP #include #include namespace cybozu { // Seed . // // This function presets a 128bit key for . // Call this once before using . void siphash24_seed(const char key[16]); // Calculate 64bit keyed secure hash using SipHash algorithm. // @src Pointer to a memory region. // @src_sz Region size in bytes. // // This function calculates a 64bit hash value using // [SipHash](https://131002.net/siphash/) algorithm. // // @return 64bit hash value. std::uint64_t siphash24(const void *src, std::size_t src_sz); } // namespace cybozu #endif // CYBOZU_SIPHASH_HPP yrmcds-1.0.3/cybozu/spinlock.hpp000066400000000000000000000013631226611677100166760ustar00rootroot00000000000000// Spinlock. // (C) 2013 Cybozu. #ifndef CYBOZU_SPINLOCK_HPP #define CYBOZU_SPINLOCK_HPP #include #if defined(__i386__) || defined(__x86_64__) # include #endif namespace cybozu { // A simple spinlock. class spinlock { public: spinlock(): m_flag(ATOMIC_FLAG_INIT) {} void lock() { while( m_flag.test_and_set(std::memory_order_acquire) ) pause(); } void unlock() { m_flag.clear(std::memory_order_release); } private: std::atomic_flag m_flag; void pause() { #if defined(__i386__) || defined(__x86_64__) # if defined(__SSE__) _mm_pause(); # else __asm__ __volatile__ ("rep; nop"); # endif #endif } }; } // namespace cybozu #endif // CYBOZU_SPINLOCK_HPP yrmcds-1.0.3/cybozu/tcp.cpp000066400000000000000000000352221226611677100156360ustar00rootroot00000000000000// (C) 2013 Cybozu. #include "tcp.hpp" #include "logger.hpp" #ifdef USE_TCMALLOC # include # define MALLOC tc_malloc # define FREE tc_free #else # include # define MALLOC std::malloc # define FREE std::free #endif #include #include #include #include #include #include #include namespace { const unsigned int MAX_BUFCNT = 100; const int KEEPALIVE_IDLE = 300; // 5 min before keep alive probe const int KEEPALIVE_INTERVAL = 5; // 5 seconds between keep alive probes } // anonymous namespace namespace cybozu { int tcp_connect(const char* node, std::uint16_t port, unsigned int timeout) { std::string s_port = std::to_string(port); struct addrinfo hint, *res; std::memset(&hint, 0, sizeof(hint)); hint.ai_family = AF_INET; // prefer IPv4 hint.ai_socktype = SOCK_STREAM; hint.ai_flags = AI_NUMERICSERV|AI_ADDRCONFIG; int e = getaddrinfo(node, s_port.c_str(), &hint, &res); if( e == EAI_FAMILY || e == EAI_ADDRFAMILY || e == EAI_NODATA ) { hint.ai_family = AF_INET6; hint.ai_flags |= AI_V4MAPPED; e = getaddrinfo(node, s_port.c_str(), &hint, &res); } if( e == EAI_SYSTEM ) { throw_unix_error(errno, "getaddrinfo"); } else if( e != 0 ) { throw std::runtime_error(std::string("getaddrinfo: ") + gai_strerror(e)); } int s = ::socket(res->ai_family, res->ai_socktype | SOCK_NONBLOCK | SOCK_CLOEXEC, res->ai_protocol); if( s == -1 ) { freeaddrinfo(res); throw_unix_error(errno, "socket"); } e = ::connect(s, res->ai_addr, res->ai_addrlen); freeaddrinfo(res); if( e == 0 ) return s; if( errno != EINPROGRESS ) { ::close(s); throw_unix_error(errno, "connect"); } struct pollfd fds; fds.fd = s; fds.events = POLLOUT; int poll_timeout = timeout ? timeout*1000 : -1; int n = ::poll(&fds, 1, poll_timeout); if( n == 0 ) { // timeout ::close(s); return -1; } if( n == -1 ) { ::close(s); throw_unix_error(errno, "poll"); } if( fds.revents & (POLLERR|POLLHUP|POLLNVAL) ) { ::close(s); return -1; } socklen_t l = sizeof(e); if( getsockopt(s, SOL_SOCKET, SO_ERROR, &e, &l) == -1 ) { ::close(s); throw_unix_error(errno, "getsockopt(SO_ERROR)"); } if( e != 0 ) { ::close(s); return -1; } return s; } const std::size_t tcp_socket::SENDBUF_SIZE; tcp_socket::tcp_socket(int fd, unsigned int bufcnt): resource(fd) { if( bufcnt > MAX_BUFCNT ) throw std::logic_error("tcp_socket: Too many buffers"); m_free_buffers.reserve(bufcnt); m_pending.reserve(bufcnt); int v = 1; if( setsockopt(fd, SOL_SOCKET, SO_KEEPALIVE, &v, sizeof(v)) == -1 ) throw_unix_error(errno, "setsockopt(SO_KEEPALIVE)"); v = 1; if( setsockopt(fd, IPPROTO_TCP, TCP_CORK, &v, sizeof(v)) == -1 ) throw_unix_error(errno, "setsockopt(TCP_CORK)"); v = KEEPALIVE_IDLE; if( setsockopt(fd, IPPROTO_TCP, TCP_KEEPIDLE, &v, sizeof(v)) == -1 ) throw_unix_error(errno, "setsockopt(TCP_KEEPIDLE)"); v = KEEPALIVE_INTERVAL; if( setsockopt(fd, IPPROTO_TCP, TCP_KEEPINTVL, &v, sizeof(v)) == -1 ) throw_unix_error(errno, "setsockopt(TCP_KEEPINTVL)"); for( unsigned int i = 0; i < bufcnt; ++i ) { char* p = (char*)MALLOC(SENDBUF_SIZE); if( p == NULL ) { for( char* p2: m_free_buffers ) FREE(p2); m_free_buffers.clear(); throw std::runtime_error("tcp_socket: failed to allocate buffers"); } // reserved, hence no throw m_free_buffers.push_back(p); } } void tcp_socket::free_buffers() { lock_guard g(m_lock); for( auto& t: m_pending ) { char* p; std::tie(p, std::ignore, std::ignore) = t; FREE(p); } m_pending.clear(); for( char* p: m_free_buffers ) { FREE(p); } m_free_buffers.clear(); m_tmpbuf.clear(); m_tmpbuf.shrink_to_fit(); m_shutdown = true; } bool tcp_socket::_send(const char* p, std::size_t len, lock_guard& g) { while( ! can_send(len) ) m_cond_write.wait(g); if( m_shutdown ) return false; if( m_pending.empty() ) { while( len > 0 ) { ssize_t n = ::send(m_fd, p, len, 0); if( n == -1 ) { if( errno == EAGAIN || errno == EWOULDBLOCK ) break; if( errno == EINTR ) continue; auto ecnd = std::system_category().default_error_condition(errno); if( ecnd.value() != EPIPE ) logger::error() << ": (" << ecnd.value() << ") " << ecnd.message(); g.unlock(); invalidate_and_close(); return false; } p += n; len -= n; } if( len == 0 ) return true; } // put data in the pending request queue. if( capacity() < len ) { // here, m_pending.empty() and m_tmpbuf.empty() holds true. logger::debug() << " buffering " << len << " bytes data."; m_tmpbuf.resize(len); std::memcpy(m_tmpbuf.data(), p, len); return true; } if( ! m_pending.empty() ) { auto& t = m_pending.back(); char* t_p; std::size_t t_len; std::tie(t_p, t_len, std::ignore) = t; std::size_t room = SENDBUF_SIZE - t_len; if( room > 0 ) { std::size_t to_write = std::min(room, len); std::memcpy(t_p + t_len, p, to_write); p += to_write; len -= to_write; std::get<1>(t) = t_len + to_write; if( len == 0 ) return true; } } while( len > 0 ) { char* t_p = m_free_buffers.back(); m_free_buffers.pop_back(); std::size_t to_write = std::min(len, SENDBUF_SIZE); std::memcpy(t_p, p, to_write); p += to_write; len -= to_write; m_pending.emplace_back(t_p, to_write, 0); } return true; } bool tcp_socket::_sendv(const iovec* iov, const int iovcnt, lock_guard& g) { std::size_t total = 0; for( int i = 0; i < iovcnt; ++i ) { total += iov[i].len; } while( ! can_send(total) ) m_cond_write.wait(g); if( m_shutdown ) return false; ::iovec v[MAX_IOVCNT]; int v_size = 0; for( int i = 0; i < iovcnt; ++i ) { if( iov[i].len == 0 ) continue; v[v_size].iov_base = const_cast(iov[i].p); v[v_size].iov_len = iov[i].len; ++v_size; } int ind = 0; if( m_pending.empty() ) { while( ind < v_size ) { ssize_t n = ::writev(m_fd, &(v[ind]), v_size - ind); if( n == -1 ) { if( errno == EAGAIN || errno == EWOULDBLOCK ) break; if( errno == EINTR ) continue; auto ecnd = std::system_category().default_error_condition(errno); if( ecnd.value() != EPIPE ) logger::error() << ": (" << ecnd.value() << ") " << ecnd.message(); g.unlock(); invalidate_and_close(); return false; } while( n > 0 ) { if( static_cast(n) < v[ind].iov_len ) { v[ind].iov_base = ((char*)v[ind].iov_base) + n; v[ind].iov_len = v[ind].iov_len - n; break; } n -= v[ind].iov_len; ++ind; } } if( ind == v_size ) return true; } // recalculate total length total = 0; for( int i = ind; i < v_size; ++i ) { total += v[i].iov_len; } // put data in the pending request queue. if( capacity() < total ) { // here, m_pending.empty() and m_tmpbuf.empty() holds true. logger::debug() << " buffering " << total << " bytes data."; m_tmpbuf.resize(total); char* p = m_tmpbuf.data(); for( int i = ind; i < v_size; ++i ) { std::memcpy(p, v[i].iov_base, v[i].iov_len); p += v[i].iov_len; } return true; } while( ind < v_size ) { char* t_p; std::size_t t_len; if(m_pending.empty() || std::get<1>(m_pending.back()) == SENDBUF_SIZE) { t_p = m_free_buffers.back(); t_len = 0; m_free_buffers.pop_back(); m_pending.emplace_back(t_p, t_len, 0); } else { std::tie(t_p, t_len, std::ignore) = m_pending.back(); } std::size_t room = SENDBUF_SIZE - t_len; while( room > 0 ) { std::size_t to_write = std::min(room, v[ind].iov_len); std::memcpy(t_p + t_len, v[ind].iov_base, to_write); room -= to_write; t_len += to_write; std::get<1>(m_pending.back()) = t_len; if( to_write == v[ind].iov_len ) { ++ind; if( ind == v_size ) break; continue; } v[ind].iov_base = ((char*)v[ind].iov_base) + to_write; v[ind].iov_len -= to_write; } } return true; } bool tcp_socket::write_pending_data() { lock_guard g(m_lock); while( ! m_tmpbuf.empty() ) { ssize_t n = ::send(m_fd, m_tmpbuf.data(), m_tmpbuf.size(), 0); if( n == -1 ) { if( errno == EINTR ) continue; if( errno == EAGAIN || errno == EWOULDBLOCK ) return true; auto ecnd = std::system_category().default_error_condition(errno); if( ecnd.value() != EPIPE ) logger::error() << ": (" << ecnd.value() << ") " << ecnd.message(); return false; } m_tmpbuf.erase(m_tmpbuf.begin(), m_tmpbuf.begin() + n); } m_tmpbuf.shrink_to_fit(); while( ! m_pending.empty() ) { auto& t = m_pending.front(); char* p; std::size_t len; std::size_t sent; std::tie(p, len, sent) = t; while( len != sent ) { ssize_t n = ::send(m_fd, p+sent, len-sent, 0); if( n == -1 ) { if( errno == EINTR ) continue; if( errno == EAGAIN || errno == EWOULDBLOCK ) break; auto ecnd = std::system_category().default_error_condition(errno); logger::error() << ": (" << ecnd.value() << ") " << ecnd.message(); return false; } sent += n; } if( len == sent ) { m_pending.erase(m_pending.begin()); m_free_buffers.push_back(p); } else { std::get<2>(t) = sent; g.unlock(); // notify other threads of the new free space m_cond_write.notify_all(); return true; } } // all data have been sent. _flush(); if( ! m_shutdown ) { g.unlock(); m_cond_write.notify_all(); return true; } // invalidate and close this socket when m_shutdown==true. return false; } int setup_server_socket(const char* bind_addr, std::uint16_t port) { struct addrinfo hint, *res; std::string s_port = std::to_string(port); std::memset(&hint, 0, sizeof(hint)); hint.ai_family = AF_INET6; // can accept IPv4 address hint.ai_socktype = SOCK_STREAM; hint.ai_flags = AI_PASSIVE | AI_NUMERICSERV; int e = getaddrinfo(bind_addr, s_port.c_str(), &hint, &res); if( e == EAI_FAMILY || e == EAI_ADDRFAMILY || e == EAI_NODATA ) { logger::info() << "Binding to IPv6 fails, trying IPv4..."; hint.ai_family = AF_INET; e = getaddrinfo(bind_addr, s_port.c_str(), &hint, &res); } if( e == EAI_SYSTEM ) { throw_unix_error(errno, "getaddrinfo"); } else if( e != 0 ) { throw std::runtime_error(std::string("getaddrinfo: ") + gai_strerror(e)); } int s = ::socket(res->ai_family, res->ai_socktype | SOCK_NONBLOCK | SOCK_CLOEXEC, res->ai_protocol); if( s == -1 ) { freeaddrinfo(res); throw_unix_error(errno, "socket"); } // intentionally ignore errors int ok = 1; setsockopt(s, SOL_SOCKET, SO_REUSEADDR, &ok, sizeof(ok)); if( bind(s, res->ai_addr, res->ai_addrlen) == -1 ) { ::close(s); freeaddrinfo(res); throw_unix_error(errno, "bind"); } freeaddrinfo(res); if( listen(s, 128) == -1 ) { ::close(s); throw_unix_error(errno, "listen"); } return s; } bool tcp_server_socket::on_readable() { while( true ) { union { struct sockaddr sa; struct sockaddr_storage ss; } addr; socklen_t addrlen = sizeof(addr); #ifdef _GNU_SOURCE int s = ::accept4(m_fd, &(addr.sa), &addrlen, SOCK_NONBLOCK|SOCK_CLOEXEC); #else int s = ::accept(m_fd, &(addr.sa), &addrlen); if( s != -1 ) { int fl = fcntl(s, F_GETFL, 0); if( fl == -1 ) fl = 0; if( fcntl(s, F_SETFL, fl | O_NONBLOCK) == -1 ) { ::close(s); throw_unix_error(errno, "fcntl(F_SETFL)"); } fl = fcntl(s, F_GETFD, 0); if( fl == -1 ) fl = 0; if( fcntl(s, F_SETFD, fl | FD_CLOEXEC) == -1 ) { ::close(s); throw_unix_error(errno, "fcntl(F_SETFD)"); } } #endif if( s == -1 ) { if( errno == EINTR || errno == ECONNABORTED ) continue; if( errno == EMFILE || errno == ENFILE ) { logger::error() << "accept: Too many open files."; continue; } if( errno == EAGAIN || errno == EWOULDBLOCK ) break; throw_unix_error(errno, "accept"); } try { std::unique_ptr t = m_wrapper(s, ip_address(&(addr.sa))); if( t.get() == nullptr ) { ::close(s); } else { m_reactor->add_resource( std::move(t), reactor::EVENT_IN|reactor::EVENT_OUT ); } } catch( ... ) { ::close(s); throw; } } return true; } } // namespace cybozu yrmcds-1.0.3/cybozu/tcp.hpp000066400000000000000000000225071226611677100156450ustar00rootroot00000000000000// Asynchronous socket abstraction. // (C) 2013 Cybozu. #ifndef CYBOZU_TCP_HPP #define CYBOZU_TCP_HPP #include "ip_address.hpp" #include "reactor.hpp" #include "util.hpp" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include namespace cybozu { // Create a TCP socket and connect to `node:port`. // @node The node name or IP address to connect. // @port The TCP port number to connect. // @timeout Seconds before timeout. // // This creates a TCP socket and connect to a remote node. // The returned socket is set non-blocking. If `timeout` is 0, // this will wait until the kernel gives up. If `node` is `NULL`, // this will try to connect to the local loopback interface. // // This function may return `-1` when it cannot establish a connection // within `timeout` seconds or when the peer node denies connection. // For other (serious) errors, exceptions will be thrown. // // @return A valid UNIX file descriptor, or `-1` for non-fatal errors. int tcp_connect(const char* node, std::uint16_t port, unsigned int timeout=10); // A subclass for connected TCP sockets. // // This is an abstract base class implementing // and provides , , , and member // functions for connected TCP sockets. // // Derived classes still need to implement . class tcp_socket: public resource { static const std::size_t SENDBUF_SIZE = 1 << 20; public: // Construct an already connected socket. // @fd A file descriptor of a connected socket. // @bufcnt The number of 1 MiB buffers for pending send data. // // Construct a socket resource with a connected socket file descriptor. // The socket should already be set non-blocking. explicit tcp_socket(int fd, unsigned int bufcnt = 0); virtual ~tcp_socket() { free_buffers(); } // The maximum size of array for . static const int MAX_IOVCNT = 20; // struct for and . struct iovec { const char* p; std::size_t len; }; // Atomically send data. // @p Data to be sent. // @len Length of data starting from `p`. // @flush If `true`, the kernel send buffer will be flushed. // // This function sends a chunk of data atomically. The reactor // thread should not call this, or it may be blocked forever. // // @return `true` if this socket is valid, `false` otherwise. bool send(const char* p, std::size_t len, bool flush=false) { lock_guard g(m_lock); if( ! _send(p, len, g) ) return false; if( flush && empty() ) _flush(); return true; } // Atomically send multiple data. // @iov Array of . // @iovcnt Number of elements in `iov`. // @flush If `true`, the kernel send buffer will be flushed. // // This function sends a chunk of data atomically. The reactor // thread should not call this, or it may be blocked forever. // // @return `true` if this socket is valid, `false` otherwise. bool sendv(const iovec* iov, int iovcnt, bool flush=false) { if( iovcnt >= MAX_IOVCNT ) throw std::logic_error(" too many iovec."); lock_guard g(m_lock); if( ! _sendv(iov, iovcnt, g) ) return false; if( flush && empty() ) _flush(); return true; } // Atomically send data, then close the socket. // @p Data to be sent. // @len Length of data starting from `p`. // // This function sends a chunk of data atomically. The socket // will be closed after data are sent. The reactor thread should // not call this, or it may be blocked forever. // // @return `true` if this socket is valid, `false` otherwise. bool send_close(const char* p, std::size_t len) { lock_guard g(m_lock); if( ! _send(p, len, g) ) return false; m_shutdown = true; if( empty() ) { _flush(); g.unlock(); invalidate_and_close(); return true; } g.unlock(); m_cond_write.notify_all(); return true; } // Atomically send multiple data, then close the socket. // @iov Array of . // @iovcnt Number of elements in `iov`. // // This function sends a chunk of data atomically. The socket // will be closed after data are sent. The reactor thread should // not call this, or it may be blocked forever. // // @return `true` if this socket is valid, `false` otherwise. bool sendv_close(const iovec* iov, int iovcnt) { if( iovcnt >= MAX_IOVCNT ) throw std::logic_error(" too many iov."); lock_guard g(m_lock); if( ! _sendv(iov, iovcnt, g) ) return false; m_shutdown = true; if( empty() ) { _flush(); g.unlock(); invalidate_and_close(); return true; } g.unlock(); m_cond_write.notify_all(); return true; } protected: // Write out pending data. // // This method tries to send pending data as much as possible. // // @return `false` if some error happened, `true` otherwise. bool write_pending_data(); // Just call . // // The default implementation just invoke . // You may override this to dispatch the job to another thread. virtual bool on_writable() override { if( write_pending_data() ) return true; return invalidate(); } virtual void on_invalidate() override { ::shutdown(m_fd, SHUT_RDWR); free_buffers(); m_cond_write.notify_all(); } private: std::vector m_free_buffers; // tuple of std::vector> m_pending; std::vector m_tmpbuf; bool m_shutdown = false; typedef std::unique_lock lock_guard; mutable std::mutex m_lock; mutable std::condition_variable m_cond_write; std::size_t capacity() const { std::size_t c = m_free_buffers.size() * SENDBUF_SIZE; if( m_pending.empty() ) return c; return c + SENDBUF_SIZE - std::get<1>(m_pending.back()); } bool can_send(std::size_t len) const { if( m_shutdown ) return true; // in fact, fail if( ! m_tmpbuf.empty() ) return false; if( m_pending.empty() ) return true; return capacity() >= len; } bool _send(const char* p, std::size_t len, lock_guard& g); bool _sendv(const iovec* iov, const int iovcnt, lock_guard& g); bool empty() const { return m_pending.empty() && m_tmpbuf.empty(); } void _flush() { // with TCP_CORK, setting TCP_NODELAY effectively flushes // the kernel send buffer. int v = 1; if( setsockopt(m_fd, IPPROTO_TCP, TCP_NODELAY, &v, sizeof(v)) == -1 ) throw_unix_error(errno, "setsockopt(TCP_NODELAY)"); } void free_buffers(); }; // A helper function to create a server socket. // @bind_addr A numeric IP address to be bound, or `NULL`. // @port TCP port number to be bound. // // This is a helper function for template. // The socket is set non-blocking before return. // // @return A UNIX file descriptor of a listening socket. int setup_server_socket(const char* bind_addr, std::uint16_t port); // A subclass to accept new TCP connections. class tcp_server_socket: public resource { public: using wrapper = std::function (int s, const ip_address&)>; // Construct a server socket. // @bind_addr A numeric IP address to be bound, or `NULL`. // @port TCP port number to be bound. // @on_accept Callback function. // // This creates a socket and bind it to the given address and port. // If `bind_addr` is `NULL`, the socket will listen on any address. // Both IPv4 and IPv6 addresses are supported. // // For each new connection, `w` is called to determine if the new // connection need to be closed immediately or to be added to the // reactor. If `w` returns an empty , the new // connection is closed immediately. Otherwise, the new connection // is added to the reactor. tcp_server_socket(const char* bind_addr, std::uint16_t port, wrapper w): resource( setup_server_socket(bind_addr, port) ), m_wrapper(w) {} virtual ~tcp_server_socket() {} private: wrapper m_wrapper; virtual bool on_readable() override final; virtual bool on_writable() override final { return true; } }; // Utility function to create a of . inline std::unique_ptr make_server_socket( const char* bind_addr, std::uint16_t port, tcp_server_socket::wrapper w) { return std::unique_ptr( new tcp_server_socket(bind_addr, port, w) ); } } // namespace cybozu #endif // CYBOZU_SOCKET_HPP yrmcds-1.0.3/cybozu/test.hpp000066400000000000000000000236101226611677100160320ustar00rootroot00000000000000// Unit test class. // (C) 2008 Cybozu Labs, Inc., all rights reserved. // (C) 2013 Cybozu. All rights reserved. #ifndef CYBOZU_TEST_HPP #define CYBOZU_TEST_HPP #include #include #include #include #include #include #include #include #include #include namespace cybozu { namespace test { class AutoRun { typedef void (*Func)(); typedef std::list > UnitTestList; public: AutoRun() : init_(0) , term_(0) , okCount_(0) , ngCount_(0) , exceptionCount_(0) { } void setup(Func init, Func term) { init_ = init; term_ = term; } void append(const char *name, Func func) { list_.push_back(std::make_pair(name, func)); } void set(bool isOK) { if (isOK) { okCount_++; } else { ngCount_++; } } std::string getBaseName(const std::string& name) const { #ifdef _WIN32 const char sep = '\\'; #else const char sep = '/'; #endif size_t pos = name.find_last_of(sep); std::string ret = name.substr(pos + 1); pos = ret.find('.'); return ret.substr(0, pos); } using ret_t = std::tuple; ret_t run(const char* arg0) { using namespace std::chrono; std::string msg; steady_clock::duration elapsed; try { if (init_) init_(); for (UnitTestList::const_iterator i = list_.begin(), ie = list_.end(); i != ie; ++i) { std::cout << "ctest:module=" << i->first << std::endl; try { auto t1 = steady_clock::now(); (i->second)(); auto t2 = steady_clock::now(); elapsed += t2 - t1; } catch (std::exception& e) { exceptionCount_++; std::cout << "ctest: " << i->first << " is stopped by std::exception " << e.what() << std::endl; } catch (...) { exceptionCount_++; std::cout << "ctest: " << i->first << " is stopped by an exception" << std::endl; } } if (term_) term_(); } catch (...) { msg = "ctest:err: catch unexpected exception"; } fflush(stdout); std::uint64_t elapsed_us = duration_cast(elapsed).count(); if (msg.empty()) { std::cout << "ctest:name=" << getBaseName(arg0) << ", module=" << list_.size() << ", total=" << (okCount_ + ngCount_ + exceptionCount_) << ", ok=" << okCount_ << ", ng=" << ngCount_ << ", exception=" << exceptionCount_ << std::endl; return (ngCount_>0) ? ret_t(1, elapsed_us) : ret_t(0, elapsed_us); } else { std::cout << msg << std::endl; return ret_t(1, elapsed_us); } } static inline AutoRun& getInstance() { static AutoRun instance; return instance; } private: Func init_; Func term_; int okCount_; int ngCount_; int exceptionCount_; std::uint64_t elapsed_; UnitTestList list_; }; static AutoRun& autoRun = AutoRun::getInstance(); inline void test(bool ret, const std::string& msg, const std::string& param, const char *file, int line) { autoRun.set(ret); if (!ret) { std::cout << "ctest:" << file << ":" << line << " " << msg << param << ";" << std::endl; } } template bool isEqual(const T& lhs, const U& rhs) { return lhs == rhs; } inline bool isEqual(const char *lhs, const char *rhs) { return strcmp(lhs, rhs) == 0; } inline bool isEqual(char *lhs, const char *rhs) { return strcmp(lhs, rhs) == 0; } inline bool isEqual(const char *lhs, char *rhs) { return strcmp(lhs, rhs) == 0; } inline bool isEqual(char *lhs, char *rhs) { return strcmp(lhs, rhs) == 0; } // avoid to compare float directly inline bool isEqual(float lhs, float rhs) { union fi { float f; uint32_t i; } lfi, rfi; lfi.f = lhs; rfi.f = rhs; return lfi.i == rfi.i; } // avoid to compare double directly inline bool isEqual(double lhs, double rhs) { union di { double d; uint64_t i; } ldi, rdi; ldi.d = lhs; rdi.d = rhs; return ldi.i == rdi.i; } } } // cybozu::test #define TEST_MAIN(arg_parser) \ int main(int argc, char** argv) { \ if( ! arg_parser(argc, argv) ) \ return 0; \ int r; \ std::uint64_t elapsed; \ std::tie(r, elapsed) = cybozu::test::autoRun.run(argv[0]); \ std::cerr << "ctest:elapsed=" << elapsed << "us" << std::endl; \ return r; \ } #ifndef TEST_DISABLE_AUTO_RUN bool null_parser(int argc, char** argv) { return true; } TEST_MAIN(null_parser); #endif /** alert if !x @param x [in] */ #define cybozu_assert(...) cybozu_assert_((__VA_ARGS__)) #define cybozu_assert_(x) cybozu::test::test(!!(x), "cybozu_assert", #x, __FILE__, __LINE__) /** alert if x != y @param x [in] @param y [in] */ #define cybozu_equal(x, y) { \ bool eq = cybozu::test::isEqual(x, y); \ cybozu::test::test(eq, "cybozu_equal", "(" #x ", " #y ")", __FILE__, __LINE__); \ if (!eq) { \ std::cout << "ctest: lhs=" << (x) << std::endl; \ std::cout << "ctest: rhs=" << (y) << std::endl; \ } \ } /** alert if fabs(x, y) >= eps @param x [in] @param y [in] */ #define cybozu_near(x, y, eps) { \ bool isNear = fabs((x) - (y)) < eps; \ cybozu::test::test(isNear, "cybozu_near", "(" #x ", " #y ")", __FILE__, __LINE__); \ if (!isNear) { \ std::cout << "ctest: lhs=" << (x) << std::endl; \ std::cout << "ctest: rhs=" << (y) << std::endl; \ } \ } #define cybozu_equal_pointer(x, y) { \ bool eq = x == y; \ cybozu::test::test(eq, "cybozu_equal_pointer", "(" #x ", " #y ")", __FILE__, __LINE__); \ if (!eq) { \ std::cout << "ctest: lhs=" << (const void*)(x) << std::endl; \ std::cout << "ctest: rhs=" << (const void*)(y) << std::endl; \ } \ } /** always alert @param msg [in] */ #define cybozu_fail(msg) cybozu::test::test(false, "cybozu_fail", "(" msg ")", __FILE__, __LINE__) /** verify message in exception */ #define cybozu_test_exception_message(statement, Exception, msg) \ { \ int ret = 0; \ std::string errMsg; \ try { \ statement; \ ret = 1; \ } catch (const Exception& e) { \ errMsg = e.what(); \ if (errMsg.find(msg) == std::string::npos) { \ ret = 2; \ } \ } catch (...) { \ ret = 3; \ } \ if (ret) { \ cybozu::test::test(false, "cybozu_test_exception_message", "(" #statement ", " #Exception ", " #msg ")", __FILE__, __LINE__); \ if (ret == 1) { \ std::cout << "ctest: no exception" << std::endl; \ } else if (ret == 2) { \ std::cout << "ctest: bad exception msg:" << errMsg << std::endl; \ } else { \ std::cout << "ctest: unexpected exception" << std::endl; \ } \ } else { \ cybozu::test::autoRun.set(true); \ } \ } #define cybozu_test_exception(statement, Exception) \ { \ int ret = 0; \ try { \ statement; \ ret = 1; \ } catch (const Exception&) { \ } catch (...) { \ ret = 2; \ } \ if (ret) { \ cybozu::test::test(false, "cybozu_test_exception", "(" #statement ", " #Exception ")", __FILE__, __LINE__); \ if (ret == 1) { \ std::cout << "ctest: no exception" << std::endl; \ } else { \ std::cout << "ctest: unexpected exception" << std::endl; \ } \ } else { \ cybozu::test::autoRun.set(true); \ } \ } /** verify statement does not throw */ #define cybozu_test_no_exception(statement) \ try { \ statement; \ cybozu::test::autoRun.set(true); \ } catch (...) { \ cybozu::test::test(false, "cybozu_test_no_exception", "(" #statement ")", __FILE__, __LINE__); \ } /** append auto unit test @param name [in] module name */ #define AUTOTEST(name) \ void cybozu_test_ ## name(); \ struct cybozu_test_local_ ## name { \ cybozu_test_local_ ## name() \ { \ cybozu::test::autoRun.append(#name, cybozu_test_ ## name); \ } \ } cybozu_test_local_instance_ ## name; \ void cybozu_test_ ## name() /** append auto unit test with fixture @param name [in] module name */ #define AUTOTEST_WITH_FIXTURE(name, Fixture) \ void cybozu_test_ ## name(); \ void cybozu_test_real_ ## name() \ { \ Fixture f; \ cybozu_test_ ## name(); \ } \ struct cybozu_test_local_ ## name { \ cybozu_test_local_ ## name() \ { \ cybozu::test::autoRun.append(#name, cybozu_test_real_ ## name); \ } \ } cybozu_test_local_instance_ ## name; \ void cybozu_test_ ## name() /** setup fixture @param Fixture [in] class name of fixture @note cstr of Fixture is called before test and dstr of Fixture is called after test */ #define SETUP_FIXTURE(Fixture) \ Fixture *cybozu_test_local_fixture; \ void cybozu_test_local_init() \ { \ cybozu_test_local_fixture = new Fixture(); \ } \ void cybozu_test_local_term() \ { \ delete cybozu_test_local_fixture; \ } \ struct cybozu_test_local_fixture_setup_ { \ cybozu_test_local_fixture_setup_() \ { \ cybozu::test::autoRun.setup(cybozu_test_local_init, cybozu_test_local_term); \ } \ } cybozu_test_local_fixture_setup_instance_; #endif // CYBOZU_TEST_HPP yrmcds-1.0.3/cybozu/thread.hpp000066400000000000000000000036151226611677100163250ustar00rootroot00000000000000// CRTP thread wrapper. // (C) 2013 Cybozu. #ifndef CYBOZU_THREAD_HPP #define CYBOZU_THREAD_HPP #include "logger.hpp" #include "util.hpp" #include #include #include #include namespace cybozu { // A CRTP template wrapping . // // A curiously-recurring-template-pattern (CRTP) template to wrap // with associated data structures. Derived classes // need to implement `void run()`. // // and will be caught and logged. // If `exit_on_exception` is not 0, then `exit(exit_on_exception)` is // called to terminate the program. template class thread_base { public: void start() { if( m_thread.joinable() ) throw std::logic_error("Thread already started!"); m_done.store(false, std::memory_order_relaxed); m_thread = std::thread([this]() { run_in_try(); }); } void run_in_try() { try { static_cast(this)->run(); m_done.store(true, std::memory_order_release); } catch( const std::system_error& e ) { demangler t(typeid(e).name()); logger::error() << "[" << t.name() << "] " << "(" << e.code() << ") " << e.what(); if( exit_on_exception != 0 ) ::exit(exit_on_exception); } catch( const std::exception& e ) { demangler t(typeid(e).name()); logger::error() << "[" << t.name() << "] " << e.what(); if( exit_on_exception != 0 ) ::exit(exit_on_exception); } } bool done() const noexcept { return m_done.load(std::memory_order_acquire); } protected: std::atomic m_done; std::thread m_thread; }; } // namespace cybozu #endif // CYBOZU_THREAD_HPP yrmcds-1.0.3/cybozu/util.cpp000066400000000000000000000014251226611677100160230ustar00rootroot00000000000000// (C) 2013 Cybozu. #include "util.hpp" #include #include #include #include #include #include #include #include #include namespace cybozu { demangler::demangler(const char* name) { int status; char* demangled = abi::__cxa_demangle(name, 0, 0, &status); if( status == 0 ) { m_name = demangled; std::free(demangled); } else { m_name = name; } } void dump_stack() noexcept { char buf[32]; std::time_t t = std::time(nullptr); std::size_t len = std::strlen(ctime_r(&t, buf)); write(STDERR_FILENO, buf, len); void* bt[100]; int n = backtrace(bt, 100); backtrace_symbols_fd(bt, n, STDERR_FILENO); } } // namespace cybozu yrmcds-1.0.3/cybozu/util.hpp000066400000000000000000000045141226611677100160320ustar00rootroot00000000000000// Utilities // (C) 2013 Cybozu. #ifndef CYBOZU_UTIL_HPP #define CYBOZU_UTIL_HPP #include #include #include #include namespace cybozu { // Demangle C++ symbol names. class demangler { public: // Constructor. // @name Mangled symbol name. explicit demangler(const char* name); // Return the demangled symbol name. const std::string& name() const { return m_name; } private: std::string m_name; }; // Dump stack trace to the standard error. void dump_stack() noexcept; // Throw for an `errno`. // @e `errno` value. // @func System call name that returns the error number. inline void throw_unix_error [[noreturn]] (int e, const char* func) { dump_stack(); throw std::system_error(e, std::system_category(), func); } // Throw for an `errno`. // @e `errno` value. // @s A detailed string to describe the context of the error. inline void throw_unix_error [[noreturn]] (int e, const std::string& s) { dump_stack(); throw std::system_error(e, std::system_category(), s); } // Convert an integer from the network byte order. // @p Pointer to a memory region. // @n Reference to an integer to store the converted value. template inline void ntoh(const char* p, UInt& n) noexcept { UInt d; static_assert( sizeof(d) == 2 || sizeof(d) == 4 || sizeof(d) == 8, "Wrong integer type" ); std::memcpy(&d, p, sizeof(d)); switch( sizeof(d) ) { case 2: n = be16toh(d); return; case 4: n = be32toh(d); return; case 8: n = be64toh(d); return; } } // Convert an integer into the network byte order. // @d An integer to be converted. // @p Pointer to a memory region for the converted integer. template inline void hton(UInt d, char* p) noexcept { static_assert( sizeof(d) == 2 || sizeof(d) == 4 || sizeof(d) == 8, "Wrong integer type" ); UInt n; switch( sizeof(d) ) { case 2: n = htobe16(d); std::memcpy(p, &n, 2); return; case 4: n = htobe32(d); std::memcpy(p, &n, 4); return; case 8: n = htobe64(d); std::memcpy(p, &n, 8); return; } } } // namespace cybozu #endif // CYBOZU_UTIL_HPP yrmcds-1.0.3/cybozu/worker.hpp000066400000000000000000000056151226611677100163710ustar00rootroot00000000000000// General worker thread. // (C) 2013 Cybozu. #ifndef CYBOZU_WORKER_HPP #define CYBOZU_WORKER_HPP #include #include #include #include #include #include #include #include #include namespace cybozu { // General worker thread implementation. // // Every worker thread has a pre-allocated internal memory buffer. // A worker receives a new `job` through as a callback function, // then invokes the function with that buffer. // // To stop the worker, call . class worker final: public thread_base { public: // Constructor. // @bufsiz The size of the internal buffer. worker(std::size_t bufsiz) : m_running(false), m_exit(false), m_event(eventfd(0, EFD_CLOEXEC)), m_buffer(bufsiz) { if( m_event == -1 ) throw_unix_error(errno, "eventfd"); } ~worker() { ::close(m_event); } // forbid copy & assignment worker(const worker&) = delete; worker& operator=(const worker&) = delete; worker(worker&&) = delete; worker& operator=(worker&&) = delete; // Jobs for workers are this kind of functions. // `buf` is a memory buffer that can be used freely for temporary data. using job = std::function; // Return `true` while this worker thread is busy for a job. bool is_running() const noexcept { return m_running.load(std::memory_order_acquire); } // Ask this worker thread to execute a new job. // @job_ A callback function to be executed by the worker thread. void post_job(job job_) { m_job = job_; m_running.store(true, std::memory_order_release); notify(); } // Stop this worker thread. // // The thread will be joined automatically. void stop() { m_exit = true; m_running.store(true, std::memory_order_release); notify(); m_thread.join(); } // CRTP method for . void run() { while ( true ) { // wait a new job or an exit signal m_running.store(false, std::memory_order_release); std::uint64_t i; ssize_t n = ::read(m_event, &i, sizeof(i)); if( n == -1 ) throw_unix_error(errno, "read(eventfd)"); while( ! m_running.load(std::memory_order_acquire) ); if( m_exit ) return; m_job(m_buffer); m_buffer.reset(); } } private: alignas(CACHELINE_SIZE) std::atomic m_running; bool m_exit; const int m_event; dynbuf m_buffer; job m_job; void notify() { std::uint64_t i = 1; ssize_t n = ::write(m_event, &i, sizeof(i)); if( n == -1 ) throw_unix_error(errno, "write(eventfd)"); } }; } // namespace cybozu #endif // CYBOZU_WORKER_HPP yrmcds-1.0.3/docs/000077500000000000000000000000001226611677100137555ustar00rootroot00000000000000yrmcds-1.0.3/docs/bench.md000066400000000000000000000016551226611677100153650ustar00rootroot00000000000000# Benchmark results. Environment ----------- * CPU Intel Xeon E5507 2.27 GHz * 2 (total 8 physical cores) * OS Ubuntu 12.04 * Threads 8 worker threads for yrmcds / memcached * Memory 1 GiB for yrmcds / memcached * Buckets 5 million for yrmcds Benchmark Tool -------------- [memslap][] from [libmemcached][]. Parameters: `--threads=8 --concurrency=80` Other parameters are left default, i.e., memslap runs for 600 seconds with set/get ratio = 1:9. memslap ran on a different machine. Versions -------- * memcached 1.4.13 * yrmcds 0.9.7 Results ------- * memcached Ops: 71584331 TPS: 119298 * yrmcds (0 slave) Ops: **81028171** TPS: **135037** * yrmcds (1 slave) Ops: 69657147 TPS: 116086 * yrmcds (2 slaves) Ops: 63479541 TPS: 105791 [memslap]: http://docs.libmemcached.org/bin/memslap.html [libmemcached]: http://libmemcached.org/libMemcached.html yrmcds-1.0.3/docs/design.md000066400000000000000000000320151226611677100155510ustar00rootroot00000000000000# Design Notes ============ Automatic master promotion -------------------------- The standard set of yrmcds consists of a master server and two slave servers. A set of yrmcds will be prepared for each server farm. To elect the master server automatically, keepalived or similar virtual IP address allocation tool is used. The server owning the virtual IP address becomes the master server. The others are slaves. Slaves connect to the master via the virtual IP address. The master will *not* connect to slaves. A new slave will receive all data from the master server once the connection has been established. For some reason, if the master suddenly notices it is no longer the master (i.e. the server has lost the virtual IP), it kills itself to reborn as a new slave (restarted by upstart). If a slave gets disconnected from the master, it waits for some seconds to see if the virtual IP address is assigned. If assigned, the slave promotes to the new master. If not, the slave forgets all replicated data then try to connect to the new master. The new master should close the established connection to the old master, if any. Replication protocol -------------------- The replication protocol is the same as [the binary protocol of memcached][1]. Specifically, slaves receive only "SetQ" and "DeleteQ" requests. This allows any memcached compatible programs can become yrmcds slaves with slight modifications. The number of connections ------------------------- The master will have tens of thousands of client connections from every PHP-fpm processes in a server farm. A thread per connection model is therefore not very efficient. This encourages us to adopt asynchronous sockets and [the reactor pattern][2]. The reactor and workers ----------------------- So we adopt [the reactor pattern with a thread-pool][3] for the best performance. The reactor is implemented with asynchronous sockets and [epoll][] in edge-triggered mode. Basically, with edge-triggered mode, a socket need to be read until [recv][] returns an `EWOULDBLOCK` error. If, for some reason like fairness, not all available data are received from a socket, the reactor need to remember that socket to manually handle it later. To remember such sockets, the reactor manages a list of _readable_ socket internally. When the reactor detects some data are available for a socket, it dispatches the socket to an **idle** _worker_ thread. The worker then receives and processes the data. While a worker is handling a socket, that socket must not be passed to another worker to keep the protocol semantics. To implement these, following properties are required: * Every worker thread has an **idle** flag. * Every socket has a **busy** (in-use) flag. Both flags are set by the reactor thread, and cleared by a worker thread. If the reactor finds a socket is readable but the socket is busy or there are no idle workers, the socket is remembered in the previously stated readable socket list. This keeps the reactor thread from being blocked. ### Efficient allocation of readable lists Readable lists used in the reactor can be implemented with very rare memory allocations by preparing two lists that pre-allocate a certain amount of memory. Swap the list with another for each epoll loop. ### Use of eventfd to dispatch jobs To notify an idle worker a new job (or to wait a new job), we use Linux-specific [eventfd][] rather than pipes or pthread's condition variables. eventfd is lighter than pipe. Unlike condition variables, eventfd remembers events so that workers never fail to catch events. The hash -------- The biggest and the most important data structure in yrmcds is clearly the hash map of objects. To reduce contention between worker threads, the hash does not use a single lock to protect the data. Instead, each bucket has a lock to protect objects in the bucket and the bucket itself. One drawback is that resizing the hash is almost impossible. In the real implementation, yrmcds statically allocates a large number of buckets. Housekeeping ------------ yrmcds is essentially an object _cache_ system. Objects whose life-time have been expired need to be removed from the cache. If the cache is full of objects, least-recently-used (LRU) objects should be removed. For such housekeeping, a dedicated thread called **GC thread** is used. It scans the whole hash to detect and remove expired objects. To implement LRU cache, objects have a counter that increments at every GC. The counter is reset to zero when a worker accesses the object. When the cache is full, GC thread removes objects whose LRU counter value is high. A GC thread is started by the reactor thread only when there is no running GC thread. The reactor thread passes the current list of slaves to the GC thread in order to replicate object removals. If a slave is added while a GC thread is running, that slave may fail to remove some objects, which is *not* a big problem. Replication ----------- When a slave connect to the master, it need to receive all data in the master. yrmcds reuses the GC thread for this initial replication because the GC thread scans the entire hash. While GC is running, the reactor and worker threads also keep running. The GC thread and worker threads therefore need to be synchronized when sending data of the same object. This can be achieved by keeping the lock of a hash bucket acquired while an object is being sent. Sockets for replication ----------------------- Sockets connected to slaves are used to send a lot of replication data. Further, worker threads serving client requests need to access replication sockets to send updates to slaves. Another difference from client sockets is the possible number of sockets. The number of replication sockets can be limited to a fairly small value, say 5. Worker threads need to have references to all replication sockets. This can be done without any locks; the reactor passes a copy of the references along with the socket and received data. However, this can introduce a race between workers and the GC thread. If a new GC thread starts the initial replication process for a new slave while some worker threads having old copies of replication sockets are executing its job, the workers would fail to send some objects to the new slave. To resolve this race, we need to guarantee that all worker threads are idle or have the latest copies of replication sockets. This can be implemented as: 1. Accept a new replication socket and add it to the list of replication sockets. At this point, the GC thread must not begin the initial replication for this new slave. 2. The reactor thread puts a synchronization request. The request will be satisfied once the reactor thread observes every worker thread gets idle. 3. The reactor thread adds the new replication socket to the confirmed list. 4. At the next GC, the reactor thread requests initial replication for sockets stored in the confirmed list. Sending data ------------ As yrmcds need to handle a lot of clients, having every socket has a large send buffer is not a good idea. Sockets connected to clients therefore can have temporarily allocated memory for pending data only. On the other hand, sockets connected to slaves are few, hence they can statically allocate large memory for pending send data. Anyways, a sending thread need to wait for the reactor to send pending data and clear the buffer when there is not enough room. To achieve best TCP performance, we use [`TCP_CORK`][4] option although it is Linux-specific. With `TCP_CORK` turned on, the TCP send buffer need to be flushed explicitly. The buffer should be flushed only when all data are successfully sent without seeing `EWOULDBLOCK`. To put a complete response data with one call, the socket object should provide an API similar to [writev][]. Reclamation strategy of shared sockets -------------------------------------- ### Sockets are shared Sockets connected to clients may be shared by the reactor thread and a worker thread. Sockets connected to slaves may be shared by the reactor thread, worker threads, and the initial replication thread. ### Socket can be closed only by the reactor thread This is because the reactor thread manages a mapping between file descriptors and resource objects including sockets. If another thread closes a file descriptor of a resource, the same file descriptor number may be reused by the operating system, which would break the mapping. ### Strategy to reclaim shared sockets 1. Sockets can be invalidated by any thread. Invalidated sockets refuse further access to them. 2. Inform the reactor thread of invalidated sockets. 3. The reactor thread 1. removes invalidated sockets from the internal mapping, 2. closes the file descriptor, then 3. adds them a list of pending destruction resources. At some point when there is no running GC thread, the reactor thread can put a new synchronization request. To optimize memory allocations, the reactor thread will not put such a request if there are any pending requests. This way, the reactor can save the current pending destruction list by swapping contents with a pre-allocated save list. Once the reactor observes idle state of all worker threads, resources in the save list can be destructed safely. No blocking job queues, no barrier synchronization -------------------------------------------------- To avoid excessive contention between the reactor and worker threads, a blocking job queue is not used between them. Instead, the reactor thread loops over atomic flags for each worker thread to identify if a worker is idle or busy. For the same reason, we do not use barrier synchronization between the reactor and worker threads. Instead, the reactor thread checks every worker thread is idle or becomes idle when there are any synchronization request. Once it confirms all worker threads gets idle at least once after a synchronization request, it starts a job associated with the synchronization request. Slaves are essentially single-threaded -------------------------------------- Slaves do accept connection requests from memcache clients, but they close the connections immediately. This way, slaves can essentially be single-threaded. As long as being a slave, the internal hash need not be guarded by locks. For this reason, the internal hash should provide lock-free version of methods. Joining threads --------------- All threads are started by the reactor thread. The reactor thread is therefore responsible for those threads to be joined. All threads except for the reactor thread may block on either a condition variable of a socket or the [eventfd][] file descriptor. For sockets, the reactor thread signals the condition variable to unblock threads when it closes the socket. For eventfd descriptors, the reactor thread simply writes to it. When the program exits, the reactor thread executes the following termination process: 1. Sets termination flags of all worker threads. 2. Writes to eventfd descriptors to unblock worker threads. 3. invalidates all resources to unblock worker threads. 4. joins with the GC thread, if any, then 5. destructs resources. Large data ---------- Too large data should be stored in a temporary file. The temporary file shall soon be unlinked for automatic removal upon the program exit. Summary ------- ### Threads In the master, * the reactor thread, * worker threads to process client requests, and * a GC thread. Slaves run the reactor (main) thread only. ### Sockets In the master, * a listening socket for clients, * a listening socket for slaves, * sockets connected to clients, and * sockets connected to slaves. In a slave, * a listening socket for clients, and * a socket connected to the master. ### Shared data structures * Connected sockets. * The object hash map. * Close request queue in the reactor. * Other less significant resources such as statistics counters. ### Locks and ordering * The reactor thread * acquires a lock of a socket to send data. * acquires a spinlock of socket close request queue. * A worker thread * acquires a lock of a hash bucket. * acquires a lock of a socket. * acquires a lock of a socket to send data independent of cached objects. * acquires a spinlock of the reactor to put socket close request. * A GC thread * acquires a lock of a hash bucket. * acquires locks of sockets connected to slaves. * acquires a spinlock of the reactor to put socket close request. [1]: https://code.google.com/p/memcached/wiki/BinaryProtocolRevamped [2]: http://en.wikipedia.org/wiki/Reactor_pattern [3]: http://stackoverflow.com/questions/14317992/thread-per-connection-vs-reactor-pattern-with-a-thread-pool [4]: http://manpages.ubuntu.com/manpages/precise/en/man7/tcp.7.html [epoll]: http://manpages.ubuntu.com/manpages/precise/en/man7/epoll.7.html [eventfd]: http://manpages.ubuntu.com/manpages/precise/en/man2/eventfd.2.html [recv]: http://manpages.ubuntu.com/manpages/precise/en/man2/recv.2.html [signalfd]: http://manpages.ubuntu.com/manpages/precise/en/man2/signalfd.2.html [writev]: http://manpages.ubuntu.com/manpages/precise/en/man2/writev.2.html yrmcds-1.0.3/docs/diffs.md000066400000000000000000000015171226611677100153760ustar00rootroot00000000000000# Differences from memcached. Differences from memcached 1.4.15 --------------------------------- * Implementations of binary protocol command `GaT` and `Get` are identical. Both may have an optional expiration time. Objects will be touched only when the command is attended with an expiration time. * `stats` returns different items. `stats slabs` is not implemented. `stats ops` returns ops counts for each text/binary command. * `slabs automove` and `slabs reassign` are not implemented. These always return "OK". * `verbosity` takes a string argument rather than an integer. Valid values are `error`, `warning`, `info`, and `debug`. * UDP transport is not implemented. * UNIX domain transport is not, and will not be, implemented. * SASL authentication is not implemented. Who cares? yrmcds-1.0.3/docs/future.md000066400000000000000000000012761226611677100156170ustar00rootroot00000000000000# List of future plans. Future Plans ------------ * Replicate CAS unique values. Currently, yrmcds does not replicate CAS values, which may lead to **false-positives** as well as (harmless) false-negatives.

* Allow eviction to be disabled and honor the memory limit. This is a requirement for in-memory database.

* Save/load cached objects at program exit/startup. This is another requirement for in-memory database.

* Save snapshots periodically. This can be done by [forking][fork] a child process when all worker threads are idle. [fork]: http://manpages.ubuntu.com/manpages/precise/en/man2/fork.2.html yrmcds-1.0.3/docs/index.md000066400000000000000000000003141226611677100154040ustar00rootroot00000000000000# Index. yrmcds documents ---------------- yrmcds is an object cache system featuring master-slave replication. # General utilities. # Implementing yrmcds. yrmcds-1.0.3/docs/locking.md000066400000000000000000000115221226611677100157260ustar00rootroot00000000000000# Server-side locking support. Motivation ---------- Locking is one of the things people often want to have with memcached. Though `add` has been used to implement locks at client-side [1][] [2][], this approach is fragile if clients are not very stable; if a client acquires a lock then dies, the lock will never be released. Moreover, objects used to implement locks may be evicted. yrmcds introduces a server-side locking mechanism to resolve this problem. Overview -------- A lock associates an existing object and a client connection. Locked objects will neither be expired nor be evicted. Modification commands such as `set`, `replace`, or `delete` will fail if the object is locked, unless the client is holding the lock. `get` succeeds even though the object is locked by another client. If a client is disconnected, all locks held by the client will be released. Although lock commands are available in both ASCII and binary protocols, use of binary protocol is strongly recommended. ASCII protocol -------------- * `lock KEY\r\n` acquires a server-side lock of the object associated with KEY. The response will be one of: - "OK\r\n", to indicate success. - "LOCKED\r\n", if another client has already locked the object. - "NOT_FOUND\r\n", if the object does not exist. * `unlock KEY\r\n` releases a server-side lock. The response will be one of: - "OK\r\n", to indicate success. - "CLIENT_ERROR MESSAGE\r\n", if the object is not locked by the client, or if the object does not exist. * `unlock_all\r\n` releases all locks held by the client. The response will be "OK\r\n". Additionally, modification commands such as `set` and `delete` will fail when the object is locked by another client. The response will be "LOCKED\r\n" in this case. Binary protocol --------------- The binary protocol is enhanced with new `Lock`, `Unlock`, `UnlockAll`, `LaG` (lock and get), `LaGK` (lock and get with key), and `RaU` (replace and unlock) commands. New opcodes are: - 0x40 (Lock) - 0x41 (LockQ) - 0x42 (Unlock) - 0x43 (UnlockQ) - 0x44 (UnlockAll) - 0x45 (UnlockAllQ) - 0x46 (LaG) - 0x47 (LaGQ) - 0x48 (LaGK) - 0x49 (LaGKQ) - 0x4a (RaU) - 0x4b (RaUQ) New response statuses are: - 0x0010 Locked - 0x0011 Not locked ### Lock, Lock Quietly Request: - MUST NOT have extras. - MUST have key. - MUST NOT have value. Lock an object. The response-packet contains no extra data, and the result of the operation is signaled through the status code. Specifically, 0x0001 (not found) is returned when the object does not exist, and 0x0010 (locked) is returned when the object has been locked by another client. ### Unlock, Unlock Quietly Request: - MUST NOT have extras. - MUST have key. - MUST NOT have value. Unlock a locked object. The response-packet contains no extra data, and the result of the operation is signaled through the status code. Specifically, 0x0001 (not found) is returned when the object does not exist, and 0x0011 (not locked) is returned when the object is not locked by the client. ### Unlock All, Unlock All Quietly Request: - MUST NOT have extras. - MUST NOT have key. - MUST NOT have value. Unlock all locked objects held by the client. The response-packet contains no extra data, and the result of the operation is signaled through the status code. In fact, this command will succeed unconditionally. ### Lock and Get, Lock and Get Quietly, Lock and Get Key, Lock and Get Key Quietly Request: - MAY have extras. - MUST have key. - MUST NOT have value. Response (if found): - MUST have extras. - MAY have key. - MAY have value. Lock an object and get the data atomically. Optional extra data for Lock and Get request is 4 byte expiration time. If extra data exists, the object's expiration time will be renewed. The response is the same as that of `Get` or `GetK` command. Specifically, if the object is locked, the response with status code = 0x0010 (locked) will be returned. `LaGQ` and `LaGKQ` will return the response with status code = 0x0001 (not found) if the object is not found. ### Replace and Unlock, Replace and Unlock Quietly Request: - MUST have extras. - MUST have key. - MUST have value. Replace a locked object then unlock it atomically. The client need to lock the object in advance. The extra data are the same as that of `Replace` binary command, i.e., 4 byte flags and 4 byte expiration time. The response is almost the same as that of `Replace` binary command. Specifically, 0x0011 (not locked) is returned when the object is not locked by the client. [1]: http://www.regexprn.com/2010/05/using-memcached-as-distributed-locking.html [2]: http://russellneufeld.wordpress.com/2012/05/24/using-memcached-as-a-distributed-lock-from-within-django/ yrmcds-1.0.3/docs/usage.md000066400000000000000000000067631226611677100154170ustar00rootroot00000000000000# Configuration and execution of `yrmcdsd`. `yrmcdsd` is the server program of yrmcds. Its built-in configurations are reasonably good to run as a stand-alone server. For replication, you need to specify a virtual IP address and configure a clustering software such as [keepalived][]. Files ----- Usually, yrmcds is installed under `/usr/local`. * /usr/local/etc/yrmcds.conf The default configuration file. * /usr/local/sbin/yrmcdsd The program. Configuration ------------- You can change any of these configuration options through the configuration file: * `user` (Default: none) If set, the program will try to `setuid` to the given user. * `group` (Default: none) If set, the program will try to `setgid` to the given group. * `virtual_ip` (Default: 127.0.0.1) The master's virtual IP address. Both IPv4 and IPv6 addresses are supported. * `port` (Default: 11211) memcache protocol port number. * `repl_port` (Default: 11213) The replication protocol port number. * `max_connections` (Default: 0) Maximum number of client connections. 0 means unlimited. * `temp_dir` (Default: /var/tmp) Directory to store temporary files for large objects. * `log.threshold` (Default: info) Log threshold. Possible values: `error`, `warning`, `info`, `debug`. * `log.file` (Default: standard error) Logs will be written to this file. * `buckets` (Default: 1000000) Hash table size. * `max_data_size` (Default: 1M) The maximum object size. * `heap_data_limit` (Default: 256K) Objects larger than this will be stored in temporary files. * `memory_limit` (Default: 1024M) The amount of memory allowed for yrmcdsd. * `workers` (Default: 8) The number of worker threads. * `gc_interval` (Default: 10) The interval between garbage collections in seconds. Running yrmcdsd --------------- Just run `yrmcdsd`. The program does not run as traditional UNIX daemons. Instead, use [upstart][] or [systemd][] to run `yrmcdsd` as a background service. Do not forget to increase the maximum file descriptors. A sample upstart script is available at [etc/upstart](https://github.com/cybozu/yrmcds/blob/master/etc/upstart). Replication ----------- The replication of yrmcds servers is based on a virtual IP address. To assign a virtual IP address to a live server, a clustering software such as [keepalived][] or [Pacemaker][pacemaker] is used. Our recommendation is to use [keepalived][]. ### keepalived For stability, keepalived should be configured in non-preemptive mode. A sample configuration file is available at [etc/keepalived.conf](https://github.com/cybozu/yrmcds/blob/master/etc/keepalived.conf). ### How replication works `yrmcdsd` **periodically watches** if the server owns the virtual IP address or not. If `yrmcdsd` finds it holds the virtual IP address, that server will promote itself as the new master. All others become slaves. Slaves connect to the master through the virtual IP address. If the connection resets, `yrmcdsd` waits some seconds to see if the virtual IP address is assigned. If assigned, the slave becomes the new master. If not, the slave drops all objects then try to connect to the new master. Since the master node is elected dynamically by [keepalived][], each `yrmcdsd` should have the same configuration parameters. [keepalived]: http://www.keepalived.org/ [pacemaker]: http://clusterlabs.org/wiki/Main_Page [upstart]: http://upstart.ubuntu.com/ [systemd]: http://www.freedesktop.org/wiki/Software/systemd/ yrmcds-1.0.3/etc/000077500000000000000000000000001226611677100136005ustar00rootroot00000000000000yrmcds-1.0.3/etc/keepalived.conf000066400000000000000000000004321226611677100165570ustar00rootroot00000000000000### VRRP Configurations vrrp_instance VI_YRMCDS { # to disable preemption, the initial state must be BACKUP. state BACKUP nopreempt priority 100 interface eth0 virtual_router_id 10 virtual_ipaddress { 10.0.160.2/20 dev eth0 label eth0:1 } } yrmcds-1.0.3/etc/logrotate000066400000000000000000000003241226611677100155220ustar00rootroot00000000000000# place this file as /etc/logrotate.d/yrmcds /var/log/yrmcds.log { notifempty weekly rotate 4 compress missingok create 0644 nobody nogroup postrotate reload yrmcds >/dev/null 2>&1 || true endscript } yrmcds-1.0.3/etc/upstart000066400000000000000000000011621226611677100152250ustar00rootroot00000000000000# Upstart script for yrmcds description "yrmcds" start on net-device-up stop on runlevel [!2345] respawn umask 077 limit nofile 100000 100000 chdir /tmp pre-start script [ -d /var/tmp/yrmcds ] || mkdir /var/tmp/yrmcds chmod 700 /var/tmp/yrmcds chown nobody:nogroup /var/tmp/yrmcds rm -f /var/tmp/yrmcds/* # make sure keepalived init script was disabled. # ex) sudo update-rc.d keepalived disable /etc/init.d/keepalived start sleep 5 end script post-stop script /etc/init.d/keepalived stop # sleep a while to hand-off VIP sleep 5 end script exec /usr/local/sbin/yrmcdsd yrmcds-1.0.3/etc/valgrind.suppress000066400000000000000000000002571226611677100172200ustar00rootroot00000000000000{ reactor::epoll_ctl Memcheck:Param epoll_ctl(event) fun:epoll_ctl fun:_ZN6cybozu7reactor12add_resourceESt10unique_ptrINS_8resourceESt14default_deleteIS2_EEi } yrmcds-1.0.3/etc/yrmcds.conf000066400000000000000000000023171226611677100157530ustar00rootroot00000000000000# Configuration file for yrmcdsd # setuid user user = nobody # setgid group group = nogroup # To become the master, virtual_ip address must be owned. virtual_ip = 127.0.0.1 # memcache protocol port number. port = 11211 # yrmcds replication protocol port number. repl_port = 11213 # max number of client connections. 0 means unlimited. max_connections = 10000 # large objects are saved in this directory as temporary files. temp_dir = "/var/tmp/yrmcds" # possible values: error, warning, info, debug log.threshold = info # logs are recorded to this file. # If log.file is not defined, logs are printed to standard error. log.file = "/var/log/yrmcds.log" # Hash table size. 1 million is the sane default. buckets = 1000000 # The maximum object size. This is a soft-limit. # There is a compile-time hard-limit around 30 MiB. max_data_size = 10M # Objects larger than this will be stored in temporary files. heap_data_limit = 256K # The amount of memory allowed for the entire yrmcds. # This is by no means a hard limit; rather, this is just a hint for # the garbage collection. memory_limit = 1024M # The number of worker threads. workers = 10 # The interval between garbage collections in seconds. gc_interval = 10 yrmcds-1.0.3/src/000077500000000000000000000000001226611677100136145ustar00rootroot00000000000000yrmcds-1.0.3/src/config.cpp000066400000000000000000000114701226611677100155700ustar00rootroot00000000000000// (C) 2013 Cybozu. #include "config.hpp" #include #include #include #include namespace { const char VIRTUAL_IP[] = "virtual_ip"; const char PORT[] = "port"; const char REPL_PORT[] = "repl_port"; const char MAX_CONNECTIONS[] = "max_connections"; const char TEMP_DIR[] = "temp_dir"; const char USER[] = "user"; const char GROUP[] = "group"; const char LOG_THRESHOLD[] = "log.threshold"; const char LOG_FILE[] = "log.file"; const char BUCKETS[] = "buckets"; const char MAX_DATA_SIZE[] = "max_data_size"; const char HEAP_DATA_LIMIT[] = "heap_data_limit"; const char MEMORY_LIMIT[] = "memory_limit"; const char WORKERS[] = "workers"; const char GC_INTERVAL[] = "gc_interval"; std::unordered_map THRESHOLDS { {"error", cybozu::severity::error}, {"warning", cybozu::severity::warning}, {"info", cybozu::severity::info}, {"debug", cybozu::severity::debug} }; inline std::size_t parse_unit(std::string& s, const char* cmd) { std::size_t base = 1; switch( s.back() ) { case 'k': case 'K': base <<= 10; break; case 'm': case 'M': base <<=20; break; case 'g': case 'G': base <<=30; break; } if( base != 1 ) s.pop_back(); int n = std::stoi( s ); if( n < 1 ) throw yrmcds::config::bad_config(cmd + std::string(" must be > 0")); return base * static_cast(n); } } // anonymous namespace namespace yrmcds { void config::load(const std::string& path) { cybozu::config_parser cp(path); if( cp.exists(VIRTUAL_IP) ) m_vip.parse(cp.get(VIRTUAL_IP)); if( cp.exists(PORT) ) { int n = cp.get_as_int(PORT); if( n < 1 || n > 65535 ) throw bad_config("Bad port: " + cp.get(PORT)); m_port = static_cast(n); } if( cp.exists(REPL_PORT) ) { int n = cp.get_as_int(REPL_PORT); if( n < 1 || n > 65535 ) throw bad_config("Bad repl port: " + cp.get(REPL_PORT)); m_repl_port = static_cast(n); } if( cp.exists(MAX_CONNECTIONS) ) { int conns = cp.get_as_int(MAX_CONNECTIONS); if( conns < 0 ) throw bad_config("max_connections must be >= 0"); m_max_connections = conns; } if( cp.exists(TEMP_DIR) ) { m_tempdir = cp.get(TEMP_DIR); if( ! cybozu::is_dir(m_tempdir) ) throw bad_config("Not a directory: " + m_tempdir); if( ! cybozu::is_writable(m_tempdir) ) throw bad_config("Directory not writable: " + m_tempdir); } if( cp.exists(USER) ) { m_user = cp.get(USER); } if( cp.exists(GROUP) ) { m_group = cp.get(GROUP); } if( cp.exists(LOG_THRESHOLD) ) { auto it = THRESHOLDS.find(cp.get(LOG_THRESHOLD)); if( it == THRESHOLDS.end() ) throw bad_config("Invalid threshold: " + cp.get(LOG_THRESHOLD)); m_threshold = it->second; } if( cp.exists(LOG_FILE) ) { m_logfile = cp.get(LOG_FILE); if( m_logfile.size() == 0 || m_logfile[0] != '/' ) throw bad_config("Invalid log file: " + m_logfile); } if( cp.exists(BUCKETS) ) { int buckets = cp.get_as_int(BUCKETS); if( buckets < 1 ) throw bad_config("buckets must be > 0"); if( buckets < 10000 ) cybozu::logger::warning() << "Too small bucket count!"; m_buckets = buckets; } if( cp.exists(MAX_DATA_SIZE) ) { std::string t = cp.get(MAX_DATA_SIZE); if( t.empty() ) throw bad_config("max_data_size must not be empty"); m_max_data_size = parse_unit(t, MAX_DATA_SIZE); } if( cp.exists(HEAP_DATA_LIMIT) ) { std::string t = cp.get(HEAP_DATA_LIMIT); if( t.empty() ) throw bad_config("heap_data_limit must not be empty"); m_heap_data_limit = parse_unit(t, HEAP_DATA_LIMIT); if( m_heap_data_limit < 4096 ) throw bad_config("too small heap_data_limit"); } if( cp.exists(MEMORY_LIMIT) ) { std::string t = cp.get(MEMORY_LIMIT); if( t.empty() ) throw bad_config("memory_limit must not be empty"); m_memory_limit = parse_unit(t, MEMORY_LIMIT); } if( cp.exists(WORKERS) ) { int n = cp.get_as_int(WORKERS); if( n < 1 ) throw bad_config("workers must be > 0"); if( n > MAX_WORKERS ) throw bad_config("workers must be <= " + std::to_string(MAX_WORKERS)); m_workers = n; } if( cp.exists(GC_INTERVAL) ) { int n = cp.get_as_int(GC_INTERVAL); if( n < 1 ) throw bad_config("gc_interval must be > 0"); m_gc_interval = n; } } config g_config; } // namespace yrmcds yrmcds-1.0.3/src/config.hpp000066400000000000000000000060041226611677100155720ustar00rootroot00000000000000// The server configurations. // (C) 2013 Cybozu. #ifndef YRMCDS_CONFIG_HPP #define YRMCDS_CONFIG_HPP #include "constants.hpp" #include #include #include #include namespace yrmcds { // Configurations for yrmcds. // // Configurations for yrmcds. // Defaults are configured properly without calling . class config { public: // Setup default configurations. config(): m_vip("127.0.0.1"), m_tempdir(DEFAULT_TMPDIR) { static_assert( sizeof(std::size_t) >= 4, "std::size_t is too small" ); } struct bad_config: public std::runtime_error { bad_config(const std::string& s): std::runtime_error(s) {} }; // Load configurations from `path`. // // This loads configurations from a file at `path`. // This may throw miscellaneous exceptions. void load(const std::string& path); const cybozu::ip_address& vip() const noexcept { return m_vip; } std::uint16_t port() const noexcept { return m_port; } std::uint16_t repl_port() const noexcept { return m_repl_port; } unsigned int max_connections() const noexcept { return m_max_connections; } const std::string& tempdir() const noexcept { return m_tempdir; } const std::string& user() const noexcept { return m_user; } const std::string& group() const noexcept { return m_group; } cybozu::severity threshold() const noexcept { return m_threshold; } const std::string& logfile() const noexcept { return m_logfile; } unsigned int buckets() const noexcept { return m_buckets; } std::size_t max_data_size() const noexcept { return m_max_data_size; } std::size_t heap_data_limit() const noexcept { return m_heap_data_limit; } std::size_t memory_limit() const noexcept { return m_memory_limit; } unsigned int workers() const noexcept { return m_workers; } unsigned int gc_interval() const noexcept { return m_gc_interval; } void set_heap_data_limit(std::size_t new_limit) noexcept { m_heap_data_limit = new_limit; } private: alignas(CACHELINE_SIZE) cybozu::ip_address m_vip; std::uint16_t m_port = DEFAULT_MEMCACHE_PORT; std::uint16_t m_repl_port = DEFAULT_REPL_PORT; unsigned int m_max_connections = 0; std::string m_tempdir; std::string m_user; std::string m_group; cybozu::severity m_threshold = cybozu::severity::info; std::string m_logfile; unsigned int m_buckets = DEFAULT_BUCKETS; std::size_t m_max_data_size = DEFAULT_MAX_DATA_SIZE; std::size_t m_heap_data_limit = DEFAULT_HEAP_DATA_LIMIT; std::size_t m_memory_limit = DEFAULT_MEMORY_LIMIT; unsigned int m_workers = DEFAULT_WORKER_THREADS; unsigned int m_gc_interval = DEFAULT_GC_INTERVAL; }; // Global configuration object. extern config g_config; } // namespace yrmcds #endif // YRMCDS_CONFIG_HPP yrmcds-1.0.3/src/constants.hpp000066400000000000000000000024361226611677100163460ustar00rootroot00000000000000// Constants used in yrmcds. // (C) 2013 Cybozu. #ifndef YRMCDS_CONSTANTS_HPP #define YRMCDS_CONSTANTS_HPP #include #include namespace yrmcds { const std::uint16_t DEFAULT_MEMCACHE_PORT = 11211; const std::uint16_t DEFAULT_REPL_PORT = 11213; const unsigned int DEFAULT_BUCKETS = 1000000; const std::size_t DEFAULT_MAX_DATA_SIZE = static_cast(1) << 20; const std::size_t DEFAULT_HEAP_DATA_LIMIT= 256 << 10; const std::size_t DEFAULT_MEMORY_LIMIT = static_cast(1) << 30; const int DEFAULT_WORKER_THREADS = 8; const unsigned int DEFAULT_GC_INTERVAL = 10; const char DEFAULT_TMPDIR[] = "/var/tmp"; const std::size_t MAX_KEY_LENGTH = 250; // 250 bytes const int MASTER_CHECKS = 50; // wait 50 * 100ms = 5 seconds const int FLUSH_AGE = 10; const std::size_t MAX_RECVSIZE = 2 << 20; // 2 MiB const std::size_t WORKER_BUFSIZE = 5 << 20; // 5 MiB const int MAX_WORKERS = 64; const std::size_t MAX_REQUEST_LENGTH = 30 << 20; // 30 MiB const int MAX_SLAVES = 5; const int MAX_CONSECUTIVE_GCS = 3; const char VERSION[] = "yrmcds version 1.0.3"; } // namespace yrmcds #endif // YRMCDS_CONSTANTS_HPP yrmcds-1.0.3/src/gc.cpp000066400000000000000000000122421226611677100147120ustar00rootroot00000000000000// (C) 2013 Cybozu. #include "config.hpp" #include "gc.hpp" #include "replication.hpp" #include "stats.hpp" #include #include namespace yrmcds { void gc_thread::run() { if( ! cybozu::has_ip_address(g_config.vip()) ) { cybozu::logger::error() << "VIP has been lost. Exiting quickly..."; std::quick_exit(2); } using namespace std::chrono; auto t1 = steady_clock::now(); gc(); if( ! m_new_slaves.empty() ) cybozu::logger::info() << "Initial replication completed for " << m_new_slaves.size() << " new slave(s)."; g_stats.objects.store(m_objects, std::memory_order_relaxed); g_stats.objects_under_1k.store(m_objects_under_1k, std::memory_order_relaxed); g_stats.objects_under_4k.store(m_objects_under_4k, std::memory_order_relaxed); g_stats.objects_under_16k.store(m_objects_under_16k, std::memory_order_relaxed); g_stats.objects_under_64k.store(m_objects_under_64k, std::memory_order_relaxed); g_stats.objects_under_256k.store(m_objects_under_256k, std::memory_order_relaxed); g_stats.objects_under_1m.store(m_objects_under_1m, std::memory_order_relaxed); g_stats.objects_under_4m.store(m_objects_under_4m, std::memory_order_relaxed); g_stats.objects_huge.store(m_objects_huge, std::memory_order_relaxed); g_stats.used_memory.store(m_used_memory, std::memory_order_relaxed); g_stats.conflicts.store(m_conflicts, std::memory_order_relaxed); g_stats.gc_count.fetch_add(1, std::memory_order_relaxed); g_stats.oldest_age.store(m_oldest_age, std::memory_order_relaxed); g_stats.largest_object_size.store(m_largest_object_size, std::memory_order_relaxed); g_stats.last_expirations.store(m_last_expirations, std::memory_order_relaxed); g_stats.last_evictions.store(m_last_evictions, std::memory_order_relaxed); g_stats.total_evictions.fetch_add(m_last_evictions, std::memory_order_relaxed); auto t2 = steady_clock::now(); std::uint64_t us = static_cast( duration_cast(t2-t1).count() ); g_stats.last_gc_elapsed.store(us, std::memory_order_relaxed); g_stats.total_gc_elapsed.fetch_add(us, std::memory_order_relaxed); cybozu::logger::debug() << "GC end: elapsed=" << us << "us, expired=" << m_last_expirations << ", evicted=" << m_last_evictions << ", survived=" << m_objects; } void gc_thread::gc() { std::time_t t = g_stats.flush_time.load(); bool flush = (t != 0) && (std::time(nullptr) >= t); unsigned int evict_age = 0; if( (! flush) && (g_stats.used_memory.load(std::memory_order_relaxed) > g_config.memory_limit()) ) { unsigned int oldest_age = g_stats.oldest_age.load(std::memory_order_relaxed); unsigned int one_hour = 3600U / g_config.gc_interval() + 1; if( oldest_age < (one_hour * 2) ) { evict_age = std::max(1U, oldest_age / 2); } else { evict_age = oldest_age - one_hour; } cybozu::logger::warning() << "Evicting object of " << evict_age << " gc old"; } auto pred = [this,flush,evict_age](const cybozu::hash_key& k, object& obj) ->bool { if( flush && (! obj.locked()) ) { if( ! m_slaves.empty() ) repl_delete(m_slaves, k); return true; } if( evict_age > 0 && obj.age() >= evict_age && (! obj.locked()) ) { ++ m_last_evictions; if( ! m_slaves.empty() ) repl_delete(m_slaves, k); return true; } if( obj.expired() ) { ++ m_last_expirations; if( ! m_slaves.empty() ) repl_delete(m_slaves, k); return true; } obj.survive(m_flushers); if( ++m_objects_in_bucket == 2 ) ++ m_conflicts; ++ m_objects; std::size_t size = obj.size(); if( size < 1024 ) { ++ m_objects_under_1k; } else if( size < 4096 ) { ++ m_objects_under_4k; } else if( size < (16 <<10) ) { ++ m_objects_under_16k; } else if( size < (64 <<10) ) { ++ m_objects_under_64k; } else if( size < (256 <<10) ) { ++ m_objects_under_256k; } else if( size < (1 <<20) ) { ++ m_objects_under_1m; } else if( size < (4 <<20) ) { ++ m_objects_under_4m; } else { ++ m_objects_huge; } m_used_memory += sizeof(cybozu::hash_key) + k.length() + sizeof(object); if( g_config.heap_data_limit() >= obj.size() ) m_used_memory += obj.size(); m_oldest_age = std::max(m_oldest_age, obj.age()); m_largest_object_size = std::max(m_largest_object_size, obj.size()); if( ! m_new_slaves.empty() ) repl_object(m_new_slaves, k, obj, false); return false; }; for( auto it = m_hash.begin(); it != m_hash.end(); ++it ) { m_objects_in_bucket = 0; it->gc(pred); m_flushers.clear(); } if( flush ) g_stats.flush_time.store(0); } } // namespace yrmcds yrmcds-1.0.3/src/gc.hpp000066400000000000000000000037461226611677100147300ustar00rootroot00000000000000// Garbage object collection. // (C) 2013 Cybozu. #ifndef YRMCDS_GC_HPP #define YRMCDS_GC_HPP #include "object.hpp" #include "stats.hpp" #include #include #include #include #include #include #include #include #include namespace yrmcds { class gc_thread final: public cybozu::thread_base { public: gc_thread(cybozu::hash_map& m, const std::vector& slaves, const std::vector& new_slaves): m_hash(m), m_slaves(slaves), m_new_slaves(new_slaves) { for( cybozu::tcp_socket* s: new_slaves ) { if( std::find(slaves.begin(), slaves.end(), s) == slaves.end() ) m_slaves.push_back(s); } m_flushers.reserve(10); } gc_thread(const gc_thread&) = delete; gc_thread& operator=(const gc_thread&) = delete; gc_thread(gc_thread&&) = delete; gc_thread& operator=(gc_thread&&) = delete; ~gc_thread() { m_thread.join(); } void run(); private: void gc(); cybozu::hash_map& m_hash; std::vector m_slaves; std::vector m_new_slaves; std::uint32_t m_objects = 0; std::uint32_t m_objects_under_1k = 0; std::uint32_t m_objects_under_4k = 0; std::uint32_t m_objects_under_16k = 0; std::uint32_t m_objects_under_64k = 0; std::uint32_t m_objects_under_256k = 0; std::uint32_t m_objects_under_1m = 0; std::uint32_t m_objects_under_4m = 0; std::uint32_t m_objects_huge = 0; std::size_t m_used_memory = 0; std::uint32_t m_conflicts = 0; std::uint32_t m_oldest_age = 0; std::size_t m_largest_object_size = 0; std::uint32_t m_last_expirations = 0; std::uint32_t m_last_evictions = 0; int m_objects_in_bucket = 0; std::vector m_flushers; }; } // namespace yrmcds #endif // YRMCDS_GC_HPP yrmcds-1.0.3/src/main.cpp000066400000000000000000000104141226611677100152440ustar00rootroot00000000000000// The entry point of yrmcdsd. // (C) 2013 Cybozu. #include "config.hpp" #include "constants.hpp" #include "server.hpp" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #define QUOTE(str) #str #define EXPAND_AND_QUOTE(str) QUOTE(str) namespace { #include "../COPYING.hpp" void print_help() { std::cout << "Usage: yrmcdsd [-v] [-h] [-f FILE]" << std::endl; } void print_version() { std::cout << yrmcds::VERSION << std::endl << std::endl << COPYING; } void seed_siphash() { union { char key[16]; std::uint64_t ikey[2]; } k; std::random_device rd; std::uniform_int_distribution dis; k.ikey[0] = dis(rd); k.ikey[1] = dis(rd); cybozu::siphash24_seed(k.key); } bool load_config(const std::vector& args) { auto it = std::find(args.begin(), args.end(), "-f"); if( it != args.end() ) { ++it; if( it == args.end() ) { std::cerr << "missing filename after -f" << std::endl; return false; } yrmcds::g_config.load(*it); } else { std::string config_path = EXPAND_AND_QUOTE(DEFAULT_CONFIG); struct stat st; if( cybozu::get_stat(config_path, st) ) yrmcds::g_config.load( config_path ); } return true; } } // anonymous namespace int main(int argc, char** argv) { std::vector args; for( int i = 1; i < argc; ++i ) args.push_back(argv[i]); if( std::find(args.begin(), args.end(), "-h") != args.end() ) { print_help(); return 0; } if( std::find(args.begin(), args.end(), "-v") != args.end() ) { print_version(); return 0; } using cybozu::logger; seed_siphash(); try { if( ! load_config(args) ) return 1; // first, set group id (as root) const std::string& g = yrmcds::g_config.group(); if( ! g.empty() ) { if( getuid() != 0 ) { std::cerr << "Ignored group configuration." << std::endl; } else { struct group* grp = getgrnam(g.c_str()); if( grp == nullptr ) { std::cerr << "No such group: " << g << std::endl; return 1; } if( setgid(grp->gr_gid) == -1 ) { std::cerr << "Failed to set group id!" << std::endl; return 1; } } } const std::string& u = yrmcds::g_config.user(); if( ! u.empty() ) { if( getuid() != 0 ) { std::cerr << "Ignored user configuration." << std::endl; } else { struct passwd* p = getpwnam(u.c_str()); if( p == nullptr ) { std::cerr << "No such user: " << u << std::endl; return 1; } if( setuid(p->pw_uid) == -1 ) { std::cerr << "Failed to set user id!" << std::endl; return 1; } } } // make it core dumpable if( prctl(PR_SET_DUMPABLE, 1) == -1 ) { std::cerr << "WARNING: failed to enable core dump!" << std::endl; } // setup logger logger::set_threshold(yrmcds::g_config.threshold()); if( ! yrmcds::g_config.logfile().empty() ) logger::instance().open(yrmcds::g_config.logfile()); yrmcds::server().serve(); } catch( const std::system_error& e ) { cybozu::demangler t( typeid(e).name() ); logger::error() << "[" << t.name() << "] (" << e.code() << ") " << e.what(); std::cerr << "Exception [" << t.name() << "] (" << e.code() << ") " << e.what() << std::endl; return 1; } catch( const std::exception& e ) { cybozu::demangler t( typeid(e).name() ); logger::error() << "[" << t.name() << "] " << e.what(); std::cerr << "Exception [" << t.name() << "] " << e.what() << std::endl; return 1; } return 0; } yrmcds-1.0.3/src/memcache.cpp000066400000000000000000001167121226611677100160720ustar00rootroot00000000000000// (C) 2013 Cybozu. #include "config.hpp" #include "memcache.hpp" #include "stats.hpp" #include #include #include #include #include #include #include #include #include #include namespace { using namespace yrmcds; enum std::memory_order relaxed = std::memory_order_relaxed; const char CRLF[] = "\x0d\x0a"; const char CR = '\x0d'; const char LF = '\x0a'; const char SP = '\x20'; const char VALUE[] = "VALUE "; const char STATUS_NOT_FOUND[] = "Not found"; const char STATUS_EXISTS[] = "Exists"; const char STATUS_TOO_LARGE[] = "Too large value"; const char STATUS_INVALID[] = "Invalid request"; const char STATUS_NOT_STORED[] = "Not stored"; const char STATUS_NON_NUMERIC[] = "Non-numeric value"; const char STATUS_LOCKED[] = "Locked"; const char STATUS_NOT_LOCKED[] = "Not locked"; const char STATUS_UNKNOWN[] = "Unknown command"; const char STATUS_OOM[] = "Out of memory"; const std::time_t EXPTIME_THRESHOLD = 60*60*24*30; const std::size_t BINARY_HEADER_SIZE = 24; inline const char* cfind(const char* p, char c, std::size_t len) { return (const char*)std::memchr(p, c, len); } template inline UInt to_uint(const char* p, bool& result) { result = false; char* end; unsigned long long i = strtoull(p, &end, 10); if( i == 0 && p == end ) return 0; if( i == std::numeric_limits::max() && errno == ERANGE ) return 0; char c = *end; if( c != CR && c != LF && c != SP ) return 0; if( i > std::numeric_limits::max() ) return 0; result = true; return static_cast(i); } inline std::time_t binary_exptime(const char* p) noexcept { std::uint32_t t; cybozu::ntoh(p, t); if( t == 0 ) return 0; if( t == 0xffffffffUL ) return memcache::binary_request::EXPTIME_NONE; if( t > EXPTIME_THRESHOLD ) return t; return g_stats.current_time.load(relaxed) + t; } } // anonymous namespace namespace yrmcds { namespace memcache { constexpr item text_request::eos; inline void text_request::parse_flushall(const char* b, const char* e) noexcept { m_exptime = g_stats.current_time.load(relaxed); while( *b == SP ) ++b; if( b == e ) { m_valid = true; return; } if( '0' <= *b && *b <= '9' ) { const char* exp_end = cfind(b, SP, e-b); if( exp_end == nullptr ) exp_end = e; bool result; std::time_t t = to_uint(b, result); if( ! result ) return; m_exptime = (t > EXPTIME_THRESHOLD) ? t : (g_stats.current_time.load(relaxed) + t); b = exp_end; } while( *b == SP ) ++b; if( b != e ) { // noreply ? const char* we = cfind(b, SP, e-b); std::size_t noreply_len = (we != nullptr) ? (we - b) : (e - b); if( noreply_len != 7 ) return; if( std::memcmp(b, "noreply", 7) != 0 ) return; m_no_reply = true; b = (we != nullptr) ? we : e; } while( *b == SP ) ++b; if( b != e ) return; // garbage left m_valid = true; } inline void text_request::parse_verbosity(const char* b, const char* e) noexcept { while( *b == SP ) ++b; if( b == e ) return; const char* vb_end = cfind(b, SP, e-b); if( vb_end == nullptr ) vb_end = e; std::size_t len = vb_end - b; if( len == 0 ) return; // invalid if( len == 5 ) { if( std::memcmp("error", b, 5) == 0 ) { m_verbosity = cybozu::severity::error; } else if( std::memcmp("debug", b, 5) == 0 ) { m_verbosity = cybozu::severity::debug; } } if( len == 7 && std::memcmp("warning", b, 7) == 0 ) m_verbosity = cybozu::severity::warning; if( len == 4 && std::memcmp("info", b, 4) == 0 ) m_verbosity = cybozu::severity::info; b = vb_end; while( *b == SP ) ++b; if( b != e ) { // noreply ? const char* we = cfind(b, SP, e-b); std::size_t noreply_len = (we != nullptr) ? (we - b) : (e - b); if( noreply_len != 7 ) return; if( std::memcmp(b, "noreply", 7) != 0 ) return; m_no_reply = true; b = (we != nullptr) ? we : e; } while( *b == SP ) ++b; if( b != e ) return; // garbage left m_valid = true; } inline void text_request::parse_storage(const char* b, const char* e, bool is_cas) noexcept { while( *b == SP ) ++b; if( b == e ) return; const char* key_end = cfind(b, SP, e-b); if( key_end == nullptr ) return; m_key = item(b, key_end-b); b = key_end; while( *b == SP ) ++b; if( b == e ) return; const char* flags_end = cfind(b, SP, e-b); if( flags_end == nullptr ) return; bool result; m_flags = to_uint(b, result); if( ! result ) return; b = flags_end; while( *b == SP ) ++b; if( b == e ) return; const char* exptime_end = cfind(b, SP, e-b); if( exptime_end == nullptr ) return; std::time_t t = to_uint(b, result); if( ! result ) return; if( t != 0 ) m_exptime = (t > EXPTIME_THRESHOLD) ? t : (g_stats.current_time.load(relaxed) + t); b = exptime_end; while( *b == SP ) ++b; if( b == e ) return; std::uint32_t nbytes = to_uint(b, result); if( ! result ) return; const char* bytes_end = cfind(b, SP, e-b); b = (bytes_end == nullptr) ? e : bytes_end; if( is_cas ) { while( *b == SP ) ++b; if( b == e ) return; m_cas_unique = to_uint(b, result); if( ! result ) return; const char* cas_end = cfind(b, SP, e-b); b = (cas_end == nullptr) ? e : cas_end; } while( *b == SP ) ++b; if( b != e ) { // noreply ? const char* we = cfind(b, SP, e-b); std::size_t noreply_len = (we != nullptr) ? (we - b) : (e - b); if( noreply_len != 7 ) return; if( std::memcmp(b, "noreply", 7) != 0 ) return; m_no_reply = true; b = (we != nullptr) ? we : e; } while( *b == SP ) ++b; if( b != e ) return; // garbage left // here, parsing was successful. check data length. b += 2; // CRLF m_request_len = (b - m_p) + nbytes + 2; if( m_len < m_request_len ) { m_request_len = 0; return; } // check CRLF following the data if( *(b+nbytes) != CR || *(b+nbytes+1) != LF ) return; // passed all checks. m_data = item(b, nbytes); m_valid = true; } inline void text_request::parse_delete(const char* b, const char* e) noexcept { while( *b == SP ) ++b; if( b == e ) return; const char* key_end = cfind(b, SP, e-b); if( key_end == nullptr ) key_end = e; m_key = item(b, key_end-b); b = key_end; while( *b == SP ) ++b; if( b != e ) { // noreply ? const char* we = cfind(b, SP, e-b); std::size_t noreply_len = (we != nullptr) ? (we - b) : (e - b); if( noreply_len != 7 ) return; if( std::memcmp(b, "noreply", 7) != 0 ) return; m_no_reply = true; b = (we != nullptr) ? we : e; } while( *b == SP ) ++b; if( b != e ) return; // garbage left m_valid = true; } inline void text_request::parse_touch(const char* b, const char* e) noexcept { while( *b == SP ) ++b; if( b == e ) return; const char* key_end = cfind(b, SP, e-b); if( key_end == nullptr ) return; m_key = item(b, key_end-b); b = key_end; while( *b == SP ) ++b; if( b == e ) return; const char* exptime_end = cfind(b, SP, e-b); if( exptime_end == nullptr ) exptime_end = e; bool result; std::time_t t = to_uint(b, result); if( ! result ) return; if( t != 0 ) m_exptime = (t > EXPTIME_THRESHOLD) ? t : (g_stats.current_time.load(relaxed) + t); b = exptime_end; while( *b == SP ) ++b; if( b != e ) { // noreply ? const char* we = cfind(b, SP, e-b); std::size_t noreply_len = (we != nullptr) ? (we - b) : (e - b); if( noreply_len != 7 ) return; if( std::memcmp(b, "noreply", 7) != 0 ) return; m_no_reply = true; b = (we != nullptr) ? we : e; } while( *b == SP ) ++b; if( b != e ) return; // garbage left m_valid = true; } inline void text_request::parse_lock(const char* b, const char* e) noexcept { while( *b == SP ) ++b; if( b == e ) return; const char* key_end = cfind(b, SP, e-b); if( key_end == nullptr ) key_end = e; m_key = item(b, key_end-b); b = key_end; while( *b == SP ) ++b; if( b != e ) return; // garbage left m_valid = true; } inline void text_request::parse_unlock(const char* b, const char* e) noexcept { while( *b == SP ) ++b; if( b == e ) return; const char* key_end = cfind(b, SP, e-b); if( key_end == nullptr ) key_end = e; m_key = item(b, key_end-b); b = key_end; while( *b == SP ) ++b; if( b != e ) return; // garbage left m_valid = true; } inline void text_request::parse_stats(const char* b, const char* e) noexcept { while( *b == SP ) ++b; if( b == e ) { m_valid = true; return; } const char* value_end = cfind(b, SP, e-b); if( value_end == nullptr ) value_end = e; std::size_t value_len = value_end - b; if( value_len == 3 && std::memcmp(b, "ops", 3) == 0 ) { m_stats = stats_t::OPS; m_valid = true; return; } if( value_len == 5 ) { if( std::memcmp(b, "items", 5) == 0 ) { m_stats = stats_t::ITEMS; m_valid = true; return; } if( std::memcmp(b, "sizes", 5) == 0 ) { m_stats = stats_t::SIZES; m_valid = true; return; } } if( value_len == 8 && std::memcmp(b, "settings", 8) == 0 ) { m_stats = stats_t::SETTINGS; m_valid = true; return; } } inline void text_request::parse_incdec(const char* b, const char* e) noexcept { while( *b == SP ) ++b; if( b == e ) return; const char* key_end = cfind(b, SP, e-b); if( key_end == nullptr ) return; m_key = item(b, key_end-b); b = key_end; while( *b == SP ) ++b; if( b == e ) return; const char* value_end = cfind(b, SP, e-b); if( value_end == nullptr ) value_end = e; bool result; m_value = to_uint(b, result); if( ! result ) return; b = value_end; while( *b == SP ) ++b; if( b != e ) { // noreply ? const char* we = cfind(b, SP, e-b); std::size_t noreply_len = (we != nullptr) ? (we - b) : (e - b); if( noreply_len != 7 ) return; if( std::memcmp(b, "noreply", 7) != 0 ) return; m_no_reply = true; b = (we != nullptr) ? we : e; } while( *b == SP ) ++b; if( b != e ) return; // garbage left m_valid = true; } inline void text_request::parse_get(const char* b, const char* e) noexcept { while( *b == SP ) ++b; if( b == e ) return; const char* key_end = cfind(b, SP, e-b); if( key_end == nullptr ) key_end = e; m_key = item(b, key_end-b); m_valid = true; } void text_request::parse() noexcept { const char* eol = cfind(m_p, LF, m_len); if( eol == nullptr ) return; // incomplete m_request_len = eol - m_p + 1; if( m_request_len == 1 ) return; // invalid if( *(--eol) != CR ) return; const char* cmd_start = m_p; while( *cmd_start == SP ) ++cmd_start; const char* cmd_end = cfind(cmd_start, SP, (eol - cmd_start)); if( cmd_end == nullptr ) cmd_end = eol; std::size_t cmd_len = (cmd_end - cmd_start); if( cmd_len == 10 ) { if( std::memcmp(cmd_start, "unlock_all", 10) == 0 ) { m_command = text_command::UNLOCK_ALL; m_valid = true; return; } } if( cmd_len == 9 ) { if( std::memcmp(cmd_start, "flush_all", 9) == 0 ) { m_command = text_command::FLUSH_ALL; parse_flushall(cmd_end, eol); return; } if( std::memcmp(cmd_start, "verbosity", 9) == 0 ) { m_command = text_command::VERBOSITY; parse_verbosity(cmd_end, eol); return; } } if( cmd_len == 7 ) { if( std::memcmp(cmd_start, "version", 7) == 0 ) { m_command = text_command::VERSION; m_valid = true; return; } if( std::memcmp(cmd_start, "prepend", 7) == 0 ) { m_command = text_command::PREPEND; parse_storage(cmd_end, eol, false); return; } if( std::memcmp(cmd_start, "replace", 7) == 0 ) { m_command = text_command::REPLACE; parse_storage(cmd_end, eol, false); return; } } if( cmd_len == 6 ) { if( std::memcmp(cmd_start, "append", 6) == 0 ) { m_command = text_command::APPEND; parse_storage(cmd_end, eol, false); return; } if( std::memcmp(cmd_start, "delete", 6) == 0 ) { m_command = text_command::DELETE; parse_delete(cmd_end, eol); return; } if( std::memcmp(cmd_start, "unlock", 6) == 0 ) { m_command = text_command::UNLOCK; parse_unlock(cmd_end, eol); return; } } if( cmd_len == 5 ) { if( std::memcmp(cmd_start, "touch", 5) == 0 ) { m_command = text_command::TOUCH; parse_touch(cmd_end, eol); return; } if( std::memcmp(cmd_start, "slabs", 5) == 0 ) { m_command = text_command::SLABS; m_valid = true; // ignore. return; } if( std::memcmp(cmd_start, "stats", 5) == 0 ) { m_command = text_command::STATS; parse_stats(cmd_end, eol); return; } } if( cmd_len == 4 ) { if( std::memcmp(cmd_start, "gets", 4) == 0 ) { m_command = text_command::GETS; parse_get(cmd_end, eol); return; } if( std::memcmp(cmd_start, "incr", 4) == 0 ) { m_command = text_command::INCR; parse_incdec(cmd_end, eol); return; } if( std::memcmp(cmd_start, "decr", 4) == 0 ) { m_command = text_command::DECR; parse_incdec(cmd_end, eol); return; } if( std::memcmp(cmd_start, "lock", 4) == 0 ) { m_command = text_command::LOCK; parse_lock(cmd_end, eol); return; } if( std::memcmp(cmd_start, "quit", 4) == 0 ) { m_command = text_command::QUIT; m_valid = true; return; } } if( cmd_len == 3 ) { if( std::memcmp(cmd_start, "set", 3) == 0 ) { m_command = text_command::SET; parse_storage(cmd_end, eol, false); return; } if( std::memcmp(cmd_start, "add", 3) == 0 ) { m_command = text_command::ADD; parse_storage(cmd_end, eol, false); return; } if( std::memcmp(cmd_start, "get", 3) == 0 ) { m_command = text_command::GET; parse_get(cmd_end, eol); return; } if( std::memcmp(cmd_start, "cas", 3) == 0 ) { m_command = text_command::CAS; parse_storage(cmd_end, eol, true); return; } } } void text_response::value(const cybozu::hash_key& key, std::uint32_t flags, const cybozu::dynbuf& data) { if( key.length() > MAX_KEY_LENGTH ) throw std::logic_error("MAX_KEY_LENGTH over bug"); m_iov[0] = {VALUE, sizeof(VALUE) - 1}; m_iov[1] = {key.data(), key.length()}; char buf[MAX_KEY_LENGTH + 100]; static_assert( sizeof(unsigned int) >= sizeof(std::uint32_t), "unsigned int is smaller than std::uint32_t" ); int length = snprintf(buf, sizeof(buf), " %u %llu\x0d\x0a", (unsigned int)flags, (long long unsigned int)data.size()); m_iov[2] = {buf, (std::size_t)length}; m_iov[3] = {data.data(), data.size()}; m_iov[4] = {CRLF, sizeof(CRLF) - 1}; m_socket.sendv(m_iov, 5, false); } void text_response::value(const cybozu::hash_key& key, std::uint32_t flags, const cybozu::dynbuf& data, std::uint64_t cas) { if( key.length() > MAX_KEY_LENGTH ) throw std::logic_error("MAX_KEY_LENGTH over bug"); m_iov[0] = {VALUE, sizeof(VALUE) - 1}; m_iov[1] = {key.data(), key.length()}; char buf[MAX_KEY_LENGTH + 100]; static_assert( sizeof(unsigned int) >= sizeof(std::uint32_t), "unsigned int is smaller than std::uint32_t" ); int length = snprintf(buf, sizeof(buf), " %u %llu %llu\x0d\x0a", (unsigned int)flags, (long long unsigned int)data.size(), (long long unsigned int)cas); m_iov[2] = {buf, (std::size_t)length}; m_iov[3] = {data.data(), data.size()}; m_iov[4] = {CRLF, sizeof(CRLF) - 1}; m_socket.sendv(m_iov, 5, false); } void text_response::stats_settings() { std::ostringstream os; os << "STAT maxbytes " << g_config.memory_limit() << CRLF; os << "STAT tcpport " << g_config.port() << CRLF; os << "STAT replport " << g_config.repl_port() << CRLF; os << "STAT virtual_ip " << g_config.vip().str() << CRLF; os << "STAT evictions on" << CRLF; os << "STAT cas_enabled on" << CRLF; os << "STAT locking on" << CRLF; os << "STAT tmp_dir " << g_config.tempdir() << CRLF; os << "STAT buckets " << g_config.buckets() << CRLF; os << "STAT item_size_max " << g_config.max_data_size() << CRLF; os << "STAT num_threads " << g_config.workers() << CRLF; os << "STAT gc_interval " << g_config.gc_interval() << CRLF; std::string s = os.str(); m_socket.send(s.data(), s.size()); } void text_response::stats_items() { std::ostringstream os; os << "STAT items:1:number " << g_stats.objects.load(relaxed) << CRLF; os << "STAT items:1:age " << g_stats.oldest_age.load(relaxed) << CRLF; os << "STAT items:1:evicted " << g_stats.total_evictions.load(relaxed) << CRLF; os << "STAT items:1:conflicts " << g_stats.conflicts.load(relaxed) << CRLF; os << "STAT items:1:largest " << g_stats.largest_object_size.load(relaxed) << CRLF; std::string s = os.str(); m_socket.send(s.data(), s.size()); } void text_response::stats_sizes() { std::ostringstream os; os << "STAT 1024 " << g_stats.objects_under_1k.load(relaxed) << CRLF; os << "STAT 4096 " << g_stats.objects_under_4k.load(relaxed) << CRLF; os << "STAT 16384 " << g_stats.objects_under_16k.load(relaxed) << CRLF; os << "STAT 65536 " << g_stats.objects_under_64k.load(relaxed) << CRLF; os << "STAT 262144 " << g_stats.objects_under_256k.load(relaxed) << CRLF; os << "STAT 1048576 " << g_stats.objects_under_1m.load(relaxed) << CRLF; os << "STAT 4194304 " << g_stats.objects_under_4m.load(relaxed) << CRLF; os << "STAT huge " << g_stats.objects_huge.load(relaxed) << CRLF; std::string s = os.str(); m_socket.send(s.data(), s.size()); } void text_response::stats_ops() { std::ostringstream os; #define SEND_TEXT_OPS(n,i) \ os << "STAT text:" n " " \ << g_stats.text_ops[(std::size_t)text_command::i].load(relaxed) << CRLF; SEND_TEXT_OPS("set", SET); SEND_TEXT_OPS("add", ADD); SEND_TEXT_OPS("replace", REPLACE); SEND_TEXT_OPS("append", APPEND); SEND_TEXT_OPS("prepend", PREPEND); SEND_TEXT_OPS("cas", CAS); SEND_TEXT_OPS("get", GET); SEND_TEXT_OPS("gets", GETS); SEND_TEXT_OPS("delete", DELETE); SEND_TEXT_OPS("incr", INCR); SEND_TEXT_OPS("decr", DECR); SEND_TEXT_OPS("touch", TOUCH); SEND_TEXT_OPS("lock", LOCK); SEND_TEXT_OPS("unlock", UNLOCK); SEND_TEXT_OPS("unlock_all", UNLOCK_ALL); SEND_TEXT_OPS("slabs", SLABS); SEND_TEXT_OPS("stats", STATS); SEND_TEXT_OPS("flush_all", FLUSH_ALL); SEND_TEXT_OPS("version", VERSION); SEND_TEXT_OPS("verbosity", VERBOSITY); SEND_TEXT_OPS("quit", QUIT); #undef SEND_TEXT_OPS #define SEND_BINARY_OPS(n) \ os << "STAT binary:" #n " " \ << g_stats.bin_ops[(std::size_t)binary_command::n].load(relaxed) << CRLF; SEND_BINARY_OPS(Get); SEND_BINARY_OPS(Set); SEND_BINARY_OPS(Add); SEND_BINARY_OPS(Replace); SEND_BINARY_OPS(Delete); SEND_BINARY_OPS(Increment); SEND_BINARY_OPS(Decrement); SEND_BINARY_OPS(Quit); SEND_BINARY_OPS(Flush); SEND_BINARY_OPS(GetQ); SEND_BINARY_OPS(Noop); SEND_BINARY_OPS(Version); SEND_BINARY_OPS(GetK); SEND_BINARY_OPS(GetKQ); SEND_BINARY_OPS(Append); SEND_BINARY_OPS(Prepend); SEND_BINARY_OPS(Stat); SEND_BINARY_OPS(SetQ); SEND_BINARY_OPS(AddQ); SEND_BINARY_OPS(ReplaceQ); SEND_BINARY_OPS(DeleteQ); SEND_BINARY_OPS(IncrementQ); SEND_BINARY_OPS(DecrementQ); SEND_BINARY_OPS(QuitQ); SEND_BINARY_OPS(FlushQ); SEND_BINARY_OPS(AppendQ); SEND_BINARY_OPS(PrependQ); SEND_BINARY_OPS(Touch); SEND_BINARY_OPS(GaT); SEND_BINARY_OPS(GaTQ); SEND_BINARY_OPS(GaTK); SEND_BINARY_OPS(GaTKQ); SEND_BINARY_OPS(Lock); SEND_BINARY_OPS(LockQ); SEND_BINARY_OPS(Unlock); SEND_BINARY_OPS(UnlockQ); SEND_BINARY_OPS(UnlockAll); SEND_BINARY_OPS(UnlockAllQ); SEND_BINARY_OPS(LaG); SEND_BINARY_OPS(LaGQ); SEND_BINARY_OPS(LaGK); SEND_BINARY_OPS(LaGKQ); SEND_BINARY_OPS(RaU); SEND_BINARY_OPS(RaUQ); #undef SEND_BINARY_OPS std::string s = os.str(); m_socket.send(s.data(), s.size()); } void text_response::stats_general(std::size_t n_slaves) { struct rusage ru; if( ::getrusage(RUSAGE_SELF, &ru) == -1 ) cybozu::throw_unix_error(errno, "getrusage"); std::ostringstream os; os << "STAT pid " << ::getpid() << CRLF; os << "STAT time " << g_stats.current_time.load(relaxed) << CRLF; os << "STAT version " << VERSION << CRLF; os << "STAT pointer_size " << sizeof(void*)*8 << CRLF; os << "STAT rusage_user " << ru.ru_utime.tv_sec << ":" << ru.ru_utime.tv_usec << CRLF; os << "STAT rusage_system " << ru.ru_stime.tv_sec << ":" << ru.ru_stime.tv_usec << CRLF; os << "STAT curr_connections " << g_stats.curr_connections.load(relaxed) << CRLF; os << "STAT total_connections " << g_stats.total_connections.load(relaxed) << CRLF; os << "STAT curr_items " << g_stats.objects.load(relaxed) << CRLF; os << "STAT total_items " << g_stats.total_objects.load(relaxed) << CRLF; os << "STAT bytes " << g_stats.used_memory.load(relaxed) << CRLF; os << "STAT limit_maxbytes " << g_config.memory_limit() << CRLF; os << "STAT threads " << g_config.workers() << CRLF; os << "STAT gc_count " << g_stats.gc_count.load(relaxed) << CRLF; os << "STAT slaves " << n_slaves << CRLF; os << "STAT last_expirations " << g_stats.last_expirations.load(relaxed) << CRLF; os << "STAT last_evictions " << g_stats.last_evictions.load(relaxed) << CRLF; os << "STAT evictions " << g_stats.total_evictions.load(relaxed) << CRLF; os << "STAT last_gc_elapsed " << g_stats.last_gc_elapsed.load(relaxed) << CRLF; os << "STAT total_gc_elapsed " << g_stats.total_gc_elapsed.load(relaxed) << CRLF; std::string s = os.str(); m_socket.send(s.data(), s.size()); } void text_response::version() { m_iov[0] = {TEXT_VERSION, sizeof(TEXT_VERSION) -1}; m_iov[1] = {VERSION, sizeof(VERSION) - 1}; m_iov[2] = {CRLF, sizeof(CRLF) - 1}; m_socket.sendv(m_iov, 3, true); } /* * Binary protocol. */ void binary_request::parse() noexcept { if( m_len < BINARY_HEADER_SIZE ) return; // incomplete std::uint32_t total_len; cybozu::ntoh(m_p + 8, total_len); if( m_len < (BINARY_HEADER_SIZE + total_len) ) return; // incomplete m_request_len = BINARY_HEADER_SIZE + total_len; // Opcode parsing m_command = (binary_command)*(const unsigned char*)(m_p+1); m_quiet = ( m_command == binary_command::GetQ || m_command == binary_command::GetKQ || (binary_command::SetQ <= m_command && m_command <= binary_command::PrependQ) || m_command == binary_command::GaTQ || m_command == binary_command::GaTKQ || (binary_command::LockQ <= m_command && m_command <= binary_command::RaUQ && ((unsigned char)m_command) & 1) ); std::uint16_t key_len; cybozu::ntoh(m_p + 2, key_len); if( key_len > MAX_KEY_LENGTH ) return; // invalid std::uint8_t extras_len = *(const unsigned char*)(m_p + 4); if( total_len < (key_len + extras_len) ) return; // invalid if( key_len > 0 ) m_key = item(m_p + (BINARY_HEADER_SIZE + extras_len), key_len); cybozu::ntoh(m_p + 16, m_cas_unique); std::size_t data_len = total_len - key_len - extras_len; if( data_len > MAX_REQUEST_LENGTH ) { m_status = binary_status::TooLargeValue; return; } if( data_len > 0 ) m_data = item(m_p + (BINARY_HEADER_SIZE + extras_len + key_len), data_len); const char* const p_extra = m_p + BINARY_HEADER_SIZE; switch( m_command ) { case binary_command::Get: case binary_command::GetQ: case binary_command::GetK: case binary_command::GetKQ: case binary_command::GaT: case binary_command::GaTQ: case binary_command::GaTK: case binary_command::GaTKQ: case binary_command::LaG: case binary_command::LaGQ: case binary_command::LaGK: case binary_command::LaGKQ: if( extras_len != 0 && extras_len != 4 ) return; // invalid if( key_len == 0 || data_len > 0 ) return; // invalid if( extras_len == 4 ) { m_exptime = binary_exptime(p_extra); } else { m_exptime = EXPTIME_NONE; } break; case binary_command::Set: case binary_command::SetQ: case binary_command::Add: case binary_command::AddQ: case binary_command::Replace: case binary_command::ReplaceQ: case binary_command::RaU: case binary_command::RaUQ: if( extras_len != 8 || key_len == 0 || data_len == 0 ) return; // invalid cybozu::ntoh(p_extra, m_flags); m_exptime = binary_exptime(p_extra + 4); break; case binary_command::Delete: case binary_command::DeleteQ: if( extras_len != 0 || key_len == 0 || data_len > 0 ) return; // invalid break; case binary_command::Increment: case binary_command::IncrementQ: case binary_command::Decrement: case binary_command::DecrementQ: if( extras_len != 20 || key_len == 0 || data_len > 0 ) return; // invalid cybozu::ntoh(p_extra, m_value); cybozu::ntoh(p_extra + sizeof(std::uint64_t), m_initial); m_exptime = binary_exptime(p_extra + sizeof(std::uint64_t)*2); break; case binary_command::Touch: if( key_len == 0 || extras_len != 4 || data_len != 0 ) return; // invalid m_exptime = binary_exptime(p_extra); break; case binary_command::Flush: case binary_command::FlushQ: if( extras_len != 0 && extras_len != 4 ) return; // invalid if( key_len != 0 || data_len > 0 ) return; // invalid if( extras_len == 4 ) { // yrmcds extension m_exptime = binary_exptime(p_extra); } else { m_exptime = g_stats.current_time.load(relaxed); } break; case binary_command::Append: case binary_command::AppendQ: case binary_command::Prepend: case binary_command::PrependQ: if( extras_len != 0 || key_len == 0 || data_len == 0 ) return; // invalid break; case binary_command::Lock: case binary_command::LockQ: case binary_command::Unlock: case binary_command::UnlockQ: if( extras_len != 0 || key_len == 0 || data_len > 0 ) return; // invalid break; case binary_command::Stat: if( extras_len != 0 || data_len != 0 ) return; // invalid if( key_len == 8 && std::memcmp(p_extra, "settings", 8) == 0 ) { m_stats = stats_t::SETTINGS; } else if( key_len == 5 ) { if( std::memcmp(p_extra, "items", 5) == 0 ) { m_stats = stats_t::ITEMS; } else if( std::memcmp(p_extra, "sizes", 5) == 0 ) { m_stats = stats_t::SIZES; } } else if( key_len == 3 && std::memcmp(p_extra, "ops", 3) == 0 ) { m_stats = stats_t::OPS; } break; case binary_command::Quit: case binary_command::QuitQ: case binary_command::Version: case binary_command::Noop: case binary_command::UnlockAll: case binary_command::UnlockAllQ: // no validation. // this is intentional violation against the protocol definition. break; default: m_command = binary_command::Unknown; m_status = binary_status::UnknownCommand; return; } m_status = binary_status::OK; } inline void binary_response::send_error(binary_status status, const char* msg, std::size_t len) { if( len > 100 ) throw std::logic_error(" Too long message"); char header[BINARY_HEADER_SIZE + 100]; fill_header(header, 0, 0, len, 0, status); std::memcpy(&header[BINARY_HEADER_SIZE], msg, len); m_socket.send(header, BINARY_HEADER_SIZE + len, true); } void binary_response::error(binary_status status) { switch( status ) { case binary_status::NotFound: send_error(status, STATUS_NOT_FOUND, sizeof(STATUS_NOT_FOUND) - 1); break; case binary_status::Exists: send_error(status, STATUS_EXISTS, sizeof(STATUS_EXISTS) - 1); break; case binary_status::TooLargeValue: send_error(status, STATUS_TOO_LARGE, sizeof(STATUS_TOO_LARGE) - 1); break; case binary_status::Invalid: send_error(status, STATUS_INVALID, sizeof(STATUS_INVALID) - 1); break; case binary_status::NotStored: send_error(status, STATUS_NOT_STORED, sizeof(STATUS_NOT_STORED) - 1); break; case binary_status::NonNumeric: send_error(status, STATUS_NON_NUMERIC, sizeof(STATUS_NON_NUMERIC) - 1); break; case binary_status::Locked: send_error(status, STATUS_LOCKED, sizeof(STATUS_LOCKED) - 1); break; case binary_status::NotLocked: send_error(status, STATUS_NOT_LOCKED, sizeof(STATUS_NOT_LOCKED) - 1); break; case binary_status::UnknownCommand: send_error(status, STATUS_UNKNOWN, sizeof(STATUS_UNKNOWN) - 1); break; case binary_status::OutOfMemory: send_error(status, STATUS_OOM, sizeof(STATUS_OOM) - 1); break; default: throw std::logic_error(" bug"); } } void binary_response::success() { char header[BINARY_HEADER_SIZE]; fill_header(header, 0, 0, 0, 0); m_socket.send(header, BINARY_HEADER_SIZE, true); } void binary_response::get(std::uint32_t flags, const cybozu::dynbuf& data, std::uint64_t cas, bool flush, const char* key, std::size_t key_len) { char header[BINARY_HEADER_SIZE]; fill_header(header, key_len, sizeof(flags), data.size(), cas); char b_flags[sizeof(flags)]; cybozu::hton(flags, b_flags); m_iov[0] = {header, BINARY_HEADER_SIZE}; m_iov[1] = {b_flags, sizeof(b_flags)}; if( key == nullptr ) { m_iov[2] = {data.data(), data.size()}; m_socket.sendv(m_iov, 3, flush); } else { m_iov[2] = {key, key_len}; m_iov[3] = {data.data(), data.size()}; m_socket.sendv(m_iov, 4, flush); } } void binary_response::set(std::uint64_t cas) { char header[BINARY_HEADER_SIZE]; fill_header(header, 0, 0, 0, cas); m_socket.send(header, BINARY_HEADER_SIZE, true); } void binary_response::incdec(std::uint64_t value, std::uint64_t cas) { char header[BINARY_HEADER_SIZE + sizeof(value)]; fill_header(header, 0, 0, sizeof(value), cas); cybozu::hton(value, &header[BINARY_HEADER_SIZE]); m_socket.send(header, sizeof(header), true); } void binary_response::quit() { char header[BINARY_HEADER_SIZE]; fill_header(header, 0, 0, 0, 0); m_socket.send_close(header, sizeof(header)); } inline void binary_response::send_stat(const std::string& key, const std::string& value) { char header[BINARY_HEADER_SIZE]; fill_header(header, key.size(), 0, value.size(), 0); m_iov[0] = {header, sizeof(header)}; m_iov[1] = {key.data(), key.size()}; m_iov[2] = {value.data(), value.size()}; m_socket.sendv(m_iov, 3, false); } void binary_response::stats_settings() { send_stat("maxbytes", std::to_string(g_config.memory_limit())); send_stat("tcpport", std::to_string(g_config.port())); send_stat("replport", std::to_string(g_config.repl_port())); send_stat("virtual_ip", g_config.vip().str()); send_stat("evictions", "on"); send_stat("cas_enabled", "on"); send_stat("locking", "on"); send_stat("tmp_dir", g_config.tempdir()); send_stat("buckets", std::to_string(g_config.buckets())); send_stat("item_size_max", std::to_string(g_config.max_data_size())); send_stat("num_threads", std::to_string(g_config.workers())); send_stat("gc_interval", std::to_string(g_config.gc_interval())); success(); } void binary_response::stats_items() { send_stat("items:1:number", std::to_string(g_stats.objects.load(relaxed))); send_stat("items:1:age", std::to_string(g_stats.oldest_age.load(relaxed))); send_stat("items:1:evicted", std::to_string(g_stats.total_evictions.load(relaxed))); send_stat("items:1:conflicts", std::to_string(g_stats.conflicts.load(relaxed))); send_stat("items:1:largest", std::to_string(g_stats.largest_object_size.load(relaxed))); success(); } void binary_response::stats_sizes() { send_stat("1024", std::to_string(g_stats.objects_under_1k.load(relaxed))); send_stat("4096", std::to_string(g_stats.objects_under_4k.load(relaxed))); send_stat("16384", std::to_string(g_stats.objects_under_16k.load(relaxed))); send_stat("65536", std::to_string(g_stats.objects_under_64k.load(relaxed))); send_stat("262144", std::to_string(g_stats.objects_under_256k.load(relaxed))); send_stat("1048576", std::to_string(g_stats.objects_under_1m.load(relaxed))); send_stat("4194304", std::to_string(g_stats.objects_under_4m.load(relaxed))); send_stat("huge", std::to_string(g_stats.objects_huge.load(relaxed))); success(); } void binary_response::stats_ops() { #define SEND_TEXT_OPS(n,i) \ send_stat("text:" n, \ std::to_string(g_stats.text_ops[(std::size_t)text_command::i].load(relaxed))) SEND_TEXT_OPS("set", SET); SEND_TEXT_OPS("add", ADD); SEND_TEXT_OPS("replace", REPLACE); SEND_TEXT_OPS("append", APPEND); SEND_TEXT_OPS("prepend", PREPEND); SEND_TEXT_OPS("cas", CAS); SEND_TEXT_OPS("get", GET); SEND_TEXT_OPS("gets", GETS); SEND_TEXT_OPS("delete", DELETE); SEND_TEXT_OPS("incr", INCR); SEND_TEXT_OPS("decr", DECR); SEND_TEXT_OPS("touch", TOUCH); SEND_TEXT_OPS("lock", LOCK); SEND_TEXT_OPS("unlock", UNLOCK); SEND_TEXT_OPS("unlock_all", UNLOCK_ALL); SEND_TEXT_OPS("slabs", SLABS); SEND_TEXT_OPS("stats", STATS); SEND_TEXT_OPS("flush_all", FLUSH_ALL); SEND_TEXT_OPS("version", VERSION); SEND_TEXT_OPS("verbosity", VERBOSITY); SEND_TEXT_OPS("quit", QUIT); #undef SEND_TEXT_OPS #define SEND_BINARY_OPS(n) \ send_stat("binary:" #n, \ std::to_string(g_stats.bin_ops[(std::size_t)binary_command::n].load(relaxed))) SEND_BINARY_OPS(Get); SEND_BINARY_OPS(Set); SEND_BINARY_OPS(Add); SEND_BINARY_OPS(Replace); SEND_BINARY_OPS(Delete); SEND_BINARY_OPS(Increment); SEND_BINARY_OPS(Decrement); SEND_BINARY_OPS(Quit); SEND_BINARY_OPS(Flush); SEND_BINARY_OPS(GetQ); SEND_BINARY_OPS(Noop); SEND_BINARY_OPS(Version); SEND_BINARY_OPS(GetK); SEND_BINARY_OPS(GetKQ); SEND_BINARY_OPS(Append); SEND_BINARY_OPS(Prepend); SEND_BINARY_OPS(Stat); SEND_BINARY_OPS(SetQ); SEND_BINARY_OPS(AddQ); SEND_BINARY_OPS(ReplaceQ); SEND_BINARY_OPS(DeleteQ); SEND_BINARY_OPS(IncrementQ); SEND_BINARY_OPS(DecrementQ); SEND_BINARY_OPS(QuitQ); SEND_BINARY_OPS(FlushQ); SEND_BINARY_OPS(AppendQ); SEND_BINARY_OPS(PrependQ); SEND_BINARY_OPS(Touch); SEND_BINARY_OPS(GaT); SEND_BINARY_OPS(GaTQ); SEND_BINARY_OPS(GaTK); SEND_BINARY_OPS(GaTKQ); SEND_BINARY_OPS(Lock); SEND_BINARY_OPS(LockQ); SEND_BINARY_OPS(Unlock); SEND_BINARY_OPS(UnlockQ); SEND_BINARY_OPS(UnlockAll); SEND_BINARY_OPS(UnlockAllQ); SEND_BINARY_OPS(LaG); SEND_BINARY_OPS(LaGQ); SEND_BINARY_OPS(LaGK); SEND_BINARY_OPS(LaGKQ); SEND_BINARY_OPS(RaU); SEND_BINARY_OPS(RaUQ); #undef SEND_BINARY_OPS success(); } void binary_response::stats_general(std::size_t n_slaves) { struct rusage ru; if( ::getrusage(RUSAGE_SELF, &ru) == -1 ) cybozu::throw_unix_error(errno, "getrusage"); send_stat("pid", std::to_string(::getpid())); send_stat("time", std::to_string(g_stats.current_time.load(relaxed))); send_stat("version", VERSION); send_stat("pointer_size", std::to_string(sizeof(void*)*8)); send_stat("rusage_user", std::to_string(ru.ru_utime.tv_sec) + ":" + std::to_string(ru.ru_utime.tv_usec)); send_stat("rusage_system", std::to_string(ru.ru_stime.tv_sec) + ":" + std::to_string(ru.ru_stime.tv_usec)); send_stat("curr_connections", std::to_string(g_stats.curr_connections.load(relaxed))); send_stat("total_connections", std::to_string(g_stats.total_connections.load(relaxed))); send_stat("curr_items", std::to_string(g_stats.objects.load(relaxed))); send_stat("total_items", std::to_string(g_stats.total_objects.load(relaxed))); send_stat("bytes", std::to_string(g_stats.used_memory.load(relaxed))); send_stat("limit_maxbytes", std::to_string(g_config.memory_limit())); send_stat("threads", std::to_string(g_config.workers())); send_stat("gc_count", std::to_string(g_stats.gc_count.load(relaxed))); send_stat("slaves", std::to_string(n_slaves)); send_stat("last_expirations", std::to_string(g_stats.last_expirations.load(relaxed))); send_stat("last_evictions", std::to_string(g_stats.last_evictions.load(relaxed))); send_stat("evictions", std::to_string(g_stats.total_evictions.load(relaxed))); send_stat("last_gc_elapsed", std::to_string(g_stats.last_gc_elapsed.load(relaxed))); send_stat("total_gc_elapsed", std::to_string(g_stats.total_gc_elapsed.load(relaxed))); success(); } void binary_response::version() { char header[BINARY_HEADER_SIZE + sizeof(VERSION) - 1]; fill_header(header, 0, 0, sizeof(VERSION) - 1, 0); std::memcpy(&header[BINARY_HEADER_SIZE], VERSION, sizeof(VERSION) - 1); m_socket.send(header, sizeof(header), true); } }} // namespace yrmcds::memcache yrmcds-1.0.3/src/memcache.hpp000066400000000000000000000301361226611677100160720ustar00rootroot00000000000000// memcached server-side protocol. // (C) 2013 Cybozu. #ifndef YRMCDS_MEMCACHE_HPP #define YRMCDS_MEMCACHE_HPP #include "constants.hpp" #include #include #include #include #include #include #include #include #include #include namespace yrmcds { namespace memcache { inline bool is_binary_request(const char* p) { return *p == '\x80'; } // Possible stats categories. enum class stats_t { GENERAL, SETTINGS, ITEMS, SIZES, OPS }; using item = std::tuple; // Memcache text commands. enum class text_command { UNKNOWN, SET, ADD, REPLACE, APPEND, PREPEND, CAS, GET, GETS, DELETE, INCR, DECR, TOUCH, LOCK, UNLOCK, UNLOCK_ALL, SLABS, STATS, FLUSH_ALL, VERSION, VERBOSITY, QUIT, END_OF_COMMAND // must be defined the last }; // Text request parser. class text_request final { public: text_request(const char* p, std::size_t len): m_p(p), m_len(len) { if( len == 0 || p == nullptr ) throw std::logic_error(" bad ctor arguments"); parse(); } // End of the key stream. static constexpr item eos{}; // Return length of the request. // // Return length of the request. // If the request is incomplete, zero is returned. std::size_t length() const noexcept { return m_request_len; } // Return the command type. text_command command() const noexcept { return m_command; } // Return `true` if the request can be parsed successfully. bool valid() const noexcept { return m_valid; } // `true` if the command has "noreply" option. bool no_reply() const noexcept { return m_no_reply; } // Return `key` sent with various commands. item key() const noexcept { return m_key; } // Return `flags` sent with storage commands. std::uint32_t flags() const noexcept { return m_flags; } // Return `exptime` sent with storage commands and TOUCH and FLUSH_ALL. std::time_t exptime() const noexcept { return m_exptime; } // Return `cas unique` sent with CAS command. std::uint64_t cas_unique() const noexcept { return m_cas_unique; } // Return data block sent with storage commands. item data() const noexcept { return m_data; } // Return `item` for the first key sent with GET or GETS. item first_key() const noexcept { return m_key; } // Return `item` next to the `prev` key in GET or GETS commands. // If the item == , no more key is available. static item next_key(const item& prev) noexcept { const char* b; std::size_t len; std::tie(b, len) = prev; b += len; while( *b == '\x20' ) ++b; const char* e = (const char*)::rawmemchr(b, '\x0d'); if( b == e ) return eos; const char* key_end = (const char*)std::memchr(b, '\x20', e-b); if( key_end == nullptr ) key_end = e; return item(b, key_end-b); } // Return an unsigned 64bit integer value sent with INCR or DECR. std::uint64_t value() const noexcept { return m_value; } // Return the log verbosity sent with VERBOSITY. cybozu::severity verbosity() const noexcept { return m_verbosity; } // Return memcache stats category sent with STATS. stats_t stats() const noexcept { return m_stats; } private: const char* const m_p; const std::size_t m_len; std::size_t m_request_len = 0; text_command m_command = text_command::UNKNOWN; bool m_valid = false; bool m_no_reply = false; item m_key; std::uint32_t m_flags = 0; std::time_t m_exptime = 0; std::uint64_t m_cas_unique = 0; item m_data; std::uint64_t m_value = 0; cybozu::severity m_verbosity = cybozu::severity::info; stats_t m_stats = stats_t::GENERAL; void parse() noexcept; void parse_flushall(const char* b, const char* e) noexcept; void parse_verbosity(const char* b, const char* e) noexcept; void parse_storage(const char* b, const char* e, bool is_cas) noexcept; void parse_delete(const char* b, const char* e) noexcept; void parse_touch(const char* b, const char* e) noexcept; void parse_lock(const char* b, const char* e) noexcept; void parse_unlock(const char* b, const char* e) noexcept; void parse_stats(const char* b, const char* e) noexcept; void parse_incdec(const char* b, const char* e) noexcept; void parse_get(const char* b, const char* e) noexcept; }; const char TEXT_ERROR[] = "ERROR\x0d\x0a"; const char TEXT_OK[] = "OK\x0d\x0a"; const char TEXT_STORED[] = "STORED\x0d\x0a"; const char TEXT_NOT_STORED[] = "NOT_STORED\x0d\x0a"; const char TEXT_EXISTS[] = "EXISTS\x0d\x0a"; const char TEXT_NOT_FOUND[] = "NOT_FOUND\x0d\x0a"; const char TEXT_TOUCHED[] = "TOUCHED\x0d\x0a"; const char TEXT_DELETED[] = "DELETED\x0d\x0a"; const char TEXT_END[] = "END\x0d\x0a"; const char TEXT_LOCKED[] = "LOCKED\x0d\x0a"; const char TEXT_VERSION[] = "VERSION "; // Text response sender. class text_response final { public: text_response(cybozu::tcp_socket& sock): m_socket(sock) {} void error() { m_socket.send(TEXT_ERROR, sizeof(TEXT_ERROR) - 1, true); } void ok() { m_socket.send(TEXT_OK, sizeof(TEXT_OK) - 1, true); } void end() { m_socket.send(TEXT_END, sizeof(TEXT_END) - 1, true); } void stored() { m_socket.send(TEXT_STORED, sizeof(TEXT_STORED) - 1, true); } void not_stored() { m_socket.send(TEXT_NOT_STORED, sizeof(TEXT_NOT_STORED) - 1, true); } void exists() { m_socket.send(TEXT_EXISTS, sizeof(TEXT_EXISTS) - 1, true); } void touched() { m_socket.send(TEXT_TOUCHED, sizeof(TEXT_TOUCHED) - 1, true); } void deleted() { m_socket.send(TEXT_DELETED, sizeof(TEXT_DELETED) - 1, true); } void not_found() { m_socket.send(TEXT_NOT_FOUND, sizeof(TEXT_NOT_FOUND) - 1, true); } void locked() { m_socket.send(TEXT_LOCKED, sizeof(TEXT_LOCKED) - 1, true); } void value(const cybozu::hash_key& key, std::uint32_t flags, const cybozu::dynbuf& data); void value(const cybozu::hash_key& key, std::uint32_t flags, const cybozu::dynbuf& data, std::uint64_t cas); void send(const char* p, std::size_t len, bool flush) { m_socket.send(p, len, flush); } void stats_settings(); void stats_items(); void stats_sizes(); void stats_ops(); void stats_general(std::size_t n_slaves); void version(); private: cybozu::tcp_socket& m_socket; cybozu::tcp_socket::iovec m_iov[cybozu::tcp_socket::MAX_IOVCNT]; }; // Memcache binary commands. enum class binary_command: unsigned char { Get = '\x00', Set = '\x01', Add = '\x02', Replace = '\x03', Delete = '\x04', Increment = '\x05', Decrement = '\x06', Quit = '\x07', Flush = '\x08', GetQ = '\x09', Noop = '\x0a', Version = '\x0b', GetK = '\x0c', GetKQ = '\x0d', Append = '\x0e', Prepend = '\x0f', Stat = '\x10', SetQ = '\x11', AddQ = '\x12', ReplaceQ = '\x13', DeleteQ = '\x14', IncrementQ = '\x15', DecrementQ = '\x16', QuitQ = '\x17', FlushQ = '\x18', AppendQ = '\x19', PrependQ = '\x1a', Touch = '\x1c', GaT = '\x1d', GaTQ = '\x1e', GaTK = '\x23', GaTKQ = '\x24', Lock = '\x40', LockQ = '\x41', Unlock = '\x42', UnlockQ = '\x43', UnlockAll = '\x44', UnlockAllQ = '\x45', LaG = '\x46', LaGQ = '\x47', LaGK = '\x48', LaGKQ = '\x49', RaU = '\x4a', RaUQ = '\x4b', Unknown, END_OF_COMMAND // must be defined the last }; // Binary protocol response status enum class binary_status: std::uint16_t { OK = 0x0000, NotFound = 0x0001, Exists = 0x0002, TooLargeValue = 0x0003, Invalid = 0x0004, NotStored = 0x0005, NonNumeric = 0x0006, Locked = 0x0010, NotLocked = 0x0011, UnknownCommand = 0x0081, OutOfMemory = 0x0082 }; // Binary request parser class binary_request final { public: binary_request(const char* p, std::size_t len): m_p(p), m_len(len) { if( len == 0 || p == nullptr ) throw std::logic_error(" bad ctor arguments"); parse(); } // Return length of the request. // // Return length of the request. // If the request is incomplete, zero is returned. std::size_t length() const noexcept { return m_request_len; } // Response status, if determined by the request. binary_status status() const noexcept { return m_status; } // Return the command type. binary_command command() const noexcept { return m_command; } // The command is quiet or not. bool quiet() const noexcept { return m_quiet; } // Return `key`. item key() const noexcept { return m_key; } // Return `opaque` sent with the request. const char* opaque() const noexcept { return m_p + 12; } // Return `cas unique` sent with CAS command. std::uint64_t cas_unique() const noexcept { return m_cas_unique; } // Return `flags` sent with storage commands. std::uint32_t flags() const noexcept { return m_flags; } // Special expiration time for increment/decrement. static const std::time_t EXPTIME_NONE = (std::time_t)-1; // Return `exptime`. std::time_t exptime() const noexcept { return m_exptime; } // Return data block sent with storage commands. item data() const noexcept { return m_data; } // Return an unsigned 64bit integer value for increment or decrement. std::uint64_t value() const noexcept { return m_value; } // Return an unsigned 64bit integer value to initialize an object. std::uint64_t initial() const noexcept { return m_initial; } // Return memcache stats category sent with STATS. stats_t stats() const noexcept { return m_stats; } private: const char* const m_p; const std::size_t m_len; std::size_t m_request_len = 0; binary_status m_status = binary_status::Invalid; binary_command m_command; bool m_quiet; item m_key; std::uint64_t m_cas_unique; std::uint32_t m_flags = 0; std::time_t m_exptime = 0; item m_data; std::uint64_t m_value = 0; std::uint64_t m_initial = 0; stats_t m_stats = stats_t::GENERAL; void parse() noexcept; }; // Binary response sender. class binary_response final { public: binary_response(cybozu::tcp_socket& sock, const binary_request& req): m_socket(sock), m_request(req) {} void error(binary_status status); void success(); void get(std::uint32_t flags, const cybozu::dynbuf& data, std::uint64_t cas, bool flush, const char* key = nullptr, std::size_t key_len = 0); void set(std::uint64_t cas); void incdec(std::uint64_t value, std::uint64_t cas); void quit(); void stats_settings(); void stats_items(); void stats_sizes(); void stats_ops(); void stats_general(std::size_t n_slaves); void version(); private: cybozu::tcp_socket& m_socket; const binary_request& m_request; cybozu::tcp_socket::iovec m_iov[cybozu::tcp_socket::MAX_IOVCNT]; void send_error(binary_status status, const char* msg, std::size_t len); void send_stat(const std::string& key, const std::string& value); void fill_header(char* buf, std::uint16_t key_len, std::uint8_t extras_len, std::uint32_t data_len, std::uint64_t cas, binary_status status = binary_status::OK) const noexcept { buf[0] = '\x81'; buf[1] = (char)m_request.command(); cybozu::hton(key_len, buf+2); buf[4] = (char)extras_len; buf[5] = '\x00'; cybozu::hton((std::uint16_t)status, buf+6); std::uint32_t total_len = key_len + extras_len + data_len; cybozu::hton(total_len, buf+8); std::memcpy(buf+12, m_request.opaque(), 4); cybozu::hton(cas, buf+16); } }; }} // namespace yrmcds::memcache #endif // YRMCDS_MEMCACHE_HPP yrmcds-1.0.3/src/object.cpp000066400000000000000000000115621226611677100155730ustar00rootroot00000000000000// (C) 2013 Cybozu. #include "config.hpp" #include "object.hpp" #include "stats.hpp" #include #include #include #include #include #include namespace { std::uint64_t to_uint64(const char* p, std::size_t len) { unsigned long long ull = std::stoull( std::string(p, len) ); if( ull > std::numeric_limits::max() ) throw std::out_of_range("out of range for uint64"); return static_cast(ull); } } // anonymous namespace namespace yrmcds { thread_local int g_context = -1; file_flusher::~file_flusher() { if( m_fd == -1 ) return; if( ::fdatasync(m_fd) == 0 ) ::posix_fadvise(m_fd, 0, 0, POSIX_FADV_DONTNEED); ::close(m_fd); } object::object(const char* p, std::size_t len, std::uint32_t flags_, std::time_t exptime): m_length(len), m_data(0), m_file(nullptr), m_flags(flags_), m_exptime(exptime) { if( len > g_config.heap_data_limit() ) { m_file = std::unique_ptr(new tempfile); m_file->write(p, len); } else { if( len > 0 ) m_data.append(p, len); } g_stats.total_objects.fetch_add(1, std::memory_order_relaxed); } object::object(std::uint64_t initial, std::time_t exptime): m_length(0), m_data(24), m_file(nullptr), m_flags(0), m_exptime(exptime) { char s_value[24]; // uint64 can be as large as 20 byte decimal string. m_length = ::snprintf(s_value, sizeof(s_value), "%llu", (unsigned long long)initial); m_data.append(s_value, m_length); g_stats.total_objects.fetch_add(1, std::memory_order_relaxed); } void object::set(const char* p, std::size_t len, std::uint32_t flags_, std::time_t exptime) { m_flags = flags_; m_exptime = exptime; ++ m_cas; m_gc_old = 0; m_data.reset(); if( len > g_config.heap_data_limit() ) { if( m_file.get() == nullptr ) { m_file = std::unique_ptr(new tempfile); } else { m_file->clear(); } m_file->write(p, len); } else { m_file = nullptr; if( len > 0 ) m_data.append(p, len); } m_length = len; } void object::append(const char* p, std::size_t len) { ++ m_cas; m_gc_old = 0; if( len == 0 ) return; std::size_t new_size = m_length + len; if( new_size > g_config.heap_data_limit() ) { if( m_file.get() == nullptr ) { m_file = std::unique_ptr(new tempfile); if( m_length > 0 ) m_file->write(m_data.data(), m_length); m_data.reset(); m_file->write(p, len); } else { m_file->write(p, len); } } else { m_data.append(p, len); } m_length = new_size; } void object::prepend(const char* p, std::size_t len) { ++ m_cas; m_gc_old = 0; if( len == 0 ) return; std::size_t new_size = m_length + len; if( new_size > g_config.heap_data_limit() ) { if( m_file.get() == nullptr ) { m_file = std::unique_ptr(new tempfile); m_file->write(p, len); if( m_length > 0 ) m_file->write(m_data.data(), m_length); m_data.reset(); } else { cybozu::dynbuf buf(new_size); buf.append(p, len); m_file->read_contents(buf); m_file->clear(); m_file->write(buf.data(), new_size); } } else { cybozu::dynbuf buf(new_size); buf.append(p, len); buf.append(m_data.data(), m_length); m_data.swap(buf); } m_length = new_size; } std::uint64_t object::incr(std::uint64_t n) { if( m_file.get() != nullptr ) throw not_a_number{}; std::uint64_t u64_value; try { u64_value = to_uint64(m_data.data(), m_length); } catch( const std::logic_error& ) { throw not_a_number{}; } u64_value += n; char s_value[24]; // uint64 can be as large as 20 byte decimal string. m_length = ::snprintf(s_value, sizeof(s_value), "%llu", (unsigned long long)u64_value); m_data.reset(); m_data.append(s_value, m_length); ++ m_cas; m_gc_old = 0; return u64_value; } std::uint64_t object::decr(std::uint64_t n) { if( m_file.get() != nullptr ) throw not_a_number{}; std::uint64_t u64_value; try { u64_value = to_uint64(m_data.data(), m_length); } catch( const std::logic_error& ) { throw not_a_number{}; } u64_value = (u64_value < n) ? 0 : (u64_value - n); char s_value[24]; // uint64 can be as large as 20 byte decimal string. m_length = ::snprintf(s_value, sizeof(s_value), "%llu", (unsigned long long)u64_value); m_data.reset(); m_data.append(s_value, m_length); ++ m_cas; m_gc_old = 0; return u64_value; } } // namespace yrmcds yrmcds-1.0.3/src/object.hpp000066400000000000000000000115711226611677100156000ustar00rootroot00000000000000// The cache object. // (C) 2013 Cybozu. #ifndef YRMCDS_OBJECT_HPP #define YRMCDS_OBJECT_HPP #include "stats.hpp" #include "tempfile.hpp" #include #include #include #include #include #include #include #include #include #include #include namespace yrmcds { // Context for object locking. // // The context is in fact the file descriptor of a client connection. extern thread_local int g_context; // Purge temporary file contents from the page cache at dtor. class file_flusher final { public: explicit file_flusher(int fd): m_fd(fd) {} file_flusher(file_flusher&& rhs) { std::swap(m_fd, rhs.m_fd); } file_flusher(const file_flusher&) = delete; file_flusher& operator=(file_flusher&&) = delete; file_flusher& operator=(const file_flusher&) = delete; ~file_flusher(); private: int m_fd = -1; }; // Object in the hash table. // // This class represents an object in the hash table. // Large objects are stored in temporary files. class object final { public: object(const char* p, std::size_t len, std::uint32_t flags_, std::time_t exptime); object(std::uint64_t initial, std::time_t exptime); object(const object&) = delete; object(object&& rhs) noexcept: m_length(rhs.m_length), m_data(std::move(rhs.m_data)), m_file(std::move(rhs.m_file)), m_flags(rhs.m_flags), m_exptime(rhs.m_exptime), m_cas(rhs.m_cas) {} object& operator=(const object&) = delete; object& operator=(object&&) = delete; // Exception thrown by or . struct not_a_number: public std::runtime_error { not_a_number(): std::runtime_error("") {} }; void set(const char* p, std::size_t len, std::uint32_t flags_, std::time_t exptime); void append(const char* p, std::size_t len); void prepend(const char* p, std::size_t len); std::uint64_t incr(std::uint64_t n); std::uint64_t decr(std::uint64_t n); void touch(std::time_t exptime) { m_exptime = exptime; m_gc_old = 0; } const cybozu::dynbuf& data(cybozu::dynbuf& buf) const { m_gc_old = 0; if( m_file.get() == nullptr ) return m_data; buf.reset(); m_file->read_contents(buf); return buf; } std::size_t size() const noexcept { if( m_file.get() == nullptr ) return m_data.size(); return m_file->length(); } std::uint32_t flags() const noexcept { return m_flags; } std::uint64_t cas_unique() const noexcept { return m_cas; } std::uint32_t exptime() const noexcept { return (std::uint32_t)m_exptime; } bool expired() const noexcept { if( locked() ) return false; std::time_t t = g_stats.flush_time.load(std::memory_order_relaxed); std::time_t now = g_stats.current_time.load(std::memory_order_relaxed); if( t != 0 && t <= now ) return true; if( m_exptime == 0 ) return false; return m_exptime <= now; } unsigned int age() const noexcept { return m_gc_old; } void survive(std::vector& flushers) const { ++ m_gc_old; if( m_gc_old != FLUSH_AGE || m_file.get() == nullptr ) return; int new_fd = ::dup(m_file->fileno()); if( new_fd == -1 ) { cybozu::logger::warning() << "Failed to dup a file descriptor"; return; } flushers.emplace_back(new_fd); } void lock() { if( locked() ) throw std::logic_error("object::lock bug"); m_lock = g_context; } void unlock(bool force = false) { if( ! force && ! locked_by_self() ) { cybozu::dump_stack(); std::ostringstream os; os << "object::unlock bug (m_lock=" << m_lock << ", g_context=" << g_context << ", unlocked thread=" << m_unlocker << ", this thread=" << std::this_thread::get_id(); throw std::logic_error(os.str()); } m_unlocker = std::this_thread::get_id(); m_lock = -1; } // Return `true` if this object is locked. bool locked() const noexcept { return m_lock != -1; } // Return `true` if this object is locked by the current context. bool locked_by_self() const noexcept { return m_lock == g_context; } // Return `true` if this object is locked by another context. bool locked_by_other() const noexcept { return locked() && (! locked_by_self()); } private: std::size_t m_length; cybozu::dynbuf m_data; std::unique_ptr m_file; std::uint32_t m_flags; std::time_t m_exptime; std::uint64_t m_cas = 1; mutable unsigned int m_gc_old = 0; int m_lock = -1; std::thread::id m_unlocker; }; } // namespace yrmcds #endif // YRMCDS_OBJECT_HPP yrmcds-1.0.3/src/replication.cpp000066400000000000000000000077411226611677100166420ustar00rootroot00000000000000// (C) 2013 Cybozu. #include "memcache.hpp" #include "replication.hpp" #include #include #include #include #include #include namespace { using namespace yrmcds::memcache; const int BINARY_HEADER_SIZE = 24; inline void fill_header(char* buf, std::uint16_t key_len, std::uint8_t extras_len, std::uint32_t data_len, binary_command cmd) noexcept { std::memset(buf, 0, BINARY_HEADER_SIZE); buf[0] = '\x80'; buf[1] = (char)cmd; cybozu::hton(key_len, buf+2); buf[4] = (char)extras_len; std::uint32_t total_len = key_len + extras_len + data_len; cybozu::hton(total_len, buf+8); } } // anonymous namespace namespace yrmcds { void repl_object(const std::vector& slaves, const cybozu::hash_key& key, const object& obj, bool flush) { char header[BINARY_HEADER_SIZE]; fill_header(header, key.length(), 8, obj.size(), binary_command::SetQ); char extras[8]; cybozu::hton(obj.flags(), extras); cybozu::hton(obj.exptime(), &extras[4]); cybozu::dynbuf buf(0); const cybozu::dynbuf& data = obj.data(buf); cybozu::tcp_socket::iovec iov[4] = { {header, sizeof(header)}, {extras, sizeof(extras)}, {key.data(), key.length()}, {data.data(), data.size()} }; for( cybozu::tcp_socket* s: slaves ) s->sendv(iov, 4, flush); } void repl_delete(const std::vector& slaves, const cybozu::hash_key& key) { char header[BINARY_HEADER_SIZE]; fill_header(header, key.length(), 0, 0, binary_command::DeleteQ); cybozu::tcp_socket::iovec iov[2] = { {header, sizeof(header)}, {key.data(), key.length()} }; for( cybozu::tcp_socket* s: slaves ) s->sendv(iov, 2, false); } std::size_t repl_recv(const char* p, std::size_t len, cybozu::hash_map& hash) { namespace mc = yrmcds::memcache; std::size_t consumed = 0; while( len > 0 ) { if( ! mc::is_binary_request(p) ) throw std::runtime_error("Invalid replication data"); mc::binary_request parser(p, len); std::size_t n = parser.length(); if( n == 0 ) break; p += n; len -= n; consumed += n; const char* key_data; std::size_t key_len; cybozu::hash_map::handler h = nullptr; cybozu::hash_map::creator c = nullptr; switch( parser.command() ) { case mc::binary_command::SetQ: h = [&parser](const cybozu::hash_key&, object& obj) -> bool { const char* p2; std::size_t len2; std::tie(p2, len2) = parser.data(); obj.set(p2, len2, parser.flags(), parser.exptime()); return true; }; c = [&parser](const cybozu::hash_key&) -> object { const char* p2; std::size_t len2; std::tie(p2, len2) = parser.data(); return object(p2, len2, parser.flags(), parser.exptime()); }; std::tie(key_data, key_len) = parser.key(); cybozu::logger::debug() << "repl: set " << std::string(key_data, key_len); hash.apply_nolock(cybozu::hash_key(key_data, key_len), h, c); break; case mc::binary_command::DeleteQ: std::tie(key_data, key_len) = parser.key(); cybozu::logger::debug() << "repl: remove " << std::string(key_data, key_len); hash.remove_nolock(cybozu::hash_key(key_data, key_len), nullptr); break; default: cybozu::logger::error() << "Unknown replication command" << std::hex << (unsigned int)parser.command(); } } return consumed; } } // namespace yrmcds yrmcds-1.0.3/src/replication.hpp000066400000000000000000000012441226611677100166370ustar00rootroot00000000000000// Replication protocol. // (C) 2013 Cybozu. #ifndef YRMCDS_REPLICATION_HPP #define YRMCDS_REPLICATION_HPP #include "object.hpp" #include #include #include namespace yrmcds { void repl_object(const std::vector& slaves, const cybozu::hash_key& key, const object& obj, bool flush = true); void repl_delete(const std::vector& slaves, const cybozu::hash_key& key); std::size_t repl_recv(const char* p, std::size_t len, cybozu::hash_map& hash); } // namespace yrmcds #endif // YRMCDS_REPLICATION_HPP yrmcds-1.0.3/src/server.cpp000066400000000000000000000167251226611677100156410ustar00rootroot00000000000000// (C) 2013 Cybozu. #include "constants.hpp" #include "server.hpp" #include "sockets.hpp" #include "stats.hpp" #include #include #include #include #include namespace yrmcds { server::server(): m_is_slave( ! is_master() ), m_hash(g_config.buckets()), m_syncer(m_workers) { m_slaves.reserve(MAX_SLAVES); m_new_slaves.reserve(MAX_SLAVES); m_finder = [this]() ->cybozu::worker* { std::size_t n_workers = m_workers.size(); for( std::size_t i = 0; i < n_workers; ++i ) { cybozu::worker* pw = m_workers[m_worker_index].get(); m_worker_index = (m_worker_index + 1) % n_workers; if( ! pw->is_running() ) return pw; } return nullptr; }; } inline bool server::gc_ready() { std::time_t now = std::time(nullptr); if( m_gc_thread.get() != nullptr ) { if( ! m_gc_thread->done() ) return false; m_last_gc = now; m_gc_thread = nullptr; } std::time_t t = g_stats.flush_time.load(std::memory_order_relaxed); if( t != 0 && now >= t ) return true; // run GC immediately if the heap is over used. if( g_stats.used_memory.load(std::memory_order_relaxed) > g_config.memory_limit() ) return true; // Run GC when there are new slaves. // In case there are unstable slaves that try to connect to // the master too frequently, the number of consecutive GCs is limited. if( ! m_new_slaves.empty() && m_consecutive_gcs < MAX_CONSECUTIVE_GCS ) { ++m_consecutive_gcs; return true; } if( now > (m_last_gc + g_config.gc_interval()) ) { m_consecutive_gcs = 0; return true; } return false; } inline bool server::reactor_gc_ready() { if( ! m_reactor.has_garbage() ) return false; if( ! m_syncer.empty() ) return false; if( m_gc_thread.get() != nullptr ) return false; if( ! m_new_slaves.empty() ) return false; return true; } inline void server::clear_everything() { for( auto& bucket: m_hash ) bucket.clear_nolock(); g_stats.total_objects.store(0, std::memory_order_relaxed); } void server::serve() { auto res = cybozu::signal_setup({SIGHUP, SIGQUIT, SIGTERM, SIGINT}); res->set_handler([this](const struct signalfd_siginfo& si, cybozu::reactor& r) { switch(si.ssi_signo) { case SIGHUP: cybozu::logger::instance().reopen(); cybozu::logger::info() << "got SIGHUP."; break; case SIGQUIT: case SIGTERM: case SIGINT: m_signaled = true; r.quit(); break; } }); m_reactor.add_resource(std::move(res), cybozu::reactor::EVENT_IN); using cybozu::make_server_socket; cybozu::tcp_server_socket::wrapper w = [this](int s, const cybozu::ip_address&) { return make_memcache_socket(s); }; m_reactor.add_resource(make_server_socket(NULL, g_config.port(), w), cybozu::reactor::EVENT_IN); while( m_is_slave ) { clear_everything(); serve_slave(); if( m_signaled ) return; // disconnected from the master for( int i = 0; i < MASTER_CHECKS; ++i ) { if( is_master() ) { m_is_slave = false; break; } std::this_thread::sleep_for( std::chrono::milliseconds(100) ); } } cybozu::tcp_server_socket::wrapper w2 = [this](int s, const cybozu::ip_address&) { return make_repl_socket(s); }; m_reactor.add_resource(make_server_socket(NULL, g_config.repl_port(), w2), cybozu::reactor::EVENT_IN); serve_master(); } void server::serve_slave() { int fd = cybozu::tcp_connect(g_config.vip().str().c_str(), g_config.repl_port()); if( fd == -1 ) { m_reactor.run_once(); return; } repl_client_socket* rs = new repl_client_socket(fd, m_hash); m_reactor.add_resource(std::unique_ptr(rs), cybozu::reactor::EVENT_IN|cybozu::reactor::EVENT_OUT ); cybozu::logger::info() << "Slave start"; m_reactor.run([rs](cybozu::reactor& r) { std::time_t now = std::time(nullptr); g_stats.current_time.store(now, std::memory_order_relaxed); if( is_master() ) { if( rs->valid() ) r.remove_resource(*rs); r.quit(); return; } // ping to the master char c = '\0'; rs->send(&c, sizeof(c), true); r.fix_garbage(); r.gc(); }); cybozu::logger::info() << "Slave end"; } void server::serve_master() { cybozu::logger::info() << "Entering master mode"; auto callback = [this](cybozu::reactor&) { std::time_t now = std::time(nullptr); g_stats.current_time.store(now, std::memory_order_relaxed); for( auto it = m_slaves.begin(); it != m_slaves.end(); ) { if( ! (*it)->valid() ) { it = m_slaves.erase(it); } else { ++it; } } if( ! m_syncer.empty() ) m_syncer.check(); if( gc_ready() ) { m_gc_thread = std::unique_ptr( new gc_thread(m_hash, m_slaves, m_new_slaves)); m_new_slaves.clear(); m_gc_thread->start(); } if( reactor_gc_ready() ) { m_reactor.fix_garbage(); m_syncer.add_request( std::unique_ptr( new sync_request([this]{ m_reactor.gc(); }) )); } }; for( unsigned int i = 0; i < g_config.workers(); ++i ) m_workers.emplace_back(new cybozu::worker(WORKER_BUFSIZE)); for( auto& w: m_workers ) w->start(); auto stop = [this] { m_reactor.invalidate(); for( auto& w: m_workers ) w->stop(); if( m_gc_thread.get() != nullptr ) m_gc_thread = nullptr; // join }; try { cybozu::logger::info() << "Reactor thread id=" << std::this_thread::get_id(); m_reactor.run(callback); cybozu::logger::info() << "Exiting"; } catch( ... ) { stop(); throw; } stop(); } std::unique_ptr server::make_memcache_socket(int s) { if( m_is_slave ) return nullptr; unsigned int mc = g_config.max_connections(); if( mc != 0 && (g_stats.curr_connections.load(std::memory_order_relaxed) >= mc) ) return nullptr; return std::unique_ptr( new memcache_socket(s, m_finder, m_hash, m_slaves) ); } std::unique_ptr server::make_repl_socket(int s) { if( m_slaves.size() == MAX_SLAVES ) return nullptr; std::unique_ptr t( new repl_socket(s, m_finder) ); cybozu::tcp_socket* pt = t.get(); m_slaves.push_back(pt); m_syncer.add_request( std::unique_ptr( new sync_request([this,pt]{ m_new_slaves.push_back(pt); }) )); return std::move(t); } } // namespace yrmcds yrmcds-1.0.3/src/server.hpp000066400000000000000000000025661226611677100156440ustar00rootroot00000000000000// Logics and data structures for the main thread. // (C) 2013 Cybozu. #ifndef YRMCDS_SERVER_HPP #define YRMCDS_SERVER_HPP #include "config.hpp" #include "gc.hpp" #include "object.hpp" #include "sync.hpp" #include #include #include #include #include #include #include namespace yrmcds { // The yrmcds server class server { public: server(); static bool is_master() { return cybozu::has_ip_address( g_config.vip() ); } void serve(); private: bool m_is_slave; bool m_signaled = false; cybozu::hash_map m_hash; cybozu::reactor m_reactor; std::vector> m_workers; int m_worker_index = 0; std::time_t m_last_gc = 0; std::unique_ptr m_gc_thread = nullptr; int m_consecutive_gcs = 0; std::vector m_slaves; std::vector m_new_slaves; syncer m_syncer; std::function m_finder; bool gc_ready(); bool reactor_gc_ready(); void clear_everything(); void serve_slave(); void serve_master(); std::unique_ptr make_memcache_socket(int s); std::unique_ptr make_repl_socket(int s); }; } // namespace yrmcds #endif // YRMCDS_SERVER_HPP yrmcds-1.0.3/src/sockets.cpp000066400000000000000000001003131226611677100157710ustar00rootroot00000000000000// (C) 2013 Cybozu. #include "replication.hpp" #include "sockets.hpp" #include #include #include #include #include namespace mc = yrmcds::memcache; using mc::text_command; using mc::binary_command; using mc::binary_status; namespace { const char NON_NUMERIC[] = "CLIENT_ERROR cannot increment or decrement non-numeric value\x0d\x0a"; const char NOT_LOCKED[] = "CLIENT_ERROR object is not locked or not found\x0d\x0a"; } // anonymous namespace namespace yrmcds { using hash_map = cybozu::hash_map; memcache_socket::memcache_socket(int fd, const std::function& finder, cybozu::hash_map& hash, const std::vector& slaves) : cybozu::tcp_socket(fd), m_busy(false), m_finder(finder), m_hash(hash), m_pending(0), m_slaves_origin(slaves) { m_slaves.reserve(MAX_SLAVES); g_stats.curr_connections.fetch_add(1); g_stats.total_connections.fetch_add(1); m_recvjob = [this](cybozu::dynbuf& buf) { // set lock context for objects. g_context = m_fd; // load pending data if( ! m_pending.empty() ) { buf.append(m_pending.data(), m_pending.size()); m_pending.reset(); } while( true ) { char* p = buf.prepare(MAX_RECVSIZE); ssize_t n = ::recv(m_fd, p, MAX_RECVSIZE, 0); if( n == -1 ) { if( errno == EAGAIN || errno == EWOULDBLOCK ) break; if( errno == EINTR ) continue; if( errno == ECONNRESET ) { buf.reset(); unlock_all(); invalidate_and_close(); break; } cybozu::throw_unix_error(errno, "recv"); } if( n == 0 ) { buf.reset(); unlock_all(); invalidate_and_close(); break; } // if (n != -1) && (n != 0) buf.consume(n); const char* head = buf.data(); std::size_t len = buf.size(); while( len > 0 ) { if( mc::is_binary_request(head) ) { mc::binary_request parser(head, len); std::size_t c = parser.length(); if( c == 0 ) break; head += c; len -= c; cmd_bin(parser); } else { mc::text_request parser(head, len); std::size_t c = parser.length(); if( c == 0 ) break; head += c; len -= c; cmd_text(parser); } } if( len > MAX_REQUEST_LENGTH ) { cybozu::logger::warning() << "denied too large request of " << len << " bytes."; buf.reset(); unlock_all(); invalidate_and_close(); break; } buf.erase(head - buf.data()); } // recv returns EAGAIN, or some error happens. if( buf.size() > 0 ) m_pending.append(buf.data(), buf.size()); m_busy.store(false, std::memory_order_release); }; m_sendjob = [this](cybozu::dynbuf&) { if( ! write_pending_data() ) invalidate_and_close(); }; } memcache_socket::~memcache_socket() { // the destructor is the safe place to release remaining locks. for( auto& ref: m_locks ) { m_hash.apply(ref.get(), [](const cybozu::hash_key&, object& obj) -> bool { obj.unlock(true); return true; }, nullptr); } } bool memcache_socket::on_readable() { if( m_busy.load(std::memory_order_acquire) ) { m_reactor->add_readable(*this); return true; } // find an idle worker. cybozu::worker* w = m_finder(); if( w == nullptr ) { m_reactor->add_readable(*this); return true; } // copy the current list of slaves m_slaves = m_slaves_origin; m_busy.store(true, std::memory_order_release); w->post_job(m_recvjob); return true; } bool memcache_socket::on_writable() { cybozu::worker* w = m_finder(); if( w == nullptr ) { // if there is no idle worker, fallback to the default. return cybozu::tcp_socket::on_writable(); } w->post_job(m_sendjob); return true; } void memcache_socket::cmd_bin(const memcache::binary_request& cmd) { mc::binary_response r(*this, cmd); if( cmd.status() != binary_status::OK ) { r.error( cmd.status() ); return; } g_stats.bin_ops[(std::size_t)cmd.command()].fetch_add(1); const char* p; std::size_t len; hash_map::handler h = nullptr; hash_map::creator c = nullptr; std::function repl = nullptr; std::function pred; switch( cmd.command() ) { case binary_command::Get: case binary_command::GetQ: case binary_command::GetK: case binary_command::GetKQ: case binary_command::GaT: case binary_command::GaTQ: case binary_command::GaTK: case binary_command::GaTKQ: case binary_command::LaG: case binary_command::LaGQ: case binary_command::LaGK: case binary_command::LaGKQ: h = [this,&cmd,&r](const cybozu::hash_key& k, object& obj) -> bool { if( (binary_command::LaG <= cmd.command()) && (cmd.command() <= binary_command::LaGKQ) ) { if( obj.locked() ) { r.error( binary_status::Locked ); return true; } obj.lock(); add_lock(k); } if( obj.expired() ) return false; if( cmd.exptime() != mc::binary_request::EXPTIME_NONE ) obj.touch( cmd.exptime() ); cybozu::dynbuf buf(0); const cybozu::dynbuf& data = obj.data(buf); if( cmd.command() == binary_command::Get || cmd.command() == binary_command::GetQ || cmd.command() == binary_command::GaT || cmd.command() == binary_command::GaTQ || cmd.command() == binary_command::LaG || cmd.command() == binary_command::LaGQ ) { r.get(obj.flags(), data, obj.cas_unique(), ! cmd.quiet()); } else { r.get(obj.flags(), data, obj.cas_unique(), ! cmd.quiet(), k.data(), k.length()); } return true; }; std::tie(p, len) = cmd.key(); if( ! m_hash.apply(cybozu::hash_key(p, len), h, c) ) { if( ! cmd.quiet() || cmd.command() == binary_command::LaGQ ) r.error( binary_status::NotFound ); } break; case binary_command::Set: case binary_command::SetQ: case binary_command::Add: case binary_command::AddQ: case binary_command::Replace: case binary_command::ReplaceQ: std::tie(p, len) = cmd.key(); if( len > MAX_KEY_LENGTH ) { r.error( binary_status::Invalid ); return; } if( std::get<1>(cmd.data()) > g_config.max_data_size() ) { r.error( binary_status::TooLargeValue ); return; } h = [this,&cmd,&r](const cybozu::hash_key& k, object& obj) -> bool { if( obj.locked_by_other() ) { r.error( binary_status::Locked ); return true; } if( obj.expired() ) { if( cmd.cas_unique() != 0 || cmd.command() == binary_command::Replace || cmd.command() == binary_command::ReplaceQ ) { r.error( binary_status::NotFound ); return true; } } else if( cmd.command() == binary_command::Add || cmd.command() == binary_command::AddQ ) { r.error( binary_status::Exists ); return true; } if( cmd.cas_unique() != 0 && cmd.cas_unique() != obj.cas_unique() ) { r.error( binary_status::Exists ); return true; } const char* p2; std::size_t len2; std::tie(p2, len2) = cmd.data(); obj.set(p2, len2, cmd.flags(), cmd.exptime()); if( ! cmd.quiet() ) r.set( obj.cas_unique() ); if( ! m_slaves.empty() ) repl_object(m_slaves, k, obj); return true; }; if( cmd.command() != binary_command::Replace && cmd.command() != binary_command::ReplaceQ && cmd.cas_unique() == 0 ) { c = [this,&cmd,&r](const cybozu::hash_key& k) -> object { const char* p2; std::size_t len2; std::tie(p2, len2) = cmd.data(); object o(p2, len2, cmd.flags(), cmd.exptime()); if( ! cmd.quiet() ) r.set( o.cas_unique() ); if( ! m_slaves.empty() ) repl_object(m_slaves, k, o); return std::move(o); }; } if( ! m_hash.apply(cybozu::hash_key(p, len), h, c) ) r.error( binary_status::NotFound ); break; case binary_command::RaU: case binary_command::RaUQ: std::tie(p, len) = cmd.key(); if( len > MAX_KEY_LENGTH ) { r.error( binary_status::Invalid ); return; } if( std::get<1>(cmd.data()) > g_config.max_data_size() ) { r.error( binary_status::TooLargeValue ); return; } h = [this,&cmd,&r](const cybozu::hash_key& k, object& obj) -> bool { if( ! obj.locked_by_self() ) { r.error( binary_status::NotLocked ); return true; } if( cmd.cas_unique() != 0 && cmd.cas_unique() != obj.cas_unique() ) { r.error( binary_status::Exists ); return true; } const char* p2; std::size_t len2; std::tie(p2, len2) = cmd.data(); obj.set(p2, len2, cmd.flags(), cmd.exptime()); obj.unlock(); remove_lock(k); if( ! cmd.quiet() ) r.set( obj.cas_unique() ); if( ! m_slaves.empty() ) repl_object(m_slaves, k, obj); return true; }; if( ! m_hash.apply(cybozu::hash_key(p, len), h, c) ) r.error( binary_status::NotFound ); break; case binary_command::Append: case binary_command::AppendQ: std::tie(p, len) = cmd.key(); h = [this,&cmd,&r](const cybozu::hash_key& k, object& obj) -> bool { if( obj.locked_by_other() ) { r.error( binary_status::Locked ); return true; } if( obj.expired() ) return false; if( cmd.cas_unique() != 0 && cmd.cas_unique() != obj.cas_unique() ) { r.error( binary_status::Exists ); return true; } const char* p2; std::size_t len2; std::tie(p2, len2) = cmd.data(); obj.append(p2, len2); if( ! cmd.quiet() ) r.set( obj.cas_unique() ); if( ! m_slaves.empty() ) repl_object(m_slaves, k, obj); return true; }; if( ! m_hash.apply(cybozu::hash_key(p, len), h, c) ) r.error( binary_status::NotFound ); break; case binary_command::Prepend: case binary_command::PrependQ: std::tie(p, len) = cmd.key(); h = [this,&cmd,&r](const cybozu::hash_key& k, object& obj) -> bool { if( obj.locked_by_other() ) { r.error( binary_status::Locked ); return true; } if( obj.expired() ) return false; if( cmd.cas_unique() != 0 && cmd.cas_unique() != obj.cas_unique() ) { r.error( binary_status::Exists ); return true; } const char* p2; std::size_t len2; std::tie(p2, len2) = cmd.data(); obj.prepend(p2, len2); if( ! cmd.quiet() ) r.set( obj.cas_unique() ); if( ! m_slaves.empty() ) repl_object(m_slaves, k, obj); return true; }; if( ! m_hash.apply(cybozu::hash_key(p, len), h, c) ) r.error( binary_status::NotFound ); break; case binary_command::Delete: case binary_command::DeleteQ: std::tie(p, len) = cmd.key(); pred = [this,&cmd,&r](const cybozu::hash_key& k, object& obj) -> bool { if( obj.locked_by_other() ) { r.error( binary_status::Locked ); return false; } if( cmd.cas_unique() != 0 && cmd.cas_unique() != obj.cas_unique() ) { r.error( binary_status::Exists ); return false; } if( obj.locked_by_self() ) remove_lock(k); if( ! cmd.quiet() ) r.success(); if( ! m_slaves.empty() ) repl_delete(m_slaves, k); return true; }; if( ! m_hash.remove_if(cybozu::hash_key(p, len), pred) && ! cmd.quiet() ) r.error( binary_status::NotFound ); break; case binary_command::Increment: case binary_command::IncrementQ: std::tie(p, len) = cmd.key(); h = [this,&cmd,&r](const cybozu::hash_key& k, object& obj) -> bool { if( obj.locked_by_other() ) { r.error( binary_status::Locked ); return true; } if( obj.expired() ) return false; if( cmd.cas_unique() != 0 && cmd.cas_unique() != obj.cas_unique() ) { r.error( binary_status::Exists ); return true; } try { std::uint64_t n = obj.incr( cmd.value() ); if( ! cmd.quiet() ) r.incdec( n, obj.cas_unique() ); if( ! m_slaves.empty() ) repl_object(m_slaves, k, obj); } catch( const object::not_a_number& ) { r.error( binary_status::NonNumeric ); } return true; }; if( cmd.exptime() != mc::binary_request::EXPTIME_NONE ) { c = [this,&cmd,&r](const cybozu::hash_key& k) -> object { object o(cmd.initial(), cmd.exptime()); if( ! cmd.quiet() ) r.incdec( cmd.initial(), o.cas_unique() ); if( ! m_slaves.empty() ) repl_object(m_slaves, k, o); return std::move(o); }; } if( ! m_hash.apply(cybozu::hash_key(p, len), h, c) ) r.error( binary_status::NotFound ); break; case binary_command::Decrement: case binary_command::DecrementQ: std::tie(p, len) = cmd.key(); h = [this,&cmd,&r](const cybozu::hash_key& k, object& obj) -> bool { if( obj.locked_by_other() ) { r.error( binary_status::Locked ); return true; } if( obj.expired() ) return false; if( cmd.cas_unique() != 0 && cmd.cas_unique() != obj.cas_unique() ) { r.error( binary_status::Exists ); return true; } try { std::uint64_t n = obj.decr( cmd.value() ); if( ! cmd.quiet() ) r.incdec( n, obj.cas_unique() ); if( ! m_slaves.empty() ) repl_object(m_slaves, k, obj); } catch( const object::not_a_number& ) { r.error( binary_status::NonNumeric ); } return true; }; if( cmd.exptime() != mc::binary_request::EXPTIME_NONE ) { c = [this,&cmd,&r](const cybozu::hash_key& k) -> object { object o(cmd.initial(), cmd.exptime()); if( ! cmd.quiet() ) r.incdec( cmd.initial(), o.cas_unique() ); if( ! m_slaves.empty() ) repl_object(m_slaves, k, o); return std::move(o); }; } if( ! m_hash.apply(cybozu::hash_key(p, len), h, c) ) r.error( binary_status::NotFound ); break; case binary_command::Touch: std::tie(p, len) = cmd.key(); h = [this,&cmd,&r](const cybozu::hash_key& k, object& obj) -> bool { if( obj.expired() ) return false; obj.touch( cmd.exptime() ); if( ! m_slaves.empty() ) repl_object(m_slaves, k, obj, false); r.set( obj.cas_unique() ); return true; }; if( ! m_hash.apply(cybozu::hash_key(p, len), h, c) ) r.error( binary_status::NotFound ); break; case binary_command::Lock: case binary_command::LockQ: h = [this,&cmd,&r](const cybozu::hash_key& k, object& obj) -> bool { if( obj.expired() ) return false; if( obj.locked() ) { r.error( binary_status::Locked ); return true; } obj.lock(); add_lock(k); if( ! cmd.quiet() ) r.success(); return true; }; std::tie(p, len) = cmd.key(); if( ! m_hash.apply(cybozu::hash_key(p, len), h, c) ) r.error( binary_status::NotFound ); break; case binary_command::Unlock: case binary_command::UnlockQ: h = [this,&cmd,&r](const cybozu::hash_key& k, object& obj) -> bool { if( ! obj.locked_by_self() ) { r.error( binary_status::NotLocked ); return true; } obj.unlock(); remove_lock(k); if( ! cmd.quiet() ) r.success(); return true; }; std::tie(p, len) = cmd.key(); if( ! m_hash.apply(cybozu::hash_key(p, len), h, c) ) r.error( binary_status::NotFound ); break; case binary_command::UnlockAll: case binary_command::UnlockAllQ: unlock_all(); if( ! cmd.quiet() ) r.success(); break; case binary_command::Quit: case binary_command::QuitQ: unlock_all(); if( cmd.quiet() ) { invalidate_and_close(); } else { r.quit(); } break; case binary_command::Flush: case binary_command::FlushQ: g_stats.flush_time.store( cmd.exptime() ); if( ! cmd.quiet() ) r.success(); break; case binary_command::Noop: r.success(); break; case binary_command::Version: r.version(); break; case binary_command::Stat: switch( cmd.stats() ) { case mc::stats_t::SETTINGS: r.stats_settings(); break; case mc::stats_t::ITEMS: r.stats_items(); break; case mc::stats_t::SIZES: r.stats_sizes(); break; case mc::stats_t::OPS: r.stats_ops(); break; default: r.stats_general(m_slaves.size()); } break; default: cybozu::logger::info() << "not implemented"; r.error( binary_status::UnknownCommand ); } } void memcache_socket::cmd_text(const memcache::text_request& cmd) { mc::text_response r(*this); if( ! cmd.valid() ) { r.error(); return; } g_stats.text_ops[(std::size_t)cmd.command()].fetch_add(1); const char* p; std::size_t len; hash_map::handler h = nullptr; hash_map::creator c = nullptr; std::function pred; switch( cmd.command() ) { case text_command::SET: case text_command::ADD: case text_command::REPLACE: std::tie(p, len) = cmd.key(); if( len > MAX_KEY_LENGTH ) { r.error(); return; } if( std::get<1>(cmd.data()) > g_config.max_data_size() ) { r.error(); return; } h = [this,&cmd,&r](const cybozu::hash_key& k, object& obj) -> bool { if( obj.locked_by_other() ) { if( ! cmd.no_reply() ) r.locked(); return true; } if( obj.expired() ) { if( cmd.command() == text_command::REPLACE ) return false; } else if( cmd.command() == text_command::ADD ) { return false; } const char* p2; std::size_t len2; std::tie(p2, len2) = cmd.data(); obj.set(p2, len2, cmd.flags(), cmd.exptime()); if( ! cmd.no_reply() ) r.stored(); if( ! m_slaves.empty() ) repl_object(m_slaves, k, obj); return true; }; if( cmd.command() != text_command::REPLACE ) { c = [this,&cmd,&r](const cybozu::hash_key& k) -> object { const char* p2; std::size_t len2; std::tie(p2, len2) = cmd.data(); object o(p2, len2, cmd.flags(), cmd.exptime()); if( ! cmd.no_reply() ) r.stored(); if( ! m_slaves.empty() ) repl_object(m_slaves, k, o); return std::move(o); }; } if( ! m_hash.apply(cybozu::hash_key(p, len), h, c) && ! cmd.no_reply() ) r.not_stored(); break; case text_command::APPEND: std::tie(p, len) = cmd.key(); h = [this,&cmd,&r](const cybozu::hash_key& k, object& obj) -> bool { if( obj.locked_by_other() ) { if( ! cmd.no_reply() ) r.locked(); return true; } if( obj.expired() ) return false; const char* p2; std::size_t len2; std::tie(p2, len2) = cmd.data(); obj.append(p2, len2); if( ! cmd.no_reply() ) r.stored(); if( ! m_slaves.empty() ) repl_object(m_slaves, k, obj); return true; }; if( ! m_hash.apply(cybozu::hash_key(p, len), h, c) && ! cmd.no_reply() ) r.not_stored(); break; case text_command::PREPEND: std::tie(p, len) = cmd.key(); h = [this,&cmd,&r](const cybozu::hash_key& k, object& obj) -> bool { if( obj.locked_by_other() ) { if( ! cmd.no_reply() ) r.locked(); return true; } if( obj.expired() ) return false; const char* p2; std::size_t len2; std::tie(p2, len2) = cmd.data(); obj.prepend(p2, len2); if( ! cmd.no_reply() ) r.stored(); if( ! m_slaves.empty() ) repl_object(m_slaves, k, obj); return true; }; if( ! m_hash.apply(cybozu::hash_key(p, len), h, c) && ! cmd.no_reply() ) r.not_stored(); break; case text_command::CAS: std::tie(p, len) = cmd.key(); h = [this,&cmd,&r](const cybozu::hash_key& k, object& obj) -> bool { if( obj.locked_by_other() ) { if( ! cmd.no_reply() ) r.locked(); return true; } if( obj.expired() ) return false; if( obj.cas_unique() != cmd.cas_unique() ) { if( ! cmd.no_reply() ) r.exists(); return true; } const char* p2; std::size_t len2; std::tie(p2, len2) = cmd.data(); obj.set(p2, len2, cmd.flags(), cmd.exptime()); if( ! cmd.no_reply() ) r.stored(); if( ! m_slaves.empty() ) repl_object(m_slaves, k, obj); return true; }; if( ! m_hash.apply(cybozu::hash_key(p, len), h, c) && ! cmd.no_reply() ) r.not_found(); break; case text_command::INCR: std::tie(p, len) = cmd.key(); h = [this,&cmd,&r](const cybozu::hash_key& k, object& obj) -> bool { if( obj.locked_by_other() ) { if( ! cmd.no_reply() ) r.locked(); return true; } if( obj.expired() ) return false; try { std::uint64_t n = obj.incr( cmd.value() ); if( ! cmd.no_reply() ) { char buf[24]; int numlen = snprintf(buf, sizeof(buf), "%llu\x0d\x0a", (unsigned long long)n); r.send(buf, numlen, true); } if( ! m_slaves.empty() ) repl_object(m_slaves, k, obj); } catch( const object::not_a_number& ) { if( ! cmd.no_reply() ) r.send(NON_NUMERIC, sizeof(NON_NUMERIC) - 1, true); } return true; }; if( ! m_hash.apply(cybozu::hash_key(p, len), h, c) && ! cmd.no_reply() ) r.not_found(); break; case text_command::DECR: std::tie(p, len) = cmd.key(); h = [this,&cmd,&r](const cybozu::hash_key& k, object& obj) -> bool { if( obj.locked_by_other() ) { if( ! cmd.no_reply() ) r.locked(); return true; } if( obj.expired() ) return false; try { std::uint64_t n = obj.decr( cmd.value() ); if( ! cmd.no_reply() ) { char buf[24]; int numlen = snprintf(buf, sizeof(buf), "%llu\x0d\x0a", (unsigned long long)n); r.send(buf, numlen, true); } if( ! m_slaves.empty() ) repl_object(m_slaves, k, obj); } catch( const object::not_a_number& ) { if( ! cmd.no_reply() ) r.send(NON_NUMERIC, sizeof(NON_NUMERIC) - 1, true); } return true; }; if( ! m_hash.apply(cybozu::hash_key(p, len), h, c) && ! cmd.no_reply() ) r.not_found(); break; case text_command::TOUCH: std::tie(p, len) = cmd.key(); h = [this,&cmd](const cybozu::hash_key& k, object& obj) -> bool { if( obj.expired() ) return false; obj.touch( cmd.exptime() ); if( ! m_slaves.empty() ) repl_object(m_slaves, k, obj, false); return true; }; if( m_hash.apply(cybozu::hash_key(p, len), h, c) ) { if( ! cmd.no_reply() ) r.touched(); } else { if( ! cmd.no_reply() ) r.not_found(); } break; case text_command::DELETE: std::tie(p, len) = cmd.key(); pred = [this,&cmd,&r](const cybozu::hash_key& k, object& obj) -> bool { if( obj.locked_by_other() ) { if( ! cmd.no_reply() ) r.locked(); return false; } if( obj.locked_by_self() ) remove_lock(k); if( ! cmd.no_reply() ) r.deleted(); if( ! m_slaves.empty() ) repl_delete(m_slaves, k); return true; }; if( ! m_hash.remove_if(cybozu::hash_key(p, len), pred) && ! cmd.no_reply() ) r.not_found(); break; case text_command::LOCK: h = [this,&cmd,&r](const cybozu::hash_key& k, object& obj) -> bool { if( obj.expired() ) return false; if( obj.locked() ) { r.locked(); return true; } obj.lock(); add_lock(k); r.ok(); return true; }; std::tie(p, len) = cmd.key(); if( ! m_hash.apply(cybozu::hash_key(p, len), h, c) ) r.not_found(); break; case text_command::UNLOCK: h = [this,&cmd,&r](const cybozu::hash_key& k, object& obj) -> bool { if( ! obj.locked_by_self() ) return false; obj.unlock(); remove_lock(k); r.ok(); return true; }; std::tie(p, len) = cmd.key(); if( ! m_hash.apply(cybozu::hash_key(p, len), h, c) ) r.send(NOT_LOCKED, sizeof(NOT_LOCKED) - 1, true); break; case text_command::UNLOCK_ALL: unlock_all(); r.ok(); break; case text_command::GET: case text_command::GETS: h = [&cmd,&r](const cybozu::hash_key& k, object& obj) -> bool { if( obj.expired() ) return false; cybozu::dynbuf buf(0); const cybozu::dynbuf& data = obj.data(buf); if( cmd.command() == text_command::GETS ) { r.value(k, obj.flags(), data, obj.cas_unique()); } else { r.value(k, obj.flags(), data); } return true; }; for( mc::item it = cmd.first_key(); it != mc::text_request::eos; it = cmd.next_key(it) ) { std::tie(p, len) = it; m_hash.apply(cybozu::hash_key(p, len), h, nullptr); } r.end(); break; case text_command::SLABS: r.ok(); break; case text_command::STATS: switch( cmd.stats() ) { case mc::stats_t::SETTINGS: r.stats_settings(); break; case mc::stats_t::ITEMS: r.stats_items(); break; case mc::stats_t::SIZES: r.stats_sizes(); break; case mc::stats_t::OPS: r.stats_ops(); break; default: r.stats_general(m_slaves.size()); } r.end(); break; case text_command::FLUSH_ALL: g_stats.flush_time.store( cmd.exptime() ); if( ! cmd.no_reply() ) r.ok(); break; case text_command::VERSION: r.version(); break; case text_command::VERBOSITY: cybozu::logger::set_threshold(cmd.verbosity()); if( ! cmd.no_reply() ) r.ok(); break; case text_command::QUIT: unlock_all(); invalidate_and_close(); break; default: cybozu::logger::info() << "not implemented"; r.error(); } } bool repl_socket::on_readable() { // recv and drop. while( true ) { ssize_t n = ::recv(m_fd, &m_recvbuf[0], MAX_RECVSIZE, 0); if( n == -1 ) { if( errno == EAGAIN || errno == EWOULDBLOCK ) break; if( errno == EINTR ) continue; if( errno == ECONNRESET ) return invalidate(); cybozu::throw_unix_error(errno, "recv"); } if( n == 0 ) return invalidate(); } return true; } bool repl_socket::on_writable() { cybozu::worker* w = m_finder(); if( w == nullptr ) { // if there is no idle worker, fallback to the default. return cybozu::tcp_socket::on_writable(); } w->post_job(m_sendjob); return true; } bool repl_client_socket::on_readable() { while( true ) { char* p = m_recvbuf.prepare(MAX_RECVSIZE); ssize_t n = ::recv(m_fd, p, MAX_RECVSIZE, 0); if( n == -1 ) { if( errno == EAGAIN || errno == EWOULDBLOCK ) break; if( errno == EINTR ) continue; if( errno == ECONNRESET ) { m_reactor->quit(); return invalidate(); } cybozu::throw_unix_error(errno, "recv"); } if( n == 0 ) { m_reactor->quit(); return invalidate(); } m_recvbuf.consume(n); std::size_t c = repl_recv(m_recvbuf.data(), m_recvbuf.size(), m_hash); m_recvbuf.erase(c); } return true; } } // namespace yrmcds yrmcds-1.0.3/src/sockets.hpp000066400000000000000000000075751226611677100160160ustar00rootroot00000000000000// Defines sockets for yrmcds. // (C) 2013 Cybozu. #ifndef YRMCDS_SOCKETS_HPP #define YRMCDS_SOCKETS_HPP #include "constants.hpp" #include "memcache.hpp" #include "object.hpp" #include "stats.hpp" #include #include #include #include #include #include #include namespace yrmcds { class memcache_socket: public cybozu::tcp_socket { public: memcache_socket(int fd, const std::function& finder, cybozu::hash_map& hash, const std::vector& slaves); virtual ~memcache_socket(); void add_lock(const cybozu::hash_key& k) { // m_locks are accessed by at most one worker thread, hence // there is no need to guard it with a mutex. m_locks.emplace_back(k); } void remove_lock(const cybozu::hash_key& k) { for( auto it = m_locks.begin(); it != m_locks.end(); ++it ) { if( it->get() == k ) { m_locks.erase(it); return; } } cybozu::dump_stack(); throw std::logic_error(" bug"); } void unlock_all() { for( auto& ref: m_locks ) { m_hash.apply(ref.get(), [](const cybozu::hash_key&, object& obj) -> bool { obj.unlock(false); return true; }, nullptr); } m_locks.clear(); } // Process a binary request command. // @cmd A binary request. void cmd_bin(const memcache::binary_request& cmd); // Process a test request command. // @cmd A binary request. void cmd_text(const memcache::text_request& cmd); private: alignas(CACHELINE_SIZE) std::atomic m_busy; const std::function& m_finder; cybozu::hash_map& m_hash; cybozu::dynbuf m_pending; const std::vector& m_slaves_origin; std::vector m_slaves; cybozu::worker::job m_recvjob; cybozu::worker::job m_sendjob; std::vector> m_locks; virtual void on_invalidate() override final { // In order to avoid races and deadlocks, remaining locks // are not released here. They are released in the destructor // where no other threads have access to this object. g_stats.curr_connections.fetch_sub(1); cybozu::tcp_socket::on_invalidate(); } virtual bool on_readable() override final; virtual bool on_writable() override final; }; class object; class repl_socket: public cybozu::tcp_socket { public: repl_socket(int fd, const std::function& finder) : cybozu::tcp_socket(fd, 30), m_finder(finder), m_recvbuf(MAX_RECVSIZE) { m_sendjob = [this](cybozu::dynbuf&) { if( ! write_pending_data() ) invalidate_and_close(); }; } private: const std::function& m_finder; std::vector m_recvbuf; cybozu::worker::job m_sendjob; virtual bool on_readable() override final; virtual bool on_writable() override final; }; class repl_client_socket: public cybozu::tcp_socket { public: repl_client_socket(int fd, cybozu::hash_map& m): cybozu::tcp_socket(fd), m_hash(m), m_recvbuf(30 << 20) {} private: cybozu::hash_map& m_hash; cybozu::dynbuf m_recvbuf; virtual bool on_readable() override final; virtual bool on_hangup() override final { m_reactor->quit(); return invalidate(); } virtual bool on_error() override final { m_reactor->quit(); return invalidate(); } }; } // namespace yrmcds #endif // YRMCDS_SOCKETS_HPP yrmcds-1.0.3/src/stats.cpp000066400000000000000000000017101226611677100154550ustar00rootroot00000000000000// (C) 2013 Cybozu. #include "stats.hpp" namespace yrmcds { statistics g_stats; void statistics::reset() noexcept { objects = 0; objects_under_1k = 0; objects_under_4k = 0; objects_under_16k = 0; objects_under_64k = 0; objects_under_256k = 0; objects_under_1m = 0; objects_under_4m = 0; objects_huge = 0; /* bucket statistics. Updated at every GC. */ used_memory = 0; conflicts = 0; /* GC statistics. Updated at every GC, of course. */ gc_count = 0; oldest_age = 0; largest_object_size = 0; last_expirations = 0; last_evictions = 0; total_evictions = 0; last_gc_elapsed = 0; total_gc_elapsed = 0; /* Realtime staticstics. */ total_objects = 0; current_time = std::time(nullptr); flush_time = 0; curr_connections = 0; total_connections = 0; for( auto& v: text_ops ) v = 0; for( auto& v: bin_ops ) v = 0; } } // namespace yrmcds yrmcds-1.0.3/src/stats.hpp000066400000000000000000000042171226611677100154670ustar00rootroot00000000000000// Atomic counters for statistics. // (C) 2013 Cybozu. #ifndef YRMCDS_STATS_HPP #define YRMCDS_STATS_HPP #include "memcache.hpp" #include #include #include #include namespace yrmcds { // statistics counters. struct statistics { statistics(): current_time(std::time(nullptr)) {} // Reset all counters. void reset() noexcept; /* object counters. Updated at every GC. */ alignas(CACHELINE_SIZE) std::atomic objects; std::atomic objects_under_1k; std::atomic objects_under_4k; std::atomic objects_under_16k; std::atomic objects_under_64k; std::atomic objects_under_256k; std::atomic objects_under_1m; std::atomic objects_under_4m; std::atomic objects_huge; /* bucket statistics. Updated at every GC. */ std::atomic used_memory; std::atomic conflicts; /* GC statistics. Updated at every GC, of course. */ std::atomic gc_count; std::atomic oldest_age; std::atomic largest_object_size; std::atomic last_expirations; std::atomic last_evictions; std::atomic total_evictions; std::atomic last_gc_elapsed; // micro seconds std::atomic total_gc_elapsed; // micro seconds /* Realtime staticstics. */ alignas(CACHELINE_SIZE) std::atomic total_objects; alignas(CACHELINE_SIZE) std::atomic current_time; alignas(CACHELINE_SIZE) std::atomic flush_time; // abused by "flush_all" alignas(CACHELINE_SIZE) std::atomic curr_connections; std::atomic total_connections; alignas(CACHELINE_SIZE) std::atomic text_ops[(std::size_t)memcache::text_command::END_OF_COMMAND]; std::atomic bin_ops[(std::size_t)memcache::binary_command::END_OF_COMMAND]; }; extern statistics g_stats; } // namespace yrmcds #endif // YRMCDS_STATS_HPP yrmcds-1.0.3/src/sync.hpp000066400000000000000000000035771226611677100153150ustar00rootroot00000000000000// Worker synchronizer. // (C) 2013 Cybozu. #ifndef YRMCDS_SYNC_HPP #define YRMCDS_SYNC_HPP #include "constants.hpp" #include #include #include #include #include namespace yrmcds { class sync_request final { public: explicit sync_request(std::function callback): m_callback(callback) {} void set(int n) { m_flags[n] = true; } bool check() { if( ! m_flags.all() ) return false; m_callback(); return true; } private: std::function m_callback; std::bitset m_flags; }; class syncer { public: explicit syncer(const std::vector>& workers): m_workers(workers) {} bool empty() const noexcept { return m_requests.empty(); } void add_request(std::unique_ptr req) { const std::size_t n = m_workers.size(); for( std::size_t i = 0; i < n; ++i ) { if( ! m_workers[i]->is_running() ) req->set(i); } for( std::size_t i = n; i < MAX_WORKERS; ++i ) req->set(i); if( ! req->check() ) m_requests.emplace_back( std::move(req) ); } void check() { const std::size_t n = m_workers.size(); for( std::size_t i = 0; i < n; ++i ) { if( ! m_workers[i]->is_running() ) { for( auto& req: m_requests ) req->set(i); } } for( auto it = m_requests.begin(); it != m_requests.end(); ) { if( (*it)->check() ) { it = m_requests.erase(it); } else { ++it; } } } private: const std::vector>& m_workers; std::vector> m_requests; }; } // namespace yrmcds #endif // YRMCDS_SYNC_HPP yrmcds-1.0.3/src/tempfile.cpp000066400000000000000000000021441226611677100161260ustar00rootroot00000000000000// (C) 2013 Cybozu. #include "tempfile.hpp" #include #include #include namespace yrmcds { void tempfile::write(const char* p, std::size_t len) { while( len != 0 ) { ssize_t n = ::write(m_fd, p, len); if( n == -1 ) cybozu::throw_unix_error(errno, "write"); p += n; len -= n; m_length += n; } } void tempfile::clear() { if( lseek(m_fd, 0, SEEK_SET) == (off_t)-1 ) cybozu::throw_unix_error(errno, "lseek"); if( ftruncate(m_fd, 0) == -1 ) cybozu::throw_unix_error(errno, "ftruncate"); m_length = 0; } void tempfile::read_contents(cybozu::dynbuf& buf) const { char* p = buf.prepare(m_length); std::size_t to_read = m_length; while( to_read != 0 ) { ssize_t n = pread(m_fd, p, to_read, m_length - to_read); if( n == -1 ) cybozu::throw_unix_error(errno, "pread"); if( n == 0 ) throw std::runtime_error(" unexpected EOF."); p += n; to_read -= n; buf.consume(n); } } } // namespace yrmcds yrmcds-1.0.3/src/tempfile.hpp000066400000000000000000000032111226611677100161270ustar00rootroot00000000000000// Temporary file to store object data. // (C) 2013 Cybozu. #ifndef YRMCDS_TEMPFILE_HPP #define YRMCDS_TEMPFILE_HPP #include "config.hpp" #include #include #include #include #include #include #include #include namespace yrmcds { // A helper function for . inline int mkstemp_wrap(const std::string& tempdir) { std::string tmpl = tempdir + "/XXXXXX\0"; int fd = ::mkostemp(&(tmpl[0]), O_CLOEXEC); if( fd == -1 ) cybozu::throw_unix_error(errno, "mkstemp"); ::unlink(tmpl.data()); return fd; } // A temporary file abstraction. class tempfile { public: tempfile(): m_fd(mkstemp_wrap(g_config.tempdir())) {} tempfile(const tempfile&) = delete; tempfile& operator=(const tempfile&) = delete; tempfile(tempfile&& rhs) = delete; tempfile& operator=(tempfile&&) = delete; ~tempfile() { ::close(m_fd); } // Append data to the end of the current contents. // @p Pointer to the data. // @len Length of the data void write(const char* p, std::size_t len); // Clear the current file contents. void clear(); // Return the open file descriptor. int fileno() const { return m_fd; } // Read the file contents. // @buf Storage for the file contents. // // Read the file contents and append them to `buf`. void read_contents(cybozu::dynbuf& buf) const; std::size_t length() const noexcept { return m_length; } private: const int m_fd; std::size_t m_length = 0; }; } // namespace yrmcds #endif // YRMCDS_TEMPFILE_HPP yrmcds-1.0.3/test/000077500000000000000000000000001226611677100140045ustar00rootroot00000000000000yrmcds-1.0.3/test/bitfield.cpp000066400000000000000000000003321226611677100162700ustar00rootroot00000000000000#include int main(int argc, char** argv) { struct { unsigned int n : 1; } t; t.n = 0; for( int i = 0; i < 10; ++i ) { std::cout << t.n++ << std::endl; } return 0; } yrmcds-1.0.3/test/capture-this.cpp000066400000000000000000000003051226611677100171160ustar00rootroot00000000000000#include struct A { void foo() { std::cout << "Hello" << std::endl; } void bar() { [this]() { foo(); } (); } }; int main() { A a; a.bar(); return 0; } yrmcds-1.0.3/test/chrono.cpp000066400000000000000000000022141226611677100157770ustar00rootroot00000000000000#include #include #include #include typedef std::chrono::microseconds us_t; template inline us_t::rep to_us(T start, T end) { return std::chrono::duration_cast(end-start).count(); } template inline void print_ratio() { cybozu::demangler t(typeid(T).name()); std::cout << t.name() << " ratio = " << T::period::num << "/" << T::period::den << " second." << std::endl; } int main() { auto t1 = std::chrono::steady_clock::now(); print_ratio(); print_ratio(); print_ratio(); std::cout << "std::chrono::high_resolution_clock is " << (std::chrono::high_resolution_clock::is_steady ? "" : "not ") << "steady." << std::endl; static_assert(std::chrono::steady_clock::is_steady, "std::chrono::steady_clock is not steady!"); auto t2 = std::chrono::steady_clock::now(); std::cout << "t2 - t1 = " << to_us(t1, t2) << "us" << std::endl; return 0; } yrmcds-1.0.3/test/config.cpp000066400000000000000000000014101226611677100157510ustar00rootroot00000000000000#include "../src/config.hpp" #include const std::string TEST_CONF = "test/test.conf"; AUTOTEST(config) { using yrmcds::g_config; g_config.load(TEST_CONF); cybozu_assert(g_config.port() == 1121); cybozu_assert(g_config.repl_port() == 1122); cybozu_assert(g_config.max_connections() == 10000); cybozu_assert(g_config.user() == "nobody"); cybozu_assert(g_config.group() == "nogroup"); cybozu_assert(g_config.memory_limit() == (1024 << 20)); cybozu_assert(g_config.threshold() == cybozu::severity::warning); cybozu_assert(g_config.max_data_size() == (5 << 20)); cybozu_assert(g_config.heap_data_limit() == (16 << 10)); cybozu_assert(g_config.workers() == 10); cybozu_assert(g_config.gc_interval() == 20); } yrmcds-1.0.3/test/config_parser.cpp000066400000000000000000000010411226611677100173250ustar00rootroot00000000000000#include #include int main(int argc, char** argv) { cybozu::config_parser cp; cp.set("aaa", "3"); std::cout << cp.get_as_int("aaa") << std::endl; cp.set("aaa", "5"); std::cout << cp.get_as_int("aaa") << std::endl; cp.set("bbb", "false"); std::cout << cp.get_as_bool("bbb") << std::endl; if( argc == 2 ) { cp.load(argv[1]); std::cout << cp.get_as_int("memory_limit") << std::endl; std::cout << cp.get("temp_dir") << std::endl; } return 0; } yrmcds-1.0.3/test/connect.cpp000066400000000000000000000010051226611677100161350ustar00rootroot00000000000000#include #include int main(int argc, char** argv) { if( argc != 3 ) { std::cout << "Usage: connect.exe HOST PORT" << std::endl; return 0; } const char* node = argv[1]; std::uint16_t port = static_cast( std::stoi(argv[2]) ); int s = cybozu::tcp_connect(node, port, 5); if( s == -1 ) { std::cout << "failed." << std::endl; } else { std::cout << "connected." << std::endl; close(s); } return 0; } yrmcds-1.0.3/test/cpucount.cpp000066400000000000000000000002071226611677100163470ustar00rootroot00000000000000#include #include int main() { std::cout << std::thread::hardware_concurrency() << std::endl; return 0; } yrmcds-1.0.3/test/detach.cpp000066400000000000000000000003011226611677100157320ustar00rootroot00000000000000#include #include void f() { char c; std::cin >> c; } int main() { std::thread t(f); t.detach(); std::cout << "detached" << std::endl; return 0; } yrmcds-1.0.3/test/dynbuf.cpp000066400000000000000000000023511226611677100160000ustar00rootroot00000000000000#include #include #include AUTOTEST(dynbuf0) { cybozu::dynbuf b(0); cybozu_assert( b.empty() ); b.append("abc", 3); cybozu_assert( b.size() == 3 ); cybozu_assert( ! b.empty() ); b.append("def", 4); cybozu_assert( b.size() == 7 ); cybozu_assert( std::memcmp(b.data(), "abcdef", 7) == 0 ); b.reset(); cybozu_assert( b.size() == 0 ); char* p = b.prepare(7); std::memcpy(p, "aaa", 4); b.consume(4); cybozu_assert( b.size() == 4 ); cybozu_assert( std::memcmp(b.data(), "aaa", 4) == 0 ); b.append("1111112345678", 13); b.erase(10); cybozu_assert( b.size() == 7 ); cybozu_assert( std::memcmp(b.data(), "2345678", 7) == 0 ); } AUTOTEST(dynbuf5) { cybozu::dynbuf b(0); cybozu_assert( b.empty() ); b.append("abc", 3); b.append("defghi", 6); cybozu_assert( b.size() == 9 ); cybozu_assert( std::memcmp(b.data(), "abcdefghi", 9) == 0 ); b.erase(6); cybozu_assert( b.size() == 3 ); cybozu_assert( std::memcmp(b.data(), "ghi", 3) == 0 ); b.append("0123456789", 10); b.erase(1); cybozu_assert( b.size() == 12 ); cybozu_assert( std::memcmp(b.data(), "hi0123456789", 12) == 0 ); b.reset(); } yrmcds-1.0.3/test/echo_server.cpp000066400000000000000000000043551226611677100170230ustar00rootroot00000000000000#include #include #include #include #include #include #include #include #include #include #include class echo_socket: public cybozu::tcp_socket { public: echo_socket(int fd): tcp_socket(fd, 1), m_buffer(1 <<20) {} static bool on_accept(int s) { std::cerr << "accepted." << std::endl; return true; } private: std::vector m_buffer; void echo_back(std::size_t len) { send(m_buffer.data(), len, false); send(m_buffer.data(), len, true); } virtual bool on_readable() override final { std::size_t received = 0; while( true ) { if( m_buffer.size() == received ) { std::cerr << "Too large data received." << std::endl; received = 0; } ssize_t n = ::recv(m_fd, &m_buffer[received], m_buffer.size() - received, 0); if( n == -1 ) { if( errno == EINTR ) continue; if( errno == EAGAIN || errno == EWOULDBLOCK ) { echo_back(received); return true; } auto ecnd = std::system_category().default_error_condition(errno); cybozu::logger::error() << ": (" << ecnd.value() << ") " << ecnd.message(); return false; } if( n == 0 ) return invalidate(); received += n; } } }; int main(int argc, char** argv) { if( argc == 1 ) { std::cout << "Usage: echo_server.exe PORT" << std::endl; return 0; } uint16_t port = static_cast(std::stoi(argv[1])); cybozu::reactor r; cybozu::tcp_server_socket::wrapper w = [](int s, const cybozu::ip_address&) { return std::unique_ptr( new echo_socket(s) ); }; r.add_resource( cybozu::make_server_socket(NULL, port, w), cybozu::reactor::EVENT_IN ); r.run([](cybozu::reactor& r){ std::cerr << "."; }); return 0; } yrmcds-1.0.3/test/endian.cpp000066400000000000000000000012411226611677100157440ustar00rootroot00000000000000#include #include #include int main(int argc, char** argv) { #if __BYTE_ORDER == __LITTLE_ENDIAN std::cout << "little endian" << std::endl; #elif __BYTE_ORDER == __BIG_ENDIAN std::cout << "big endian" << std::endl; #endif std::uint64_t n = 0x1234567887654321ULL; std::cout << std::hex << "n = 0x" << n << std::endl; std::cout << "htobe64(n) = 0x" << htobe64(n) << std::endl; std::uint64_t t; cybozu::hton(n, (char*)&t); std::cout << "hton(n, &t) = 0x" << t << std::endl; std::uint64_t n2; cybozu::ntoh((char*)&t, n2); std::cout << "ntoh(t, n2) = 0x" << n2 << std::endl; return 0; } yrmcds-1.0.3/test/epoll.cpp000066400000000000000000000064511226611677100156310ustar00rootroot00000000000000// test if epoll can receive hangup at clients. #include #include #include #include #include #include #include #include #include #include #include int create_socket(const struct addrinfo& ai) { int s = socket(ai.ai_family, ai.ai_socktype, ai.ai_protocol); if( s == -1 ) throw std::system_error(errno, std::system_category(), "socket"); int enable = 1; if( setsockopt(s, SOL_SOCKET, SO_REUSEADDR, &enable, sizeof(enable)) == -1 ) throw std::system_error(errno, std::system_category(), "setsocketopt"); if( bind(s, ai.ai_addr, ai.ai_addrlen) == -1 ) throw std::system_error(errno, std::system_category(), "bind"); if( listen(s, 10) == -1 ) throw std::system_error(errno, std::system_category(), "listen"); return s; } int main(int argc, char** argv) { if( argc != 2 ) { std::cout << "Usage: epoll PORT" << std::endl; return 0; } struct addrinfo hint; std::memset(&hint, 0, sizeof(hint)); hint.ai_family = AF_UNSPEC; hint.ai_socktype = SOCK_STREAM; hint.ai_protocol = 0; hint.ai_flags = AI_PASSIVE; struct addrinfo *res; int e = getaddrinfo(NULL, argv[1], &hint, &res); if( e == EAI_SYSTEM ) { throw std::system_error(errno, std::system_category(), "getaddrinfo"); } else if( e != 0 ) { throw std::runtime_error(std::string("getaddrinfo: ") + gai_strerror(e)); } const struct addrinfo* p; int s; for( p = res; p != nullptr; p = p->ai_next ) if( p->ai_family == AF_INET6 ) break; if( p != nullptr ) { s = create_socket(*p); } else { for( p = res; p != nullptr; p = p->ai_next ) if( p->ai_family == AF_INET ) break; if( p != nullptr ) { s = create_socket(*p); } else { throw std::runtime_error("No address to bind."); } } struct sockaddr addr; socklen_t slen = sizeof(addr); int clientfd = accept4(s, &addr, &slen, SOCK_NONBLOCK); if( clientfd == -1 ) throw std::system_error(errno, std::system_category(), "accept4"); close(s); int efd = epoll_create(10); if( efd == -1 ) throw std::system_error(errno, std::system_category(), "epoll_create"); struct epoll_event ev; ev.events = EPOLLIN|EPOLLET; ev.data.fd = clientfd; if( epoll_ctl(efd, EPOLL_CTL_ADD, clientfd, &ev) == -1 ) throw std::system_error(errno, std::system_category(), "epoll_ctl"); std::vector buf(1 << 20); struct epoll_event events[10]; while( true ) { int n = epoll_wait(efd, events, 10, -1); for(int i = 0; i < n; ++i ) { if( events[i].events & EPOLLIN ) { std::cout << "EPOLLIN" << std::endl; while( true ) { if( recv(events[i].data.fd, buf.data(), buf.size(), 0) == -1 ) break; } } if( events[i].events & EPOLLHUP ) { std::cout << "EPOLLHUP" << std::endl; break; } if( events[i].events & EPOLLERR ) { std::cout << "EPOLLHUP" << std::endl; } } } return 0; } yrmcds-1.0.3/test/eventfd.cpp000066400000000000000000000023111226611677100161400ustar00rootroot00000000000000#include #include #include #include #include #include #include #include void write_event(int efd, std::uint64_t ev) { ssize_t n = ::write(efd, &ev, sizeof(ev)); if( n != sizeof(ev) ) throw std::system_error(errno, std::system_category(), "write"); std::cerr << "write event of " << n << " bytes." << std::endl; } std::uint64_t read_event(int efd) { std::uint64_t i; ssize_t n = ::read(efd, &i, sizeof(i)); if( n != sizeof(i) ) throw std::system_error(errno, std::system_category(), "read"); return i; } void foo(int efd) { std::cout << "got event: " << read_event(efd) << std::endl; // see what will be returned, 10 or 16? Both can be seen. std::cout << "got event: " << read_event(efd) << std::endl; } int main(int argc, char** argv) { int efd = eventfd(0, EFD_CLOEXEC); if( efd == -1 ) throw std::system_error(errno, std::system_category(), "eventfd"); write_event(efd, 2); std::thread t(foo, efd); std::this_thread::sleep_for(std::chrono::milliseconds(200)); write_event(efd, 10); write_event(efd, 6); t.join(); return 0; } yrmcds-1.0.3/test/filerecv.cpp000066400000000000000000000071731226611677100163170ustar00rootroot00000000000000#include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include std::string g_prefix; int g_counter = 0; const std::size_t MAX_RECV = (2 << 20); // 2 MiB class filerecv_socket: public cybozu::tcp_socket { public: filerecv_socket(int fd): cybozu::tcp_socket(fd), m_file( ::creat((g_prefix+std::to_string(++g_counter)).c_str(), 0644) ), m_buffer(1 << 20) { if( m_fd == -1 ) cybozu::throw_unix_error(errno, "creat"); } ~filerecv_socket() { ::close(m_file); } private: const int m_file; std::size_t m_written = 0; std::vector m_buffer; virtual bool on_readable() override final { std::size_t received = 0; while( true ) { ssize_t n = ::recv(m_fd, m_buffer.data(), m_buffer.size(), 0); if( n == -1 ) { if( errno == EAGAIN || errno == EWOULDBLOCK ) break; cybozu::throw_unix_error(errno, "recv"); } if( n == 0 ) { std::cout << "received and stored " << m_written << " bytes." << std::endl; return invalidate(); } received += n; m_written += n; while( n > 0 ) { ssize_t nw = ::write(m_file, m_buffer.data(), n); if( nw == -1 ) cybozu::throw_unix_error(errno, "write"); n -= nw; } if( received > MAX_RECV ) { m_reactor->add_readable(*this); return true; } } return true; } }; int main(int argc, char** argv) { if( argc != 2 ) { std::cout << "Usage: filerecv.exe PREFIX" << std::endl; return 0; } g_prefix = argv[1]; cybozu::logger::set_threshold(cybozu::severity::debug); cybozu::reactor r; auto sigres = cybozu::signal_setup({SIGHUP, SIGQUIT, SIGTERM, SIGINT}); sigres->set_handler( [](const struct signalfd_siginfo& si, cybozu::reactor& r) { switch(si.ssi_signo) { case SIGHUP: std::cerr << "got SIGHUP." << std::endl; break; case SIGQUIT: std::cerr << "got SIGQUIT." << std::endl; break; case SIGTERM: std::cerr << "got SIGTERM." << std::endl; break; case SIGINT: std::cerr << "got SIGINT." << std::endl; r.quit(); break; } } ); r.add_resource( std::move(sigres), cybozu::reactor::EVENT_IN ); cybozu::tcp_server_socket::wrapper w = [](int s, const cybozu::ip_address&) { return std::unique_ptr( new filerecv_socket(s) ); }; r.add_resource( cybozu::make_server_socket(NULL, 11111, w), cybozu::reactor::EVENT_IN ); r.run([](cybozu::reactor& r){ r.fix_garbage(); r.gc();}); return 0; } yrmcds-1.0.3/test/filesend.cpp000066400000000000000000000054101226611677100163010ustar00rootroot00000000000000#include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include //const std::size_t CHUNK_SIZE = (2 << 20); // 2 MiB const std::size_t CHUNK_SIZE = (512 << 10); // 512 KiB class filesend_socket: public cybozu::tcp_socket { public: filesend_socket(int fd): cybozu::tcp_socket(fd, 2) {} private: virtual bool on_readable() override final { return true; } }; void filesend(const char* path, cybozu::tcp_socket& s) { int fd = open(path, O_RDONLY); if( fd == -1 ) return; struct stat st; if( fstat(fd, &st) == -1 ) { std::cerr << "fstat failed." << std::endl; close(fd); cybozu::throw_unix_error(errno, "fstat"); } off_t length = st.st_size; std::cout << "sending " << length << " bytes." << std::endl; std::vector buffer(CHUNK_SIZE); char* const p_buf = buffer.data(); while( length ) { off_t to_read = std::min(length, (off_t)CHUNK_SIZE); ssize_t n = read(fd, p_buf, to_read); if( n == -1 ) { std::cerr << "read failed." << std::endl; close(fd); cybozu::throw_unix_error(errno, "read"); } length -= n; if( length == 0 ) { std::cerr << "closing." << std::endl; s.send_close(p_buf, n); } else { s.send(p_buf, n); } } close(fd); } int main(int argc, char** argv) { if( argc != 3 ) { std::cout << "Usage: filesend.exe FILENAME THREADS" << std::endl; return 0; } cybozu::logger::set_threshold(cybozu::severity::debug); const char* path = argv[1]; int n_threads = std::stoi(argv[2]); if( n_threads < 0 ) { std::cout << "Invalid number of threads." << std::endl; return 1; } cybozu::reactor r; std::vector threads; for( int i = 0; i < n_threads; ++i ) { int c = cybozu::tcp_connect(NULL, 11111); if( c == -1 ) { std::cerr << "failed to connect to localhost:11111" << std::endl; return 2; } std::unique_ptr s( new filesend_socket(c) ); threads.emplace_back(filesend, path, std::ref(*s)); r.add_resource( std::move(s), cybozu::reactor::EVENT_OUT ); } r.run([&threads](cybozu::reactor& r) { r.fix_garbage(); r.gc(); if( r.size() == 0 ) { for( auto& t: threads ) t.join(); r.quit(); } }); return 0; } yrmcds-1.0.3/test/hash_map.cpp000066400000000000000000000035361226611677100162770ustar00rootroot00000000000000#include #include #include using hash_map = cybozu::hash_map; bool updater(const cybozu::hash_key& k, std::string& s) { std::cerr << "Updating " << k.str() << std::endl; s.append("+"); return true; } bool expired(const cybozu::hash_key&, std::string&) { return false; } bool reader(const cybozu::hash_key&, std::string& s) { std::cout << s << std::endl; return true; } std::string creator(const cybozu::hash_key& k) { return std::string("hoge"); } const char key1[] = "abc"; const cybozu::hash_key hkey1(key1, sizeof(key1)); const char key2[] = "def"; const cybozu::hash_key hkey2(key2, sizeof(key2)); int main(int argc, char** argv) { hash_map m(8); std::cout << "bucket count = " << m.bucket_count() << std::endl; std::cerr << std::boolalpha; std::cerr << m.apply(hkey1, updater, nullptr) << std::endl; std::cerr << m.apply(hkey1, nullptr, creator) << std::endl; std::cerr << m.remove(hkey1, nullptr) << std::endl; std::cerr << m.remove(hkey1, nullptr) << std::endl; std::cerr << m.apply(hkey1, nullptr, creator) << std::endl; std::cerr << m.apply(hkey1, nullptr, creator) << std::endl; std::cerr << m.apply(hkey1, updater, creator) << std::endl; std::cerr << m.apply(hkey1, reader, nullptr) << std::endl; std::cerr << m.apply(hkey2, nullptr, creator) << std::endl; std::cerr << m.apply(hkey2, expired, nullptr) << std::endl; std::cerr << m.apply(hkey2, expired, creator) << std::endl; for( hash_map::bucket& b: m ) { b.gc([](const cybozu::hash_key& key, std::string& s) { std::cout << "collecting " << s << std::endl; return s.size() == 4; }); } std::cerr << m.apply(hkey1, reader, nullptr) << std::endl; std::cerr << m.apply(hkey2, reader, nullptr) << std::endl; return 0; } yrmcds-1.0.3/test/hide_api.cpp000066400000000000000000000006451226611677100162570ustar00rootroot00000000000000#include class A { public: A() {} virtual ~A() {} virtual int f() = 0; }; class B: public A { public: B(): A() {} virtual int g() { return 100; } private: virtual int f() { return 3; } }; int main() { A* a = new B; std::cout << a->f() << std::endl; delete a; // This causes compilation error, as expected! // //B b; //std::cout << b->f() << std::endl; } yrmcds-1.0.3/test/init.cpp000066400000000000000000000003231226611677100154510ustar00rootroot00000000000000#include struct A { A() { std::cout << "A::ctor" << std::endl; } void f() { std::cout << "A::f" << std::endl; } }; int main() { A a{}; a.f(); return 0; } yrmcds-1.0.3/test/ip_address.cpp000066400000000000000000000003011226611677100166170ustar00rootroot00000000000000#include #include AUTOTEST(has_ip_address) { for( int i = 0; i < 1000; ++i ) cybozu::has_ip_address(cybozu::ip_address("11.11.11.11")); } yrmcds-1.0.3/test/lambda.cpp000066400000000000000000000003251226611677100157300ustar00rootroot00000000000000#include int main(int argc, char** argv) { int x = 3; auto foo = [=]() mutable { ++x; return x * x; }; std::cout << foo() << std::endl; std::cout << foo() << std::endl; return 0; } yrmcds-1.0.3/test/logger.cpp000066400000000000000000000005761226611677100157770ustar00rootroot00000000000000#include /* To see the optimization effects, * 1. run gdbtui, * 2. enter "layout asm", * 3. "b main", "run", then "stepi". */ int main(int argc, char** argv) { if( argc > 1 ) cybozu::logger::instance().open(argv[1]); cybozu::logger::debug() << "hoge, " << 3.145 << ", " << (10LL<<40); cybozu::logger::info() << "fuga"; return 0; } yrmcds-1.0.3/test/lz4.cpp000066400000000000000000000035551226611677100152310ustar00rootroot00000000000000// #include "../lz4/lz4.h" #include #include #include #include #include #include #include #include #include const std::size_t S = 1 << 20; void fill_seq(std::vector& buf) { unsigned char c = 0; char* p = buf.data(); for( std::size_t i = 0; i < S; ++i ) { *p = c++; } } void fill_random(std::vector& buf) { int fd = open("/dev/urandom", O_RDONLY); if( fd == -1 ) { std::cerr << "failed to open /dev/urandom" << std::endl; std::exit(1); } std::size_t to_read = S; char* p = buf.data(); while( to_read != 0 ) { ssize_t n = read(fd, p, to_read); if( n == -1 ) { std::cerr << "failed to read from /dev/urandom" << std::endl; std::exit(1); } p += n; to_read -= n; } close(fd); } int main(int argc, char** argv) { // std::vector buf1(S); // fill_seq(buf1); // //fill_random(buf1); // std::vector buf2(S); // std::vector cbuf(LZ4_COMPRESSBOUND(S)); // std::cout << "cbuf size = " << cbuf.size() << std::endl; // int csize = LZ4_compress(buf1.data(), cbuf.data(), S); // if( csize == 0 ) { // std::cerr << "compression failed." << std::endl; // return 1; // } // std::cout << "compressed: " << csize << " bytes." << std::endl; // // beware that the third parameter is the *uncompressed* size. // int csize2 = LZ4_decompress_fast(cbuf.data(), buf2.data(), S); // if( csize != csize2 ) { // std::cerr << "wtf!?" << std::endl; // return 1; // } // if( std::memcmp(buf1.data(), buf2.data(), S) != 0 ) { // std::cerr << "verify failed." << std::endl; // return 1; // } // std::cout << "verify ok." << std::endl; return 0; } yrmcds-1.0.3/test/macro_stringify.cpp000066400000000000000000000002631226611677100177100ustar00rootroot00000000000000#include #define QUOTE(str) #str #define EXPAND_AND_QUOTE(str) QUOTE(str) int main(int argc, char** argv) { puts(EXPAND_AND_QUOTE(DEFAULT_CONFIG)); return 0; } yrmcds-1.0.3/test/memcache_binary.cpp000066400000000000000000000663061226611677100176310ustar00rootroot00000000000000#include "../src/memcache.hpp" #include #include #include using namespace yrmcds::memcache; #define REQ(n, data) \ const char i##n[] = data; \ binary_request r##n(i##n, sizeof(i##n) - 1); #define ITEMCMP(t, s) \ cybozu_assert( std::get<1>(t) == (sizeof(s) - 1) ); \ cybozu_assert( std::memcmp(std::get<0>(t), s, sizeof(s) - 1) == 0 ); AUTOTEST(get) { REQ(1, "\x80\x00\x00\x00"); cybozu_assert( r1.length() == 0 ); REQ(2, "\x80\x00\x00\x05\x00\x00\x00\x00" "\x00\x00\x00\x05" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x01" // CAS "Hello" // key ); cybozu_assert( r2.length() == (24 + 5) ); cybozu_assert( r2.status() == binary_status::OK ); cybozu_assert( r2.command() == binary_command::Get ); cybozu_assert( ! r2.quiet() ); cybozu_assert( r2.key() == item(i2 + 24, 5) ); cybozu_assert( r2.exptime() == binary_request::EXPTIME_NONE ); ITEMCMP(r2.key(), "Hello"); cybozu_assert( std::memcmp(r2.opaque(), "\x12\x34\x56\x78", 4) == 0 ); cybozu_assert( r2.cas_unique() == 1 ); REQ(3, "\x80\x00\x00\x05\x00\x00\x00\x00" "\x00\x00\x00\x05" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x01" // CAS "Hell" // key ); cybozu_assert( r3.length() == 0 ); REQ(4, "\x80\x00\x00\x06\x00\x00\x00\x00" "\x00\x00\x00\x05" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x01" // CAS "Hello " // key ); cybozu_assert( r4.length() == (24 + 5) ); cybozu_assert( r4.status() == binary_status::Invalid ); REQ(5, "\x80\x00\x00\x05\x04\x00\x00\x00" "\x00\x00\x00\x09" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x01" // CAS "\x00\x00\x01\x00" // extra (exptime) "Hello " // key ); cybozu_assert( r5.length() == (24 + 9) ); cybozu_assert( r5.status() == binary_status::OK ); cybozu_assert( r5.exptime() != 0 && r5.exptime() != binary_request::EXPTIME_NONE ); const char data6[] = "\x80\x00\x00\x05\x04\x00\x00\x00" "\x00\x00\x00\x09" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x01" // CAS "\x00\x00\x01\x00" // extra (exptime) "Hello" // key "\x80\x09\x00\x05\x00\x00\x00\x00" "\x00\x00\x00\x05" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x01" // CAS "Hello "; // key binary_request r6(data6, sizeof(data6) - 1); binary_request r6_2(data6 + r6.length(), sizeof(data6) - r6.length() - 1); cybozu_assert( r6_2.status() == binary_status::OK ); cybozu_assert( r6_2.command() == binary_command::GetQ ); cybozu_assert( r6_2.quiet() ); REQ(7, "\x80\x81\x00\x00\x00\x00\x00\x00" "\x00\x00\x00\x00" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x00" // CAS ); cybozu_assert( r7.length() == 24 ); cybozu_assert( r7.status() == binary_status::UnknownCommand ); REQ(8, "\x80\x0c\x00\x05\x04\x00\x00\x00" "\x00\x00\x00\x09" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x01" // CAS "\x00\x00\x01\x00" // extra (exptime) "Hello " // key ); cybozu_assert( r8.length() != 0 ); cybozu_assert( r8.status() == binary_status::OK ); cybozu_assert( r8.command() == binary_command::GetK ); cybozu_assert( ! r8.quiet() ); REQ(9, "\x80\x0d\x00\x05\x04\x00\x00\x00" "\x00\x00\x00\x09" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x01" // CAS "\x00\x00\x01\x00" // extra (exptime) "Hello " // key ); cybozu_assert( r9.length() != 0 ); cybozu_assert( r9.status() == binary_status::OK ); cybozu_assert( r9.command() == binary_command::GetKQ ); cybozu_assert( r9.quiet() ); REQ(10, "\x80\x1d\x00\x05\x04\x00\x00\x00" "\x00\x00\x00\x09" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x01" // CAS "\x00\x00\x01\x00" // extra (exptime) "Hello" // key ); cybozu_assert( r10.length() != 0 ); cybozu_assert( r10.status() == binary_status::OK ); cybozu_assert( r10.command() == binary_command::GaT ); cybozu_assert( ! r10.quiet() ); cybozu_assert( r10.exptime() != binary_request::EXPTIME_NONE ); REQ(11, "\x80\x1e\x00\x05\x04\x00\x00\x00" "\x00\x00\x00\x09" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x01" // CAS "\x00\x00\x01\x00" // extra (exptime) "Hello" // key ); cybozu_assert( r11.length() != 0 ); cybozu_assert( r11.status() == binary_status::OK ); cybozu_assert( r11.command() == binary_command::GaTQ ); cybozu_assert( r11.quiet() ); REQ(12, "\x80\x23\x00\x05\x04\x00\x00\x00" "\x00\x00\x00\x09" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x01" // CAS "\x00\x00\x01\x00" // extra (exptime) "Hello" // key ); cybozu_assert( r12.length() != 0 ); cybozu_assert( r12.status() == binary_status::OK ); cybozu_assert( r12.command() == binary_command::GaTK ); cybozu_assert( ! r12.quiet() ); REQ(13, "\x80\x24\x00\x05\x04\x00\x00\x00" "\x00\x00\x00\x09" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x01" // CAS "\x00\x00\x01\x00" // extra (exptime) "Hello" // key ); cybozu_assert( r13.length() != 0 ); cybozu_assert( r13.status() == binary_status::OK ); cybozu_assert( r13.command() == binary_command::GaTKQ ); cybozu_assert( r13.quiet() ); REQ(14, "\x80\x46\x00\x05\x04\x00\x00\x00" "\x00\x00\x00\x09" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x01" // CAS "\x00\x00\x01\x00" // extra (exptime) "Hello" // key ); cybozu_assert( r14.length() == 24 + 9 ); cybozu_assert( r14.status() == binary_status::OK ); cybozu_assert( r14.command() == binary_command::LaG ); cybozu_assert( r14.cas_unique() == 1UL ); cybozu_assert( ! r14.quiet() ); cybozu_assert( r14.exptime() != binary_request::EXPTIME_NONE ); REQ(15, "\x80\x47\x00\x05\x00\x00\x00\x00" "\x00\x00\x00\x05" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x01" // CAS "Hello" // key ); cybozu_assert( r15.length() != 0 ); cybozu_assert( r15.status() == binary_status::OK ); cybozu_assert( r15.command() == binary_command::LaGQ ); cybozu_assert( r15.quiet() ); cybozu_assert( r15.exptime() == binary_request::EXPTIME_NONE ); REQ(16, "\x80\x48\x00\x05\x04\x00\x00\x00" "\x00\x00\x00\x09" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x01" // CAS "\x00\x00\x01\x00" // extra (exptime) "Hello" // key ); cybozu_assert( r16.length() != 0 ); cybozu_assert( r16.status() == binary_status::OK ); cybozu_assert( r16.command() == binary_command::LaGK ); cybozu_assert( ! r16.quiet() ); REQ(17, "\x80\x49\x00\x05\x04\x00\x00\x00" "\x00\x00\x00\x09" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x01" // CAS "\x00\x00\x01\x00" // extra (exptime) "Hello" // key ); cybozu_assert( r17.length() != 0 ); cybozu_assert( r17.status() == binary_status::OK ); cybozu_assert( r17.command() == binary_command::LaGKQ ); cybozu_assert( r17.quiet() ); } AUTOTEST(set_add_replace) { REQ(1, "\x80\x01\x00\x05\x08\x00\x00\x00" "\x00\x00\x00\x12" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x10" // CAS "\x00\x00\x00\x20" // flags "\x11\x11\x22\x33" // extra (exptime) "Hello" // key "World" // body ); cybozu_assert( r1.length() == (24 + 18) ); cybozu_assert( r1.status() == binary_status::OK ); cybozu_assert( r1.command() == binary_command::Set ); cybozu_assert( ! r1.quiet() ); cybozu_assert( r1.cas_unique() == 0x10 ); cybozu_assert( r1.flags() == 0x20 ); cybozu_assert( r1.exptime() == 0x11112233UL ); ITEMCMP(r1.key(), "Hello"); ITEMCMP(r1.data(), "World"); REQ(2, "\x80\x01\x00\x00\x08\x00\x00\x00" "\x00\x00\x00\x0d" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x10" // CAS "\x00\x00\x00\x20" // flags "\x00\x00\x00\x00" // extra (exptime) "World" // body ); cybozu_assert( r2.length() == (24 + 13) ); cybozu_assert( r2.status() == binary_status::Invalid ); cybozu_assert( r2.exptime() == 0 ); REQ(3, "\x80\x02\x00\x06\x08\x00\x00\x00" "\x00\x00\x00\x13" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x10" // CAS "\x00\x00\x00\x20" // flags "\x11\x11\x22\x33" // extra (exptime) "Hello " // key "World" // body "\x80 " ); cybozu_assert( r3.length() == (24 + 19) ); cybozu_assert( r3.status() == binary_status::OK ); cybozu_assert( r3.command() == binary_command::Add ); cybozu_assert( ! r3.quiet() ); REQ(4, "\x80\x03\x00\x06\x08\x00\x00\x00" "\x00\x00\x00\x13" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x10" // CAS "\x00\x00\x00\x20" // flags "\x11\x11\x22\x33" // extra (exptime) "Hello " // key "World" // body "\x80 " ); cybozu_assert( r4.length() == (24 + 19) ); cybozu_assert( r4.status() == binary_status::OK ); cybozu_assert( r4.command() == binary_command::Replace ); cybozu_assert( ! r4.quiet() ); REQ(5, "\x80\x11\x00\x06\x08\x00\x00\x00" "\x00\x00\x00\x13" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x10" // CAS "\x00\x00\x00\x20" // flags "\x11\x11\x22\x33" // extra (exptime) "Hello " // key "World" // body ); cybozu_assert( r5.length() != 0 ); cybozu_assert( r5.status() == binary_status::OK ); cybozu_assert( r5.command() == binary_command::SetQ ); cybozu_assert( r5.quiet() ); REQ(6, "\x80\x12\x00\x06\x08\x00\x00\x00" "\x00\x00\x00\x13" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x10" // CAS "\x00\x00\x00\x20" // flags "\x11\x11\x22\x33" // extra (exptime) "Hello " // key "World" // body ); cybozu_assert( r6.length() != 0 ); cybozu_assert( r6.status() == binary_status::OK ); cybozu_assert( r6.command() == binary_command::AddQ ); cybozu_assert( r6.quiet() ); REQ(7, "\x80\x13\x00\x06\x08\x00\x00\x00" "\x00\x00\x00\x13" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x10" // CAS "\x00\x00\x00\x20" // flags "\x11\x11\x22\x33" // extra (exptime) "Hello " // key "World" // body ); cybozu_assert( r7.length() != 0 ); cybozu_assert( r7.status() == binary_status::OK ); cybozu_assert( r7.command() == binary_command::ReplaceQ ); cybozu_assert( r7.quiet() ); REQ(8, "\x80\x02\x00\x06\x04\x00\x00\x00" "\x00\x00\x00\x0f" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x10" // CAS "\x00\x00\x00\x20" // flags "Hello " // key "World" // body "\x80 " ); cybozu_assert( r8.length() == (24 + 15) ); cybozu_assert( r8.status() == binary_status::Invalid ); REQ(9, "\x80\x13\x00\x00\x08\x00\x00\x00" "\x00\x00\x00\x0d" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x10" // CAS "\x00\x00\x00\x20" // flags "\x11\x11\x22\x33" // extra (exptime) "World" // body ); cybozu_assert( r9.length() != 0 ); cybozu_assert( r9.status() == binary_status::Invalid ); REQ(10, "\x80\x4a\x00\x06\x08\x00\x00\x00" "\x00\x00\x00\x13" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x10" // CAS "\x00\x00\x00\x20" // flags "\x11\x11\x22\x33" // extra (exptime) "Hello " // key "World" // body "\x80 " ); cybozu_assert( r10.length() == (24 + 19) ); cybozu_assert( r10.status() == binary_status::OK ); cybozu_assert( r10.command() == binary_command::RaU ); cybozu_assert( ! r10.quiet() ); ITEMCMP(r10.key(), "Hello "); ITEMCMP(r10.data(), "World"); REQ(11, "\x80\x4b\x00\x06\x08\x00\x00\x00" "\x00\x00\x00\x13" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x10" // CAS "\x00\x00\x00\x20" // flags "\x11\x11\x22\x33" // extra (exptime) "Hello " // key "World" // body "\x80 " ); cybozu_assert( r11.length() != 0 ); cybozu_assert( r11.status() == binary_status::OK ); cybozu_assert( r11.command() == binary_command::RaUQ ); cybozu_assert( r11.quiet() ); } AUTOTEST(delete) { REQ(1, "\x80\x04\x00\x05\x00\x00\x00\x00" "\x00\x00\x00\x05" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x10" // CAS "Hello" // key ); cybozu_assert( r1.length() != 0 ); cybozu_assert( r1.status() == binary_status::OK ); cybozu_assert( r1.command() == binary_command::Delete ); cybozu_assert( ! r1.quiet() ); ITEMCMP(r1.key(), "Hello"); REQ(2, "\x80\x14\x00\x05\x00\x00\x00\x00" "\x00\x00\x00\x05" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x10" // CAS "Hello" // key ); cybozu_assert( r2.length() != 0 ); cybozu_assert( r2.status() == binary_status::OK ); cybozu_assert( r2.command() == binary_command::DeleteQ ); cybozu_assert( r2.quiet() ); REQ(3, "\x80\x14\x00\x00\x00\x00\x00\x00" "\x00\x00\x00\x00" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x10" // CAS ); cybozu_assert( r3.length() != 0 ); cybozu_assert( r3.status() == binary_status::Invalid ); } AUTOTEST(inc_dec) { REQ(1, "\x80\x05\x00\x03\x14\x00\x00\x00" "\x00\x00\x00\x17" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x10" // CAS "\x00\x00\x00\x00\x00\x00\x00\x20" // value "\x00\x00\x00\x00\x00\x00\x00\x01" // inital "\xff\xff\xff\xff" // extra (exptime) "abc" // key ); cybozu_assert( r1.length() == (24 + 23) ); cybozu_assert( r1.status() == binary_status::OK ); cybozu_assert( r1.command() == binary_command::Increment ); cybozu_assert( ! r1.quiet() ); cybozu_assert( r1.value() == 0x20 ); cybozu_assert( r1.initial() == 0x01 ); cybozu_assert( r1.exptime() == binary_request::EXPTIME_NONE ); REQ(2, "\x80\x06\x00\x03\x14\x00\x00\x00" "\x00\x00\x00\x17" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x10" // CAS "\x00\x00\x00\x00\x00\x00\x00\x20" // value "\x00\x00\x00\x00\x00\x00\x00\x01" // inital "\x00\x00\x00\x00" // extra (exptime) "abc" // key ); cybozu_assert( r2.length() == (24 + 23) ); cybozu_assert( r2.status() == binary_status::OK ); cybozu_assert( r2.command() == binary_command::Decrement ); cybozu_assert( ! r2.quiet() ); cybozu_assert( r2.exptime() == 0 ); REQ(3, "\x80\x06\x00\x03\x0c\x00\x00\x00" "\x00\x00\x00\x0f" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x10" // CAS "\x00\x00\x00\x00\x00\x00\x00\x20" // value "\x00\x00\x00\x00" // extra (exptime) "abc" // key ); cybozu_assert( r3.length() != 0 ); cybozu_assert( r3.status() == binary_status::Invalid ); REQ(4, "\x80\x15\x00\x03\x14\x00\x00\x00" "\x00\x00\x00\x17" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x10" // CAS "\x00\x00\x00\x00\x00\x00\x00\x20" // value "\x00\x00\x00\x00\x00\x00\x00\x01" // inital "\x00\x00\x00\x00" // extra (exptime) "abc" // key ); cybozu_assert( r4.length() != 0 ); cybozu_assert( r4.status() == binary_status::OK ); cybozu_assert( r4.command() == binary_command::IncrementQ ); cybozu_assert( r4.quiet() ); REQ(5, "\x80\x16\x00\x03\x14\x00\x00\x00" "\x00\x00\x00\x17" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x10" // CAS "\x00\x00\x00\x00\x00\x00\x00\x20" // value "\x00\x00\x00\x00\x00\x00\x00\x01" // inital "\x00\x00\x00\x00" // extra (exptime) "abc" // key ); cybozu_assert( r5.length() != 0 ); cybozu_assert( r5.status() == binary_status::OK ); cybozu_assert( r5.command() == binary_command::DecrementQ ); cybozu_assert( r5.quiet() ); } AUTOTEST(flush) { REQ(1, "\x80\x08\x00\x00\x00\x00\x00\x00" "\x00\x00\x00\x00" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x10" // CAS ); cybozu_assert( r1.length() != 0 ); cybozu_assert( r1.status() == binary_status::OK ); cybozu_assert( r1.command() == binary_command::Flush ); cybozu_assert( ! r1.quiet() ); cybozu_assert( r1.exptime() != 0 || r1.exptime() != binary_request::EXPTIME_NONE ); REQ(2, "\x80\x08\x00\x00\x04\x00\x00\x00" "\x00\x00\x00\x04" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x10" // CAS "\x11\x22\x33\x44" // extra (exptime) ); cybozu_assert( r2.length() != 0 ); cybozu_assert( r2.status() == binary_status::OK ); cybozu_assert( r2.command() == binary_command::Flush ); cybozu_assert( r2.exptime() == 0x11223344UL ); REQ(3, "\x80\x18\x00\x00\x00\x00\x00\x00" "\x00\x00\x00\x00" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x10" // CAS ); cybozu_assert( r3.length() != 0 ); cybozu_assert( r3.status() == binary_status::OK ); cybozu_assert( r3.command() == binary_command::FlushQ ); cybozu_assert( r3.quiet() ); } AUTOTEST(append_prepend) { REQ(1, "\x80\x0e\x00\x05\x00\x00\x00\x00" "\x00\x00\x00\x06" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x10" // CAS "Hello" // key "!" // body ); cybozu_assert( r1.length() != 0 ); cybozu_assert( r1.status() == binary_status::OK ); cybozu_assert( r1.command() == binary_command::Append ); cybozu_assert( ! r1.quiet() ); ITEMCMP(r1.key(), "Hello"); ITEMCMP(r1.data(), "!"); REQ(2, "\x80\x0f\x00\x05\x00\x00\x00\x00" "\x00\x00\x00\x08" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x10" // CAS "Hello" // key "!!!" // body ); cybozu_assert( r2.length() != 0 ); cybozu_assert( r2.status() == binary_status::OK ); cybozu_assert( r2.command() == binary_command::Prepend ); cybozu_assert( ! r2.quiet() ); ITEMCMP(r2.key(), "Hello"); ITEMCMP(r2.data(), "!!!"); REQ(3, "\x80\x19\x00\x05\x00\x00\x00\x00" "\x00\x00\x00\x08" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x10" // CAS "Hello" // key "!!!" // body ); cybozu_assert( r3.length() != 0 ); cybozu_assert( r3.status() == binary_status::OK ); cybozu_assert( r3.command() == binary_command::AppendQ ); cybozu_assert( r3.quiet() ); REQ(4, "\x80\x1a\x00\x05\x00\x00\x00\x00" "\x00\x00\x00\x08" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x10" // CAS "Hello" // key "!!!" // body ); cybozu_assert( r4.length() != 0 ); cybozu_assert( r4.status() == binary_status::OK ); cybozu_assert( r4.command() == binary_command::PrependQ ); cybozu_assert( r4.quiet() ); } AUTOTEST(lock) { REQ(1, "\x80\x40\x00\x05\x00\x00\x00\x00" "\x00\x00\x00\x05" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x00" // CAS "Hello" // key ); cybozu_assert( r1.length() == 24 + 5 ); cybozu_assert( r1.status() == binary_status::OK ); cybozu_assert( r1.command() == binary_command::Lock ); cybozu_assert( ! r1.quiet() ); ITEMCMP(r1.key(), "Hello"); REQ(2, "\x80\x40\x00\x00\x00\x00\x00\x00" "\x00\x00\x00\x00" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x00" // CAS ); cybozu_assert( r2.length() == 24 ); cybozu_assert( r2.status() == binary_status::Invalid ); REQ(3, "\x80\x41\x00\x05\x00\x00\x00\x00" "\x00\x00\x00\x05" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x00" // CAS "Hello" // key ); cybozu_assert( r3.length() != 0 ); cybozu_assert( r3.status() == binary_status::OK ); cybozu_assert( r3.command() == binary_command::LockQ ); cybozu_assert( r3.quiet() ); } AUTOTEST(unlock) { REQ(1, "\x80\x42\x00\x05\x00\x00\x00\x00" "\x00\x00\x00\x05" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x00" // CAS "Hello" // key ); cybozu_assert( r1.length() == 24 + 5 ); cybozu_assert( r1.status() == binary_status::OK ); cybozu_assert( r1.command() == binary_command::Unlock ); cybozu_assert( ! r1.quiet() ); ITEMCMP(r1.key(), "Hello"); REQ(2, "\x80\x42\x00\x00\x00\x00\x00\x00" "\x00\x00\x00\x00" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x00" // CAS ); cybozu_assert( r2.length() == 24 ); cybozu_assert( r2.status() == binary_status::Invalid ); REQ(3, "\x80\x43\x00\x05\x00\x00\x00\x00" "\x00\x00\x00\x05" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x00" // CAS "Hello" // key ); cybozu_assert( r3.length() != 0 ); cybozu_assert( r3.status() == binary_status::OK ); cybozu_assert( r3.command() == binary_command::UnlockQ ); cybozu_assert( r3.quiet() ); } AUTOTEST(stat) { REQ(1, "\x80\x10\x00\x03\x00\x00\x00\x00" "\x00\x00\x00\x03" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x10" // CAS "foo" // key ); cybozu_assert( r1.length() != 0 ); cybozu_assert( r1.status() == binary_status::OK ); cybozu_assert( r1.command() == binary_command::Stat ); cybozu_assert( ! r1.quiet() ); ITEMCMP(r1.key(), "foo"); cybozu_assert( r1.stats() == stats_t::GENERAL ); REQ(2, "\x80\x10\x00\x00\x00\x00\x00\x00" "\x00\x00\x00\x00" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x10" // CAS ); cybozu_assert( r2.length() != 0 ); cybozu_assert( r2.status() == binary_status::OK ); cybozu_assert( r2.stats() == stats_t::GENERAL ); REQ(3, "\x80\x10\x00\x05\x00\x00\x00\x00" "\x00\x00\x00\x05" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x10" // CAS "items" // key ); cybozu_assert( r3.length() != 0 ); cybozu_assert( r3.status() == binary_status::OK ); cybozu_assert( r3.stats() == stats_t::ITEMS ); REQ(4, "\x80\x10\x00\x05\x00\x00\x00\x00" "\x00\x00\x00\x05" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x10" // CAS "sizes" // key ); cybozu_assert( r4.length() != 0 ); cybozu_assert( r4.status() == binary_status::OK ); cybozu_assert( r4.stats() == stats_t::SIZES ); REQ(5, "\x80\x10\x00\x08\x00\x00\x00\x00" "\x00\x00\x00\x08" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x10" // CAS "settings" // key ); cybozu_assert( r5.length() != 0 ); cybozu_assert( r5.status() == binary_status::OK ); cybozu_assert( r5.stats() == stats_t::SETTINGS ); } AUTOTEST(touch) { REQ(1, "\x80\x1c\x00\x05\x04\x00\x00\x00" "\x00\x00\x00\x09" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x01" // CAS "\x00\x00\x01\x00" // extra (exptime) "Hello " // key ); cybozu_assert( r1.length() == (24 + 9) ); cybozu_assert( r1.status() == binary_status::OK ); cybozu_assert( r1.command() == binary_command::Touch ); cybozu_assert( r1.exptime() != 0 && r1.exptime() != binary_request::EXPTIME_NONE ); REQ(2, "\x80\x1c\x00\x05\x00\x00\x00\x00" "\x00\x00\x00\x05" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x01" // CAS "Hello " // key ); cybozu_assert( r2.length() != 0 ); cybozu_assert( r2.status() == binary_status::Invalid ); } AUTOTEST(misc) { REQ(1, "\x80\x07\x00\x00\x00\x00\x00\x00" "\x00\x00\x00\x00" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x10" // CAS ); cybozu_assert( r1.length() != 0 ); cybozu_assert( r1.status() == binary_status::OK ); cybozu_assert( r1.command() == binary_command::Quit ); cybozu_assert( ! r1.quiet() ); REQ(2, "\x80\x17\x00\x00\x00\x00\x00\x00" "\x00\x00\x00\x00" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x10" // CAS ); cybozu_assert( r2.length() != 0 ); cybozu_assert( r2.status() == binary_status::OK ); cybozu_assert( r2.command() == binary_command::QuitQ ); cybozu_assert( r2.quiet() ); REQ(3, "\x80\x0a\x00\x00\x00\x00\x00\x00" "\x00\x00\x00\x00" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x10" // CAS ); cybozu_assert( r3.length() != 0 ); cybozu_assert( r3.status() == binary_status::OK ); cybozu_assert( r3.command() == binary_command::Noop ); REQ(4, "\x80\x0b\x00\x00\x00\x00\x00\x00" "\x00\x00\x00\x00" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x10" // CAS ); cybozu_assert( r4.length() != 0 ); cybozu_assert( r4.status() == binary_status::OK ); cybozu_assert( r4.command() == binary_command::Version ); REQ(5, "\x80\x44\x00\x00\x00\x00\x00\x00" "\x00\x00\x00\x00" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x10" // CAS ); cybozu_assert( r5.length() != 0 ); cybozu_assert( r5.status() == binary_status::OK ); cybozu_assert( r5.command() == binary_command::UnlockAll ); cybozu_assert( ! r5.quiet() ); REQ(6, "\x80\x45\x00\x00\x00\x00\x00\x00" "\x00\x00\x00\x00" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x10" // CAS ); cybozu_assert( r6.length() != 0 ); cybozu_assert( r6.status() == binary_status::OK ); cybozu_assert( r6.command() == binary_command::UnlockAllQ ); cybozu_assert( r6.quiet() ); } yrmcds-1.0.3/test/memcache_text.cpp000066400000000000000000000325351226611677100173260ustar00rootroot00000000000000#include "../src/memcache.hpp" #include #include #include #include #include #include #include #include #include using namespace yrmcds; using namespace yrmcds::memcache; const char CR = '\x0d'; const char NL = '\x0a'; const char SP = '\x20'; template inline UInt to_uint(const char* p, bool& result) { result = false; char* end; unsigned long long i = strtoull(p, &end, 10); if( i == 0 && p == end ) return 0; if( i == std::numeric_limits::max() && errno == ERANGE ) return 0; char c = *end; if( c != CR && c != NL && c != SP ) return 0; if( i > std::numeric_limits::max() ) return 0; result = true; return static_cast(i); } AUTOTEST(to_uint) { bool r; to_uint("\n", r); cybozu_assert( ! r ); to_uint("4294967295", r); cybozu_assert( ! r ); to_uint("4294967295 ", r); cybozu_assert( r ); to_uint("4294967295\r", r); cybozu_assert( r ); to_uint("4294967295\n", r); cybozu_assert( r ); to_uint("4294967296\n", r); cybozu_assert( ! r ); to_uint("4294967296\n", r); cybozu_assert( r ); cybozu_assert( to_uint("4294967296\n", r) == 4294967296ULL ); to_uint("-1\n", r); cybozu_assert( ! r ); to_uint(" 429496729\n", r); cybozu_assert( r ); to_uint(" 429496729\n", r); cybozu_assert( r ); to_uint(" 429496729\n", r); cybozu_assert( r ); cybozu_assert( to_uint("18446744073709551615 ", r) == 18446744073709551615ULL ); to_uint("18446744073709551616 ", r); cybozu_assert( ! r ); } #define MEMCACHE_TEST(n, d) \ char data##n[] = d; \ text_request t##n(data##n, sizeof(data##n) - 1); #define VERIFY(n, k, f) \ cybozu_assert( std::memcmp(std::get<0>(t##n.f()), k, sizeof(k)-1) == 0 ) AUTOTEST(empty) { MEMCACHE_TEST(1, "\r\n"); cybozu_assert( t1.length() == 2 ); cybozu_assert( ! t1.valid() ); MEMCACHE_TEST(2, " \n"); cybozu_assert( t2.length() == 2 ); cybozu_assert( ! t2.valid() ); } AUTOTEST(verbosity) { MEMCACHE_TEST(1, " verbosity"); cybozu_assert( t1.length() == 0 ); MEMCACHE_TEST(2, " verbosity\naaa"); cybozu_assert( t2.length() == 11 ); cybozu_assert( ! t2.valid() ); MEMCACHE_TEST(3, " verbosity \r\n"); cybozu_assert( t3.command() == text_command::VERBOSITY ); cybozu_assert( t3.length() != 0 ); cybozu_assert( ! t3.valid() ); cybozu_assert( ! t3.no_reply() ); MEMCACHE_TEST(4, " verbosity warning \r\naaa"); cybozu_assert( t4.valid() ); cybozu_assert( t4.verbosity() == cybozu::severity::warning ); cybozu_assert( ! t4.no_reply() ); MEMCACHE_TEST(5, " verbosity warning \r\nverbosity error\r\n"); cybozu_assert( t5.valid() ); cybozu_assert( t5.verbosity() == cybozu::severity::warning ); text_request t5_2(&data5[t5.length()], sizeof(data5) - 1 - t5.length()); cybozu_assert( t5_2.valid() ); cybozu_assert( t5_2.verbosity() == cybozu::severity::error ); MEMCACHE_TEST(6, " verbosity 1\r\n"); cybozu_assert( t6.valid() ); MEMCACHE_TEST(7, " verbosity error noreply\r\n"); cybozu_assert( t7.valid() ); cybozu_assert( t7.no_reply() ); } AUTOTEST(set) { MEMCACHE_TEST(1, "set \r\n "); cybozu_assert( t1.length() == 6 ); cybozu_assert( t1.command() == text_command::SET ); MEMCACHE_TEST(2, "set abcdef \r\n"); cybozu_assert( std::get<1>(t2.key()) == 6 ); VERIFY(2, "abcdef", key); cybozu_assert( ! t2.valid() ); MEMCACHE_TEST(3, "set abcdef 65536 \r\n"); cybozu_assert( t3.flags() == 65536 ); MEMCACHE_TEST(4, "set abcdef 0 \r\n"); cybozu_assert( t4.flags() == 0 ); MEMCACHE_TEST(5, "set abcdef 4294967295 \r\n"); cybozu_assert( t5.flags() == 4294967295ULL ); MEMCACHE_TEST(6, "set abcdef 4294967296 \r\n"); cybozu_assert( t6.flags() != 4294967296ULL ); MEMCACHE_TEST(7, "set abcdef 3 0 \r\n"); cybozu_assert( t7.flags() == 3 ); cybozu_assert( t7.exptime() == 0 ); cybozu_assert( ! t7.valid() ); MEMCACHE_TEST(8, "set abcdef 3 100 \r\n"); cybozu_assert( t8.exptime() < std::time(nullptr) + 101 ); cybozu_assert( t8.exptime() > 100 ); MEMCACHE_TEST(9, "set abcdef 3 3000000 \r\n"); cybozu_assert( t9.exptime() == 3000000 ); MEMCACHE_TEST(10, "set aaa 100 0 5\r\nabcde\r\nset aaa"); cybozu_assert( t10.valid() ); cybozu_assert( std::get<1>(t10.data()) == 5 ); VERIFY(10, "abcde", data); MEMCACHE_TEST(11, "set aaa 100 0 10\r\nabcdefghija\r\nset aaa"); cybozu_assert( ! t11.valid() ); MEMCACHE_TEST(12, "set aaa 100 0 10\r\nabcdefghij\r"); cybozu_assert( t12.length() == 0 ); MEMCACHE_TEST(13, "set aaa 100 0 10\r\nabcdefghij"); cybozu_assert( t13.length() == 0 ); MEMCACHE_TEST(14, "set aaa 100 0 10 \r\nabcdefghij\r\naaa"); cybozu_assert( t14.valid() ); cybozu_assert( ! t14.no_reply() ); MEMCACHE_TEST(15, "set aaa 100 0 10 noreply \r\nabcdefghij\r\naaa"); cybozu_assert( t15.valid() ); cybozu_assert( t15.no_reply() ); MEMCACHE_TEST(16, "set aaa 100 0 10 norepry\r\nabcdefghij\r\naaa"); cybozu_assert( ! t16.valid() ); MEMCACHE_TEST(17, "set aaa 100 0 10 noreply aaa \r\nabcdefghij\r\naaa"); cybozu_assert( ! t17.valid() ); } AUTOTEST(add) { MEMCACHE_TEST(1, " add aaa 100 0 10 noreply\r\nabcdefghij\r\n"); cybozu_assert( t1.valid() ); cybozu_assert( t1.command() == text_command::ADD ); cybozu_assert( t1.no_reply() ); } AUTOTEST(replace) { MEMCACHE_TEST(1, " replace aaa 100 0 10 noreply \r\nabcdefghij\r\n"); cybozu_assert( t1.valid() ); cybozu_assert( t1.command() == text_command::REPLACE ); cybozu_assert( t1.no_reply() ); } AUTOTEST(append) { MEMCACHE_TEST(1, " append aaa 100 0 10 noreply \r\nabcdefghij\r\n"); cybozu_assert( t1.valid() ); cybozu_assert( t1.command() == text_command::APPEND ); cybozu_assert( t1.no_reply() ); } AUTOTEST(prepend) { MEMCACHE_TEST(1, " prepend aaa 100 0 10 noreply \r\nabcdefghij\r\n"); cybozu_assert( t1.valid() ); cybozu_assert( t1.command() == text_command::PREPEND ); cybozu_assert( t1.no_reply() ); } AUTOTEST(cas) { MEMCACHE_TEST(1, " cas aaa 100 0 10 noreply \r\nabcdefghij\r\n"); cybozu_assert( ! t1.valid() ); cybozu_assert( t1.command() == text_command::CAS ); MEMCACHE_TEST(2, " cas aaa 100 0 10 3 noreply \r\nabcdefghij\r\n"); cybozu_assert( t2.valid() ); cybozu_assert( t2.cas_unique() == 3 ); cybozu_assert( t2.no_reply() ); MEMCACHE_TEST(3, " cas aaa 100 0 10 4\r\nabcdefghij\r\n"); cybozu_assert( t3.valid() ); cybozu_assert( t3.cas_unique() == 4 ); cybozu_assert( ! t3.no_reply() ); } AUTOTEST(delete) { MEMCACHE_TEST(1, "delete aaa noreply \r\nabc"); cybozu_assert( t1.valid() ); cybozu_assert( t1.command() == text_command::DELETE ); cybozu_assert( std::memcmp(std::get<0>(t1.key()), "aaa", 3) == 0 ); cybozu_assert( std::get<1>(t1.key()) == 3 ); cybozu_assert( t1.no_reply() ); MEMCACHE_TEST(2, "delete aaa garbage\r\nabc"); cybozu_assert( ! t2.valid() ); MEMCACHE_TEST(3, "delete aaagarbage\r\nabc"); cybozu_assert( t3.valid() ); cybozu_assert( std::get<1>(t3.key()) == 10 ); cybozu_assert( ! t3.no_reply() ); } AUTOTEST(touch) { MEMCACHE_TEST(1, "touch\r\nabc"); cybozu_assert( ! t1.valid() ); cybozu_assert( t1.command() == text_command::TOUCH ); MEMCACHE_TEST(2, "touch abcd \r\nabc"); cybozu_assert( ! t2.valid() ); cybozu_assert( std::get<1>(t2.key()) == 4 ); MEMCACHE_TEST(3, "touch abcd qqq\r\nabc"); cybozu_assert( ! t3.valid() ); MEMCACHE_TEST(4, "touch abcd 0 \r\nabc"); cybozu_assert( t4.valid() ); cybozu_assert( ! t4.no_reply() ); cybozu_assert( t4.exptime() == 0 ); MEMCACHE_TEST(5, "touch abcd 100 noreply\r\nabc"); cybozu_assert( t5.valid() ); cybozu_assert( t5.no_reply() ); MEMCACHE_TEST(6, "touch abcd 100 noreply 38\r\nabc"); cybozu_assert( ! t6.valid() ); cybozu_assert( t6.length() == 29 ); } AUTOTEST(stats) { MEMCACHE_TEST(1, "stats\r\nabc"); cybozu_assert( t1.valid() ); cybozu_assert( t1.command() == text_command::STATS ); cybozu_assert( t1.stats() == stats_t::GENERAL ); MEMCACHE_TEST(2, "stats hoge \r\nabc"); cybozu_assert( ! t2.valid() ); MEMCACHE_TEST(3, "stats items \r\nabc"); cybozu_assert( t3.valid() ); cybozu_assert( t3.stats() == stats_t::ITEMS ); MEMCACHE_TEST(4, "stats items allow garbage\r\nabc"); cybozu_assert( t4.valid() ); cybozu_assert( t4.stats() == stats_t::ITEMS ); MEMCACHE_TEST(5, "stats settings\r\nabc"); cybozu_assert( t5.valid() ); cybozu_assert( t5.stats() == stats_t::SETTINGS ); MEMCACHE_TEST(6, "stats sizes\r\nabc"); cybozu_assert( t6.valid() ); cybozu_assert( t6.stats() == stats_t::SIZES ); } AUTOTEST(incr) { MEMCACHE_TEST(1, "incr\r\nabc"); cybozu_assert( ! t1.valid() ); cybozu_assert( t1.command() == text_command::INCR ); MEMCACHE_TEST(2, "incr aaaaaa \r\nabc"); cybozu_assert( ! t2.valid() ); cybozu_assert( std::get<1>(t2.key()) == 6 ); MEMCACHE_TEST(3, "incr a 429496729500 \r\nabc"); cybozu_assert( t3.valid() ); cybozu_assert( t3.value() == 429496729500ULL ); cybozu_assert( ! t3.no_reply() ); MEMCACHE_TEST(4, "incr a noreply \r\nabc"); cybozu_assert( ! t4.valid() ); MEMCACHE_TEST(5, "incr a 3 garbagenowallowed\r\nabc"); cybozu_assert( ! t5.valid() ); MEMCACHE_TEST(6, "incr a 3 noreply garbagenowallowed\r\nabc"); cybozu_assert( ! t6.valid() ); MEMCACHE_TEST(7, "incr a 3 noreply \r\nabc"); cybozu_assert( t7.valid() ); cybozu_assert( t7.no_reply() ); } AUTOTEST(decr) { MEMCACHE_TEST(1, "decr tttt 100 noreply \r\nabc"); cybozu_assert( t1.valid() ); cybozu_assert( t1.command() == text_command::DECR ); cybozu_assert( std::get<1>(t1.key()) == 4 ); cybozu_assert( t1.value() == 100 ); cybozu_assert( t1.no_reply() ); } AUTOTEST(lock) { MEMCACHE_TEST(1, "lock hoge \r\n"); cybozu_assert( t1.valid() ); cybozu_assert( t1.command() == text_command::LOCK ); cybozu_assert( std::get<1>(t1.key()) == 4 ); VERIFY(1, "hoge", key); MEMCACHE_TEST(2, "lock hoge leftover\r\n"); cybozu_assert( ! t2.valid() ); cybozu_assert( t2.length() == 20 ); } AUTOTEST(unlock) { MEMCACHE_TEST(1, "unlock hoge \r\n"); cybozu_assert( t1.valid() ); cybozu_assert( t1.command() == text_command::UNLOCK ); cybozu_assert( std::get<1>(t1.key()) == 4 ); VERIFY(1, "hoge", key); MEMCACHE_TEST(2, "unlock hoge leftover\r\n"); cybozu_assert( ! t2.valid() ); cybozu_assert( t2.length() == 22 ); } AUTOTEST(unlockall) { MEMCACHE_TEST(1, "unlock_all \r\n"); cybozu_assert( t1.valid() ); cybozu_assert( t1.command() == text_command::UNLOCK_ALL ); } AUTOTEST(get) { MEMCACHE_TEST(1, "get \r\nabc"); cybozu_assert( ! t1.valid() ); cybozu_assert( t1.length() == 7 ); MEMCACHE_TEST(2, "get 38338 \r\nabc"); cybozu_assert( t2.valid() ); cybozu_assert( t2.command() == text_command::GET ); cybozu_assert( std::get<1>(t2.first_key()) == 5 ); VERIFY(2, "38338", first_key); cybozu_assert( t2.next_key(t2.first_key()) == text_request::eos ); MEMCACHE_TEST(3, "get 38338 4\r\nabc"); cybozu_assert( t3.valid() ); cybozu_assert( std::get<1>(t3.first_key()) == 5 ); VERIFY(3, "38338", first_key); auto next_key = t3.next_key(t3.first_key()); cybozu_assert( next_key != text_request::eos ); cybozu_assert( std::get<1>(next_key) == 1 ); cybozu_assert( *std::get<0>(next_key) == '4' ); next_key = t3.next_key(next_key); cybozu_assert( next_key == text_request::eos ); MEMCACHE_TEST(4, "get 38338 4 \r\nabc"); cybozu_assert( t4.valid() ); next_key = t4.next_key(t4.first_key()); cybozu_assert( std::get<1>(next_key) == 1 ); cybozu_assert( *std::get<0>(next_key) == '4' ); next_key = t4.next_key(next_key); cybozu_assert( next_key == text_request::eos ); } AUTOTEST(gets) { MEMCACHE_TEST(1, "gets 38338 \r\nabc"); cybozu_assert( t1.valid() ); cybozu_assert( t1.command() == text_command::GETS ); } AUTOTEST(flush_all) { MEMCACHE_TEST(1, "flush_all \r\nabc"); cybozu_assert( t1.valid() ); cybozu_assert( t1.command() == text_command::FLUSH_ALL ); MEMCACHE_TEST(2, "flush_all 100 \r\nabc"); cybozu_assert( t2.valid() ); cybozu_assert( ! t2.no_reply() ); cybozu_assert( t2.exptime() != 0 ); cybozu_assert( t2.exptime() < (time(NULL) + 101) ); MEMCACHE_TEST(3, "flush_all 100a \r\nabc"); cybozu_assert( ! t3.valid() ); MEMCACHE_TEST(4, "flush_all 100 noreply \r\nabc"); cybozu_assert( t4.valid() ); cybozu_assert( t4.no_reply() ); } AUTOTEST(singles) { MEMCACHE_TEST(1, "slabs 38338 \r\nabc"); cybozu_assert( t1.valid() ); cybozu_assert( t1.command() == text_command::SLABS ); MEMCACHE_TEST(2, "version 38338 \r\nabc"); cybozu_assert( t2.valid() ); cybozu_assert( t2.command() == text_command::VERSION ); MEMCACHE_TEST(3, "quit 38338 \r\nabc"); cybozu_assert( t3.valid() ); cybozu_assert( t3.command() == text_command::QUIT ); } yrmcds-1.0.3/test/move.cpp000066400000000000000000000021511226611677100154550ustar00rootroot00000000000000// move and copy test #include #include #include #include struct A { A() { std::cout << "A ctor." << std::endl; } A(A&& rhs) noexcept { std::cout << "A move." << std::endl; moved = true; } A(const A&) = delete; ~A() { if( ! moved ) std::cout << "A dtor." << std::endl; } private: bool moved = false; }; std::vector v; void f(A a) { v.push_back( std::move(a) ); } void g() { std::unique_ptr pa( new A ); typedef std::unordered_map > map_type; map_type m; m.emplace( 3, std::move(pa) ); std::cout << "get A." << std::endl; auto it = m.find(3); it->second.get(); std::cout << "got A." << std::endl; std::cout << "get A take2." << std::endl; std::unique_ptr& alias = m.at(3); A a( std::move(*alias) ); std::cout << "got A take2." << std::endl; } int main(int argc, char** argv) { A a; f( std::move(a) ); std::vector iv = {1, 2, 3}; std::vector iv2 = iv; for( auto i : iv2 ) { std::cout << i << std::endl; } g(); return 0; } yrmcds-1.0.3/test/mutex.cpp000066400000000000000000000006611226611677100156550ustar00rootroot00000000000000#include #include #include std::mutex lock; int g_counter = 0; void foo() { for (int i = 0; i < 10000; ++i) { std::lock_guard g(lock); ++g_counter; } return; } int main() { std::thread t1(foo); std::thread t2(foo); std::thread t3(foo); foo(); t1.join(); t2.join(); t3.join(); std::cout << g_counter << std::endl; return 0; } yrmcds-1.0.3/test/netiface.cpp000066400000000000000000000013741226611677100162730ustar00rootroot00000000000000#include #include int main(int argc, char** argv) { if( argc < 2 ) { std::cout << "Usage: netiface ADDRESS" << std::endl; return 0; } cybozu::ip_address a(argv[1]); if( a.is_v4() ) { std::cout << "v4!" << std::endl; } else if( a.is_v6() ) { std::cout << "v6!" << std::endl; } cybozu::ip_address b("127.0.0.1"); if( a == b ) { std::cout << a.str() << " == 127.0.0.1" << std::endl; } else { std::cout << a.str() << " != 127.0.0.1" << std::endl; } if( has_ip_address(a) ) { std::cout << a.str() << " is assigned." << std::endl; } else { std::cout << a.str() << " is not assigned." << std::endl; } return 0; } yrmcds-1.0.3/test/object.cpp000066400000000000000000000055051226611677100157630ustar00rootroot00000000000000#include "../src/config.hpp" #include "../src/object.hpp" #include #include using yrmcds::object; using cybozu::dynbuf; std::size_t reset_heap_limit() { const char* p = std::getenv("HEAP_LIMIT"); std::size_t n = 0; if( p != nullptr ) n = (std::size_t)std::atoi(p); if( n == 0 ) n = yrmcds::DEFAULT_HEAP_DATA_LIMIT; yrmcds::g_config.set_heap_data_limit(n); return n; } AUTOTEST(set) { reset_heap_limit(); object o1("abcde", 5, 100, 0); cybozu_assert( o1.size() == 5 ); cybozu_assert( o1.flags() == 100 ); dynbuf d1(0); d1.append("abcde", 5); dynbuf d1_(0); std::uint64_t cas = o1.cas_unique(); cybozu_assert( o1.data(d1_) == d1 ); cybozu_assert( o1.cas_unique() == cas ); o1.set("123", 3, 200, 200); cybozu_assert( o1.size() == 3 ); cybozu_assert( o1.flags() == 200 ); d1.reset(); d1.append("123", 3); cybozu_assert( o1.data(d1_) == d1 ); cybozu_assert( o1.cas_unique() != cas ); } AUTOTEST(append_prepend) { reset_heap_limit(); object o1("abcde", 5, 100, 0); std::uint64_t cas = o1.cas_unique(); o1.append("12345", 5); std::uint64_t cas2 = o1.cas_unique(); cybozu_assert( o1.size() == 10 ); cybozu_assert( cas != cas2 ); dynbuf d1(0); d1.append("abcde12345", 10); dynbuf d1_(0); cybozu_assert( o1.data(d1_) == d1 ); o1.prepend("#*A", 3); std::uint64_t cas3 = o1.cas_unique(); cybozu_assert( o1.size() == 13 ); cybozu_assert( cas != cas3 ); cybozu_assert( cas2 != cas3 ); d1.reset(); d1.append("#*Aabcde12345", 13); cybozu_assert( o1.data(d1_) == d1 ); } AUTOTEST(touch) { reset_heap_limit(); object o1("abcde", 5, 100, 0); std::uint64_t cas = o1.cas_unique(); o1.touch(1111); dynbuf d1(0); d1.append("abcde", 5); dynbuf d1_(0); cybozu_assert( o1.size() == 5 ); cybozu_assert( o1.data(d1_) == d1 ); cybozu_assert( o1.cas_unique() == cas ); } AUTOTEST(incr_decr) { if( reset_heap_limit() < 24 ) return; object o1("abcde", 5, 100, 0); cybozu_test_exception( o1.incr(3), object::not_a_number ); object o2("123ab", 5, 100, 0); std::uint64_t cas = o2.cas_unique(); std::uint64_t n = 0; cybozu_test_no_exception( n = o2.incr(3) ); std::uint64_t cas2 = o2.cas_unique(); cybozu_assert( n == 126 ); cybozu_assert( cas != cas2 ); n = o2.decr(3); std::uint64_t cas3 = o2.cas_unique(); cybozu_assert( n == 123 ); cybozu_assert( cas2 != cas3 ); n = o2.decr(200); cybozu_assert( n == 0 ); object o3(" 18446744073709551615", 22, 100, 0); n = o3.decr(1); cybozu_assert( n == 18446744073709551614ULL ); n = o3.incr(2); cybozu_assert( n == 0 ); o3.set(" 111", 4, 0, 0); dynbuf d3(0); d3.append(" 111", 4); dynbuf d3_(0); cybozu_assert( o3.data(d3_) == d3 ); } yrmcds-1.0.3/test/override.cpp000066400000000000000000000002231226611677100163240ustar00rootroot00000000000000struct A { virtual void f() {} virtual ~A() {} }; struct B: public A { void f() override {} }; int main() { B b; return 0; } yrmcds-1.0.3/test/prime.cpp000066400000000000000000000006061226611677100156260ustar00rootroot00000000000000#include #include void show_nearest_prime(unsigned int n) { std::cout << "nearest_prime(" << n << ") = " << cybozu::nearest_prime(n) << std::endl; } int main() { for( int i = 0; i <= 20; ++i ) show_nearest_prime(i); show_nearest_prime(1024); show_nearest_prime(2<<20); show_nearest_prime(2<<30); return 0; } yrmcds-1.0.3/test/protocol_binary.cpp000066400000000000000000001040551226611677100177220ustar00rootroot00000000000000#include "../src/memcache.hpp" #include #include #define TEST_DISABLE_AUTO_RUN #include #include #include #include #include #include #include #include #include #include #include #include #include using yrmcds::memcache::binary_command; using yrmcds::memcache::binary_status; using yrmcds::memcache::item; const char* g_server = nullptr; std::uint16_t g_port = 11211; typedef char opaque_t[4]; const std::size_t BINARY_HEADER_SIZE = 24; int connect_server() { int s = cybozu::tcp_connect(g_server, g_port); if( s == -1 ) return -1; ::fcntl(s, F_SETFL, ::fcntl(s, F_GETFL, 0) & ~O_NONBLOCK); struct timeval tv; tv.tv_sec = 0; tv.tv_usec = 300000; ::setsockopt(s, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv)); int ok = 1; ::setsockopt(s, IPPROTO_TCP, TCP_NODELAY, &ok, sizeof(ok)); return s; } // compose binary request class request { public: request(binary_command cmd, std::uint16_t key_len, const char* key, char extras_len, const char* extras, std::uint32_t data_len, const char* data, opaque_t* opaque = nullptr, std::uint64_t cas = 0): m_data(BINARY_HEADER_SIZE + key_len + extras_len + data_len), m_p(&m_data[0]) { m_p[0] = '\x80'; m_p[1] = (char)cmd; cybozu::hton(key_len, m_p+2); m_p[4] = extras_len; m_p[5] = 0; m_p[6] = 0; m_p[7] = 0; std::uint32_t total_len = key_len + extras_len + data_len; cybozu::hton(total_len, m_p+8); if( opaque != nullptr ) std::memcpy(m_p+12, opaque, sizeof(opaque_t)); cybozu::hton(cas, m_p+16); char* p = m_p + BINARY_HEADER_SIZE; std::memcpy(p, extras, extras_len); p += extras_len; std::memcpy(p, key, key_len); p += key_len; std::memcpy(p, data, data_len); } const char* data() { return m_p; } std::size_t length() const noexcept { return m_data.size(); } private: std::vector m_data; char* const m_p; }; // parse binary response class response { public: response() {} response(response&&) noexcept = default; // Return length of the response. // // Return length of the response. // If the response is incomplete, zero is returned. std::size_t length() const noexcept { return m_response_len; } // Response status, if determined by the request. binary_status status() const noexcept { return m_status; } // Return `true` if the response status was not OK. bool error() { return m_status != binary_status::OK; } // Return the command type. binary_command command() const noexcept { return m_command; } // Return `key`. item key() const noexcept { return m_key; } // Return `opaque` sent with the request. const char* opaque() const noexcept { return m_p + 12; } // Return `cas unique` sent with CAS command. std::uint64_t cas_unique() const noexcept { return m_cas_unique; } // Return `flags` sent with storage commands. std::uint32_t flags() const noexcept { return m_flags; } // Return data block sent with storage commands. item data() const noexcept { return m_data; } // Return an unsigned 64bit integer value for increment or decrement. std::uint64_t value() const noexcept { return m_value; } bool parse(const char* p, std::size_t len); private: const char* m_p = nullptr; std::size_t m_len = 0; std::size_t m_response_len = 0; binary_status m_status; binary_command m_command; item m_key; std::uint64_t m_cas_unique; std::uint32_t m_flags = 0; item m_data; std::uint64_t m_value = 0; }; bool response::parse(const char* p, std::size_t len) { m_p = p; m_len = len; if( m_len < BINARY_HEADER_SIZE ) return false; cybozu_assert( *m_p == '\x81' ); std::uint32_t total_len; cybozu::ntoh(m_p + 8, total_len); if( m_len < (BINARY_HEADER_SIZE + total_len) ) return false; m_response_len = BINARY_HEADER_SIZE + total_len; m_command = (binary_command)*(unsigned char*)(m_p + 1); std::uint16_t key_len; cybozu::ntoh(m_p + 2, key_len); std::uint8_t extras_len = *(unsigned char*)(m_p + 4); cybozu_assert( total_len >= (key_len + extras_len) ); if( key_len > 0 ) { m_key = item(m_p + (BINARY_HEADER_SIZE + extras_len), key_len); } else { m_key = item(nullptr, 0); } std::uint16_t i_status; cybozu::ntoh(m_p + 6, i_status); m_status = (binary_status)i_status; cybozu::ntoh(m_p + 16, m_cas_unique); std::size_t data_len = total_len - key_len - extras_len; if( data_len > 0 ) { m_data = item(m_p + (BINARY_HEADER_SIZE + extras_len + key_len), data_len); } else { m_data = item(nullptr, 0); } if( extras_len > 0 ) { cybozu_assert( extras_len == 4 ); cybozu::ntoh(m_p + BINARY_HEADER_SIZE, m_flags); } if( (m_command == binary_command::Increment || m_command == binary_command::Decrement) && m_status == binary_status::OK ) { cybozu_assert( data_len == 8 ); if( data_len == 8 ) { cybozu::ntoh(m_p + BINARY_HEADER_SIZE + key_len + extras_len, m_value); } } return true; } class client { public: client(): m_socket(connect_server()), m_buffer(1 << 20) { cybozu_assert( m_socket != -1 ); } ~client() { ::close(m_socket); } bool get_response(response& resp) { m_buffer.erase(m_last_response_size); m_last_response_size = 0; if( m_buffer.empty() ) { if( ! recv() ) { cybozu_assert( m_buffer.empty() ); return false; } } while( true ) { if( resp.parse(m_buffer.data(), m_buffer.size()) ) { m_last_response_size = resp.length(); return true; } cybozu_assert( recv() ); } } void noop(opaque_t* opaque) { request req(binary_command::Noop, 0, nullptr, 0, nullptr, 0, nullptr, opaque); send(req.data(), req.length()); } void get(const std::string& key, bool q) { request req(q ? binary_command::GetQ : binary_command::Get, (std::uint16_t)key.size(), key.data(), 0, nullptr, 0, nullptr); send(req.data(), req.length()); } void getk(const std::string& key, bool q) { request req(q ? binary_command::GetKQ : binary_command::GetK, (std::uint16_t)key.size(), key.data(), 0, nullptr, 0, nullptr); send(req.data(), req.length()); } void touch(const std::string& key, std::uint32_t expire) { char extra[4]; cybozu::hton(expire, extra); request req(binary_command::Touch, (std::uint16_t)key.size(), key.data(), sizeof(extra), extra, 0, nullptr); send(req.data(), req.length()); } void get_and_touch(const std::string& key, std::uint32_t expire, bool q) { char extra[4]; cybozu::hton(expire, extra); request req(q ? binary_command::GaTQ : binary_command::GaT, (std::uint16_t)key.size(), key.data(), sizeof(extra), extra, 0, nullptr); send(req.data(), req.length()); } void getk_and_touch(const std::string& key, std::uint32_t expire, bool q) { char extra[4]; cybozu::hton(expire, extra); request req(q ? binary_command::GaTKQ : binary_command::GaTK, (std::uint16_t)key.size(), key.data(), sizeof(extra), extra, 0, nullptr); send(req.data(), req.length()); } void lock_and_get(const std::string& key, bool q) { request req(q ? binary_command::LaGQ : binary_command::LaG, (std::uint16_t)key.size(), key.data(), 0, nullptr, 0, nullptr); send(req.data(), req.length()); } void lock_and_getk(const std::string& key, bool q) { request req(q ? binary_command::LaGKQ : binary_command::LaGK, (std::uint16_t)key.size(), key.data(), 0, nullptr, 0, nullptr); send(req.data(), req.length()); } void set(const std::string& key, const std::string& data, bool q, std::uint32_t flags, std::uint32_t expire, std::uint64_t cas = 0) { char extra[8]; cybozu::hton(flags, extra); cybozu::hton(expire, &extra[4]); request req(q ? binary_command::SetQ : binary_command::Set, (std::uint16_t)key.size(), key.data(), sizeof(extra), extra, (std::uint32_t)data.length(), data.data(), nullptr, cas); send(req.data(), req.length()); } void replace(const std::string& key, const std::string& data, bool q, std::uint32_t flags, std::uint32_t expire, std::uint64_t cas = 0) { char extra[8]; cybozu::hton(flags, extra); cybozu::hton(expire, &extra[4]); request req(q ? binary_command::ReplaceQ : binary_command::Replace, (std::uint16_t)key.size(), key.data(), sizeof(extra), extra, (std::uint32_t)data.length(), data.data(), nullptr, cas); send(req.data(), req.length()); } void add(const std::string& key, const std::string& data, bool q, std::uint32_t flags, std::uint32_t expire, std::uint64_t cas = 0) { char extra[8]; cybozu::hton(flags, extra); cybozu::hton(expire, &extra[4]); request req(q ? binary_command::AddQ : binary_command::Add, (std::uint16_t)key.size(), key.data(), sizeof(extra), extra, (std::uint32_t)data.length(), data.data(), nullptr, cas); send(req.data(), req.length()); } void replace_and_unlock(const std::string& key, const std::string& data, bool q, std::uint32_t flags, std::uint32_t expire, std::uint64_t cas = 0) { char extra[8]; cybozu::hton(flags, extra); cybozu::hton(expire, &extra[4]); request req(q ? binary_command::RaUQ : binary_command::RaU, (std::uint16_t)key.size(), key.data(), sizeof(extra), extra, (std::uint32_t)data.length(), data.data(), nullptr, cas); send(req.data(), req.length()); } void incr(const std::string& key, std::uint64_t value, bool q, std::uint32_t expire = ~(std::uint32_t)0, std::uint64_t initial = 0, std::uint64_t cas = 0) { char extra[20]; cybozu::hton(value, extra); cybozu::hton(initial, &extra[8]); cybozu::hton(expire, &extra[16]); request req(q ? binary_command::IncrementQ : binary_command::Increment, (std::uint16_t)key.size(), key.data(), sizeof(extra), extra, 0, nullptr, nullptr, cas); send(req.data(), req.length()); } void decr(const std::string& key, std::uint64_t value, bool q, std::uint32_t expire = ~(std::uint32_t)0, std::uint64_t initial = 0, std::uint64_t cas = 0) { char extra[20]; cybozu::hton(value, extra); cybozu::hton(initial, &extra[8]); cybozu::hton(expire, &extra[16]); request req(q ? binary_command::DecrementQ : binary_command::Decrement, (std::uint16_t)key.size(), key.data(), sizeof(extra), extra, 0, nullptr, nullptr, cas); send(req.data(), req.length()); } void append(const std::string& key, const std::string& value, bool q, std::uint64_t cas = 0) { request req(q ? binary_command::AppendQ : binary_command::Append, (std::uint16_t)key.size(), key.data(), 0, nullptr, (std::uint32_t)value.size(), value.data(), nullptr, cas); send(req.data(), req.length()); } void prepend(const std::string& key, const std::string& value, bool q, std::uint64_t cas = 0) { request req(q ? binary_command::PrependQ : binary_command::Prepend, (std::uint16_t)key.size(), key.data(), 0, nullptr, (std::uint32_t)value.size(), value.data(), nullptr, cas); send(req.data(), req.length()); } void remove(const std::string& key, bool q, std::uint64_t cas = 0) { request req(q ? binary_command::DeleteQ : binary_command::Delete, (std::uint16_t)key.size(), key.data(), 0, nullptr, 0, nullptr, nullptr, cas); send(req.data(), req.length()); } void lock(const std::string& key, bool q) { request req(q ? binary_command::LockQ : binary_command::Lock, (std::uint16_t)key.size(), key.data(), 0, nullptr, 0, nullptr); send(req.data(), req.length()); } void unlock(const std::string& key, bool q) { request req(q ? binary_command::UnlockQ : binary_command::Unlock, (std::uint16_t)key.size(), key.data(), 0, nullptr, 0, nullptr); send(req.data(), req.length()); } void unlock_all(bool q) { request req(q ? binary_command::UnlockAllQ : binary_command::UnlockAll, 0, nullptr, 0, nullptr, 0, nullptr); send(req.data(), req.length()); } void flush_all(bool q, std::uint32_t expire = 0) { if( expire == 0 ) { request req(q ? binary_command::FlushQ : binary_command::Flush, 0, nullptr, 0, nullptr, 0, nullptr); send(req.data(), req.length()); return; } char extra[4]; cybozu::hton(expire, extra); request req(q ? binary_command::FlushQ : binary_command::Flush, 0, nullptr, sizeof(extra), extra, 0, nullptr); send(req.data(), req.length()); } void version() { request req(binary_command::Version, 0, nullptr, 0, nullptr, 0, nullptr); send(req.data(), req.length()); } void stat() { request req(binary_command::Stat, 0, nullptr, 0, nullptr, 0, nullptr); send(req.data(), req.length()); } void stat(const std::string& key) { request req(binary_command::Stat, (std::uint16_t)key.size(), key.data(), 0, nullptr, 0, nullptr); send(req.data(), req.length()); } void quit(bool q) { request req(q ? binary_command::QuitQ : binary_command::Quit, 0, nullptr, 0, nullptr, 0, nullptr); send(req.data(), req.length()); } private: const int m_socket; cybozu::dynbuf m_buffer; std::size_t m_last_response_size = 0; void send(const char* p, std::size_t len) { while( len > 0 ) { ssize_t n = ::send(m_socket, p, len, 0); cybozu_assert( n != -1 ); p += n; len -= n; } } bool recv() { char* p = m_buffer.prepare(256 << 10); ssize_t n = ::recv(m_socket, p, 256<<10, 0); cybozu_assert( n != -1 ); if( n == -1 || n == 0 ) return false; m_buffer.consume(n); return true; } }; void print_item(const item& i) { std::cout << std::string(std::get<0>(i), std::get<1>(i)) << std::endl; } bool itemcmp(const item& i, const std::string& s) { if( s.size() != std::get<1>(i) ) return false; return std::memcmp(s.data(), std::get<0>(i), s.size()) == 0; } // tests #define ASSERT_COMMAND(r,cmd) cybozu_assert( r.command() == binary_command::cmd ) #define ASSERT_OK(r) cybozu_assert( r.status() == binary_status::OK ) AUTOTEST(opaque) { client c; response r; c.noop(nullptr); cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, Noop); ASSERT_OK(r); opaque_t zero = {'\0', '\0', '\0', '\0'}; cybozu_assert( std::memcmp(r.opaque(), &zero, sizeof(opaque_t)) == 0 ); opaque_t op1 = {'\x12', '\x23', '\x45', '\x67'}; c.noop(&op1); cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, Noop); ASSERT_OK(r); cybozu_assert( std::memcmp(r.opaque(), &op1, sizeof(opaque_t)) == 0 ); } AUTOTEST(quit) { client c; response r; c.quit(false); cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, Quit); ASSERT_OK( r ); } AUTOTEST(quitq) { client c; response r; c.quit(true); cybozu_assert( ! c.get_response(r) ); } AUTOTEST(version) { client c; response r; c.version(); cybozu_assert( c.get_response(r) ); print_item( r.data() ); } AUTOTEST(set) { client c; response r; c.set("hello", "world", false, 123, 0); cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, Set); ASSERT_OK(r); c.set("hello", "world!", false, 111, 0, r.cas_unique()); cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, Set); ASSERT_OK(r); c.set("hello", "world!!", true, 111, 0, r.cas_unique() + 1); cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, SetQ); cybozu_assert( r.status() == binary_status::Exists ); c.set("not exist", "hoge", true, 111, 0, 100); cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, SetQ); cybozu_assert( r.status() == binary_status::NotFound ); c.set("abc", "def", true, 123, 0); c.noop(nullptr); cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, Noop); c.get("abc", false); cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, Get); ASSERT_OK(r); cybozu_assert( itemcmp(r.data(), "def") ); cybozu_assert( r.flags() == 123 ); c.get("not exist", false); cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, Get); cybozu_assert( r.status() == binary_status::NotFound ); c.get("not exist", true); c.noop(nullptr); cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, Noop); ASSERT_OK(r); } AUTOTEST(expire) { client c; response r; c.set("abc", "def1", true, 123, 1); c.get("abc", false); cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, Get); ASSERT_OK(r); std::this_thread::sleep_for(std::chrono::seconds(3)); c.get("abc", false); cybozu_assert( c.get_response(r) ); cybozu_assert( r.status() == binary_status::NotFound ); } AUTOTEST(touch) { client c; response r; c.remove("abc", true); c.touch("abc", 0); cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, Touch); cybozu_assert( r.status() == binary_status::NotFound ); c.set("abc", "def", true, 10, 2); c.touch("abc", 0); cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, Touch); ASSERT_OK(r); std::uint64_t cas = r.cas_unique(); cybozu_assert( cas != 0 ); std::this_thread::sleep_for(std::chrono::seconds(4)); c.get("abc", false); cybozu_assert( c.get_response(r) ); ASSERT_OK(r); cybozu_assert( r.cas_unique() == cas ); c.touch("abc", 1); cybozu_assert( c.get_response(r) ); ASSERT_OK(r); cybozu_assert( r.cas_unique() == cas ); std::this_thread::sleep_for(std::chrono::seconds(3)); c.get("abc", false); cybozu_assert( c.get_response(r) ); cybozu_assert( r.status() == binary_status::NotFound ); } AUTOTEST(add) { client c; response r; c.set("abc", "def", true, 0, 0); c.add("abc", "123", true, 0, 0); cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, AddQ); cybozu_assert( r.status() == binary_status::Exists ); c.remove("abc", true); c.add("abc", "123", false, 0, 0); cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, Add); ASSERT_OK(r); } AUTOTEST(replace) { client c; response r; c.replace("not exist", "hoge", true, 100, 0); cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, ReplaceQ); cybozu_assert( r.status() == binary_status::NotFound ); c.set("abc", "def", true, 10, 0); c.replace("abc", "123", false, 100, 0); cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, Replace); ASSERT_OK(r); c.get("abc", false); cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, Get); ASSERT_OK(r); cybozu_assert( itemcmp(r.data(), "123") ); cybozu_assert( r.flags() == 100 ); } AUTOTEST(get) { client c; response r; c.remove("unique key", true); c.add("unique key", "value", false, 10, 0); cybozu_assert( c.get_response(r) ); ASSERT_OK(r); c.replace("unique key", "value2", false, 11, 0); cybozu_assert( c.get_response(r) ); ASSERT_OK(r); std::uint64_t cas = r.cas_unique(); c.get("unique key", false); cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, Get); ASSERT_OK(r); cybozu_assert( r.key() == item(nullptr, 0) ); cybozu_assert( itemcmp(r.data(), "value2") ); cybozu_assert( r.flags() == 11 ); cybozu_assert( r.cas_unique() == cas ); c.getk("unique key", false); cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, GetK); ASSERT_OK(r); cybozu_assert( itemcmp(r.key(), "unique key") ); cybozu_assert( itemcmp(r.data(), "value2") ); cybozu_assert( r.flags() == 11 ); cybozu_assert( r.cas_unique() == cas ); } AUTOTEST(get_and_touch) { client c; response r; c.set("abc", "def", true, 10, 2); c.get_and_touch("abc", 0, false); cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, GaT); ASSERT_OK(r); cybozu_assert( std::get<1>(r.key()) == 0 ); cybozu_assert( itemcmp(r.data(), "def") ); std::this_thread::sleep_for(std::chrono::seconds(4)); c.getk_and_touch("abc", 1, false); cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, GaTK); ASSERT_OK(r); cybozu_assert( itemcmp(r.key(), "abc") ); cybozu_assert( itemcmp(r.data(), "def") ); std::this_thread::sleep_for(std::chrono::seconds(3)); c.touch("abc", 0); cybozu_assert( c.get_response(r) ); cybozu_assert( r.status() == binary_status::NotFound ); } AUTOTEST(incr_decr) { client c; response r; c.set("abc", "def", true, 10, 0); c.incr("abc", 10, true); cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, IncrementQ); cybozu_assert( r.status() == binary_status::NonNumeric ); c.remove("abc", true); c.incr("abc", 10, true); cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, IncrementQ); cybozu_assert( r.status() == binary_status::NotFound ); c.incr("abc", 10, false, 0, 12); cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, Increment); ASSERT_OK(r); cybozu_assert( r.value() == 12 ); cybozu_assert( r.flags() == 0 ); std::uint64_t cas = r.cas_unique(); cybozu_assert( cas != 0 ); c.incr("abc", 10, false); cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, Increment); ASSERT_OK(r); cybozu_assert( r.value() == 22 ); cybozu_assert( r.cas_unique() != 0 ); cybozu_assert( cas != r.cas_unique() ); cas = r.cas_unique(); c.decr("abc", 1, false); cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, Decrement); ASSERT_OK(r); cybozu_assert( r.value() == 21 ); cybozu_assert( r.cas_unique() != 0 ); cybozu_assert( cas != r.cas_unique() ); c.decr("abc", 100, false); cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, Decrement); ASSERT_OK(r); cybozu_assert( r.value() == 0 ); c.remove("abc", true); c.decr("abc", 1, true); cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, DecrementQ); cybozu_assert( r.status() == binary_status::NotFound ); c.set("abc", "def", true, 10, 0); c.decr("abc", 1, true); cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, DecrementQ); cybozu_assert( r.status() == binary_status::NonNumeric ); c.remove("abc", true); c.decr("abc", 10, false, 0, 22); cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, Decrement); ASSERT_OK(r); cybozu_assert( r.value() == 22 ); } AUTOTEST(append_prepend) { client c; response r; c.remove("ttt", true); c.append("ttt", "111", true); cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, AppendQ); cybozu_assert( r.status() == binary_status::NotFound ); c.set("ttt", "aaa", true, 0, 0); c.append("ttt", "111", false); cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, Append); ASSERT_OK(r); std::uint64_t cas = r.cas_unique(); cybozu_assert( cas != 0 ); c.get("ttt", false); cybozu_assert( c.get_response(r) ); ASSERT_OK(r); cybozu_assert( itemcmp(r.data(), "aaa111") ); c.prepend("ttt", "222", false); cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, Prepend); ASSERT_OK(r); cybozu_assert( r.cas_unique() != 0 ); cybozu_assert( cas != r.cas_unique() ); c.get("ttt", false); cybozu_assert( c.get_response(r) ); ASSERT_OK(r); cybozu_assert( itemcmp(r.data(), "222aaa111") ); c.remove("ttt", true); c.prepend("ttt", "111", true); cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, PrependQ); cybozu_assert( r.status() == binary_status::NotFound ); } AUTOTEST(delete) { client c; response r; c.set("abc", "def", true, 10, 0); c.remove("abc", false); cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, Delete); ASSERT_OK(r); c.set("abc", "def", false, 10, 0); cybozu_assert( c.get_response(r) ); ASSERT_OK(r); std::uint64_t cas = r.cas_unique(); c.remove("abc", true, cas + 1); cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, DeleteQ); cybozu_assert( r.status() == binary_status::Exists ); c.remove("abc", false, cas); cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, Delete); ASSERT_OK(r); c.remove("abc", false); cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, Delete); cybozu_assert( r.status() == binary_status::NotFound ); c.remove("abc", true); c.noop(nullptr); cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, Noop); } AUTOTEST(lock) { client c; response r; c.set("abc", "10", false, 0, 0); cybozu_assert( c.get_response(r) ); { client c2; c2.lock("abc", false); cybozu_assert( c2.get_response(r) ); ASSERT_COMMAND(r, Lock); ASSERT_OK(r); c.lock("abc", true); cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, LockQ); cybozu_assert( r.status() == binary_status::Locked ); c2.quit(false); cybozu_assert( c2.get_response(r) ); } c.lock("abc", false); cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, Lock); ASSERT_OK(r); c.quit(false); c.get_response(r); } AUTOTEST(unlock) { client c; response r; c.remove("abc", false); cybozu_assert( c.get_response(r) ); ASSERT_OK(r); c.lock("abc", true); cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, LockQ); cybozu_assert( r.status() == binary_status::NotFound ); c.unlock("abc", true); cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, UnlockQ); cybozu_assert( r.status() == binary_status::NotFound ); c.set("abc", "def", true, 10, 0); c.unlock("abc", true); cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, UnlockQ); cybozu_assert( r.status() == binary_status::NotLocked ); c.lock("abc", false); cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, Lock); ASSERT_OK(r); client c2; c2.unlock("abc", true); cybozu_assert( c2.get_response(r) ); ASSERT_COMMAND(r, UnlockQ); cybozu_assert( r.status() == binary_status::NotLocked ); c.unlock("abc", false); cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, Unlock); ASSERT_OK(r); c2.lock("abc", false); cybozu_assert( c2.get_response(r) ); ASSERT_COMMAND(r, Lock); ASSERT_OK(r); c2.quit(false); c2.get_response(r); } AUTOTEST(unlock_all) { client c; response r; c.set("abc", "1", true, 10, 0); c.set("def", "2", true, 10, 0); c.lock("abc", false); cybozu_assert( c.get_response(r) ); ASSERT_OK(r); c.lock("def", false); cybozu_assert( c.get_response(r) ); ASSERT_OK(r); client c2; c2.lock("abc", true); cybozu_assert( c2.get_response(r) ); cybozu_assert( r.status() == binary_status::Locked ); c2.lock("def", true); cybozu_assert( c2.get_response(r) ); cybozu_assert( r.status() == binary_status::Locked ); c.unlock_all(false); cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, UnlockAll); ASSERT_OK(r); c2.lock("abc", false); cybozu_assert( c2.get_response(r) ); ASSERT_OK(r); c2.lock("def", false); cybozu_assert( c2.get_response(r) ); ASSERT_OK(r); c2.quit(false); c2.get_response(r); } AUTOTEST(lock_and_get) { client c; response r; c.remove("abc", true); c.lock_and_get("abc", true); cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, LaGQ); cybozu_assert( r.status() == binary_status::NotFound ); c.set("abc", "def", true, 10, 0); c.lock_and_get("abc", false); cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, LaG); ASSERT_OK(r); cybozu_assert( std::get<1>(r.key()) == 0 ); cybozu_assert( itemcmp(r.data(), "def") ); std::uint64_t cas = r.cas_unique(); c.lock_and_get("abc", true); cybozu_assert( c.get_response(r) ); cybozu_assert( r.status() == binary_status::Locked ); client c2; c2.lock_and_get("abc", true); cybozu_assert( c2.get_response(r) ); cybozu_assert( r.status() == binary_status::Locked ); c2.replace_and_unlock("abc", "ghi", true, 20, 0); cybozu_assert( c2.get_response(r) ); ASSERT_COMMAND(r, RaUQ); cybozu_assert( r.status() == binary_status::NotLocked ); c.replace_and_unlock("abc", "ghi", false, 20, 0); cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, RaU); ASSERT_OK(r); cybozu_assert( r.cas_unique() != cas ); cybozu_assert( r.cas_unique() != 0 ); c2.lock_and_getk("abc", false); cybozu_assert( c2.get_response(r) ); ASSERT_OK(r); cybozu_assert( itemcmp(r.key(), "abc") ); cybozu_assert( itemcmp(r.data(), "ghi") ); c2.quit(false); c2.get_response(r); } AUTOTEST(flush) { client c; response r; c.set("abc", "def", true, 10, 0); c.flush_all(true, 2); c.get("abc", false); cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, Get); ASSERT_OK(r); std::this_thread::sleep_for(std::chrono::seconds(4)); c.get("abc", false); cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, Get); cybozu_assert( r.status() == binary_status::NotFound ); c.set("abc", "234", true, 10, 0); c.flush_all(false); cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, Flush); ASSERT_OK(r); c.get("abc", false); cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, Get); cybozu_assert( r.status() == binary_status::NotFound ); } AUTOTEST(stat_general) { client c; response r; c.stat(); while( true ) { cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, Stat); ASSERT_OK(r); if( std::get<1>(r.key()) == 0 ) break; continue; std::cout << std::string(std::get<0>(r.key()), std::get<1>(r.key())) << ": " << std::string(std::get<0>(r.data()), std::get<1>(r.data())) << std::endl; } } AUTOTEST(stat_settings) { client c; response r; c.stat("settings"); while( true ) { cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, Stat); ASSERT_OK(r); if( std::get<1>(r.key()) == 0 ) break; continue; std::cout << std::string(std::get<0>(r.key()), std::get<1>(r.key())) << ": " << std::string(std::get<0>(r.data()), std::get<1>(r.data())) << std::endl; } } AUTOTEST(stat_sizes) { client c; response r; c.stat("sizes"); while( true ) { cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, Stat); ASSERT_OK(r); if( std::get<1>(r.key()) == 0 ) break; continue; std::cout << std::string(std::get<0>(r.key()), std::get<1>(r.key())) << ": " << std::string(std::get<0>(r.data()), std::get<1>(r.data())) << std::endl; } } AUTOTEST(stat_items) { client c; response r; c.stat("items"); while( true ) { cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, Stat); ASSERT_OK(r); if( std::get<1>(r.key()) == 0 ) break; continue; std::cout << std::string(std::get<0>(r.key()), std::get<1>(r.key())) << ": " << std::string(std::get<0>(r.data()), std::get<1>(r.data())) << std::endl; } } AUTOTEST(stat_ops) { client c; response r; c.stat("ops"); while( true ) { cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, Stat); ASSERT_OK(r); if( std::get<1>(r.key()) == 0 ) break; continue; std::cout << std::string(std::get<0>(r.key()), std::get<1>(r.key())) << ": " << std::string(std::get<0>(r.data()), std::get<1>(r.data())) << std::endl; } } // main bool optparse(int argc, char** argv) { if( argc != 2 && argc != 3 ) { std::cout << "Usage: protocol_binary SERVER [PORT]" << std::endl; return false; } g_server = argv[1]; if( argc == 3 ) { int n = std::stoi(argv[2]); if( n <= 0 || n > 65535 ) { std::cout << "Invalid port number: " << argv[2] << std::endl; return false; } g_port = n; } int s = connect_server(); if( s == -1 ) { std::cout << "Failed to connect to " << g_server << std::endl; return false; } ::close(s); return true; } TEST_MAIN(optparse); yrmcds-1.0.3/test/range.cpp000066400000000000000000000002551226611677100156060ustar00rootroot00000000000000#include #include int main() { const int a[] = {1, 2, 3, 4, 5}; for( auto i: a ) { std::cout << i << std::endl; } return 0; } yrmcds-1.0.3/test/reactor.cpp000066400000000000000000000034511226611677100161520ustar00rootroot00000000000000#include #include #include #include #include #include #include #include int g_counter = 0; bool got_hangup = false; bool data_read = false; class in_resource: public cybozu::resource { public: in_resource(int fd): cybozu::resource(fd) {} ~in_resource() { std::cout << "stdin_resource dtor." << std::endl; } private: virtual bool on_readable() override final { char buf[512]; buf[sizeof(buf) - 1] = '\0'; ssize_t t = ::read(fileno(), buf, sizeof(buf) - 1); if( t == -1 ) cybozu::throw_unix_error(errno, "read"); std::cout << "aaa: " << t << std::endl; buf[t] = '\0'; std::cout << "read: " << buf << std::endl; data_read = true; return false; } virtual bool on_writable() override final { return true; } virtual bool on_hangup() override final { std::cout << "got hangup!" << std::endl; got_hangup = true; return true; } }; void f(cybozu::reactor& r) { g_counter++; if( g_counter > 2 ) { r.quit(); return; } std::cout << g_counter << std::endl; if( r.fix_garbage() ) r.gc(); } int main(int argc, char** argv) { cybozu::logger::set_threshold(cybozu::severity::debug); int fds[2]; if( ::pipe2(fds, O_NONBLOCK) == -1 ) cybozu::throw_unix_error(errno, "pipe"); if( ::write(fds[1], "Hello!", 6) == -1 ) cybozu::throw_unix_error(errno, "write"); ::close(fds[1]); cybozu::reactor r; std::unique_ptr res(new in_resource(fds[0])); r.add_resource(std::move(res), cybozu::reactor::EVENT_IN); r.run(f); assert(got_hangup); assert(data_read); return 0; } yrmcds-1.0.3/test/reference_wrapper.cpp000066400000000000000000000012271226611677100202100ustar00rootroot00000000000000#include #include #include struct foo { int i; }; int main() { int *p = nullptr; std::reference_wrapper a[3] = {*p, *p, *p}; int i = 3; a[0] = std::ref(i); if( &(a[2].get()) == nullptr ) { std::cout << "null!" << std::endl; } else { std::cout << "wtf" << std::endl; } a[0].get() = 5; std::cout << "i = " << i << std::endl; foo f; f.i = 3; std::vector> v; v.emplace_back(f); std::cout << "f.i = " << v[0].get().i << std::endl; f.i = 5; std::cout << "f.i = " << v[0].get().i << std::endl; return 0; } yrmcds-1.0.3/test/signalfd.cpp000066400000000000000000000064131226611677100163030ustar00rootroot00000000000000#include #include #include #include #include #include #include #include #include #include const int MAX_EVENTS = 10; bool readall(int fd, char* p, std::size_t len) { while( len != 0 ) { ssize_t n = read(fd, p, len); if( n == -1 ) { if( errno == EAGAIN || errno == EWOULDBLOCK ) { return false; } else { throw std::runtime_error("read failed."); } } len -= n; p += n; } return true; } void read_signal(int sfd) { struct signalfd_siginfo si; while( true ) { if( ! readall(sfd, (char*)&si, sizeof(si)) ) return; switch(si.ssi_signo) { case SIGHUP: std::cout << "SIGHUP" << std::endl; break; case SIGQUIT: std::cout << "SIGQUIT" << std::endl; break; case SIGTERM: std::cout << "SIGTERM" << std::endl; break; case SIGCHLD: std::cout << "SIGCHLD" << std::endl; break; default: std::cout << "Unknown signal: " << si.ssi_signo << std::endl; } } } int wait_signal(int sfd) { int efd = epoll_create(10); if( efd == -1 ) { std::cerr << "epoll_create failed." << std::endl; close(efd); return 1; } struct epoll_event ev; ev.events = EPOLLIN|EPOLLET; ev.data.fd = sfd; if( epoll_ctl(efd, EPOLL_CTL_ADD, sfd, &ev) == -1 ) { std::cerr << "epoll_ctl failed." << std::endl; close(efd); return 1; } std::cerr << "pid=" << getpid() << std::endl; std::cerr << "waiting for signals..." << std::endl; std::this_thread::sleep_for(std::chrono::seconds(10)); struct epoll_event events[MAX_EVENTS]; int n = epoll_wait(efd, events, MAX_EVENTS, -1); if( n == -1 ) { std::cerr << "epoll_wait failed." << std::endl; close(efd); return 1; } for( int i = 0; i < n; ++i ) { if( events[i].data.fd != sfd ) { std::cerr << "wtf!?" << std::endl; close(efd); return 1; } read_signal(sfd); } close(efd); return 0; } int main(int argc, char** argv) { if( argc == 1 ) return 0; // block signals and setup signalfd // Threads created by the main thread inherit the signal mask. sigset_t sigs[1]; sigemptyset(sigs); sigaddset(sigs, SIGQUIT); sigaddset(sigs, SIGHUP); sigaddset(sigs, SIGTERM); sigaddset(sigs, SIGCHLD); if( pthread_sigmask(SIG_BLOCK, sigs, NULL) != 0 ) { std::cerr << "pthread_sigmask failed." << std::endl; return 1; } int sfd = signalfd(-1, sigs, SFD_NONBLOCK); if( sfd == -1 ) { std::cerr << "signalfd failed." << std::endl; return 1; } // ignore SIGPIPE // As per man 7 signal, the signal disposition is a per-process attribute. struct sigaction act; std::memset(&act, 0, sizeof(act)); act.sa_handler = SIG_IGN; if( sigaction(SIGPIPE, &act, NULL) == -1 ) { std::cerr << "sigaction failed." << std::endl; return 1; } int ret = wait_signal(sfd); close(sfd); return ret; } yrmcds-1.0.3/test/siphash.cpp000066400000000000000000000036541226611677100161570ustar00rootroot00000000000000#include #include #include const std::uint64_t vectors[64] = { 0x726fdb47dd0e0e31ULL, 0x74f839c593dc67fdULL, 0x0d6c8009d9a94f5aULL, 0x85676696d7fb7e2dULL, 0xcf2794e0277187b7ULL, 0x18765564cd99a68dULL, 0xcbc9466e58fee3ceULL, 0xab0200f58b01d137ULL, 0x93f5f5799a932462ULL, 0x9e0082df0ba9e4b0ULL, 0x7a5dbbc594ddb9f3ULL, 0xf4b32f46226bada7ULL, 0x751e8fbc860ee5fbULL, 0x14ea5627c0843d90ULL, 0xf723ca908e7af2eeULL, 0xa129ca6149be45e5ULL, 0x3f2acc7f57c29bdbULL, 0x699ae9f52cbe4794ULL, 0x4bc1b3f0968dd39cULL, 0xbb6dc91da77961bdULL, 0xbed65cf21aa2ee98ULL, 0xd0f2cbb02e3b67c7ULL, 0x93536795e3a33e88ULL, 0xa80c038ccd5ccec8ULL, 0xb8ad50c6f649af94ULL, 0xbce192de8a85b8eaULL, 0x17d835b85bbb15f3ULL, 0x2f2e6163076bcfadULL, 0xde4daaaca71dc9a5ULL, 0xa6a2506687956571ULL, 0xad87a3535c49ef28ULL, 0x32d892fad841c342ULL, 0x7127512f72f27cceULL, 0xa7f32346f95978e3ULL, 0x12e0b01abb051238ULL, 0x15e034d40fa197aeULL, 0x314dffbe0815a3b4ULL, 0x027990f029623981ULL, 0xcadcd4e59ef40c4dULL, 0x9abfd8766a33735cULL, 0x0e3ea96b5304a7d0ULL, 0xad0c42d6fc585992ULL, 0x187306c89bc215a9ULL, 0xd4a60abcf3792b95ULL, 0xf935451de4f21df2ULL, 0xa9538f0419755787ULL, 0xdb9acddff56ca510ULL, 0xd06c98cd5c0975ebULL, 0xe612a3cb9ecba951ULL, 0xc766e62cfcadaf96ULL, 0xee64435a9752fe72ULL, 0xa192d576b245165aULL, 0x0a8787bf8ecb74b2ULL, 0x81b3e73d20b49b6fULL, 0x7fa8220ba3b2eceaULL, 0x245731c13ca42499ULL, 0xb78dbfaf3a8d83bdULL, 0xea1ad565322a1a0bULL, 0x60e61c23a3795013ULL, 0x6606d7e446282b93ULL, 0x6ca4ecb15c5f91e1ULL, 0x9f626da15c9625f3ULL, 0xe51b38608ef25f57ULL, 0x958a324ceb064572ULL, }; AUTOTEST(siphash) { const char key[16] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0xa, 0xb, 0xc, 0xd, 0xe, 0xf }; cybozu::siphash24_seed(key); char plaintext[64]; for (int i = 0; i < 64; i++) { plaintext[i] = (char)i; cybozu_assert(cybozu::siphash24(plaintext, i) == vectors[i]); } } yrmcds-1.0.3/test/sleep.cpp000066400000000000000000000003321226611677100156160ustar00rootroot00000000000000#include #include #include int main() { std::cerr << "Hello "; std::this_thread::sleep_for(std::chrono::milliseconds(300)); std::cerr << "World!" << std::endl; return 0; } yrmcds-1.0.3/test/spinlock.cpp000066400000000000000000000011011226611677100163230ustar00rootroot00000000000000#include #include #include #include alignas(CACHELINE_SIZE) cybozu::spinlock g_lock; alignas(CACHELINE_SIZE) unsigned int g_counter = 0; void accum(int n) { for(int i = 0; i < n; ++i) { std::lock_guard guard(g_lock); ++g_counter; } } int main() { std::thread t1(accum, 100000); std::thread t2(accum, 100000); std::thread t3(accum, 100000); t1.join(); t2.join(); t3.join(); std::cout << "accumulated value = " << g_counter << std::endl; return 0; } yrmcds-1.0.3/test/system_error.cpp000066400000000000000000000010701226611677100172430ustar00rootroot00000000000000#include #include #include #include #include #include int main(int argc, char** argv) { open("/hoge/fuga/faaa", O_WRONLY|O_CREAT, 0700); try { throw std::system_error(errno, std::system_category(), "open"); } catch( const std::exception& e ) { std::cout << e.what() << std::endl; } auto ec = std::system_category().default_error_condition(3); std::cout << "value=" << ec.value() << ", message=" << ec.message() << std::endl; return 0; } yrmcds-1.0.3/test/tempfile.cpp000066400000000000000000000017631226611677100163240ustar00rootroot00000000000000#include "../src/tempfile.hpp" #include #include #include AUTOTEST(mkstemp_wrap) { yrmcds::mkstemp_wrap("/var/tmp"); } AUTOTEST(tempfile) { yrmcds::tempfile t; t.write("abcde", 5); cybozu::dynbuf buf(10); t.read_contents(buf); cybozu_assert( buf.size() == 5 ); cybozu_assert( std::memcmp(buf.data(), "abcde", 5) == 0 ); buf.reset(); t.read_contents(buf); cybozu_assert( buf.size() == 5 ); cybozu_assert( std::memcmp(buf.data(), "abcde", 5) == 0 ); buf.reset(); t.write("xxx", 3); t.read_contents(buf); cybozu_assert( buf.size() == 8 ); cybozu_assert( std::memcmp(buf.data(), "abcdexxx", 8) == 0 ); buf.reset(); t.clear(); t.read_contents(buf); cybozu_assert( buf.size() == 0 ); buf.append("123", 3); t.write("789", 3); t.read_contents(buf); cybozu_assert( buf.size() == 6 ); cybozu_assert( std::memcmp(buf.data(), "123789", 6) == 0 ); buf.reset(); } yrmcds-1.0.3/test/test.conf000066400000000000000000000005151226611677100156330ustar00rootroot00000000000000# Configuration file for yrmcds virtual_ip = 127.0.0.1 port = 1121 repl_port = 1122 max_connections = 10000 temp_dir = "/var/tmp" user = nobody group = nogroup log.threshold = warning log.file = "/var/log/yrmcds.log" buckets = 1000000 max_data_size = 5M heap_data_limit = 16K memory_limit = 1024M workers = 10 gc_interval = 20 yrmcds-1.0.3/test/thread.cpp000066400000000000000000000011701226611677100157560ustar00rootroot00000000000000#include #include #include #include #include #include struct ttt: public cybozu::thread_base { ttt(): cybozu::thread_base() {} int i = 3; void run() { std::cout << "i = " << i << std::endl; throw std::runtime_error("hoge"); } void stop() { m_thread.join(); } }; int main() { std::unique_ptr p; std::once_flag f; std::call_once(f, [&]{ p.reset(new int); }); std::cout << "*p=" << *p << std::endl; ttt t; t.start(); t.stop(); return 0; } yrmcds-1.0.3/test/tuple.cpp000066400000000000000000000010431226611677100156370ustar00rootroot00000000000000#include #include #include #include int main(int argc, char** argv) { std::vector> p; void* vv = std::malloc(10); std::cout << vv << std::endl; p.emplace_back(vv, 3); for( auto t: p ) { void* v; std::tie(v, std::ignore) = t; std::cout << v << std::endl; } std::free(vv); auto tt = std::make_tuple(1, "abc"); std::get<0>(tt) = 3; int n; std::tie(n, std::ignore) = tt; std::cout << n << std::endl; return 0; } yrmcds-1.0.3/test/unordered_map.cpp000066400000000000000000000006761226611677100173450ustar00rootroot00000000000000#include #include #include int main(int argc, char** argv) { std::unordered_map m; std::cout << "Default max load factor: " << m.max_load_factor() << std::endl; std::cout << "Default bucket count: " << m.bucket_count() << std::endl; m.max_load_factor(1.0); m.reserve(65536); std::cout << "Bucket count now: " << m.bucket_count() << std::endl; return 0; } yrmcds-1.0.3/test/vector.cpp000066400000000000000000000027401226611677100160150ustar00rootroot00000000000000#include #include #include #include #include #include #if DEBUG_NEW void* operator new(std::size_t n) { std::cerr << "allocating " << n << " bytes." << std::endl; return std::malloc(n); } void operator delete(void* p) { std::cerr << "deallocating" << std::endl; return std::free(p); } #endif int main(int argc, char** argv) { std::cout << "sizeof(std::vector) = " << sizeof(std::vector) << std::endl; std::vector v {'a', 'b', 'c', 'd', 'e', 'f'}; std::cout << "v.size() = " << v.size() << std::endl; auto it = v.begin(); while( it != v.end() ) { if( *it == 'b' || *it == 'c' ) { it = v.erase(it); } else { ++it; } } for( char c: v ) std::cout << c; std::cout << std::endl; const char* hello = "Hello World"; std::vector hv(hello+3, hello+7); for( char c: hv ) std:: cerr << c; std::cerr << std::endl; // without v.erase, the vector size will not be changed. v.erase(std::remove_if(v.begin(), v.end(), [](char c) { return (c&1) == 1; }), v.end()); for( char c: v ) std::cout << c; std::cout << std::endl; std::vector v2; v2.reserve(5); v2.push_back('1'); v2.push_back('2'); std::vector v3 = {'a', 'b', 'c', 'd'}; v2 = v3; std::cout << v2.capacity() << std::endl; return 0; } yrmcds-1.0.3/test/vector_bool.cpp000066400000000000000000000012701226611677100170250ustar00rootroot00000000000000#include #include #include #include void foo() { std::vector vb(30); //std::fill(vb.begin(), vb.end(), true); assert( std::none_of(vb.begin(), vb.end(), [](bool b){return b;}) ); vb[3] = true; assert( ! std::none_of(vb.begin(), vb.end(), [](bool b){return b;}) ); assert( std::any_of(vb.begin(), vb.end(), [](bool b){return b;}) ); assert( ! std::all_of(vb.begin(), vb.end(), [](bool b){return b;}) ); std::fill(vb.begin(), vb.end(), true); assert( std::all_of(vb.begin(), vb.end(), [](bool b){return b;}) ); } int main(int argc, char** argv) { for( int i = 0; i < 10; ++i ) foo(); return 0; }