pax_global_header00006660000000000000000000000064126225464560014526gustar00rootroot0000000000000052 comment=2fcc59a94f1f7505f858ed120c15e24aeca1ebeb yrmcds-1.1.5/000077500000000000000000000000001262254645600130335ustar00rootroot00000000000000yrmcds-1.1.5/.gitignore000066400000000000000000000001671262254645600150270ustar00rootroot00000000000000# C/C++ *.[oa] # Editors *~ .*.swp # Directories html lz4 # binaries yrmcdsd yrmcdsd.map *.exe # misc. COPYING.hpp yrmcds-1.1.5/COPYING000066400000000000000000000024161262254645600140710ustar00rootroot00000000000000Copyright (c) 2013-2015 Cybozu. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. yrmcds-1.1.5/ChangeLog000066400000000000000000000046271262254645600146160ustar00rootroot00000000000000version 1.1.5 * New config option "repl_buffer_size". * New config option "slave_timeout". * fixed bugs: #52, #59 version 1.1.4 * add "keys" memcached extension command (#51). version 1.1.3 * auto-detect libtcmalloc. version 1.1.2 * "stats" becomes more compatible with memcached. version 1.1.1 * New config option "secure_erase" for confidential data. * New config option "lock_memory" to prevent swapping. version 1.1.0 * update documents. version 1.1.0-rc2 * semaphores are renamed to (resource) counters. version 1.1.0-rc1 * add a new protocol for distributed semaphores. version 1.0.4 * fix "make install" to property install yrmcdsd. * fixed bugs: #31, #34 version 1.0.3 * fix a compilation problem on non-SSE Intel 32bit CPU. * remove test/async as it is not related to any part of yrmcds. version 1.0.2 * fix siphash implementation to avoid unaligned memory access. * improve Makefile for better platform portability. * fix test/reactor to use pipes instead of stdin. version 1.0.1 * fix an infinite loop when sending 0 byte object via binary protocol. * improve binary protocol compatibility with memcached. Specifically, Delete/Append/Prepend/Increment/Decrement with CAS are supported now. * improve flush logic of temporary files. * fix some potential bugs: #23, #24 version 1.0.0 * [security] MurmurHash was replaced with SipHash. version 0.9.7 * add a new statistics option "stats ops" to report the total count of command executions for each command. version 0.9.6 * improve scalability by reducing the reactor thread's load. * add "max_connections" config option. * fixed some memory corruption bugs. version 0.9.5 * change UID if "user" is defined in the configuration file. * change GID if "group" is defined in the configuration file. * fixed a memory synchronization bug. version 0.9.4 * add proper memory fences. version 0.9.3 * enhances STATS to include the number of connections. * replication starts quickly by eliminating wasteful wait. * moves recv(2) from the reactor to workers, reducing the reactor load. * fixed bugs: #10 version 0.9.2 * fixed bugs: #4, #6 version 0.9.1 * implements the server-side locking mechanism. * new command line option "-v" shows the version and copyrights. * fixed bugs: #2 version 0.9.0 * The first public release. * implements memcached ASCII and binary protocols. * implements master-slave replication. yrmcds-1.1.5/Makefile000066400000000000000000000055531262254645600145030ustar00rootroot00000000000000# Makefile for yrmcds # Prerequisites: gcc 4.8+ or clang 3.3+ PREFIX = /usr/local DEFAULT_CONFIG = $(PREFIX)/etc/yrmcds.conf CACHELINE_SIZE := $(shell getconf LEVEL1_DCACHE_LINESIZE) ifeq ($(CACHELINE_SIZE), 0) CACHELINE_SIZE = 32 endif CC = gcc CXX = g++ CPPFLAGS = -I. -DCACHELINE_SIZE=$(CACHELINE_SIZE) $(TCMALLOC_FLAGS) CPPFLAGS += -DDEFAULT_CONFIG=$(DEFAULT_CONFIG) OPTFLAGS = -O2 #-flto DEBUGFLAGS = -gdwarf-3 #-fsanitize=address WARNFLAGS = -Wall -Wnon-virtual-dtor -Woverloaded-virtual CPUFLAGS = #-march=core2 -mtune=corei7 CXXFLAGS = -std=gnu++11 $(OPTFLAGS) $(DEBUGFLAGS) $(shell getconf LFS_CFLAGS) $(WARNFLAGS) $(CPUFLAGS) LDFLAGS = -L. $(shell getconf LFS_LDFLAGS) LDLIBS = $(shell getconf LFS_LIBS) -lyrmcds $(LIBTCMALLOC) -latomic -lpthread CLDOC := LD_LIBRARY_PATH=$(shell llvm-config --libdir 2>/dev/null) cldoc HEADERS = $(wildcard src/*.hpp src/*/*.hpp cybozu/*.hpp) SOURCES = $(wildcard src/*.cpp src/*/*.cpp cybozu/*.cpp) OBJECTS = $(patsubst %.cpp,%.o,$(SOURCES)) EXE = yrmcdsd TESTS = $(patsubst %.cpp,%,$(wildcard test/*.cpp)) LIB = libyrmcds.a LIB_OBJECTS = $(filter-out src/main.o,$(OBJECTS)) PACKAGES = build-essential libgoogle-perftools-dev python-pip TCMALLOC_HEADER := $(shell sh ./detect_tcmalloc.sh $(CXX) $(CPPFLAGS)) ifeq ($(TCMALLOC_HEADER), gperftools/tcmalloc.h) TCMALLOC_FLAGS = -DUSE_TCMALLOC LIBTCMALLOC = -ltcmalloc_minimal export TCMALLOC_FLAGS LIBTCMALLOC else ifeq ($(TCMALLOC_HEADER), google/tcmalloc.h) TCMALLOC_FLAGS = -DUSE_TCMALLOC -DTCMALLOC_IN_GOOGLE LIBTCMALLOC = -ltcmalloc_minimal export TCMALLOC_FLAGS LIBTCMALLOC endif all: $(EXE) lib: $(LIB) tests: $(TESTS) strip: $(EXE) nm -n $(EXE) | grep -v '\( [aNUw] \)\|\(__crc_\)\|\( \$$[adt]\)' > $(EXE).map strip $(EXE) install: $(EXE) cp etc/upstart /etc/init/yrmcds.conf cp etc/logrotate /etc/logrotate.d/yrmcds cp etc/yrmcds.conf $(DEFAULT_CONFIG) cp $(EXE) $(PREFIX)/sbin/yrmcdsd install -o nobody -g nogroup -m 644 /dev/null /var/log/yrmcds.log COPYING.hpp: COPYING echo -n 'static char COPYING[] = R"(' > $@ cat $< >>$@ echo ')";' >>$@ src/main.o: COPYING.hpp $(OBJECTS): $(HEADERS) $(LIB): $(LIB_OBJECTS) $(AR) rcs $@ $^ $(EXE): src/main.o $(LIB) $(CXX) $(CXXFLAGS) $(LDFLAGS) -o $@ $< $(LDLIBS) test/%.exe: test/%.o $(LIB) $(CXX) $(CXXFLAGS) $(LDFLAGS) -o $@ $< $(LDLIBS) $(TESTS): $(LIB) @$(MAKE) -s $@.exe ./$@.exe @echo html: @clang++ -v 2>/dev/null || (echo "No clang++ in PATH." && false) rm -rf html $(CLDOC) generate $(CPPFLAGS) $(CXXFLAGS) -w -D__STRICT_ANSI__ -- --output html --merge docs $(HEADERS) serve: html @cd html; python -m SimpleHTTPServer 8888 || true clean: rm -f src/*.o src/*/*.o cybozu/*.o test/*.o test/*.exe COPYING.hpp $(EXE) $(EXE).map $(LIB) setup: sudo apt-get install -y --install-recommends $(PACKAGES) sudo pip install cldoc --upgrade .PHONY: all strip lib tests install html serve clean setup test_env $(TESTS) yrmcds-1.1.5/README.md000066400000000000000000000062401262254645600143140ustar00rootroot00000000000000yrmcds ====== yrmcds is a memory object caching system with master/slave replication. Currently, yrmcds supports two protocols: the first is an enhanced [memcached][], and another is a protocol to implement [distributed resource counters](docs/counter.md). Since the memcached protocol is perfectly compatible with the [original implementation][memcached], yrmcds can be used as a drop-in replacement for memcached. Thanks to its virtual-IP based replication system, existing applications can obtain high-available memcached-compatible service without any modifications. A companion client library [libyrmcds][] and a [PHP extension][php-yrmcds] are also available. yrmcds was developed originally for [kintone.com][kintone]. License ------- yrmcds is licensed under [the BSD 2-clause license][bsd2]. The source code contains a [SipHash][] implementation borrowed from [csiphash][] which is licensed under [the MIT license][mit]. Features -------- * Complete memcached text and binary protocols with these extensions: * [Server-side locking](docs/locking.md) * [Dump keys](docs/keys.md) * [Distributed resource counter](docs/counter.md) protocol. * [Optional memory security](docs/usage.md#secure_erase) to store confidential information like SSL session data. * Large objects can be stored in temporary files, not in memory. * Virtual-IP based master-slave replication. * Automatic fail-over * Automatic recovery of redundancy. * Global LRU eviction / no slab distribution problem. * Unlike memcached, yrmcds is not involved with slabs problems. ([1][slab1], [2][slab2]) A [companion client library][libyrmcds] and a [PHP extension][php-yrmcds] are also available. See also [usage guide](docs/usage.md), [future plans](docs/future.md), [differences from memcached](docs/diffs.md), [design notes](docs/design.md) and some [benchmark results](docs/bench.md). Prerequisites ------------- * **Linux**. * C++11 compiler (gcc 4.8.1+ or clang 3.3+). * GNU make. Build ----- Just run `make`. yrmcds runs faster when linked with [TCMalloc](tcmalloc). On Debian/Ubuntu, install `libgoogle-perftools-dev` package to prepare TCMalloc. The makefile automatically detects TCMalloc if available. Install ------- On Ubuntu, `sudo make install` installs yrmcds under `/usr/local`. An upstart script and a logrotate configuration file are installed too. About the name -------------- The name yrmcds was taken from "Ymmt's Replicating MemCacheD for Sessions". As it reads, yrmcds was developed mainly for session storage. The correct pronunciation sounds like: "Yo-Ru-Mac-Do" (夜マクド in Japanese). [memcached]: http://memcached.org/ [bsd2]: http://opensource.org/licenses/BSD-2-Clause [SipHash]: https://131002.net/siphash/ [csiphash]: https://github.com/majek/csiphash [mit]: http://opensource.org/licenses/MIT [libyrmcds]: http://cybozu.github.io/libyrmcds/ [php-yrmcds]: http://cybozu.github.io/php-yrmcds/ [slab1]: http://nosql.mypopescu.com/post/13506116892/memcached-internals-memory-allocation-eviction [slab2]: https://groups.google.com/forum/#!topic/memcached/DuJNy5gbQ0o [kintone]: https://www.kintone.com/ [tcmalloc]: http://goog-perftools.sourceforge.net/doc/tcmalloc.html yrmcds-1.1.5/cybozu/000077500000000000000000000000001262254645600143465ustar00rootroot00000000000000yrmcds-1.1.5/cybozu/config_parser.cpp000066400000000000000000000022751262254645600177010ustar00rootroot00000000000000// (C) 2013 Cybozu. #include "config_parser.hpp" #include namespace { const char white_spaces[] = " \f\n\r\t\v"; inline void rtrim(std::string& s) { s.erase( s.find_last_not_of(white_spaces) + 1 ); } inline void ltrim(std::string& s) { s.erase( 0, s.find_first_not_of(white_spaces) ); } } // anonymous namespace namespace cybozu { void config_parser::load(const std::string& path) { m_config.clear(); std::ifstream is(path); if( !is ) throw std::runtime_error("failed to open " + path); unsigned int lineno = 0; for( std::string l; std::getline(is, l); ) { lineno++; ltrim(l); rtrim(l); if( l.empty() || l[0] == '#' ) continue; std::size_t n = l.find('='); if( n == std::string::npos ) throw parse_error(path, lineno); std::string key = l.substr(0, n); rtrim(key); std::string value = l.substr(n+1); ltrim(value); // unquote the value std::size_t len = value.size(); if( len >= 2 && value[0] == '"' && value[len-1] == '"' ) { value = value.substr(1, len-2); } set(key, value); } } } // namespace cybozu yrmcds-1.1.5/cybozu/config_parser.hpp000066400000000000000000000075161262254645600177110ustar00rootroot00000000000000// Read and parse configuration files. // (C) 2013 Cybozu. #ifndef CYBOZU_CONFIG_PARSER_HPP #define CYBOZU_CONFIG_PARSER_HPP #include #include #include namespace cybozu { // Parse configuration files. class config_parser { std::unordered_map m_config; public: // Create an empty config_parser object. config_parser() {} // Create a and load the file at `path`. // @path The path to the configuration file. // // Create a and load the file at `path`. // Raise exception if the file contains an invalid line. explicit config_parser(const std::string& path) { load(path); } // exception for invalid file format. struct parse_error: public std::runtime_error { parse_error(const std::string& path, unsigned int lineno): std::runtime_error("Parse error in " + path + " at line " + std::to_string(lineno)) {} }; // exception for key not found error. struct not_found: public std::runtime_error { not_found(const std::string& key): std::runtime_error("Key not found: " + key) {} }; // exception for illegal value error. struct illegal_value: public std::runtime_error { illegal_value(const std::string& key): std::runtime_error("Illegal value for " + key) {} }; // Load a configuration file. // @path The path to the configuration file. // // Load a configuration file. // Raise exception if the file contains an invalid line. // All previously loaded configurations will be cleared. void load(const std::string& path); // Set a configuration value. // @key A configuration key. // @value The associated value. void set(const std::string& key, const std::string& value) { auto it = m_config.find(key); if( it == m_config.end() ) { m_config.emplace(key, value); } else { it->second = value; } } // Get a value associated with `key`. // @key A configuration key. // // Get a value associated with `key`. // Raise exception if the key is not found. // // @return A reference to the associated with `key`. const std::string& get(const std::string& key) const { auto it = m_config.find(key); if( it == m_config.end() ) throw not_found(key); return it->second; } // Return `true` if `key` exists. // @key A configuration key. bool exists(const std::string& key) const { return m_config.find(key) != m_config.end(); } // Get an integer converted from the value associated with `key`. // @key A configuration key. // // Get an integer converted from the value associated with `key`. // Raise or . // // @return An integer converted from the associated value. int get_as_int(const std::string& key) const { try { return std::stoi(get(key)); } catch(const std::invalid_argument& e) { throw illegal_value(key); } catch(const std::out_of_range& e) { throw illegal_value(key); } } // Get a boolean converted from the value associated with `key`. // @key A configuration key. // // Get a boolean converted from the value associated with `key`. // Raise or . // //@return `true` or `false` converted from the associated value. bool get_as_bool(const std::string& key) const { const std::string& s = get(key); if( s == "true" ) return true; if( s == "false" ) return false; throw illegal_value(key); } }; } // namespace cybozu #endif // CYBOZU_CONFIG_PARSER_HPP yrmcds-1.1.5/cybozu/dynbuf.hpp000066400000000000000000000144161262254645600163540ustar00rootroot00000000000000// A fast, dynamic-sized char buffer. // (C) 2013 Cybozu. #ifndef CYBOZU_DYNBUF_HPP #define CYBOZU_DYNBUF_HPP #include "util.hpp" #ifdef USE_TCMALLOC # ifdef TCMALLOC_IN_GOOGLE # include # else # include # endif #else # include #endif #include #include #include #include namespace cybozu { // A fast, dynamic-sized char buffer. // // There are two motivations for this class rather than using // or . One is to provide nicer interfaces to receive data // for system calls. Another is to be efficient as much as possible. class dynbuf final { public: // Constructor. // @default_capacity The default capacity of the internal buffer. // // The constructor pre-allocates an internal buffer if `default_capacity` // is not 0. Everytime is invoked, the internal buffer will be // shrunk to `default_capacity`. explicit dynbuf(std::size_t default_capacity, bool erase=false): m_p(default_capacity ? _malloc(default_capacity) : nullptr), m_default_capacity(default_capacity), m_capacity(default_capacity), m_erase(erase) {} ~dynbuf() { if( m_p != nullptr ) { if( m_erase ) clear_memory(m_p, m_used); _free(m_p); } } dynbuf(const dynbuf&) = delete; dynbuf(dynbuf&& rhs) noexcept: m_p(nullptr), m_default_capacity(rhs.m_default_capacity), m_capacity(rhs.m_capacity), m_erase(rhs.m_erase), m_used(rhs.m_used) { std::swap(m_p, rhs.m_p); } dynbuf& operator=(const dynbuf&) = delete; dynbuf& operator=(dynbuf&&) = delete; void swap(dynbuf& other) noexcept { std::swap(m_p, other.m_p); std::swap(m_capacity, other.m_capacity); std::swap(m_used, other.m_used); } // Clear the contents and reset the internal buffer. void reset() { if( m_used && m_erase ) clear_memory(m_p, m_used); m_used = 0; if( m_default_capacity == m_capacity ) return; char* new_p = nullptr; if( m_default_capacity != 0 ) new_p = _malloc(m_default_capacity); _free(m_p); m_p = new_p; m_capacity = m_default_capacity; } // Append contents. // @p The pointer to the contents. // @len The length of the contents. void append(const char* p, std::size_t len) { if( freebytes() < len ) enlarge(len); std::memcpy(m_p+m_used, p, len); m_used += len; } // Erase contents. // @len The length to be erased. // // Erase contents from the head of the internal buffer. // Remaining data will be moved to the head. void erase(const std::size_t len) { if( m_used < len ) throw std::invalid_argument(""); if( len == 0 ) return; std::size_t remain = m_used - len; if( remain == 0 ) { reset(); return; } if( remain <= m_default_capacity && m_capacity != m_default_capacity ) { char* new_p = _malloc(m_default_capacity); std::memcpy(new_p, m_p+len, remain); if( m_erase ) clear_memory(m_p, m_used); _free(m_p); m_p = new_p; m_capacity = m_default_capacity; m_used = remain; return; } std::memmove(m_p, m_p+len, remain); if( m_erase ) clear_memory(m_p+remain, len); m_used = remain; } // Prepare enough free space in the internal buffer. // @len Required free space size. // // This prepares at least `len` byte free space in the internal buffer. // The caller should call after it wrote some data to the // free space. // // @return The pointer to the free space. char* prepare(std::size_t len) { if( freebytes() < len ) enlarge(len); return m_p + m_used; } // Consume the free space. // @len Size of memory to be consumed. void consume(std::size_t len) noexcept { m_used += len; } // Return the pointer to the head of the internal buffer. const char* data() const noexcept { return m_p; } // Same as STL's `empty()`. bool empty() const noexcept { return m_used == 0; } // Same as STL's `size()`. std::size_t size() const noexcept { return m_used; } private: char* m_p; const std::size_t m_default_capacity; std::size_t m_capacity; const bool m_erase; std::size_t m_used = 0; std::size_t freebytes() const noexcept { return m_capacity - m_used; } void enlarge(std::size_t additional) { if( additional == 0 ) return; const std::size_t new_capacity = m_capacity + additional; if( m_erase ) { char * const new_p = _malloc(new_capacity); if( m_p ) { std::memcpy(new_p, m_p, m_used); clear_memory(m_p, m_used); _free(m_p); } m_p = new_p; } else { m_p = _realloc(m_p, new_capacity); } m_capacity = new_capacity; } static char* _malloc(std::size_t len) { #ifdef USE_TCMALLOC char* p = (char*)tc_malloc(len); #else char* p = (char*)std::malloc(len); #endif if( p == nullptr ) throw std::runtime_error("failed to allocate memory."); return p; } static void _free(void* p) noexcept { #ifdef USE_TCMALLOC tc_free(p); #else std::free(p); #endif } static char* _realloc(void* p, std::size_t new_len) { #ifdef USE_TCMALLOC char* np = (char*)tc_realloc(p, new_len); #else char* np = (char*)std::realloc(p, new_len); #endif if( np == nullptr ) throw std::runtime_error("failed to re-allocate memory."); return np; } friend bool operator==(const dynbuf& lhs, const dynbuf& rhs); }; inline bool operator==(const dynbuf& lhs, const dynbuf& rhs) { if( lhs.m_p == rhs.m_p ) return true; if( lhs.m_used != rhs.m_used ) return false; return std::memcmp(lhs.m_p, rhs.m_p, lhs.m_used) == 0; } } // namespace cybozu #endif // CYBOZU_DYNBUF_HPP yrmcds-1.1.5/cybozu/filesystem.cpp000066400000000000000000000011221262254645600172320ustar00rootroot00000000000000// (C) 2013 Cybozu. #include "filesystem.hpp" #include "logger.hpp" #include #include namespace cybozu { bool get_stat(const std::string& path, struct stat& st) { if( ::stat(path.c_str(), &st) == 0 ) return true; if( errno == EFAULT || errno == ENOMEM ) throw_unix_error(errno, "stat"); auto ec = std::system_category().default_error_condition(3); logger::debug() << "stat failed. path=" << path << ", error=" << ec.message() << " (" << errno << ")"; return false; } } // namespace cybozu yrmcds-1.1.5/cybozu/filesystem.hpp000066400000000000000000000014011262254645600172370ustar00rootroot00000000000000// filesystem.hpp // (C) 2013 Cybozu. #ifndef CYBOZU_FILESYSTEM_HPP #define CYBOZU_FILESYSTEM_HPP #include "util.hpp" #include #include #include #include namespace cybozu { // stat(2) wrapper. // @return `true` if a file exists at `path`, `false` otherwise. bool get_stat(const std::string& path, struct stat& st); inline bool is_dir(const std::string& path) { struct stat st; if( ! get_stat(path, st) ) return false; return S_ISDIR(st.st_mode); } inline bool is_readable(const std::string& path) { return ::access(path.c_str(), R_OK) == 0; } inline bool is_writable(const std::string& path) { return ::access(path.c_str(), W_OK) == 0; } } // namespace cybozu #endif // CYBOZU_FILESYSTEM_HPP yrmcds-1.1.5/cybozu/hash_map.cpp000066400000000000000000000011261262254645600166320ustar00rootroot00000000000000// (C) 2013 Cybozu. #include "hash_map.hpp" #include namespace cybozu { unsigned int nearest_prime(unsigned int n) noexcept { static_assert( sizeof(n) >= 4, "Too small unsigned int." ); if( n == 2 ) return 2; for( unsigned int i = n|1; ; i += 2 ) { bool prime = true; unsigned int r = (unsigned int)std::rint( std::sqrt((double)i) ); for( unsigned int j = 3; j <= r; j += 2 ) { if( (i % j) == 0 ) { prime = false; break; } } if( prime ) return i; } } } // namespace cybozu yrmcds-1.1.5/cybozu/hash_map.hpp000066400000000000000000000303331262254645600166410ustar00rootroot00000000000000// The object hash map. // (C) 2013 Cybozu. #ifndef CYBOZU_HASH_MAP_HPP #define CYBOZU_HASH_MAP_HPP #include "siphash.hpp" #include #include #include #include #include #include #include #include namespace cybozu { // Key class for . class hash_key final { public: // Construct from a statically allocated memory. // @p Pointer to a statically allocated memory. // @len Length of the key. // // Construct from a statically allocated memory. // // As long as the constructed object lives, the memory pointed by `p` // must not be freed. hash_key(const char* p, std::size_t len) noexcept : m_p(p), m_len(len) { m_hash = siphash24(m_p, m_len); } // Construct by moving a . // // Construct by moving a . Sample usage: // ``` // hash_key( std::vector(p, p+len) ) // ``` hash_key(std::vector v): m_v(std::move(v)), m_p(m_v.data()), m_len(m_v.size()) { m_hash = siphash24(m_p, m_len); } // Copy constructor. hash_key(const hash_key& rhs): m_v(rhs.m_p, rhs.m_p+rhs.m_len), m_p(m_v.data()), m_len(m_v.size()), m_hash(rhs.m_hash) {} hash_key& operator=(const hash_key& rhs) = delete; // Move contructor and assign operator. hash_key(hash_key&& rhs) noexcept = default; hash_key& operator=(hash_key&& rhs) = default; std::uint64_t hash() const noexcept { return m_hash; } const char* data() const noexcept { return m_p; } std::size_t length() const noexcept { return m_len; } std::string str() const { return std::string(m_p, m_len); } bool has_prefix(const char* p, std::size_t len) const { if( m_len < len ) return false; return std::memcmp(m_p, p, len) == 0; } private: std::vector m_v; const char* m_p; std::size_t m_len; std::uint64_t m_hash; friend bool operator==(const hash_key&, const hash_key&) noexcept; }; inline bool operator==(const hash_key& lhs, const hash_key& rhs) noexcept { if( lhs.m_len != rhs.m_len ) return false; if( lhs.m_p == rhs.m_p ) return true; return std::memcmp(lhs.m_p, rhs.m_p, lhs.m_len) == 0; } // Return the nearest prime number. unsigned int nearest_prime(unsigned int n) noexcept; // Highly concurrent object hash map. // // Keys for this hash map are whereas objects are of type `T`. // `T` must be either move-constructible or copyable. // // Each bucket in the hash map has its unique mutex to guard itself. // This design reduces contensions between threads drastically in exchange // for some functions such as dynamic resizing of the number of buckets. template class hash_map { static_assert( std::is_move_constructible::value || std::is_copy_constructible::value, "T must be move- or copy- constructible." ); public: typedef std::function handler; typedef std::function creator; // Hash map bucket. // // Each hash value corresponds to a bucket. // Member functions whose names end with `_nolock` are not thread-safe. class bucket { struct item { const hash_key key; T object; item* next; item(const hash_key& k, const creator& c, item* next): key(k), object(c(key)), next(next) {} }; public: bucket(): m_objects(nullptr) {} ~bucket() { clear_nolock(); } // Handle or insert an object. // @key The object's key. // @h A function to handle an existing object. // @c A function to create a new object. // // This function can be used to handle an existing object, or // to insert a new object when such an object does not exist. // // if `h` is `nullptr` and there is an existing object for `key`, // `false` is returned. If `c` is `nullptr` and there is no // object for `key`, `false` is returned. // // If `h` is not `nullptr` and there is an existing object for // `key`, then `h` is called and the return value of `h` is // returned. This means `h` can return `false` if it failed // to handle the object. // // If `c` is not `nullptr` and there is no object for `key`, // an object is created by calling `c` and stored, then `true` // is returned. // // @return `true` if succeeded, `false` otherwise. bool apply_nolock(const hash_key& key, const handler& h, const creator& c) { for( item* p = m_objects; p != nullptr; p = p->next ) { if( p->key == key ) { if( ! h ) return false; return h(p->key, p->object); } } if( ! c ) return false; m_objects = new item(key, c, m_objects); return true; } // Thread-safe . bool apply(const hash_key& key, const handler& h, const creator& c) { lock_guard g(m_lock); return apply_nolock(key, h, c); } // Apply `pred` for each object. // @pred Predicate function void foreach(const std::function& pred) { lock_guard g(m_lock); for( item* p = m_objects; p != nullptr; p = p->next ) pred(p->key, p->object); } // Remove an object for `key`. // @key The object's key. // @callback A function called when an object is removed. // // This removes an object associated with `key`. If there is no // object associated with `key`, return `false`. If `callback` // is not `nullptr`, it is called when an object is removed. // // @return `true` if successfully removed, `false` otherwise. bool remove_nolock(const hash_key& key, const std::function& callback) { for( item** p = &m_objects; *p != nullptr; p = &((*p)->next) ) { if( (*p)->key == key ) { item* to_delete = *p; *p = to_delete->next; delete to_delete; if( callback ) callback(key); return true; } } return false; } // Thread-safe . bool remove(const hash_key& key, const std::function& callback) { lock_guard g(m_lock); return remove_nolock(key, callback); } // Remove an object for `key` if `pred` returns `true`. // @key The object's key. // @pred A predicate function. // // This function removes an object assiciated with `key` if a // predicate function returns `true`. This function is thread-safe. // // @return `true` if object existed, `false` otherwise. bool remove_if(const hash_key& key, const std::function& pred) { lock_guard g(m_lock); for( item** p = &m_objects; *p != nullptr; p = &((*p)->next) ) { item* to_delete = *p; if( to_delete->key == key ) { if( pred(key, to_delete->object) ) { *p = to_delete->next; delete to_delete; } return true; } } return false; } // Collect garbage objects. // @pred Predicate function. // // This function collects garbage objects. // Objects for which `pred` returns `true` will be removed. void gc(const std::function& pred) { lock_guard g(m_lock); for( item** p = &m_objects; *p != nullptr; ) { item* to_delete = *p; if( pred(to_delete->key, to_delete->object) ) { *p = to_delete->next; delete to_delete; } else { p = &(to_delete->next); } } } // Clear objects in this bucket. void clear_nolock() { while( m_objects ) { item* next = m_objects->next; delete m_objects; m_objects = next; } } private: using lock_guard = std::lock_guard; mutable std::mutex m_lock; item* m_objects; }; hash_map(unsigned int buckets): m_size(nearest_prime(buckets)), m_buckets(m_size) {} // Handle or insert an object. // @key The object's key. // @h A function to handle an existing object. // @c A function to create a new object. // // This function can be used to handle an existing object, or // to insert a new object when such an object does not exist. // // if `h` is `nullptr` and there is an existing object for `key`, // `false` is returned. If `c` is `nullptr` and there is no // object for `key`, `false` is returned. // // If `h` is not `nullptr` and there is an existing object for // `key`, then `h` is called and the return value of `h` is // returned. This means `h` can return `false` if it failed // to handle the object. // // If `c` is not `nullptr` and there is no object for `key`, // an object is created by calling `c` and stored, then `true` // is returned. // // @return `true` if succeeded, `false` otherwise. bool apply_nolock(const hash_key& key, const handler& h, const creator& c) { return get_bucket(key).apply_nolock(key, h, c); } // Thread-safe . bool apply(const hash_key& key, const handler& h, const creator& c) { return get_bucket(key).apply(key, h, c); } // Apply `pred` for each object. // @pred Predicate function void foreach(const std::function& pred) { for( auto& bucket: m_buckets ) bucket.foreach(pred); } // Remove an object for `key`. // @key The object's key. // @callback A function called when an object is removed. // // This removes an object associated with `key`. If there is no // object associated with `key`, return `false`. If `callback` // is not `nullptr`, it is called when an object is removed. // // @return `true` if successfully removed, `false` otherwise. bool remove_nolock(const hash_key& key, const std::function& callback) { return get_bucket(key).remove_nolock(key, callback); } // Thread-safe . bool remove(const hash_key& key, const std::function& callback) { return get_bucket(key).remove(key, callback); } // Remove an object for `key` if `pred` returns `true`. // @key The object's key. // @pred A predicate function. // // This function removes an object assiciated with `key` if a // predicate function returns `true`. This function is thread-safe. // // @return `true` if object existed, `false` otherwise. bool remove_if(const hash_key& key, const std::function& pred) { return get_bucket(key).remove_if(key, pred); } /* Bucket interfaces */ // Return the number of buckets in this hash map. std::size_t bucket_count() const noexcept { return m_size; } bucket& get_bucket(const hash_key& key) noexcept { return m_buckets[key.hash() % m_size]; } // iterator. using iterator = typename std::vector::iterator; iterator begin() { return m_buckets.begin(); } iterator end() { return m_buckets.end(); } private: const std::size_t m_size; std::vector m_buckets; }; } // namespace cybozu #endif // CYBOZU_HASH_MAP_HPP yrmcds-1.1.5/cybozu/ip_address.cpp000066400000000000000000000071721262254645600171760ustar00rootroot00000000000000// (C) 2013 Cybozu. #include "ip_address.hpp" #include "util.hpp" #include #include #include #include #include #include #include namespace { class ifaddrs_wrapper { struct ifaddrs* m_addr = nullptr; public: ifaddrs_wrapper() { if( getifaddrs(&m_addr) != 0 ) cybozu::throw_unix_error(errno, "getifaddrs"); } ~ifaddrs_wrapper() { if( m_addr != nullptr ) freeifaddrs(m_addr); } bool find(const cybozu::ip_address& addr) const { for(auto a = m_addr; a != nullptr; a = a->ifa_next) { if( a->ifa_addr == NULL ) continue; switch(a->ifa_addr->sa_family) { case AF_INET: case AF_INET6: if( addr == cybozu::ip_address(a->ifa_addr) ) return true; } } return false; } }; } // anonymous namespace namespace cybozu { void ip_address::parse(const std::string& s) { struct addrinfo hints; struct addrinfo* res = nullptr; std::memset(&hints, 0, sizeof(hints)); hints.ai_family = AF_INET; hints.ai_flags = AI_NUMERICHOST; int e = getaddrinfo(s.c_str(), nullptr, &hints, &res); if( e == 0 ) { af = addr_family::ipv4; std::memcpy(&addr, res->ai_addr, res->ai_addrlen); freeaddrinfo(res); return; } if( e != EAI_ADDRFAMILY && e != EAI_NONAME ) throw std::runtime_error(gai_strerror(e)); hints.ai_family = AF_INET6; e = getaddrinfo(s.c_str(), nullptr, &hints, &res); if( e == 0 ) { af = addr_family::ipv6; std::memcpy(&addr, res->ai_addr, res->ai_addrlen); freeaddrinfo(res); return; } if( e != EAI_ADDRFAMILY && e != EAI_NONAME ) throw std::runtime_error(gai_strerror(e)); throw bad_address("Invalid address: " + s); } ip_address::ip_address(const std::string& s) { parse(s); } ip_address::ip_address(const struct sockaddr* ifa_addr) { if( ifa_addr->sa_family == AF_INET ) { af = addr_family::ipv4; std::memcpy(&addr, ifa_addr, sizeof(struct sockaddr_in)); return; } if( ifa_addr->sa_family == AF_INET6 ) { af = addr_family::ipv6; std::memcpy(&addr, ifa_addr, sizeof(struct sockaddr_in6)); return; } throw bad_address("Invalid address family"); } bool ip_address::operator==(const ip_address& rhs) const { if( af != rhs.af ) return false; if( af == addr_family::none ) return true; if( is_v4() ) { return std::memcmp(v4addr(), rhs.v4addr(), sizeof(struct in_addr)) == 0; } if( v6scope() != rhs.v6scope() ) return false; return std::memcmp(v6addr(), rhs.v6addr(), sizeof(struct in6_addr)) == 0; } std::string ip_address::str() const { char host[101]; if( is_v4() || is_v6() ) { std::size_t slen = is_v4() ? sizeof(struct sockaddr_in) : sizeof(struct sockaddr_in6); int e = getnameinfo((const struct sockaddr*)&addr, (socklen_t)slen, host, sizeof(host), nullptr, 0, NI_NUMERICHOST); if( e != 0 ) throw std::runtime_error(gai_strerror(e)); return std::string(host); } return ""; } bool has_ip_address(const ip_address& addr) { ifaddrs_wrapper ifw; return ifw.find(addr); } ip_address get_peer_ip_address(int sockfd) { union { struct sockaddr sa; struct sockaddr_storage ss; } addr; socklen_t addrlen = sizeof(addr); if( ::getpeername(sockfd, &addr.sa, &addrlen) != 0 ) throw_unix_error(errno, "getpeername"); return ip_address(&addr.sa); } } // namespace cybozu yrmcds-1.1.5/cybozu/ip_address.hpp000066400000000000000000000034531262254645600172010ustar00rootroot00000000000000// Abstract IP address. // (C) 2013 Cybozu. #ifndef CYBOZU_IP_ADDRESS_HPP #define CYBOZU_IP_ADDRESS_HPP #include #include #include namespace cybozu { // Abstract IP address. // // Abstract IP address. // Both IPv4 and IPv6 addresses can be stored transparently. class ip_address { enum class addr_family {none, ipv4, ipv6} af = addr_family::none; union { struct sockaddr_in ipv4_addr; struct sockaddr_in6 ipv6_addr; } addr; public: ip_address() noexcept {} explicit ip_address(const std::string& s); explicit ip_address(const struct sockaddr* ifa_addr); struct bad_address: public std::runtime_error { explicit bad_address(const std::string& s): std::runtime_error(s) {} }; // This may throw if `s` is not a valid IP address. void parse(const std::string& s); bool is_v4() const { return af == addr_family::ipv4; } bool is_v6() const { return af == addr_family::ipv6; } // 32 bit IPv4 address const struct in_addr* v4addr() const { return &(addr.ipv4_addr.sin_addr); } // 128 bit IPv6 address const struct in6_addr* v6addr() const { return &(addr.ipv6_addr.sin6_addr); } // Link identifier for IPv6 link-local address. std::uint32_t v6scope() const { return addr.ipv6_addr.sin6_scope_id; } bool operator==(const ip_address& rhs) const; bool operator!=(const ip_address& rhs) const { return ! (*this == rhs); } std::string str() const; }; // `true` if this machine has `addr`. `false` otherwise. bool has_ip_address(const ip_address& addr); // Get the IP address of the connected socket peer. ip_address get_peer_ip_address(int sockfd); } // namespace cybozu #endif // CYBOZU_IP_ADDRESS_HPP yrmcds-1.1.5/cybozu/logger.cpp000066400000000000000000000027671262254645600163450ustar00rootroot00000000000000// (C) 2013 Cybozu. #include "logger.hpp" #include "util.hpp" #include #include #include #include #include #include #include namespace cybozu { void logstream::add_prefix(const char* level) { char buf[64]; std::time_t t = std::time(nullptr); std::tm ct; // beware that std::gmtime is not thread-safe. std::strftime(buf, sizeof(buf), "%Y-%m-%d %T ", ::gmtime_r(&t, &ct)); *m_os << buf << level << " "; } void logstream::output() { if( m_os.get() == nullptr ) return; *m_os << "\n"; logger::instance().log(m_os->str()); m_os.reset(nullptr); } std::atomic logger::_threshold{ severity::info }; void logger::open_nolock(const std::string& path) { if( m_fd != -1 && m_fd != STDERR_FILENO ) throw std::logic_error(" double open."); m_fd = ::open(path.c_str(), O_WRONLY|O_APPEND|O_CREAT, 0644); if( m_fd == -1 ) { m_fd = STDERR_FILENO; throw_unix_error(errno, ("open(" + path + ")").c_str()); } } void logger::log(const std::string& msg) { lock_guard g(m_lock); std::size_t to_write = msg.size(); const char* ptr = msg.data(); while( to_write != 0 ) { ssize_t n = ::write(m_fd, ptr, to_write); if( n == -1 ) { if( errno == EAGAIN || errno == EINTR ) continue; throw_unix_error(errno, " write"); } ptr += n; to_write -= n; } } } // namespace cybozu yrmcds-1.1.5/cybozu/logger.hpp000066400000000000000000000064271262254645600163470ustar00rootroot00000000000000// Thread-safe logger. // (C) 2013 Cybozu. #ifndef CYBOZU_LOGGER_HPP #define CYBOZU_LOGGER_HPP #include #include #include #include #include #include namespace cybozu { // Severity used for . enum class severity {error, warning, info, debug}; // Output stream to the logger. // // Just like other output streams, you can output a variety of objects // by `<<` operator. The log is written in a line from within the // destructor. class logstream { const bool m_valid; std::unique_ptr m_os; void add_prefix(const char* level); public: logstream(bool valid_, const char* level): m_valid(valid_), m_os(valid_ ? new std::ostringstream() : nullptr) { if( valid_ ) add_prefix(level); } logstream(logstream&& rhs) noexcept: m_valid(rhs.m_valid), m_os(std::move(rhs.m_os)) {} logstream(const logstream&) = delete; logstream& operator=(const logstream&) = delete; ~logstream() { if( m_valid ) output(); } // `true` if this stream is valid for logging. bool valid() const { return m_valid; } // Output the log line. The stream gets invalidated. void output(); template logstream& operator<< (const T& arg) { if( m_valid ) *m_os << arg; return *this; } }; // A thread-safe logger. // // A simple thread-safe logger. By default, this logger outputs // logs to the standard error. class logger { int m_fd; std::string m_path; mutable std::mutex m_lock; typedef std::lock_guard lock_guard; static std::atomic _threshold; logger(): m_fd(STDERR_FILENO), m_path("") {} void open_nolock(const std::string& path); void close_nolock() { if( m_fd == -1 || m_fd == STDERR_FILENO ) return; fsync(m_fd); ::close(m_fd); m_fd = -1; } public: // Return the singleton. // // @return object. static logger& instance() { static logger l; return l; } logger(const logger&) = delete; logger& operator=(const logger&) = delete; // fsync and close the log file. ~logger() { close_nolock(); } static severity threshold() { return _threshold.load(std::memory_order_relaxed); } static void set_threshold(severity new_threshold) { _threshold.store(new_threshold, std::memory_order_relaxed); } void open(const std::string& path) { lock_guard g(m_lock); m_path = path; open_nolock(path); } void reopen() { lock_guard g(m_lock); if( m_fd == STDERR_FILENO ) return; close_nolock(); open_nolock(m_path); } void close() { lock_guard g(m_lock); close_nolock(); } void log(const std::string& msg); static logstream error() { return logstream(threshold() >= severity::error, "ERROR"); } static logstream warning() { return logstream(threshold() >= severity::warning, "WARN"); } static logstream info() { return logstream(threshold() >= severity::info, "INFO"); } static logstream debug() { return logstream(threshold() >= severity::debug, "DEBUG"); } }; } // namespace cybozu #endif // CYBOZU_LOGGER_HPP yrmcds-1.1.5/cybozu/reactor.cpp000066400000000000000000000100011262254645600165010ustar00rootroot00000000000000// (C) 2013 Cybozu. #include "reactor.hpp" #include "util.hpp" #include #include #include namespace { const int EPOLL_SIZE = 128; const int POLLING_TIMEOUT = 100; // milli seconds } namespace cybozu { reactor::reactor(): m_fd( epoll_create1(EPOLL_CLOEXEC) ), m_running(true) { if( m_fd == -1 ) throw_unix_error(errno, "epoll_create1"); m_resources.max_load_factor(1.0); m_resources.reserve(100000); m_readables.reserve(256); m_readables_copy.reserve(256); m_drop_req.reserve(256); m_drop_req_copy.reserve(256); } reactor::~reactor() { ::close(m_fd); } void reactor::add_resource(std::unique_ptr res, int events) { if( res->m_reactor != nullptr ) throw std::logic_error(" already added!"); res->m_reactor = this; const int fd = res->fileno(); struct epoll_event ev; ev.events = events | EPOLLET; ev.data.fd = fd; if( epoll_ctl(m_fd, EPOLL_CTL_ADD, fd, &ev) == -1 ) throw_unix_error(errno, "epoll_ctl"); m_resources.emplace(fd, std::move(res)); } void reactor::modify_events(const resource& res, int events) { const int fd = res.fileno(); struct epoll_event ev; ev.events = events | EPOLLET; ev.data.fd = fd; if( epoll_ctl(m_fd, EPOLL_CTL_MOD, fd, &ev) == -1 ) throw_unix_error(errno, "epoll_ctl"); } void reactor::run(std::function callback, int interval) { namespace crn = std::chrono; const int milli_interval = interval * 1000; auto last = crn::steady_clock::now(); m_running = true; while( m_running ) { poll(); auto now = crn::steady_clock::now(); auto diff = crn::duration_cast(now - last).count(); if( diff > milli_interval ) { callback(*this); last = now; } } } void reactor::remove_resource(int fd) { resource_map::iterator it = m_resources.find(fd); if( it == m_resources.end() ) { dump_stack(); throw std::logic_error("bug in remove_resource"); } m_garbage.emplace_back( std::move(it->second) ); m_resources.erase(it); if( epoll_ctl(m_fd, EPOLL_CTL_DEL, fd, NULL) == -1 ) throw_unix_error(errno, "epoll_ctl(EPOLL_CTL_DEL)"); m_readables.erase(std::remove(m_readables.begin(), m_readables.end(), fd), m_readables.end()); } void reactor::poll() { // process readable resources std::sort(m_readables.begin(), m_readables.end()); std::unique_copy(m_readables.begin(), m_readables.end(), std::back_inserter(m_readables_copy)); m_readables.clear(); for( int fd: m_readables_copy ) { if( ! m_resources[fd]->on_readable() ) remove_resource(fd); } m_readables_copy.clear(); // process drop requests { lock_guard g(m_lock); m_drop_req_copy.swap(m_drop_req); } for( int fd: m_drop_req_copy ) remove_resource(fd); m_drop_req_copy.clear(); struct epoll_event events[EPOLL_SIZE]; int timeout = m_readables.empty() ? POLLING_TIMEOUT : 0; int n = epoll_wait(m_fd, events, EPOLL_SIZE, timeout); if( n == -1 ) { if( errno == EINTR ) return; throw_unix_error(errno, "epoll_wait"); } for( int i = 0; i < n; ++i ) { const struct epoll_event& ev = events[i]; const int fd = ev.data.fd; resource& r = *(m_resources[fd]); if( ev.events & EPOLLERR ) { if( ! r.on_error() ) remove_resource(fd); continue; } if( ev.events & EPOLLHUP ) { if( ! r.on_hangup() ) { remove_resource(fd); continue; } } if( ev.events & EPOLLIN ) { if( ! r.on_readable() ) { remove_resource(fd); continue; } } if( ev.events & EPOLLOUT ) { if( ! r.on_writable() ) remove_resource(fd); } } } } // namespace cybozu yrmcds-1.1.5/cybozu/reactor.hpp000066400000000000000000000202061262254645600165160ustar00rootroot00000000000000// A reactor implementation using epoll(2). // (C) 2013 Cybozu. #ifndef CYBOZU_REACTOR_HPP #define CYBOZU_REACTOR_HPP #include "logger.hpp" #include "spinlock.hpp" #include "util.hpp" #include #include #include #include #include #include #include namespace cybozu { class reactor; // An abstraction of a file descriptor. // // An abstraction of a file descriptor for use with . // The file descriptor should be set non-blocking. // // All member functions except for are for the // reactor thread. Sub classes can add methods for the other threads. class resource { public: // Constructor. // @fd A UNIX file descriptor. explicit resource(int fd): m_fd(fd) {} resource(const resource&) = delete; resource& operator=(const resource&) = delete; // Close the file descriptor. virtual ~resource() { ::close(m_fd); } // Return the UNIX file descriptor for this resource. int fileno() const { return m_fd; } // `true` if this resource is still valid. bool valid() const { lock_guard g(m_lock); return m_valid; } // Invalidate this resource. // // This is for the reactor thread only. // You may call this from within or . bool invalidate() { lock_guard g(m_lock); if( ! m_valid ) return true; invalidate_( std::move(g) ); return false; } // Invalidate this resource then request the reactor to remove this. // // This method invalidates this resource then requests the reactor // thread to remove this resource from the reactor. // // **Do not use this in the reactor thread**. void invalidate_and_close(); // A template method called from within invalidate. // // A template method called from within invalidate. // Subclasses can override this to clean up something. virtual void on_invalidate() {} // Called when the reactor finds this resource is readable. // // This method is called when the reactor finds reading from this // resource will not block. // // `on_readable` is allowed to stop reading from `m_fd` before `EAGAIN` // or `EWOULDBLOCK` for fairness. If you stop reading, make sure // to call `m_reactor->add_readable(*this)` from within `on_readable`. // // If some error happened, execute `return invalidate();`. // // `invalidate()` can be called when `recv` or `read` returns 0, // but it is up to you (or your application's protocol). In general, // the client may still be waiting for the response. // // @return `true` or return value of . virtual bool on_readable() = 0; // Called when the reactor finds this resource is writable. // // This method is called when the reactor finds this resource gets // writable. Unlike , `on_writable` must try to write // all pending data until it encounters `EAGAIN` or `EWOULDBLOCK`. // // If some error happened, execute `return invalidate();`. // // @return `true` or return value of . virtual bool on_writable() = 0; // Called when the reactor finds this resource has hanged up. // // This method is called when the reactor detects unexpected hangup. virtual bool on_hangup() { return invalidate(); } // Called when the reactor finds an error on this resource. // // This method is called when the reactor detects some error. virtual bool on_error() { return invalidate(); } protected: const int m_fd; reactor* m_reactor = nullptr; friend class reactor; private: bool m_valid = true; mutable spinlock m_lock; typedef std::unique_lock lock_guard; void invalidate_(lock_guard g) { m_valid = false; g.unlock(); on_invalidate(); } }; // The reactor. // // This reactor internally uses epoll with edge-triggered mode. class reactor { public: reactor(); ~reactor(); // Events to poll. enum reactor_event { EVENT_IN = EPOLLIN, EVENT_OUT = EPOLLOUT, }; // Add a resource to the reactor. // @events `EVENT_IN`, `EVENT_OUT`, or bitwise OR of them. // // Add a resource to the reactor. The ownership of the resource // will be moved to the reactor. Only the reactor thread can use this. void add_resource(std::unique_ptr res, int events); // Modify epoll events for a resource. // @events `EVENT_IN`, `EVENT_OUT`, or bitwise OR of them. void modify_events(const resource& res, int events); // Run the reactor loop. // @callback Function called at each interval. // @interval Interval between two callbacks in seconds. void run(std::function callback, int interval = 1); // Run the reactor once. void run_once() { poll(); } // Return the number of registered and still valid resources. std::size_t size() const { return m_resources.size(); } // Quit the run loop gracefully. // // Call this to quit the run loop gracefully. // Only the reactor thread can use this. void quit() { m_running = false; } // Invalidate all registered resources. // // Invalidate all registered resources to unblock other threads. void invalidate() { for( auto& t: m_resources ) t.second->invalidate(); } // Add a resource to the readable resource list. // // Only may call this when it stops reading // from a resource before it encounters `EAGAIN` or `EWOULDBLOCK`. void add_readable(const resource& res) { m_readables.push_back(res.fileno()); } // Add a removal request for a resource. // // Resources can be shared with threads other than the reactor thread. // When such a thread successfully invalidates a resource, the thread // need to request the reactor thread to remove the resource by // calling this. void request_removal(const resource& res) { lock_guard g(m_lock); m_drop_req.push_back(res.fileno()); } bool has_garbage() const noexcept { return ! m_garbage.empty(); } // Fix the garbage resources to be destructed at the next . // // Fix the garbage resources to be destructed at the next . // Only the reactor thread can use this. // // @return `false` if there are no garbage resources, `true` otherwise. bool fix_garbage() { if( ! m_garbage_copy.empty() ) throw std::logic_error("No gc() was called after fix_garbage()!"); std::size_t n = m_garbage.size(); if( n == 0 ) return false; logger::debug() << "reactor: collecting " << n << " resources."; m_garbage_copy.swap(m_garbage); return true; } // Destruct garbage resources fixed by . // // Destruct garbage resources fixed by . // Only the reactor thread can use this. void gc() { logger::debug() << "reactor: collected garbage resources."; m_garbage_copy.clear(); } // Remove a registered resource. // This is only for the reactor thread. void remove_resource(const resource& res) { remove_resource(res.fileno()); } private: const int m_fd; bool m_running; typedef std::unordered_map> resource_map; resource_map m_resources; std::vector m_readables; std::vector m_readables_copy; // resource close request queue and its guarding lock. mutable spinlock m_lock; typedef std::lock_guard lock_guard; std::vector m_drop_req; std::vector m_drop_req_copy; // pending destruction lists std::vector> m_garbage; std::vector> m_garbage_copy; void remove_resource(int fd); void poll(); }; inline void resource::invalidate_and_close() { lock_guard g(m_lock); if( ! m_valid ) return; invalidate_( std::move(g) ); m_reactor->request_removal(*this); } } // namespace cybozu #endif // CYBOZU_REACTOR_HPP yrmcds-1.1.5/cybozu/signal.cpp000066400000000000000000000023711262254645600163320ustar00rootroot00000000000000#include "signal.hpp" #include #include #include #include namespace { const char ABORT_MESSAGE[] = "got SIGABRT.\n"; #pragma GCC diagnostic ignored "-Wunused-result" void handle_abort [[noreturn]] (int) { ::write(STDERR_FILENO, ABORT_MESSAGE, sizeof(ABORT_MESSAGE) - 1); cybozu::dump_stack(); std::abort(); } #pragma GCC diagnostic pop } // anonymous namespace namespace cybozu { std::unique_ptr signal_setup(std::initializer_list sigs) { sigset_t mask[1]; sigemptyset(mask); for( int i: sigs ) sigaddset(mask, i); int e = pthread_sigmask(SIG_BLOCK, mask, NULL); if( e != 0 ) throw_unix_error(e, "pthread_sigmask"); // signal disposition is a per-process attribute. struct sigaction act; std::memset(&act, 0, sizeof(act)); act.sa_handler = SIG_IGN; if( sigaction(SIGPIPE, &act, NULL) == -1 ) throw_unix_error(errno, "sigaction"); std::memset(&act, 0, sizeof(act)); act.sa_handler = handle_abort; act.sa_flags = SA_RESETHAND; if( sigaction(SIGABRT, &act, NULL) == -1 ) throw_unix_error(errno, "sigaction"); return std::unique_ptr( new signal_reader(mask) ); } } // namespace cybozu yrmcds-1.1.5/cybozu/signal.hpp000066400000000000000000000053671262254645600163470ustar00rootroot00000000000000/* Signal handling resource using signalfd(2). * * (C) 2013 Cybozu. */ #ifndef CYBOZU_SIGNAL_HPP #define CYBOZU_SIGNAL_HPP #include "reactor.hpp" #include "util.hpp" #include #include #include #include namespace cybozu { // A subclass for [`signalfd`](http://manpages.ubuntu.com/manpages/precise/en/man2/signalfd.2.html). class signal_reader: public resource { public: using callback_t = std::function; // Constructor. // @mask A set of signals to be handled by this resource. // @callback Callback function to handle received signals. // // Construct a signal reading resource. Signals set in `mask` // need to be blocked on all threads. signal_reader(const sigset_t *mask, callback_t callback): resource( signalfd(-1, mask, SFD_NONBLOCK|SFD_CLOEXEC) ), m_callback(callback) { if( m_fd == -1 ) throw_unix_error(errno, "signalfd"); } // Constructor. // @mask A set of signals to be handled by this resource. // // Construct a signal reading resource. Signals set in `mask` // need to be blocked on all threads. This constructor leaves // the callback function empty. Use `set_handler` to set one. explicit signal_reader(const sigset_t *mask): signal_reader(mask, nullptr) {} virtual ~signal_reader() {} void set_handler(const callback_t& callback) { m_callback = callback; } private: callback_t m_callback; virtual bool on_readable() override final { while( true ) { struct signalfd_siginfo si; ssize_t n = read(m_fd, &si, sizeof(si)); if( n == -1 ) { if( errno == EINTR ) continue; if( errno == EAGAIN || errno ==EWOULDBLOCK ) return true; throw_unix_error(errno, "read"); } if( n != sizeof(si) ) throw std::runtime_error(" bug?"); if( m_callback ) m_callback(si, *m_reactor); } } virtual bool on_writable() override final { return true; } }; // Block signals and create for blocked signals. // @sigs List of signals to be blocked. // // This function blocks given signals and creates a // prepared to read blocked signals. In addition, this configures // `SIGPIPE` to be ignored, and `SIGABRT` produces the stack trace. // // This function should be called from the main thread before any // other threads are created. std::unique_ptr signal_setup(std::initializer_list sigs); } // namespace cybozu #endif // CYBOZU_SIGNAL_HPP yrmcds-1.1.5/cybozu/siphash.cpp000066400000000000000000000065731262254645600165240ustar00rootroot00000000000000/* Copyright (c) 2013 Marek Majkowski Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. Original location: https://github.com/majek/csiphash/ Solution inspired by code from: Samuel Neves (supercop/crypto_auth/siphash24/little) djb (supercop/crypto_auth/siphash24/little2) Jean-Philippe Aumasson (https://131002.net/siphash/siphash24.c) */ // Cosmetic changes for C++ on Linux by @ymmt2005 #include "siphash.hpp" #include #include #define ROTATE(x, b) (std::uint64_t)( ((x) << (b)) | ( (x) >> (64 - (b))) ) #define HALF_ROUND(a,b,c,d,s,t) \ a += b; c += d; \ b = ROTATE(b, s) ^ a; \ d = ROTATE(d, t) ^ c; \ a = ROTATE(a, 32); #define DOUBLE_ROUND(v0,v1,v2,v3) \ HALF_ROUND(v0,v1,v2,v3,13,16); \ HALF_ROUND(v2,v1,v0,v3,17,21); \ HALF_ROUND(v0,v1,v2,v3,13,16); \ HALF_ROUND(v2,v1,v0,v3,17,21); using std::uint8_t; using std::uint32_t; using std::uint64_t; namespace { uint64_t static_k0; uint64_t static_k1; } namespace cybozu { void siphash24_seed(const char key[16]) { uint64_t t; std::memcpy(&t, key, sizeof(t)); static_k0 = le64toh(t); std::memcpy(&t, key + 8, sizeof(t)); static_k1 = le64toh(t); } uint64_t siphash24(const void *src, std::size_t src_sz) { uint64_t b = (uint64_t)src_sz << 56; const uint8_t *m = (const uint8_t*)src; uint64_t v0 = static_k0 ^ 0x736f6d6570736575ULL; uint64_t v1 = static_k1 ^ 0x646f72616e646f6dULL; uint64_t v2 = static_k0 ^ 0x6c7967656e657261ULL; uint64_t v3 = static_k1 ^ 0x7465646279746573ULL; while (src_sz >= 8) { uint64_t mi; std::memcpy(&mi, m, sizeof(mi)); mi = le64toh(mi); m += 8; src_sz -= 8; v3 ^= mi; DOUBLE_ROUND(v0,v1,v2,v3); v0 ^= mi; } switch (src_sz) { case 7: b |= uint64_t(m[6]) << 48; case 6: b |= uint64_t(m[5]) << 40; case 5: b |= uint64_t(m[4]) << 32; case 4: b |= uint64_t(m[3]) << 24; case 3: b |= uint64_t(m[2]) << 16; case 2: b |= uint64_t(m[1]) << 8; case 1: b |= uint64_t(m[0]); } v3 ^= b; DOUBLE_ROUND(v0,v1,v2,v3); v0 ^= b; v2 ^= 0xff; DOUBLE_ROUND(v0,v1,v2,v3); DOUBLE_ROUND(v0,v1,v2,v3); return (v0 ^ v1) ^ (v2 ^ v3); } } // namespace cybozu yrmcds-1.1.5/cybozu/siphash.hpp000066400000000000000000000013041262254645600165140ustar00rootroot00000000000000// siphash.hpp // (C) 2013 Cybozu. #ifndef CYBOZU_SIPHASH_HPP #define CYBOZU_SIPHASH_HPP #include #include namespace cybozu { // Seed . // // This function presets a 128bit key for . // Call this once before using . void siphash24_seed(const char key[16]); // Calculate 64bit keyed secure hash using SipHash algorithm. // @src Pointer to a memory region. // @src_sz Region size in bytes. // // This function calculates a 64bit hash value using // [SipHash](https://131002.net/siphash/) algorithm. // // @return 64bit hash value. std::uint64_t siphash24(const void *src, std::size_t src_sz); } // namespace cybozu #endif // CYBOZU_SIPHASH_HPP yrmcds-1.1.5/cybozu/spinlock.hpp000066400000000000000000000013631262254645600167040ustar00rootroot00000000000000// Spinlock. // (C) 2013 Cybozu. #ifndef CYBOZU_SPINLOCK_HPP #define CYBOZU_SPINLOCK_HPP #include #if defined(__i386__) || defined(__x86_64__) # include #endif namespace cybozu { // A simple spinlock. class spinlock { public: spinlock(): m_flag(ATOMIC_FLAG_INIT) {} void lock() { while( m_flag.test_and_set(std::memory_order_acquire) ) pause(); } void unlock() { m_flag.clear(std::memory_order_release); } private: std::atomic_flag m_flag; void pause() { #if defined(__i386__) || defined(__x86_64__) # if defined(__SSE__) _mm_pause(); # else __asm__ __volatile__ ("rep; nop"); # endif #endif } }; } // namespace cybozu #endif // CYBOZU_SPINLOCK_HPP yrmcds-1.1.5/cybozu/tcp.cpp000066400000000000000000000363171262254645600156520ustar00rootroot00000000000000// (C) 2013 Cybozu. #include "tcp.hpp" #include "logger.hpp" #ifdef USE_TCMALLOC # ifdef TCMALLOC_IN_GOOGLE # include # else # include # endif # define MALLOC tc_malloc # define FREE tc_free #else # include # define MALLOC std::malloc # define FREE std::free #endif #include #include #include #include #include #include #include namespace { const unsigned int MAX_BUFCNT = 100; const int KEEPALIVE_IDLE = 300; // 5 min before keep alive probe const int KEEPALIVE_INTERVAL = 5; // 5 seconds between keep alive probes } // anonymous namespace namespace cybozu { int tcp_connect(const char* node, std::uint16_t port, unsigned int timeout) { std::string s_port = std::to_string(port); struct addrinfo hint, *res; std::memset(&hint, 0, sizeof(hint)); hint.ai_family = AF_INET; // prefer IPv4 hint.ai_socktype = SOCK_STREAM; hint.ai_flags = AI_NUMERICSERV|AI_ADDRCONFIG; int e = getaddrinfo(node, s_port.c_str(), &hint, &res); if( e == EAI_FAMILY || e == EAI_ADDRFAMILY || e == EAI_NODATA || e == EAI_NONAME ) { hint.ai_family = AF_INET6; // intentionally drop AI_ADDRCONFIG to support IPv6 link-local address. // see https://github.com/cybozu/yrmcds/issues/40 hint.ai_flags = AI_NUMERICSERV|AI_V4MAPPED; e = getaddrinfo(node, s_port.c_str(), &hint, &res); } if( e == EAI_SYSTEM ) { throw_unix_error(errno, "getaddrinfo"); } else if( e != 0 ) { throw std::runtime_error(std::string("getaddrinfo: ") + gai_strerror(e)); } int s = ::socket(res->ai_family, res->ai_socktype | SOCK_NONBLOCK | SOCK_CLOEXEC, res->ai_protocol); if( s == -1 ) { freeaddrinfo(res); throw_unix_error(errno, "socket"); } e = ::connect(s, res->ai_addr, res->ai_addrlen); freeaddrinfo(res); if( e == 0 ) return s; if( errno != EINPROGRESS ) { ::close(s); throw_unix_error(errno, "connect"); } struct pollfd fds; fds.fd = s; fds.events = POLLOUT; int poll_timeout = timeout ? timeout*1000 : -1; int n = ::poll(&fds, 1, poll_timeout); if( n == 0 ) { // timeout ::close(s); return -1; } if( n == -1 ) { ::close(s); throw_unix_error(errno, "poll"); } if( fds.revents & (POLLERR|POLLHUP|POLLNVAL) ) { ::close(s); return -1; } socklen_t l = sizeof(e); if( getsockopt(s, SOL_SOCKET, SO_ERROR, &e, &l) == -1 ) { ::close(s); throw_unix_error(errno, "getsockopt(SO_ERROR)"); } if( e != 0 ) { ::close(s); return -1; } return s; } const std::size_t tcp_socket::SENDBUF_SIZE; tcp_socket::tcp_socket(int fd, unsigned int bufcnt): resource(fd) { if( bufcnt > MAX_BUFCNT ) throw std::logic_error("tcp_socket: Too many buffers"); m_free_buffers.reserve(bufcnt); m_pending.reserve(bufcnt); int v = 1; if( setsockopt(fd, SOL_SOCKET, SO_KEEPALIVE, &v, sizeof(v)) == -1 ) throw_unix_error(errno, "setsockopt(SO_KEEPALIVE)"); v = 1; if( setsockopt(fd, IPPROTO_TCP, TCP_CORK, &v, sizeof(v)) == -1 ) throw_unix_error(errno, "setsockopt(TCP_CORK)"); v = KEEPALIVE_IDLE; if( setsockopt(fd, IPPROTO_TCP, TCP_KEEPIDLE, &v, sizeof(v)) == -1 ) throw_unix_error(errno, "setsockopt(TCP_KEEPIDLE)"); v = KEEPALIVE_INTERVAL; if( setsockopt(fd, IPPROTO_TCP, TCP_KEEPINTVL, &v, sizeof(v)) == -1 ) throw_unix_error(errno, "setsockopt(TCP_KEEPINTVL)"); for( unsigned int i = 0; i < bufcnt; ++i ) { char* p = (char*)MALLOC(SENDBUF_SIZE); if( p == NULL ) { for( char* p2: m_free_buffers ) FREE(p2); m_free_buffers.clear(); throw std::runtime_error("tcp_socket: failed to allocate buffers"); } // reserved, hence no throw m_free_buffers.push_back(p); } } void tcp_socket::free_buffers() { lock_guard g(m_lock); for( auto& t: m_pending ) { char* p; std::tie(p, std::ignore, std::ignore) = t; FREE(p); } m_pending.clear(); for( char* p: m_free_buffers ) { FREE(p); } m_free_buffers.clear(); m_tmpbuf.clear(); m_tmpbuf.shrink_to_fit(); m_shutdown = true; } bool tcp_socket::_send(const char* p, std::size_t len, lock_guard& g) { while( ! can_send(len) ) { on_buffer_full(); m_cond_write.wait(g); } if( m_shutdown ) return false; if( m_pending.empty() ) { while( len > 0 ) { ssize_t n = ::send(m_fd, p, len, 0); if( n == -1 ) { if( errno == EAGAIN || errno == EWOULDBLOCK ) break; if( errno == EINTR ) continue; auto ecnd = std::system_category().default_error_condition(errno); if( ecnd.value() != EPIPE ) logger::error() << ": (" << ecnd.value() << ") " << ecnd.message(); g.unlock(); invalidate_and_close(); return false; } p += n; len -= n; } if( len == 0 ) return true; } // put data in the pending request queue. if( capacity() < len ) { // here, m_pending.empty() and m_tmpbuf.empty() holds true. logger::debug() << " buffering " << len << " bytes data."; m_tmpbuf.resize(len); std::memcpy(m_tmpbuf.data(), p, len); return true; } if( ! m_pending.empty() ) { auto& t = m_pending.back(); char* t_p; std::size_t t_len; std::tie(t_p, t_len, std::ignore) = t; std::size_t room = SENDBUF_SIZE - t_len; if( room > 0 ) { std::size_t to_write = std::min(room, len); std::memcpy(t_p + t_len, p, to_write); p += to_write; len -= to_write; std::get<1>(t) = t_len + to_write; if( len == 0 ) return true; } } while( len > 0 ) { char* t_p = m_free_buffers.back(); m_free_buffers.pop_back(); std::size_t to_write = std::min(len, SENDBUF_SIZE); std::memcpy(t_p, p, to_write); p += to_write; len -= to_write; m_pending.emplace_back(t_p, to_write, 0); } return true; } bool tcp_socket::_sendv(const iovec* iov, const int iovcnt, lock_guard& g) { std::size_t total = 0; for( int i = 0; i < iovcnt; ++i ) { total += iov[i].len; } while( ! can_send(total) ) { on_buffer_full(); m_cond_write.wait(g); } if( m_shutdown ) return false; ::iovec v[MAX_IOVCNT]; int v_size = 0; for( int i = 0; i < iovcnt; ++i ) { if( iov[i].len == 0 ) continue; v[v_size].iov_base = const_cast(iov[i].p); v[v_size].iov_len = iov[i].len; ++v_size; } int ind = 0; if( m_pending.empty() ) { while( ind < v_size ) { ssize_t n = ::writev(m_fd, &(v[ind]), v_size - ind); if( n == -1 ) { if( errno == EAGAIN || errno == EWOULDBLOCK ) break; if( errno == EINTR ) continue; auto ecnd = std::system_category().default_error_condition(errno); if( ecnd.value() != EPIPE ) logger::error() << ": (" << ecnd.value() << ") " << ecnd.message(); g.unlock(); invalidate_and_close(); return false; } while( n > 0 ) { if( static_cast(n) < v[ind].iov_len ) { v[ind].iov_base = ((char*)v[ind].iov_base) + n; v[ind].iov_len = v[ind].iov_len - n; break; } n -= v[ind].iov_len; ++ind; } } if( ind == v_size ) return true; } // recalculate total length total = 0; for( int i = ind; i < v_size; ++i ) { total += v[i].iov_len; } // put data in the pending request queue. if( capacity() < total ) { // here, m_pending.empty() and m_tmpbuf.empty() holds true. logger::debug() << " buffering " << total << " bytes data."; m_tmpbuf.resize(total); char* p = m_tmpbuf.data(); for( int i = ind; i < v_size; ++i ) { std::memcpy(p, v[i].iov_base, v[i].iov_len); p += v[i].iov_len; } return true; } while( ind < v_size ) { char* t_p; std::size_t t_len; if(m_pending.empty() || std::get<1>(m_pending.back()) == SENDBUF_SIZE) { t_p = m_free_buffers.back(); t_len = 0; m_free_buffers.pop_back(); m_pending.emplace_back(t_p, t_len, 0); } else { std::tie(t_p, t_len, std::ignore) = m_pending.back(); } std::size_t room = SENDBUF_SIZE - t_len; while( room > 0 ) { std::size_t to_write = std::min(room, v[ind].iov_len); std::memcpy(t_p + t_len, v[ind].iov_base, to_write); room -= to_write; t_len += to_write; std::get<1>(m_pending.back()) = t_len; if( to_write == v[ind].iov_len ) { ++ind; if( ind == v_size ) break; continue; } v[ind].iov_base = ((char*)v[ind].iov_base) + to_write; v[ind].iov_len -= to_write; } } return true; } bool tcp_socket::write_pending_data() { lock_guard g(m_lock); while( ! m_tmpbuf.empty() ) { ssize_t n = ::send(m_fd, m_tmpbuf.data(), m_tmpbuf.size(), 0); if( n == -1 ) { if( errno == EINTR ) continue; if( errno == EAGAIN || errno == EWOULDBLOCK ) return true; auto ecnd = std::system_category().default_error_condition(errno); if( ecnd.value() != EPIPE ) logger::error() << ": (" << ecnd.value() << ") " << ecnd.message(); return false; } m_tmpbuf.erase(m_tmpbuf.begin(), m_tmpbuf.begin() + n); } m_tmpbuf.shrink_to_fit(); while( ! m_pending.empty() ) { auto& t = m_pending.front(); char* p; std::size_t len; std::size_t sent; std::tie(p, len, sent) = t; while( len != sent ) { ssize_t n = ::send(m_fd, p+sent, len-sent, 0); if( n == -1 ) { if( errno == EINTR ) continue; if( errno == EAGAIN || errno == EWOULDBLOCK ) break; auto ecnd = std::system_category().default_error_condition(errno); logger::error() << ": (" << ecnd.value() << ") " << ecnd.message(); return false; } sent += n; } if( len == sent ) { m_pending.erase(m_pending.begin()); m_free_buffers.push_back(p); } else { std::get<2>(t) = sent; g.unlock(); // notify other threads of the new free space m_cond_write.notify_all(); return true; } } // all data have been sent. _flush(); if( ! m_shutdown ) { g.unlock(); m_cond_write.notify_all(); return true; } // invalidate and close this socket when m_shutdown==true. return false; } int setup_server_socket(const char* bind_addr, std::uint16_t port) { struct addrinfo hint, *res; std::string s_port = std::to_string(port); std::memset(&hint, 0, sizeof(hint)); hint.ai_family = AF_INET6; // can accept IPv4 address hint.ai_socktype = SOCK_STREAM; hint.ai_flags = AI_PASSIVE | AI_NUMERICSERV; int e = getaddrinfo(bind_addr, s_port.c_str(), &hint, &res); if( e == EAI_FAMILY || e == EAI_ADDRFAMILY || e == EAI_NODATA ) { IPV4: logger::info() << "Binding to IPv6 fails, trying IPv4..."; hint.ai_family = AF_INET; e = getaddrinfo(bind_addr, s_port.c_str(), &hint, &res); } if( e == EAI_SYSTEM ) { throw_unix_error(errno, "getaddrinfo"); } else if( e != 0 ) { throw std::runtime_error(std::string("getaddrinfo: ") + gai_strerror(e)); } int s = ::socket(res->ai_family, res->ai_socktype | SOCK_NONBLOCK | SOCK_CLOEXEC, res->ai_protocol); if( s == -1 ) { freeaddrinfo(res); // getaddrinfo may return IPv6 though the kernel disables IPv6. // The following is the workaround for it. if( hint.ai_family == AF_INET6 && errno == EAFNOSUPPORT ) goto IPV4; throw_unix_error(errno, "socket"); } // intentionally ignore errors int ok = 1; setsockopt(s, SOL_SOCKET, SO_REUSEADDR, &ok, sizeof(ok)); if( bind(s, res->ai_addr, res->ai_addrlen) == -1 ) { ::close(s); freeaddrinfo(res); throw_unix_error(errno, "bind"); } freeaddrinfo(res); if( listen(s, 128) == -1 ) { ::close(s); throw_unix_error(errno, "listen"); } return s; } bool tcp_server_socket::on_readable() { while( true ) { union { struct sockaddr sa; struct sockaddr_storage ss; } addr; socklen_t addrlen = sizeof(addr); #ifdef _GNU_SOURCE int s = ::accept4(m_fd, &(addr.sa), &addrlen, SOCK_NONBLOCK|SOCK_CLOEXEC); #else int s = ::accept(m_fd, &(addr.sa), &addrlen); if( s != -1 ) { int fl = fcntl(s, F_GETFL, 0); if( fl == -1 ) fl = 0; if( fcntl(s, F_SETFL, fl | O_NONBLOCK) == -1 ) { ::close(s); throw_unix_error(errno, "fcntl(F_SETFL)"); } fl = fcntl(s, F_GETFD, 0); if( fl == -1 ) fl = 0; if( fcntl(s, F_SETFD, fl | FD_CLOEXEC) == -1 ) { ::close(s); throw_unix_error(errno, "fcntl(F_SETFD)"); } } #endif if( s == -1 ) { if( errno == EINTR || errno == ECONNABORTED ) continue; if( errno == EMFILE || errno == ENFILE ) { logger::error() << "accept: Too many open files."; continue; } if( errno == EAGAIN || errno == EWOULDBLOCK ) break; throw_unix_error(errno, "accept"); } try { std::unique_ptr t = m_wrapper(s, ip_address(&(addr.sa))); if( t.get() == nullptr ) { ::close(s); } else { m_reactor->add_resource( std::move(t), reactor::EVENT_IN|reactor::EVENT_OUT ); } } catch( ... ) { ::close(s); throw; } } return true; } } // namespace cybozu yrmcds-1.1.5/cybozu/tcp.hpp000066400000000000000000000231151262254645600156470ustar00rootroot00000000000000// Asynchronous socket abstraction. // (C) 2013 Cybozu. #ifndef CYBOZU_TCP_HPP #define CYBOZU_TCP_HPP #include "ip_address.hpp" #include "reactor.hpp" #include "util.hpp" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include namespace cybozu { // Create a TCP socket and connect to `node:port`. // @node The node name or IP address to connect. // @port The TCP port number to connect. // @timeout Seconds before timeout. // // This creates a TCP socket and connect to a remote node. // The returned socket is set non-blocking. If `timeout` is 0, // this will wait until the kernel gives up. If `node` is `NULL`, // this will try to connect to the local loopback interface. // // This function may return `-1` when it cannot establish a connection // within `timeout` seconds or when the peer node denies connection. // For other (serious) errors, exceptions will be thrown. // // @return A valid UNIX file descriptor, or `-1` for non-fatal errors. int tcp_connect(const char* node, std::uint16_t port, unsigned int timeout=10); // A subclass for connected TCP sockets. // // This is an abstract base class implementing // and provides , , , and member // functions for connected TCP sockets. // // Derived classes still need to implement . class tcp_socket: public resource { static const std::size_t SENDBUF_SIZE = 1 << 20; public: // Construct an already connected socket. // @fd A file descriptor of a connected socket. // @bufcnt The number of 1 MiB buffers for pending send data. // // Construct a socket resource with a connected socket file descriptor. // The socket should already be set non-blocking. explicit tcp_socket(int fd, unsigned int bufcnt = 0); virtual ~tcp_socket() { free_buffers(); } // The maximum size of array for . static const int MAX_IOVCNT = 20; // struct for and . struct iovec { const char* p; std::size_t len; }; // Atomically send data. // @p Data to be sent. // @len Length of data starting from `p`. // @flush If `true`, the kernel send buffer will be flushed. // // This function sends a chunk of data atomically. The reactor // thread should not call this, or it may be blocked forever. // // @return `true` if this socket is valid, `false` otherwise. bool send(const char* p, std::size_t len, bool flush=false) { lock_guard g(m_lock); if( ! _send(p, len, g) ) return false; if( flush && empty() ) _flush(); return true; } // Atomically send multiple data. // @iov Array of . // @iovcnt Number of elements in `iov`. // @flush If `true`, the kernel send buffer will be flushed. // // This function sends a chunk of data atomically. The reactor // thread should not call this, or it may be blocked forever. // // @return `true` if this socket is valid, `false` otherwise. bool sendv(const iovec* iov, int iovcnt, bool flush=false) { if( iovcnt >= MAX_IOVCNT ) throw std::logic_error(" too many iovec."); lock_guard g(m_lock); if( ! _sendv(iov, iovcnt, g) ) return false; if( flush && empty() ) _flush(); return true; } // Atomically send data, then close the socket. // @p Data to be sent. // @len Length of data starting from `p`. // // This function sends a chunk of data atomically. The socket // will be closed after data are sent. The reactor thread should // not call this, or it may be blocked forever. // // @return `true` if this socket is valid, `false` otherwise. bool send_close(const char* p, std::size_t len) { lock_guard g(m_lock); if( ! _send(p, len, g) ) return false; m_shutdown = true; if( empty() ) { _flush(); g.unlock(); invalidate_and_close(); return true; } g.unlock(); m_cond_write.notify_all(); return true; } // Atomically send multiple data, then close the socket. // @iov Array of . // @iovcnt Number of elements in `iov`. // // This function sends a chunk of data atomically. The socket // will be closed after data are sent. The reactor thread should // not call this, or it may be blocked forever. // // @return `true` if this socket is valid, `false` otherwise. bool sendv_close(const iovec* iov, int iovcnt) { if( iovcnt >= MAX_IOVCNT ) throw std::logic_error(" too many iov."); lock_guard g(m_lock); if( ! _sendv(iov, iovcnt, g) ) return false; m_shutdown = true; if( empty() ) { _flush(); g.unlock(); invalidate_and_close(); return true; } g.unlock(); m_cond_write.notify_all(); return true; } protected: // Write out pending data. // // This method tries to send pending data as much as possible. // // @return `false` if some error happened, `true` otherwise. bool write_pending_data(); // Just call . // // The default implementation just invoke . // You may override this to dispatch the job to another thread. virtual bool on_writable() override { if( write_pending_data() ) return true; return invalidate(); } virtual void on_invalidate() override { ::shutdown(m_fd, SHUT_RDWR); free_buffers(); m_cond_write.notify_all(); } // This method will be called everytime when , , // , or is blocked because of internal buffer full. // // Subclasses can override this to handle the buffer full event. virtual void on_buffer_full() {} private: std::vector m_free_buffers; // tuple of std::vector> m_pending; std::vector m_tmpbuf; bool m_shutdown = false; typedef std::unique_lock lock_guard; mutable std::mutex m_lock; mutable std::condition_variable m_cond_write; std::size_t capacity() const { std::size_t c = m_free_buffers.size() * SENDBUF_SIZE; if( m_pending.empty() ) return c; return c + SENDBUF_SIZE - std::get<1>(m_pending.back()); } bool can_send(std::size_t len) const { if( m_shutdown ) return true; // in fact, fail if( ! m_tmpbuf.empty() ) return false; if( m_pending.empty() ) return true; return capacity() >= len; } bool _send(const char* p, std::size_t len, lock_guard& g); bool _sendv(const iovec* iov, const int iovcnt, lock_guard& g); bool empty() const { return m_pending.empty() && m_tmpbuf.empty(); } void _flush() { // with TCP_CORK, setting TCP_NODELAY effectively flushes // the kernel send buffer. int v = 1; if( setsockopt(m_fd, IPPROTO_TCP, TCP_NODELAY, &v, sizeof(v)) == -1 ) throw_unix_error(errno, "setsockopt(TCP_NODELAY)"); } void free_buffers(); }; // A helper function to create a server socket. // @bind_addr A numeric IP address to be bound, or `NULL`. // @port TCP port number to be bound. // // This is a helper function for template. // The socket is set non-blocking before return. // // @return A UNIX file descriptor of a listening socket. int setup_server_socket(const char* bind_addr, std::uint16_t port); // A subclass to accept new TCP connections. class tcp_server_socket: public resource { public: using wrapper = std::function (int s, const ip_address&)>; // Construct a server socket. // @bind_addr A numeric IP address to be bound, or `NULL`. // @port TCP port number to be bound. // @on_accept Callback function. // // This creates a socket and bind it to the given address and port. // If `bind_addr` is `NULL`, the socket will listen on any address. // Both IPv4 and IPv6 addresses are supported. // // For each new connection, `w` is called to determine if the new // connection need to be closed immediately or to be added to the // reactor. If `w` returns an empty , the new // connection is closed immediately. Otherwise, the new connection // is added to the reactor. tcp_server_socket(const char* bind_addr, std::uint16_t port, wrapper w): resource( setup_server_socket(bind_addr, port) ), m_wrapper(w) {} virtual ~tcp_server_socket() {} private: wrapper m_wrapper; virtual bool on_readable() override final; virtual bool on_writable() override final { return true; } }; // Utility function to create a of . inline std::unique_ptr make_server_socket( const char* bind_addr, std::uint16_t port, tcp_server_socket::wrapper w) { return std::unique_ptr( new tcp_server_socket(bind_addr, port, w) ); } } // namespace cybozu #endif // CYBOZU_SOCKET_HPP yrmcds-1.1.5/cybozu/test.hpp000066400000000000000000000236101262254645600160400ustar00rootroot00000000000000// Unit test class. // (C) 2008 Cybozu Labs, Inc., all rights reserved. // (C) 2013 Cybozu. All rights reserved. #ifndef CYBOZU_TEST_HPP #define CYBOZU_TEST_HPP #include #include #include #include #include #include #include #include #include #include namespace cybozu { namespace test { class AutoRun { typedef void (*Func)(); typedef std::list > UnitTestList; public: AutoRun() : init_(0) , term_(0) , okCount_(0) , ngCount_(0) , exceptionCount_(0) { } void setup(Func init, Func term) { init_ = init; term_ = term; } void append(const char *name, Func func) { list_.push_back(std::make_pair(name, func)); } void set(bool isOK) { if (isOK) { okCount_++; } else { ngCount_++; } } std::string getBaseName(const std::string& name) const { #ifdef _WIN32 const char sep = '\\'; #else const char sep = '/'; #endif size_t pos = name.find_last_of(sep); std::string ret = name.substr(pos + 1); pos = ret.find('.'); return ret.substr(0, pos); } using ret_t = std::tuple; ret_t run(const char* arg0) { using namespace std::chrono; std::string msg; steady_clock::duration elapsed; try { if (init_) init_(); for (UnitTestList::const_iterator i = list_.begin(), ie = list_.end(); i != ie; ++i) { std::cout << "ctest:module=" << i->first << std::endl; try { auto t1 = steady_clock::now(); (i->second)(); auto t2 = steady_clock::now(); elapsed += t2 - t1; } catch (std::exception& e) { exceptionCount_++; std::cout << "ctest: " << i->first << " is stopped by std::exception " << e.what() << std::endl; } catch (...) { exceptionCount_++; std::cout << "ctest: " << i->first << " is stopped by an exception" << std::endl; } } if (term_) term_(); } catch (...) { msg = "ctest:err: catch unexpected exception"; } fflush(stdout); std::uint64_t elapsed_us = duration_cast(elapsed).count(); if (msg.empty()) { std::cout << "ctest:name=" << getBaseName(arg0) << ", module=" << list_.size() << ", total=" << (okCount_ + ngCount_ + exceptionCount_) << ", ok=" << okCount_ << ", ng=" << ngCount_ << ", exception=" << exceptionCount_ << std::endl; return (ngCount_>0) ? ret_t(1, elapsed_us) : ret_t(0, elapsed_us); } else { std::cout << msg << std::endl; return ret_t(1, elapsed_us); } } static inline AutoRun& getInstance() { static AutoRun instance; return instance; } private: Func init_; Func term_; int okCount_; int ngCount_; int exceptionCount_; std::uint64_t elapsed_; UnitTestList list_; }; static AutoRun& autoRun = AutoRun::getInstance(); inline void test(bool ret, const std::string& msg, const std::string& param, const char *file, int line) { autoRun.set(ret); if (!ret) { std::cout << "ctest:" << file << ":" << line << " " << msg << param << ";" << std::endl; } } template bool isEqual(const T& lhs, const U& rhs) { return lhs == rhs; } inline bool isEqual(const char *lhs, const char *rhs) { return strcmp(lhs, rhs) == 0; } inline bool isEqual(char *lhs, const char *rhs) { return strcmp(lhs, rhs) == 0; } inline bool isEqual(const char *lhs, char *rhs) { return strcmp(lhs, rhs) == 0; } inline bool isEqual(char *lhs, char *rhs) { return strcmp(lhs, rhs) == 0; } // avoid to compare float directly inline bool isEqual(float lhs, float rhs) { union fi { float f; uint32_t i; } lfi, rfi; lfi.f = lhs; rfi.f = rhs; return lfi.i == rfi.i; } // avoid to compare double directly inline bool isEqual(double lhs, double rhs) { union di { double d; uint64_t i; } ldi, rdi; ldi.d = lhs; rdi.d = rhs; return ldi.i == rdi.i; } } } // cybozu::test #define TEST_MAIN(arg_parser) \ int main(int argc, char** argv) { \ if( ! arg_parser(argc, argv) ) \ return 0; \ int r; \ std::uint64_t elapsed; \ std::tie(r, elapsed) = cybozu::test::autoRun.run(argv[0]); \ std::cerr << "ctest:elapsed=" << elapsed << "us" << std::endl; \ return r; \ } #ifndef TEST_DISABLE_AUTO_RUN bool null_parser(int argc, char** argv) { return true; } TEST_MAIN(null_parser); #endif /** alert if !x @param x [in] */ #define cybozu_assert(...) cybozu_assert_((__VA_ARGS__)) #define cybozu_assert_(x) cybozu::test::test(!!(x), "cybozu_assert", #x, __FILE__, __LINE__) /** alert if x != y @param x [in] @param y [in] */ #define cybozu_equal(x, y) { \ bool eq = cybozu::test::isEqual(x, y); \ cybozu::test::test(eq, "cybozu_equal", "(" #x ", " #y ")", __FILE__, __LINE__); \ if (!eq) { \ std::cout << "ctest: lhs=" << (x) << std::endl; \ std::cout << "ctest: rhs=" << (y) << std::endl; \ } \ } /** alert if fabs(x, y) >= eps @param x [in] @param y [in] */ #define cybozu_near(x, y, eps) { \ bool isNear = fabs((x) - (y)) < eps; \ cybozu::test::test(isNear, "cybozu_near", "(" #x ", " #y ")", __FILE__, __LINE__); \ if (!isNear) { \ std::cout << "ctest: lhs=" << (x) << std::endl; \ std::cout << "ctest: rhs=" << (y) << std::endl; \ } \ } #define cybozu_equal_pointer(x, y) { \ bool eq = x == y; \ cybozu::test::test(eq, "cybozu_equal_pointer", "(" #x ", " #y ")", __FILE__, __LINE__); \ if (!eq) { \ std::cout << "ctest: lhs=" << (const void*)(x) << std::endl; \ std::cout << "ctest: rhs=" << (const void*)(y) << std::endl; \ } \ } /** always alert @param msg [in] */ #define cybozu_fail(msg) cybozu::test::test(false, "cybozu_fail", "(" msg ")", __FILE__, __LINE__) /** verify message in exception */ #define cybozu_test_exception_message(statement, Exception, msg) \ { \ int ret = 0; \ std::string errMsg; \ try { \ statement; \ ret = 1; \ } catch (const Exception& e) { \ errMsg = e.what(); \ if (errMsg.find(msg) == std::string::npos) { \ ret = 2; \ } \ } catch (...) { \ ret = 3; \ } \ if (ret) { \ cybozu::test::test(false, "cybozu_test_exception_message", "(" #statement ", " #Exception ", " #msg ")", __FILE__, __LINE__); \ if (ret == 1) { \ std::cout << "ctest: no exception" << std::endl; \ } else if (ret == 2) { \ std::cout << "ctest: bad exception msg:" << errMsg << std::endl; \ } else { \ std::cout << "ctest: unexpected exception" << std::endl; \ } \ } else { \ cybozu::test::autoRun.set(true); \ } \ } #define cybozu_test_exception(statement, Exception) \ { \ int ret = 0; \ try { \ statement; \ ret = 1; \ } catch (const Exception&) { \ } catch (...) { \ ret = 2; \ } \ if (ret) { \ cybozu::test::test(false, "cybozu_test_exception", "(" #statement ", " #Exception ")", __FILE__, __LINE__); \ if (ret == 1) { \ std::cout << "ctest: no exception" << std::endl; \ } else { \ std::cout << "ctest: unexpected exception" << std::endl; \ } \ } else { \ cybozu::test::autoRun.set(true); \ } \ } /** verify statement does not throw */ #define cybozu_test_no_exception(statement) \ try { \ statement; \ cybozu::test::autoRun.set(true); \ } catch (...) { \ cybozu::test::test(false, "cybozu_test_no_exception", "(" #statement ")", __FILE__, __LINE__); \ } /** append auto unit test @param name [in] module name */ #define AUTOTEST(name) \ void cybozu_test_ ## name(); \ struct cybozu_test_local_ ## name { \ cybozu_test_local_ ## name() \ { \ cybozu::test::autoRun.append(#name, cybozu_test_ ## name); \ } \ } cybozu_test_local_instance_ ## name; \ void cybozu_test_ ## name() /** append auto unit test with fixture @param name [in] module name */ #define AUTOTEST_WITH_FIXTURE(name, Fixture) \ void cybozu_test_ ## name(); \ void cybozu_test_real_ ## name() \ { \ Fixture f; \ cybozu_test_ ## name(); \ } \ struct cybozu_test_local_ ## name { \ cybozu_test_local_ ## name() \ { \ cybozu::test::autoRun.append(#name, cybozu_test_real_ ## name); \ } \ } cybozu_test_local_instance_ ## name; \ void cybozu_test_ ## name() /** setup fixture @param Fixture [in] class name of fixture @note cstr of Fixture is called before test and dstr of Fixture is called after test */ #define SETUP_FIXTURE(Fixture) \ Fixture *cybozu_test_local_fixture; \ void cybozu_test_local_init() \ { \ cybozu_test_local_fixture = new Fixture(); \ } \ void cybozu_test_local_term() \ { \ delete cybozu_test_local_fixture; \ } \ struct cybozu_test_local_fixture_setup_ { \ cybozu_test_local_fixture_setup_() \ { \ cybozu::test::autoRun.setup(cybozu_test_local_init, cybozu_test_local_term); \ } \ } cybozu_test_local_fixture_setup_instance_; #endif // CYBOZU_TEST_HPP yrmcds-1.1.5/cybozu/thread.hpp000066400000000000000000000036151262254645600163330ustar00rootroot00000000000000// CRTP thread wrapper. // (C) 2013 Cybozu. #ifndef CYBOZU_THREAD_HPP #define CYBOZU_THREAD_HPP #include "logger.hpp" #include "util.hpp" #include #include #include #include namespace cybozu { // A CRTP template wrapping . // // A curiously-recurring-template-pattern (CRTP) template to wrap // with associated data structures. Derived classes // need to implement `void run()`. // // and will be caught and logged. // If `exit_on_exception` is not 0, then `exit(exit_on_exception)` is // called to terminate the program. template class thread_base { public: void start() { if( m_thread.joinable() ) throw std::logic_error("Thread already started!"); m_done.store(false, std::memory_order_relaxed); m_thread = std::thread([this]() { run_in_try(); }); } void run_in_try() { try { static_cast(this)->run(); m_done.store(true, std::memory_order_release); } catch( const std::system_error& e ) { demangler t(typeid(e).name()); logger::error() << "[" << t.name() << "] " << "(" << e.code() << ") " << e.what(); if( exit_on_exception != 0 ) ::exit(exit_on_exception); } catch( const std::exception& e ) { demangler t(typeid(e).name()); logger::error() << "[" << t.name() << "] " << e.what(); if( exit_on_exception != 0 ) ::exit(exit_on_exception); } } bool done() const noexcept { return m_done.load(std::memory_order_acquire); } protected: std::atomic m_done; std::thread m_thread; }; } // namespace cybozu #endif // CYBOZU_THREAD_HPP yrmcds-1.1.5/cybozu/util.cpp000066400000000000000000000021271262254645600160310ustar00rootroot00000000000000// (C) 2013 Cybozu. #include "util.hpp" #include #include #include #include #include #include #include #include #include namespace { void clear_memory_(void* s, std::size_t n) { volatile unsigned char *p = (unsigned char*)s; while( n-- ) *p++ = 0; } } // anonymous namespace namespace cybozu { demangler::demangler(const char* name) { int status; char* demangled = abi::__cxa_demangle(name, 0, 0, &status); if( status == 0 ) { m_name = demangled; std::free(demangled); } else { m_name = name; } } #pragma GCC diagnostic ignored "-Wunused-result" void dump_stack() noexcept { char buf[32]; std::time_t t = std::time(nullptr); std::size_t len = std::strlen(ctime_r(&t, buf)); ::write(STDERR_FILENO, buf, len); void* bt[100]; int n = backtrace(bt, 100); backtrace_symbols_fd(bt, n, STDERR_FILENO); } #pragma GCC diagnostic pop void (* const volatile clear_memory)(void* s, std::size_t n) = clear_memory_; } // namespace cybozu yrmcds-1.1.5/cybozu/util.hpp000066400000000000000000000050621262254645600160370ustar00rootroot00000000000000// Utilities // (C) 2013 Cybozu. #ifndef CYBOZU_UTIL_HPP #define CYBOZU_UTIL_HPP #include #include #include #include #include namespace cybozu { // Demangle C++ symbol names. class demangler { public: // Constructor. // @name Mangled symbol name. explicit demangler(const char* name); // Return the demangled symbol name. const std::string& name() const { return m_name; } private: std::string m_name; }; // Dump stack trace to the standard error. void dump_stack() noexcept; // Throw for an `errno`. // @e `errno` value. // @func System call name that returns the error number. inline void throw_unix_error [[noreturn]] (int e, const char* func) { dump_stack(); throw std::system_error(e, std::system_category(), func); } // Throw for an `errno`. // @e `errno` value. // @s A detailed string to describe the context of the error. inline void throw_unix_error [[noreturn]] (int e, const std::string& s) { dump_stack(); throw std::system_error(e, std::system_category(), s); } // Convert an integer from the network byte order. // @p Pointer to a memory region. // @n Reference to an integer to store the converted value. template inline void ntoh(const char* p, UInt& n) noexcept { UInt d; static_assert( sizeof(d) == 2 || sizeof(d) == 4 || sizeof(d) == 8, "Wrong integer type" ); std::memcpy(&d, p, sizeof(d)); switch( sizeof(d) ) { case 2: n = be16toh(d); return; case 4: n = be32toh(d); return; case 8: n = be64toh(d); return; } } // Convert an integer into the network byte order. // @d An integer to be converted. // @p Pointer to a memory region for the converted integer. template inline void hton(UInt d, char* p) noexcept { static_assert( sizeof(d) == 2 || sizeof(d) == 4 || sizeof(d) == 8, "Wrong integer type" ); UInt n; switch( sizeof(d) ) { case 2: n = htobe16(d); std::memcpy(p, &n, 2); return; case 4: n = htobe32(d); std::memcpy(p, &n, 4); return; case 8: n = htobe64(d); std::memcpy(p, &n, 8); return; } } // Clear memory securely just like memset_s in C11. // @s A pointer to a memory region. // @n The length of the memory region in bytes. extern void (* const volatile clear_memory)(void* s, std::size_t n); } // namespace cybozu #endif // CYBOZU_UTIL_HPP yrmcds-1.1.5/cybozu/worker.hpp000066400000000000000000000056151262254645600163770ustar00rootroot00000000000000// General worker thread. // (C) 2013 Cybozu. #ifndef CYBOZU_WORKER_HPP #define CYBOZU_WORKER_HPP #include #include #include #include #include #include #include #include #include namespace cybozu { // General worker thread implementation. // // Every worker thread has a pre-allocated internal memory buffer. // A worker receives a new `job` through as a callback function, // then invokes the function with that buffer. // // To stop the worker, call . class worker final: public thread_base { public: // Constructor. // @bufsiz The size of the internal buffer. worker(std::size_t bufsiz) : m_running(false), m_exit(false), m_event(eventfd(0, EFD_CLOEXEC)), m_buffer(bufsiz) { if( m_event == -1 ) throw_unix_error(errno, "eventfd"); } ~worker() { ::close(m_event); } // forbid copy & assignment worker(const worker&) = delete; worker& operator=(const worker&) = delete; worker(worker&&) = delete; worker& operator=(worker&&) = delete; // Jobs for workers are this kind of functions. // `buf` is a memory buffer that can be used freely for temporary data. using job = std::function; // Return `true` while this worker thread is busy for a job. bool is_running() const noexcept { return m_running.load(std::memory_order_acquire); } // Ask this worker thread to execute a new job. // @job_ A callback function to be executed by the worker thread. void post_job(job job_) { m_job = job_; m_running.store(true, std::memory_order_release); notify(); } // Stop this worker thread. // // The thread will be joined automatically. void stop() { m_exit = true; m_running.store(true, std::memory_order_release); notify(); m_thread.join(); } // CRTP method for . void run() { while ( true ) { // wait a new job or an exit signal m_running.store(false, std::memory_order_release); std::uint64_t i; ssize_t n = ::read(m_event, &i, sizeof(i)); if( n == -1 ) throw_unix_error(errno, "read(eventfd)"); while( ! m_running.load(std::memory_order_acquire) ); if( m_exit ) return; m_job(m_buffer); m_buffer.reset(); } } private: alignas(CACHELINE_SIZE) std::atomic m_running; bool m_exit; const int m_event; dynbuf m_buffer; job m_job; void notify() { std::uint64_t i = 1; ssize_t n = ::write(m_event, &i, sizeof(i)); if( n == -1 ) throw_unix_error(errno, "write(eventfd)"); } }; } // namespace cybozu #endif // CYBOZU_WORKER_HPP yrmcds-1.1.5/detect_tcmalloc.sh000077500000000000000000000004221262254645600165160ustar00rootroot00000000000000#!/bin/sh CXX=$1 shift CPPFLAGS="$@" T=$(mktemp --suffix=.cpp) trap "rm -f $T" INT QUIT TERM HUP 0 f() { echo "#include <$1>" >$T if $CXX "$CPPFLAGS" -E $T >/dev/null 2>&1; then echo $1 exit 0 fi } f gperftools/tcmalloc.h f google/tcmalloc.h yrmcds-1.1.5/docs/000077500000000000000000000000001262254645600137635ustar00rootroot00000000000000yrmcds-1.1.5/docs/bench.md000066400000000000000000000016551262254645600153730ustar00rootroot00000000000000# Benchmark results. Environment ----------- * CPU Intel Xeon E5507 2.27 GHz * 2 (total 8 physical cores) * OS Ubuntu 12.04 * Threads 8 worker threads for yrmcds / memcached * Memory 1 GiB for yrmcds / memcached * Buckets 5 million for yrmcds Benchmark Tool -------------- [memslap][] from [libmemcached][]. Parameters: `--threads=8 --concurrency=80` Other parameters are left default, i.e., memslap runs for 600 seconds with set/get ratio = 1:9. memslap ran on a different machine. Versions -------- * memcached 1.4.13 * yrmcds 0.9.7 Results ------- * memcached Ops: 71584331 TPS: 119298 * yrmcds (0 slave) Ops: **81028171** TPS: **135037** * yrmcds (1 slave) Ops: 69657147 TPS: 116086 * yrmcds (2 slaves) Ops: 63479541 TPS: 105791 [memslap]: http://docs.libmemcached.org/bin/memslap.html [libmemcached]: http://libmemcached.org/libMemcached.html yrmcds-1.1.5/docs/counter.md000066400000000000000000000272641262254645600157770ustar00rootroot00000000000000# Counter support Overview -------- Counter extension provides distributed counters for resource management. Values of counters represent how much a resource is consumed. Multiple clients can acquire/release the resources simultaneously. Each counter has a non-empty name. The maximum name is up to 65535 bytes. The number of resources available for a counter is represented as a 4 byte unsigned integer; this means a counter can manage up to 4294967295 resources. - The counter extension is disabled by default. - TCP port used for the counter protocol need to be different from that of the memcache protocol. - The namespace of counter objects is different from that of the memcache objects. - When a connection is closed, all resources acquired through the connection are released automatically. - A counter is created dynamically at the first time the counter is acquired. - Counters are not replicated. Protocol -------- The counter extension has its own binary protocol. ### General format of a packet ``` Byte/ 0 | 1 | 2 | 3 | / | | | | |0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7| +---------------+---------------+---------------+---------------+ 0/ HEADER / / / / / / / +---------------+---------------+---------------+---------------+ 12/ COMMAND-SPECIFIC BODY (as needed) / +---------------+---------------+---------------+---------------+ ``` ### Request header ``` Byte/ 0 | 1 | 2 | 3 | / | | | | |0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7| +---------------+---------------+---------------+---------------+ 0| Magic | Opcode | Flags | Reserved | +---------------+---------------+---------------+---------------+ 4| Body length | +---------------+---------------+---------------+---------------+ 8| Opaque | +---------------+---------------+---------------+---------------+ Total 12 bytes ``` - `Magic` is `0x90`. - `Opcode` is defined later. - `Flags` is defined later. - `Reserved` is always zero. - `Body length` is a 4 byte unsigned integer in big-endian. - `Opaque` is an arbitrary 4 byte data. ### Response header ``` Byte/ 0 | 1 | 2 | 3 | / | | | | |0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7| +---------------+---------------+---------------+---------------+ 0| Magic | Opcode | Status | Reserved | +---------------+---------------+---------------+---------------+ 4| Body length | +---------------+---------------+---------------+---------------+ 8| Opaque | +---------------+---------------+---------------+---------------+ Total 12 bytes ``` - `Magic` is `0x91`. - `Opcode` and `Opaque` are copied from the corresponding request. - `Status` is defined later. - `Reserved` is always zero. - `Body length` is a 4 byte unsigned integer in big-endian. ### Opcodes | Code | Operation |--------|----------- | `0x00` | `Noop` | `0x01` | `Get` | `0x02` | `Acquire` | `0x03` | `Release` | `0x10` | `Stats` | `0x11` | `Dump` ### Flags No flags are yet defined. ### Response status | Code | Status |--------|------------------- | `0x00` | No error | `0x01` | Not found | `0x04` | Invalid arguments | `0x21` | Resource not available | `0x22` | Not acquired | `0x81` | Unknown command | `0x82` | Out of memory Packet specifications --------------------- Unless otherwise noted, all integers are represented in big-endian. ### Error responses Errors are signaled by the `Status` field of the response header. Error messages are accompanied in the message body. ### Noop `Noop` does nothing and returns a success response. The request and response of `Noop` has no body. ### Acquire `Acquire` acquires some resources associated to a counter. In addition to the counter name, the command has two parameters; one is the number of resources to be acquired, and another is the number of total resources. Note that, unlike semaphores, counters do not keep the resource capacity; the capacity will be specified by the acquire command. This way, users can change the resource capacity dynamically anytime. Request body: ``` Byte/ 0 | 1 | 2 | 3 | / | | | | |0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7| +---------------+---------------+---------------+---------------+ 0| Resources | +---------------+---------------+---------------+---------------+ 4| Maximum | +---------------+---------------+---------------+---------------+ 8| Name length | (Name data) ... +---------------+---------------+ Total 10 + (Name length) bytes ``` Successful response body: ``` Byte/ 0 | 1 | 2 | 3 | / | | | | |0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7| +---------------+---------------+---------------+---------------+ 0| Resources | +---------------+---------------+---------------+---------------+ Total 4 bytes ``` - `Resources` is a 4-byte unsigned integer number. This request will acquire this count of resources. If this is zero, `Invalid arguments` is returned. - `Maximum` is a 4-byte unsigned integer number. This parameter represents the total number of resources. `Maximum` must be equal to or greater than `Resources`, otherwise `Invalid arguments` is returned. - `Name length` is a 2-byte unsigned integer number. If this is zero, `Invalid arguments` is returned. If the named counter does not exist, `Acquire` creates a new counter, sets its consumption to `Resources`, and returns the success. Otherwise, if `Consumption + Resources <= Maximum` where `Consumption` is the current consumption of the resource, then this request will augment `Consumption` by `Resources`, and return the success. Else, this returns `Resource not available` error. ### Release `Release` returns the acquired resources back. Request body: ``` Byte/ 0 | 1 | 2 | 3 | / | | | | |0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7| +---------------+---------------+---------------+---------------+ 0| Resources | +---------------+---------------+---------------+---------------+ 4| Name length | (Name data) ... +---------------+---------------+ Total 6 + (Name length) bytes ``` A successful response has no body. - `Resources` is the count of resources to return. It can be zero. - `Name length` is a 2-byte unsigned integer number. If this is zero, `Invalid arguments` is returned. If the named counter does not exist, `Release` returns `Not found` error. If `Resources` is greater than the current number of acquired resources, `Release` returns `Not acquired` error. Otherwise, the operation succeeds and the value of the counter is increased by `Resources`. ### Get `Get` obtains the current consumption of a counter. Request body: ``` Byte/ 0 | 1 | 2 | 3 | / | | | | |0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7| +---------------+---------------+---------------+---------------+ 0| Name length | (Name data) ... +---------------+---------------+ Total 2 + (Name length) bytes ``` Success response body: ``` Byte/ 0 | 1 | 2 | 3 | / | | | | |0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7| +---------------+---------------+---------------+---------------+ 0| Consumption | +---------------+---------------+---------------+---------------+ Total 4 bytes ``` - `Name length` is a 2-byte unsigned integer number. If this is zero, `Invalid arguments` is returned. If the named counter does not exist, `Get` returns `Not found` error. ### Stats `Stats` obtains statistics information about counters. Request body: No body. Successful response body: The body consists of a series of name-value pairs. ``` Byte/ 0 | 1 | 2 | 3 | / | | | | |0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7| +---------------+---------------+---------------+---------------+ 0| Name length 1 | Value length 1 | +---------------+---------------+---------------+---------------+ 4| Name1 ... Value1 ... +---------------+---------------+---------------+---------------+ N| Name length 2 | Value length 2 | +---------------+---------------+---------------+---------------+ N+4| Name2 ... Value2 ... ... Total (Body length) bytes. ``` - `Name` is a name of a statistics item. - `Value` is an ASCII text information. ### Dump `Dump` dumps all counters. Request body: No body. Successful responses: `Dump` command returns multiple responses for one request. Each response contains the name of a counter and the current consumption of the resource. The response also contains the maximum consumption of resources in the interval given in the configuration file. The end of the series of responses are indicated by the response whose body length is zero. ``` Byte/ 0 | 1 | 2 | 3 | / | | | | |0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7| +---------------+---------------+---------------+---------------+ 0| Current consumption | +---------------+---------------+---------------+---------------+ 4| Maximum consumption | +---------------+---------------+---------------+---------------+ 8| Name length | (Name data) ... +---------------+---------------+ Total 10 + (Name length) bytes ``` Configurations -------------- Following parameters can be specified in the configuration file: ```ini # If true, the counter extension is enabled. (default: false) counter.enable = false # TCP port used for the counter protocol. (default: 11215) counter.port = 11215 # The maximum number of connections in the counter protocol. # 0 means unlimited. (default: 0) counter.max_connections = 0 # The size of the counter hash table. (default: 1000000) counter.buckets = 1000000 # The interval of measuring the maximum number of resource consumption # in seconds. (default: 86400) counter.stats_interval = 86400 ``` yrmcds-1.1.5/docs/design.md000066400000000000000000000320151262254645600155570ustar00rootroot00000000000000# Design Notes ============ Automatic master promotion -------------------------- The standard set of yrmcds consists of a master server and two slave servers. A set of yrmcds will be prepared for each server farm. To elect the master server automatically, keepalived or similar virtual IP address allocation tool is used. The server owning the virtual IP address becomes the master server. The others are slaves. Slaves connect to the master via the virtual IP address. The master will *not* connect to slaves. A new slave will receive all data from the master server once the connection has been established. For some reason, if the master suddenly notices it is no longer the master (i.e. the server has lost the virtual IP), it kills itself to reborn as a new slave (restarted by upstart). If a slave gets disconnected from the master, it waits for some seconds to see if the virtual IP address is assigned. If assigned, the slave promotes to the new master. If not, the slave forgets all replicated data then try to connect to the new master. The new master should close the established connection to the old master, if any. Replication protocol -------------------- The replication protocol is the same as [the binary protocol of memcached][1]. Specifically, slaves receive only "SetQ" and "DeleteQ" requests. This allows any memcached compatible programs can become yrmcds slaves with slight modifications. The number of connections ------------------------- The master will have tens of thousands of client connections from every PHP-fpm processes in a server farm. A thread per connection model is therefore not very efficient. This encourages us to adopt asynchronous sockets and [the reactor pattern][2]. The reactor and workers ----------------------- So we adopt [the reactor pattern with a thread-pool][3] for the best performance. The reactor is implemented with asynchronous sockets and [epoll][] in edge-triggered mode. Basically, with edge-triggered mode, a socket need to be read until [recv][] returns an `EWOULDBLOCK` error. If, for some reason like fairness, not all available data are received from a socket, the reactor need to remember that socket to manually handle it later. To remember such sockets, the reactor manages a list of _readable_ socket internally. When the reactor detects some data are available for a socket, it dispatches the socket to an **idle** _worker_ thread. The worker then receives and processes the data. While a worker is handling a socket, that socket must not be passed to another worker to keep the protocol semantics. To implement these, following properties are required: * Every worker thread has an **idle** flag. * Every socket has a **busy** (in-use) flag. Both flags are set by the reactor thread, and cleared by a worker thread. If the reactor finds a socket is readable but the socket is busy or there are no idle workers, the socket is remembered in the previously stated readable socket list. This keeps the reactor thread from being blocked. ### Efficient allocation of readable lists Readable lists used in the reactor can be implemented with very rare memory allocations by preparing two lists that pre-allocate a certain amount of memory. Swap the list with another for each epoll loop. ### Use of eventfd to dispatch jobs To notify an idle worker a new job (or to wait a new job), we use Linux-specific [eventfd][] rather than pipes or pthread's condition variables. eventfd is lighter than pipe. Unlike condition variables, eventfd remembers events so that workers never fail to catch events. The hash -------- The biggest and the most important data structure in yrmcds is clearly the hash map of objects. To reduce contention between worker threads, the hash does not use a single lock to protect the data. Instead, each bucket has a lock to protect objects in the bucket and the bucket itself. One drawback is that resizing the hash is almost impossible. In the real implementation, yrmcds statically allocates a large number of buckets. Housekeeping ------------ yrmcds is essentially an object _cache_ system. Objects whose life-time have been expired need to be removed from the cache. If the cache is full of objects, least-recently-used (LRU) objects should be removed. For such housekeeping, a dedicated thread called **GC thread** is used. It scans the whole hash to detect and remove expired objects. To implement LRU cache, objects have a counter that increments at every GC. The counter is reset to zero when a worker accesses the object. When the cache is full, GC thread removes objects whose LRU counter value is high. A GC thread is started by the reactor thread only when there is no running GC thread. The reactor thread passes the current list of slaves to the GC thread in order to replicate object removals. If a slave is added while a GC thread is running, that slave may fail to remove some objects, which is *not* a big problem. Replication ----------- When a slave connect to the master, it need to receive all data in the master. yrmcds reuses the GC thread for this initial replication because the GC thread scans the entire hash. While GC is running, the reactor and worker threads also keep running. The GC thread and worker threads therefore need to be synchronized when sending data of the same object. This can be achieved by keeping the lock of a hash bucket acquired while an object is being sent. Sockets for replication ----------------------- Sockets connected to slaves are used to send a lot of replication data. Further, worker threads serving client requests need to access replication sockets to send updates to slaves. Another difference from client sockets is the possible number of sockets. The number of replication sockets can be limited to a fairly small value, say 5. Worker threads need to have references to all replication sockets. This can be done without any locks; the reactor passes a copy of the references along with the socket and received data. However, this can introduce a race between workers and the GC thread. If a new GC thread starts the initial replication process for a new slave while some worker threads having old copies of replication sockets are executing its job, the workers would fail to send some objects to the new slave. To resolve this race, we need to guarantee that all worker threads are idle or have the latest copies of replication sockets. This can be implemented as: 1. Accept a new replication socket and add it to the list of replication sockets. At this point, the GC thread must not begin the initial replication for this new slave. 2. The reactor thread puts a synchronization request. The request will be satisfied once the reactor thread observes every worker thread gets idle. 3. The reactor thread adds the new replication socket to the confirmed list. 4. At the next GC, the reactor thread requests initial replication for sockets stored in the confirmed list. Sending data ------------ As yrmcds need to handle a lot of clients, having every socket has a large send buffer is not a good idea. Sockets connected to clients therefore can have temporarily allocated memory for pending data only. On the other hand, sockets connected to slaves are few, hence they can statically allocate large memory for pending send data. Anyways, a sending thread need to wait for the reactor to send pending data and clear the buffer when there is not enough room. To achieve best TCP performance, we use [`TCP_CORK`][4] option although it is Linux-specific. With `TCP_CORK` turned on, the TCP send buffer need to be flushed explicitly. The buffer should be flushed only when all data are successfully sent without seeing `EWOULDBLOCK`. To put a complete response data with one call, the socket object should provide an API similar to [writev][]. Reclamation strategy of shared sockets -------------------------------------- ### Sockets are shared Sockets connected to clients may be shared by the reactor thread and a worker thread. Sockets connected to slaves may be shared by the reactor thread, worker threads, and the initial replication thread. ### Socket can be closed only by the reactor thread This is because the reactor thread manages a mapping between file descriptors and resource objects including sockets. If another thread closes a file descriptor of a resource, the same file descriptor number may be reused by the operating system, which would break the mapping. ### Strategy to reclaim shared sockets 1. Sockets can be invalidated by any thread. Invalidated sockets refuse further access to them. 2. Inform the reactor thread of invalidated sockets. 3. The reactor thread 1. removes invalidated sockets from the internal mapping, 2. closes the file descriptor, then 3. adds them a list of pending destruction resources. At some point when there is no running GC thread, the reactor thread can put a new synchronization request. To optimize memory allocations, the reactor thread will not put such a request if there are any pending requests. This way, the reactor can save the current pending destruction list by swapping contents with a pre-allocated save list. Once the reactor observes idle state of all worker threads, resources in the save list can be destructed safely. No blocking job queues, no barrier synchronization -------------------------------------------------- To avoid excessive contention between the reactor and worker threads, a blocking job queue is not used between them. Instead, the reactor thread loops over atomic flags for each worker thread to identify if a worker is idle or busy. For the same reason, we do not use barrier synchronization between the reactor and worker threads. Instead, the reactor thread checks every worker thread is idle or becomes idle when there are any synchronization request. Once it confirms all worker threads gets idle at least once after a synchronization request, it starts a job associated with the synchronization request. Slaves are essentially single-threaded -------------------------------------- Slaves do accept connection requests from memcache clients, but they close the connections immediately. This way, slaves can essentially be single-threaded. As long as being a slave, the internal hash need not be guarded by locks. For this reason, the internal hash should provide lock-free version of methods. Joining threads --------------- All threads are started by the reactor thread. The reactor thread is therefore responsible for those threads to be joined. All threads except for the reactor thread may block on either a condition variable of a socket or the [eventfd][] file descriptor. For sockets, the reactor thread signals the condition variable to unblock threads when it closes the socket. For eventfd descriptors, the reactor thread simply writes to it. When the program exits, the reactor thread executes the following termination process: 1. Sets termination flags of all worker threads. 2. Writes to eventfd descriptors to unblock worker threads. 3. invalidates all resources to unblock worker threads. 4. joins with the GC thread, if any, then 5. destructs resources. Large data ---------- Too large data should be stored in a temporary file. The temporary file shall soon be unlinked for automatic removal upon the program exit. Summary ------- ### Threads In the master, * the reactor thread, * worker threads to process client requests, and * a GC thread. Slaves run the reactor (main) thread only. ### Sockets In the master, * a listening socket for clients, * a listening socket for slaves, * sockets connected to clients, and * sockets connected to slaves. In a slave, * a listening socket for clients, and * a socket connected to the master. ### Shared data structures * Connected sockets. * The object hash map. * Close request queue in the reactor. * Other less significant resources such as statistics counters. ### Locks and ordering * The reactor thread * acquires a lock of a socket to send data. * acquires a spinlock of socket close request queue. * A worker thread * acquires a lock of a hash bucket. * acquires a lock of a socket. * acquires a lock of a socket to send data independent of cached objects. * acquires a spinlock of the reactor to put socket close request. * A GC thread * acquires a lock of a hash bucket. * acquires locks of sockets connected to slaves. * acquires a spinlock of the reactor to put socket close request. [1]: https://code.google.com/p/memcached/wiki/BinaryProtocolRevamped [2]: http://en.wikipedia.org/wiki/Reactor_pattern [3]: http://stackoverflow.com/questions/14317992/thread-per-connection-vs-reactor-pattern-with-a-thread-pool [4]: http://manpages.ubuntu.com/manpages/precise/en/man7/tcp.7.html [epoll]: http://manpages.ubuntu.com/manpages/precise/en/man7/epoll.7.html [eventfd]: http://manpages.ubuntu.com/manpages/precise/en/man2/eventfd.2.html [recv]: http://manpages.ubuntu.com/manpages/precise/en/man2/recv.2.html [signalfd]: http://manpages.ubuntu.com/manpages/precise/en/man2/signalfd.2.html [writev]: http://manpages.ubuntu.com/manpages/precise/en/man2/writev.2.html yrmcds-1.1.5/docs/diffs.md000066400000000000000000000015651262254645600154070ustar00rootroot00000000000000# Differences from memcached. Differences from memcached 1.4 ------------------------------ * Implementations of binary protocol command `GaT` and `Get` are identical. Both may have an optional expiration time. Objects will be touched only when the command is attended with an expiration time. * `stats` returns different items. `stats slabs` is not implemented. `stats cachedump` is not implemented. `stats ops` returns ops counts for each text/binary command. * `slabs automove` and `slabs reassign` are not implemented. These always return "OK". * `verbosity` takes a string argument rather than an integer. Valid values are `error`, `warning`, `info`, and `debug`. * UDP transport is not implemented. * UNIX domain transport is not, and will not be, implemented. * SASL authentication is not implemented. Who cares? yrmcds-1.1.5/docs/future.md000066400000000000000000000012761262254645600156250ustar00rootroot00000000000000# List of future plans. Future Plans ------------ * Replicate CAS unique values. Currently, yrmcds does not replicate CAS values, which may lead to **false-positives** as well as (harmless) false-negatives.

* Allow eviction to be disabled and honor the memory limit. This is a requirement for in-memory database.

* Save/load cached objects at program exit/startup. This is another requirement for in-memory database.

* Save snapshots periodically. This can be done by [forking][fork] a child process when all worker threads are idle. [fork]: http://manpages.ubuntu.com/manpages/precise/en/man2/fork.2.html yrmcds-1.1.5/docs/index.md000066400000000000000000000003141262254645600154120ustar00rootroot00000000000000# Index. yrmcds documents ---------------- yrmcds is an object cache system featuring master-slave replication. # General utilities. # Implementing yrmcds. yrmcds-1.1.5/docs/keys.md000066400000000000000000000024151262254645600152620ustar00rootroot00000000000000Key dump extesion ================= yrmcds provides an extension to memcached protocol to dump existing keys. Text protocol ------------- The general description about the text protocol is available at: https://github.com/memcached/memcached/blob/master/doc/protocol.txt The key dump command syntax is: keys []\r\n where `prefix` is the prefix string of dumping keys. If `prefix` is omitted, all keys will be dumped. After this command, the client expects zero or more values, each of which is received as a text line. After all the items have been transmitted, the server sends the string END\r\n to indicate the end of response. Each item sent by the server looks like this: VALUE \r\n where `key` is a key having the given prefix. Binary protocol --------------- The general description about the binary protocol is available at: https://code.google.com/p/memcached/wiki/MemcacheBinaryProtocol This extension introduces a new opcode: 0x50 (Keys) `Keys` Request: - MUST NOT have extras. - MAY have key. - MUST NOT have value. Successful response of `Keys` consists of a series of these responses: - MUST NOT have extras. - MUST have key. - MUST NOT have value. followed by an empty response (i.e., key length is 0). yrmcds-1.1.5/docs/locking.md000066400000000000000000000115221262254645600157340ustar00rootroot00000000000000# Server-side locking support. Motivation ---------- Locking is one of the things people often want to have with memcached. Though `add` has been used to implement locks at client-side [1][] [2][], this approach is fragile if clients are not very stable; if a client acquires a lock then dies, the lock will never be released. Moreover, objects used to implement locks may be evicted. yrmcds introduces a server-side locking mechanism to resolve this problem. Overview -------- A lock associates an existing object and a client connection. Locked objects will neither be expired nor be evicted. Modification commands such as `set`, `replace`, or `delete` will fail if the object is locked, unless the client is holding the lock. `get` succeeds even though the object is locked by another client. If a client is disconnected, all locks held by the client will be released. Although lock commands are available in both ASCII and binary protocols, use of binary protocol is strongly recommended. ASCII protocol -------------- * `lock KEY\r\n` acquires a server-side lock of the object associated with KEY. The response will be one of: - "OK\r\n", to indicate success. - "LOCKED\r\n", if another client has already locked the object. - "NOT_FOUND\r\n", if the object does not exist. * `unlock KEY\r\n` releases a server-side lock. The response will be one of: - "OK\r\n", to indicate success. - "CLIENT_ERROR MESSAGE\r\n", if the object is not locked by the client, or if the object does not exist. * `unlock_all\r\n` releases all locks held by the client. The response will be "OK\r\n". Additionally, modification commands such as `set` and `delete` will fail when the object is locked by another client. The response will be "LOCKED\r\n" in this case. Binary protocol --------------- The binary protocol is enhanced with new `Lock`, `Unlock`, `UnlockAll`, `LaG` (lock and get), `LaGK` (lock and get with key), and `RaU` (replace and unlock) commands. New opcodes are: - 0x40 (Lock) - 0x41 (LockQ) - 0x42 (Unlock) - 0x43 (UnlockQ) - 0x44 (UnlockAll) - 0x45 (UnlockAllQ) - 0x46 (LaG) - 0x47 (LaGQ) - 0x48 (LaGK) - 0x49 (LaGKQ) - 0x4a (RaU) - 0x4b (RaUQ) New response statuses are: - 0x0010 Locked - 0x0011 Not locked ### Lock, Lock Quietly Request: - MUST NOT have extras. - MUST have key. - MUST NOT have value. Lock an object. The response-packet contains no extra data, and the result of the operation is signaled through the status code. Specifically, 0x0001 (not found) is returned when the object does not exist, and 0x0010 (locked) is returned when the object has been locked by another client. ### Unlock, Unlock Quietly Request: - MUST NOT have extras. - MUST have key. - MUST NOT have value. Unlock a locked object. The response-packet contains no extra data, and the result of the operation is signaled through the status code. Specifically, 0x0001 (not found) is returned when the object does not exist, and 0x0011 (not locked) is returned when the object is not locked by the client. ### Unlock All, Unlock All Quietly Request: - MUST NOT have extras. - MUST NOT have key. - MUST NOT have value. Unlock all locked objects held by the client. The response-packet contains no extra data, and the result of the operation is signaled through the status code. In fact, this command will succeed unconditionally. ### Lock and Get, Lock and Get Quietly, Lock and Get Key, Lock and Get Key Quietly Request: - MAY have extras. - MUST have key. - MUST NOT have value. Response (if found): - MUST have extras. - MAY have key. - MAY have value. Lock an object and get the data atomically. Optional extra data for Lock and Get request is 4 byte expiration time. If extra data exists, the object's expiration time will be renewed. The response is the same as that of `Get` or `GetK` command. Specifically, if the object is locked, the response with status code = 0x0010 (locked) will be returned. `LaGQ` and `LaGKQ` will return the response with status code = 0x0001 (not found) if the object is not found. ### Replace and Unlock, Replace and Unlock Quietly Request: - MUST have extras. - MUST have key. - MUST have value. Replace a locked object then unlock it atomically. The client need to lock the object in advance. The extra data are the same as that of `Replace` binary command, i.e., 4 byte flags and 4 byte expiration time. The response is almost the same as that of `Replace` binary command. Specifically, 0x0011 (not locked) is returned when the object is not locked by the client. [1]: http://www.regexprn.com/2010/05/using-memcached-as-distributed-locking.html [2]: http://russellneufeld.wordpress.com/2012/05/24/using-memcached-as-a-distributed-lock-from-within-django/ yrmcds-1.1.5/docs/usage.md000066400000000000000000000136471262254645600154240ustar00rootroot00000000000000# Configuration and execution of `yrmcdsd`. `yrmcdsd` is the server program of yrmcds. Its built-in configurations are reasonably good to run as a stand-alone server. For replication, you need to specify a virtual IP address and configure a clustering software such as [keepalived][]. Files ----- Usually, yrmcds is installed under `/usr/local`. * /usr/local/etc/yrmcds.conf The default configuration file. * /usr/local/sbin/yrmcdsd The program. Configuration (memcache) ------------------------ These options are to configure memcache protocol: * `user` (Default: none) If set, the program will try to `setuid` to the given user. * `group` (Default: none) If set, the program will try to `setgid` to the given group. * `virtual_ip` (Default: 127.0.0.1) The master's virtual IP address. Both IPv4 and IPv6 addresses are supported. * `port` (Default: 11211) memcache protocol port number. * `repl_port` (Default: 11213) The replication protocol port number. * `max_connections` (Default: 0) Maximum number of client connections. 0 means unlimited. * `temp_dir` (Default: /var/tmp) Directory to store temporary files for large objects. * `log.threshold` (Default: info) Log threshold. Possible values: `error`, `warning`, `info`, `debug`. * `log.file` (Default: standard error) Logs will be written to this file. * `buckets` (Default: 1000000) Hash table size. * `max_data_size` (Default: 1M) The maximum object size. * `heap_data_limit` (Default: 256K) Objects larger than this will be stored in temporary files. * `repl_buffer_size` (Default: 30) The replication buffer size. Unit is MiB. * `secure_erase` (Default: false) If `true`, object memory will be cleared as soon as the object is removed. * `lock_memory` (Default: false) If `true`, prevent memory from being swapped using `mlockall`. * `memory_limit` (Default: 1024M) The amount of memory allowed for yrmcdsd. * `workers` (Default: 8) The number of worker threads. * `gc_interval` (Default: 10) The interval between garbage collections in seconds. * `slave_timeout` (Default: 10) slave_timeout specifies how many seconds to wait for heartbeats from slaves before the connection is forcibly closed. Configuration (counter) ------------------------ yrmcds has an extra protocol called **counter** in addition to memcache protocol. The counter protocol can be used to manage resource usage dynamically just like (distributed) semaphores. The counter protocol has these configuration options: * `counter.enable` (Default: `false`) If `true`, the counter extension is enabled. * `counter.port` (Default: 11215) TCP port number used for the counter protocol. * `counter.max_connections` (Default: 0) The maximum number of connections for the counter protocol. 0 means unlimited. * `counter.buckets` (Default: 1000000) The size of the counter hash table. * `counter.stat_interval` (Default: 86400) The interval of measuring the maximum resource consumption. The default is 86400 seconds = 1 day. Running yrmcdsd --------------- Just run `yrmcdsd`. The program does not run as traditional UNIX daemons. Instead, use [upstart][] or [systemd][] to run `yrmcdsd` as a background service. Do not forget to increase the maximum file descriptors. A sample upstart script is available at [etc/upstart](https://github.com/cybozu/yrmcds/blob/master/etc/upstart). Replication ----------- The replication of yrmcds servers is based on a virtual IP address. To assign a virtual IP address to a live server, a clustering software such as [keepalived][] or [Pacemaker][pacemaker] is used. Our recommendation is to use [keepalived][]. ### keepalived For stability, keepalived should be configured in non-preemptive mode. A sample configuration file is available at [etc/keepalived.conf](https://github.com/cybozu/yrmcds/blob/master/etc/keepalived.conf). ### Replication status `yrmcdsd`, either it is master or slave, dumps the current replication status to the log file when it receives `SIGUSR1` signal. ``` $ kill -USR1 ``` ### How replication works `yrmcdsd` **periodically watches** if the server owns the virtual IP address or not. If `yrmcdsd` finds it holds the virtual IP address, that server will promote itself as the new master. All others become slaves. Slaves connect to the master through the virtual IP address. If the connection resets, `yrmcdsd` waits some seconds to see if the virtual IP address is assigned. If assigned, the slave becomes the new master. If not, the slave drops all objects then try to connect to the new master. Since the master node is elected dynamically by [keepalived][], each `yrmcdsd` should have the same configuration parameters. ### counter protocol Resource counters are not replicated. Erasing confidential data in memory ----------------------------------- If `secure_erase` configuration option is enabled, `yrmcdsd` will erase memory used by objects as soon as they are removed or their lifetime expire. This ensures confidential data such as crypto keys will never be leaked after expiration. Consider setting `max_data_size` equal to `heap_data_limit` when you enable this option to avoid writing confidential data into persistent storage. Locking memory -------------- If `lock_memory` configuration option is enabled, `yrmcdsd` will lock memory with `mlockall` to prevent them from being swapped. **Use of this option may cause troubles**; locking enough memory for yrmcds will require the process having `CAP_IPC_LOCK` capability or sufficiently large `RLIMIT_MEMLOCK` resource limit. See [man page][mlockall] for details. [keepalived]: http://www.keepalived.org/ [pacemaker]: http://clusterlabs.org/wiki/Main_Page [upstart]: http://upstart.ubuntu.com/ [systemd]: http://www.freedesktop.org/wiki/Software/systemd/ [mlockall]: http://manpages.ubuntu.com/manpages/trusty/en/man2/mlockall.2.html yrmcds-1.1.5/etc/000077500000000000000000000000001262254645600136065ustar00rootroot00000000000000yrmcds-1.1.5/etc/keepalived.conf000066400000000000000000000004321262254645600165650ustar00rootroot00000000000000### VRRP Configurations vrrp_instance VI_YRMCDS { # to disable preemption, the initial state must be BACKUP. state BACKUP nopreempt priority 100 interface eth0 virtual_router_id 10 virtual_ipaddress { 10.0.160.2/20 dev eth0 label eth0:1 } } yrmcds-1.1.5/etc/logrotate000066400000000000000000000003241262254645600155300ustar00rootroot00000000000000# place this file as /etc/logrotate.d/yrmcds /var/log/yrmcds.log { notifempty weekly rotate 4 compress missingok create 0644 nobody nogroup postrotate reload yrmcds >/dev/null 2>&1 || true endscript } yrmcds-1.1.5/etc/upstart000066400000000000000000000012241262254645600152320ustar00rootroot00000000000000# Upstart script for yrmcds description "yrmcds" start on net-device-up stop on runlevel [!2345] respawn umask 077 limit nofile 100000 100000 limit memlock unlimited unlimited chdir /tmp pre-start script [ -d /var/tmp/yrmcds ] || mkdir /var/tmp/yrmcds chmod 700 /var/tmp/yrmcds chown nobody:nogroup /var/tmp/yrmcds rm -f /var/tmp/yrmcds/* # make sure keepalived init script was disabled. # ex) sudo update-rc.d keepalived disable /etc/init.d/keepalived start sleep 5 end script post-stop script /etc/init.d/keepalived stop # sleep a while to hand-off VIP sleep 5 end script exec /usr/local/sbin/yrmcdsd yrmcds-1.1.5/etc/valgrind.suppress000066400000000000000000000002571262254645600172260ustar00rootroot00000000000000{ reactor::epoll_ctl Memcheck:Param epoll_ctl(event) fun:epoll_ctl fun:_ZN6cybozu7reactor12add_resourceESt10unique_ptrINS_8resourceESt14default_deleteIS2_EEi } yrmcds-1.1.5/etc/yrmcds.conf000066400000000000000000000047341262254645600157660ustar00rootroot00000000000000# Configuration file for yrmcdsd # setuid user user = nobody # setgid group group = nogroup # To become the master, virtual_ip address must be owned. virtual_ip = 127.0.0.1 # memcache protocol port number. port = 11211 # yrmcds replication protocol port number. repl_port = 11213 # max number of client connections. 0 means unlimited. max_connections = 10000 # large objects are saved in this directory as temporary files. temp_dir = "/var/tmp/yrmcds" # possible values: error, warning, info, debug log.threshold = info # logs are recorded to this file. # If log.file is not defined, logs are printed to standard error. log.file = "/var/log/yrmcds.log" # Hash table size. 1 million is the sane default. buckets = 1000000 # The maximum object size. This is a soft-limit. # There is a compile-time hard-limit around 30 MiB. max_data_size = 10M # Objects larger than this will be stored in temporary files. heap_data_limit = 256K # The buffer size for asynchronous replication in MiB. # The value must be an integer > 0. Default is 30 (MiB). repl_buffer_size = 30 # Clear memory used by deleted or expired objects securely. # This ensures confidential data such as crypto keys will not be # leaked after expiration. # Consider setting "max_data_size" equal to "heap_data_limit" to # avoid writing confidential data into persistent storage when # you enable this option. secure_erase = false # Prevent memory from being swapped by using mlockall(2). lock_memory = false # The amount of memory allowed for the entire yrmcds. # This is by no means a hard limit; rather, this is just a hint for # the garbage collection. memory_limit = 1024M # The number of worker threads. workers = 10 # The interval between garbage collections in seconds. gc_interval = 10 # slave_timeout specifies how many seconds to wait for heartbeats from slaves # before the connection is forcibly closed. slave_timeout = 10 #---------------------------------------------------------- # configurations for counter extension # If true, the counter extension is enabled. (default: false) counter.enable = false # TCP port used for the counter protocol. (default: 11215) counter.port = 11215 # The maximum number of connections in the counter protocol. # 0 means unlimited. (default: 0) counter.max_connections = 0 # The size of the counter hash table. (default: 1000000) counter.buckets = 1000000 # The interval of measuring the maximum number of resource consumption # in seconds. (default: 86400) counter.stat_interval = 86400 yrmcds-1.1.5/src/000077500000000000000000000000001262254645600136225ustar00rootroot00000000000000yrmcds-1.1.5/src/config.cpp000066400000000000000000000160331262254645600155760ustar00rootroot00000000000000// (C) 2013 Cybozu. #include "config.hpp" #include #include #include #include namespace { const char VIRTUAL_IP[] = "virtual_ip"; const char PORT[] = "port"; const char REPL_PORT[] = "repl_port"; const char MAX_CONNECTIONS[] = "max_connections"; const char TEMP_DIR[] = "temp_dir"; const char USER[] = "user"; const char GROUP[] = "group"; const char LOG_THRESHOLD[] = "log.threshold"; const char LOG_FILE[] = "log.file"; const char BUCKETS[] = "buckets"; const char MAX_DATA_SIZE[] = "max_data_size"; const char HEAP_DATA_LIMIT[] = "heap_data_limit"; const char MEMORY_LIMIT[] = "memory_limit"; const char REPL_BUFSIZE[] = "repl_buffer_size"; const char SECURE_ERASE[] = "secure_erase"; const char LOCK_MEMORY[] = "lock_memory"; const char WORKERS[] = "workers"; const char GC_INTERVAL[] = "gc_interval"; const char SLAVE_TIMEOUT[] = "slave_timeout"; const char COUNTER_ENABLE[] = "counter.enable"; const char COUNTER_PORT[] = "counter.port"; const char COUNTER_MAX_CONNECTIONS[] = "counter.max_connections"; const char COUNTER_BUCKETS[] = "counter.buckets"; const char COUNTER_STAT_INTERVAL[] = "counter.stat_interval"; std::unordered_map THRESHOLDS { {"error", cybozu::severity::error}, {"warning", cybozu::severity::warning}, {"info", cybozu::severity::info}, {"debug", cybozu::severity::debug} }; inline std::size_t parse_unit(std::string& s, const char* cmd) { std::size_t base = 1; switch( s.back() ) { case 'k': case 'K': base <<= 10; break; case 'm': case 'M': base <<= 20; break; case 'g': case 'G': base <<= 30; break; } if( base != 1 ) s.pop_back(); int n = std::stoi( s ); if( n < 1 ) throw yrmcds::config::bad_config(cmd + std::string(" must be > 0")); return base * static_cast(n); } } // anonymous namespace namespace yrmcds { void counter_config::load(const cybozu::config_parser& cp) { if( cp.exists(COUNTER_ENABLE) ) m_enable = cp.get_as_bool(COUNTER_ENABLE); if( cp.exists(COUNTER_PORT) ) { int n = cp.get_as_int(COUNTER_PORT); if( n < 1 || n > 65535 ) throw config::bad_config("Bad port: " + cp.get(COUNTER_PORT)); m_port = static_cast(n); } if( cp.exists(COUNTER_MAX_CONNECTIONS) ) { int conns = cp.get_as_int(COUNTER_MAX_CONNECTIONS); if( conns < 0 ) throw config::bad_config("counter.max_connections must be >= 0"); m_max_connections = conns; } if( cp.exists(COUNTER_BUCKETS) ) { int buckets = cp.get_as_int(COUNTER_BUCKETS); if( buckets < 1 ) throw config::bad_config("buckets must be > 0"); if( buckets < 10000 ) cybozu::logger::warning() << "Too small bucket count!"; m_buckets = buckets; } if( cp.exists(COUNTER_STAT_INTERVAL) ) { int n = cp.get_as_int(COUNTER_STAT_INTERVAL); if( n < 10 ) throw config::bad_config("counter.stat_interval must be >= 10"); m_stat_interval = n; } } void config::load(const std::string& path) { cybozu::config_parser cp(path); if( cp.exists(VIRTUAL_IP) ) m_vip.parse(cp.get(VIRTUAL_IP)); if( cp.exists(PORT) ) { int n = cp.get_as_int(PORT); if( n < 1 || n > 65535 ) throw bad_config("Bad port: " + cp.get(PORT)); m_port = static_cast(n); } if( cp.exists(REPL_PORT) ) { int n = cp.get_as_int(REPL_PORT); if( n < 1 || n > 65535 ) throw bad_config("Bad repl port: " + cp.get(REPL_PORT)); m_repl_port = static_cast(n); } if( cp.exists(MAX_CONNECTIONS) ) { int conns = cp.get_as_int(MAX_CONNECTIONS); if( conns < 0 ) throw bad_config("max_connections must be >= 0"); m_max_connections = conns; } if( cp.exists(TEMP_DIR) ) { m_tempdir = cp.get(TEMP_DIR); if( ! cybozu::is_dir(m_tempdir) ) throw bad_config("Not a directory: " + m_tempdir); if( ! cybozu::is_writable(m_tempdir) ) throw bad_config("Directory not writable: " + m_tempdir); } if( cp.exists(USER) ) { m_user = cp.get(USER); } if( cp.exists(GROUP) ) { m_group = cp.get(GROUP); } if( cp.exists(LOG_THRESHOLD) ) { auto it = THRESHOLDS.find(cp.get(LOG_THRESHOLD)); if( it == THRESHOLDS.end() ) throw bad_config("Invalid threshold: " + cp.get(LOG_THRESHOLD)); m_threshold = it->second; } if( cp.exists(LOG_FILE) ) { m_logfile = cp.get(LOG_FILE); if( m_logfile.size() == 0 || m_logfile[0] != '/' ) throw bad_config("Invalid log file: " + m_logfile); } if( cp.exists(BUCKETS) ) { int buckets = cp.get_as_int(BUCKETS); if( buckets < 1 ) throw bad_config("buckets must be > 0"); if( buckets < 10000 ) cybozu::logger::warning() << "Too small bucket count!"; m_buckets = buckets; } if( cp.exists(MAX_DATA_SIZE) ) { std::string t = cp.get(MAX_DATA_SIZE); if( t.empty() ) throw bad_config("max_data_size must not be empty"); m_max_data_size = parse_unit(t, MAX_DATA_SIZE); } if( cp.exists(HEAP_DATA_LIMIT) ) { std::string t = cp.get(HEAP_DATA_LIMIT); if( t.empty() ) throw bad_config("heap_data_limit must not be empty"); m_heap_data_limit = parse_unit(t, HEAP_DATA_LIMIT); if( m_heap_data_limit < 4096 ) throw bad_config("too small heap_data_limit"); } if( cp.exists(MEMORY_LIMIT) ) { std::string t = cp.get(MEMORY_LIMIT); if( t.empty() ) throw bad_config("memory_limit must not be empty"); m_memory_limit = parse_unit(t, MEMORY_LIMIT); } if( cp.exists(REPL_BUFSIZE) ) { int bufs = cp.get_as_int(REPL_BUFSIZE); if( bufs < 1 ) throw bad_config("repl_buffer_size must be > 0"); m_repl_bufsize = bufs; } if( cp.exists(SECURE_ERASE) ) { m_secure_erase = cp.get_as_bool(SECURE_ERASE); } if( cp.exists(LOCK_MEMORY) ) { m_lock_memory = cp.get_as_bool(LOCK_MEMORY); } if( cp.exists(WORKERS) ) { int n = cp.get_as_int(WORKERS); if( n < 1 ) throw bad_config("workers must be > 0"); if( n > MAX_WORKERS ) throw bad_config("workers must be <= " + std::to_string(MAX_WORKERS)); m_workers = n; } if( cp.exists(GC_INTERVAL) ) { int n = cp.get_as_int(GC_INTERVAL); if( n < 1 ) throw bad_config("gc_interval must be > 0"); m_gc_interval = n; } if( cp.exists(SLAVE_TIMEOUT) ) { int n = cp.get_as_int(SLAVE_TIMEOUT); if( n < 1 ) throw bad_config("slave_timeout must be > 0"); m_slave_timeout = n; } m_counter_config.load(cp); } config g_config; } // namespace yrmcds yrmcds-1.1.5/src/config.hpp000066400000000000000000000106161262254645600156040ustar00rootroot00000000000000// The server configurations. // (C) 2013 Cybozu. #ifndef YRMCDS_CONFIG_HPP #define YRMCDS_CONFIG_HPP #include "constants.hpp" #include #include #include #include #include namespace yrmcds { // Configurations for counter extension. class counter_config { public: void load(const cybozu::config_parser&); bool enable() const noexcept { return m_enable; } std::uint16_t port() const noexcept { return m_port; } unsigned int max_connections() const noexcept { return m_max_connections; } unsigned int buckets() const noexcept { return m_buckets; } unsigned int stat_interval() const noexcept { return m_stat_interval; } private: bool m_enable = false; std::uint16_t m_port = DEFAULT_COUNTER_PORT; unsigned int m_max_connections = 0; unsigned int m_buckets = DEFAULT_BUCKETS; unsigned int m_stat_interval = DEFAULT_STAT_INTERVAL; }; // Configurations for yrmcds. // // Configurations for yrmcds. // Defaults are configured properly without calling . class config { public: // Setup default configurations. config(): m_vip("127.0.0.1"), m_tempdir(DEFAULT_TMPDIR) { static_assert( sizeof(std::size_t) >= 4, "std::size_t is too small" ); } struct bad_config: public std::runtime_error { bad_config(const std::string& s): std::runtime_error(s) {} }; // Load configurations from `path`. // // This loads configurations from a file at `path`. // This may throw miscellaneous exceptions. void load(const std::string& path); const cybozu::ip_address& vip() const noexcept { return m_vip; } std::uint16_t port() const noexcept { return m_port; } std::uint16_t repl_port() const noexcept { return m_repl_port; } unsigned int max_connections() const noexcept { return m_max_connections; } const std::string& tempdir() const noexcept { return m_tempdir; } const std::string& user() const noexcept { return m_user; } const std::string& group() const noexcept { return m_group; } cybozu::severity threshold() const noexcept { return m_threshold; } const std::string& logfile() const noexcept { return m_logfile; } unsigned int buckets() const noexcept { return m_buckets; } std::size_t max_data_size() const noexcept { return m_max_data_size; } std::size_t heap_data_limit() const noexcept { return m_heap_data_limit; } std::size_t memory_limit() const noexcept { return m_memory_limit; } unsigned int repl_bufsize() const noexcept { return m_repl_bufsize; } bool secure_erase() const noexcept { return m_secure_erase; } bool lock_memory() const noexcept { return m_lock_memory; } unsigned int workers() const noexcept { return m_workers; } unsigned int gc_interval() const noexcept { return m_gc_interval; } unsigned int slave_timeout() const noexcept { return m_slave_timeout; } const counter_config& counter() const noexcept { return m_counter_config; } void set_heap_data_limit(std::size_t new_limit) noexcept { m_heap_data_limit = new_limit; } private: alignas(CACHELINE_SIZE) cybozu::ip_address m_vip; std::uint16_t m_port = DEFAULT_MEMCACHE_PORT; std::uint16_t m_repl_port = DEFAULT_REPL_PORT; unsigned int m_max_connections = 0; std::string m_tempdir; std::string m_user; std::string m_group; cybozu::severity m_threshold = cybozu::severity::info; std::string m_logfile; unsigned int m_buckets = DEFAULT_BUCKETS; std::size_t m_max_data_size = DEFAULT_MAX_DATA_SIZE; std::size_t m_heap_data_limit = DEFAULT_HEAP_DATA_LIMIT; std::size_t m_memory_limit = DEFAULT_MEMORY_LIMIT; unsigned int m_repl_bufsize = DEFAULT_REPL_BUFSIZE; bool m_secure_erase = false; bool m_lock_memory = false; unsigned int m_workers = DEFAULT_WORKER_THREADS; unsigned int m_gc_interval = DEFAULT_GC_INTERVAL; unsigned int m_slave_timeout = DEFAULT_SLAVE_TIMEOUT; counter_config m_counter_config; }; // Global configuration object. extern config g_config; } // namespace yrmcds #endif // YRMCDS_CONFIG_HPP yrmcds-1.1.5/src/constants.hpp000066400000000000000000000027531262254645600163560ustar00rootroot00000000000000// Constants used in yrmcds. // (C) 2013-2015 Cybozu. #ifndef YRMCDS_CONSTANTS_HPP #define YRMCDS_CONSTANTS_HPP #include #include namespace yrmcds { const std::uint16_t DEFAULT_MEMCACHE_PORT = 11211; const std::uint16_t DEFAULT_REPL_PORT = 11213; const std::uint16_t DEFAULT_COUNTER_PORT = 11215; const unsigned int DEFAULT_BUCKETS = 1000000; const std::size_t DEFAULT_MAX_DATA_SIZE = static_cast(1) << 20; const std::size_t DEFAULT_HEAP_DATA_LIMIT= 256 << 10; const std::size_t DEFAULT_MEMORY_LIMIT = static_cast(1) << 30; const unsigned int DEFAULT_REPL_BUFSIZE = 30; const int DEFAULT_WORKER_THREADS = 8; const unsigned int DEFAULT_GC_INTERVAL = 10; const unsigned int DEFAULT_SLAVE_TIMEOUT = 10; const char DEFAULT_TMPDIR[] = "/var/tmp"; const unsigned int DEFAULT_STAT_INTERVAL = 86400; const std::size_t MAX_KEY_LENGTH = 250; // 250 bytes const int MASTER_CHECKS = 50; // wait 50 * 100ms = 5 seconds const int FLUSH_AGE = 10; const std::size_t MAX_RECVSIZE = 2 << 20; // 2 MiB const std::size_t WORKER_BUFSIZE = 5 << 20; // 5 MiB const int MAX_WORKERS = 64; const std::size_t MAX_REQUEST_LENGTH = 30 << 20; // 30 MiB const int MAX_SLAVES = 5; const int MAX_CONSECUTIVE_GCS = 3; const char VERSION[] = "yrmcds version 1.1.5"; } // namespace yrmcds #endif // YRMCDS_CONSTANTS_HPP yrmcds-1.1.5/src/counter/000077500000000000000000000000001262254645600153015ustar00rootroot00000000000000yrmcds-1.1.5/src/counter/counter.cpp000066400000000000000000000176751262254645600175040ustar00rootroot00000000000000// (C) 2014 Cybozu. #include "counter.hpp" #include "stats.hpp" #include #include #include namespace { const std::size_t HEADER_SIZE = 12; const std::uint8_t REQUEST_MAGIC = 0x90; const std::uint8_t RESPONSE_MAGIC = 0x91; const char STATUS_OK[] = "No error"; const char STATUS_NOT_FOUND[] = "Not found"; const char STATUS_INVALID[] = "Invalid arguments"; const char STATUS_RESOURCE_NOT_AVAILABLE[] = "Resource not available"; const char STATUS_NOT_ACQUIRED[] = "Not acquired"; const char STATUS_UNKNOWN_COMMAND[] = "Unknown command"; const char STATUS_OUT_OF_MEMORY[] = "OutOfMemory"; void add_stat(std::vector& body, const std::string& name, const std::string& value) { char buffer[4]; cybozu::hton((uint16_t)name.size(), buffer); cybozu::hton((uint16_t)value.size(), buffer + 2); body.insert(body.end(), buffer, buffer + 4); body.insert(body.end(), name.begin(), name.end()); body.insert(body.end(), value.begin(), value.end()); } template void add_stat_atomic(std::vector& body, const std::string& name, const std::atomic& value) { add_stat(body, name, std::to_string(value.load(std::memory_order_relaxed))); } void add_stat_op(std::vector& body, const std::string& name, yrmcds::counter::command c) { const auto& n = yrmcds::counter::g_stats.ops[(std::uint8_t)c]; add_stat(body, name, std::to_string(n.load(std::memory_order_relaxed))); } } // namespace anonymous namespace yrmcds { namespace counter { void request::parse(const char* p, std::size_t len) noexcept { if( len < HEADER_SIZE ) return; // incomplete std::uint32_t body_length; cybozu::ntoh(p + 4, body_length); if( len < HEADER_SIZE + body_length ) return; // incomplete m_command = (counter::command)*(const uint8_t*)(p + 1); m_flags = *(const uint8_t*)(p + 2); m_body_length = body_length; m_opaque = p + 8; m_request_length = HEADER_SIZE + body_length; const char* body = p + HEADER_SIZE; std::uint16_t name_len; if( *(const std::uint8_t*)p != REQUEST_MAGIC ) return; // invalid switch( m_command ) { case counter::command::Noop: case counter::command::Stats: case counter::command::Dump: // no body break; case counter::command::Get: if( body_length < 2 ) return; // invalid cybozu::ntoh(body, name_len); if( name_len == 0 || 2U + name_len > body_length ) return; // invalid m_name.len = name_len; m_name.p = body + 2; break; case counter::command::Acquire: if( body_length < 10 ) return; // invalid cybozu::ntoh(body, m_resources); if( m_resources == 0 ) return; // invalid cybozu::ntoh(body + 4, m_maximum); if( m_maximum < m_resources ) return; cybozu::ntoh(body + 8, name_len); if( name_len == 0 || 10U + name_len > body_length ) return; // invalid m_name.len = name_len; m_name.p = body + 10; break; case counter::command::Release: if( body_length < 6 ) return; // invalid cybozu::ntoh(body, m_resources); cybozu::ntoh(body + 4, name_len); if( name_len == 0 || 6U + name_len > body_length ) return; // invalid m_name.len = name_len; m_name.p = body + 6; break; default: m_command = counter::command::Unknown; m_status = counter::status::UnknownCommand; return; } m_status = counter::status::OK; } inline void response::fill_header(char* header, counter::status status, std::uint32_t body_length) { header[0] = (char)RESPONSE_MAGIC; header[1] = (char)m_request.command(); header[2] = (char)status; header[3] = 0; cybozu::hton(body_length, header + 4); std::memcpy(header + 8, m_request.opaque(), 4); } void response::success() { char header[HEADER_SIZE]; fill_header(header, counter::status::OK, 0); m_socket.send(header, HEADER_SIZE, true); } void response::error(counter::status status) { switch( status ) { case counter::status::NotFound: send_error(status, STATUS_NOT_FOUND, sizeof(STATUS_NOT_FOUND) - 1); break; case counter::status::Invalid: send_error(status, STATUS_INVALID, sizeof(STATUS_INVALID) - 1); break; case counter::status::ResourceNotAvailable: send_error(status, STATUS_RESOURCE_NOT_AVAILABLE, sizeof(STATUS_RESOURCE_NOT_AVAILABLE) - 1); break; case counter::status::NotAcquired: send_error(status, STATUS_NOT_ACQUIRED, sizeof(STATUS_NOT_ACQUIRED) - 1); break; case counter::status::UnknownCommand: send_error(status, STATUS_UNKNOWN_COMMAND, sizeof(STATUS_UNKNOWN_COMMAND) - 1); break; case counter::status::OutOfMemory: send_error(status, STATUS_OUT_OF_MEMORY, sizeof(STATUS_OUT_OF_MEMORY) - 1); break; default: cybozu::dump_stack(); throw std::logic_error(" invalid status: " + std::to_string((std::uint8_t)status)); } } void response::get(std::uint32_t consumption) { char header[HEADER_SIZE]; char body[4]; fill_header(header, counter::status::OK, sizeof(body)); cybozu::hton(consumption, body); cybozu::tcp_socket::iovec iov[] = { {header, HEADER_SIZE}, {body, sizeof(body)}, }; m_socket.sendv(iov, 2, true); } void response::acquire(std::uint32_t resources) { // internally same as get() get(resources); } void response::dump(const char* name, std::uint16_t name_len, std::uint32_t consumption, std::uint32_t max_conumption) { char header[HEADER_SIZE]; char body[10]; fill_header(header, counter::status::OK, sizeof(body) + name_len); cybozu::hton(consumption, body); cybozu::hton(max_conumption, body + 4); cybozu::hton(name_len, body + 8); cybozu::tcp_socket::iovec iov[] = { {header, HEADER_SIZE}, {body, sizeof(body)}, {name, name_len}, }; m_socket.sendv(iov, 3, false); } void response::stats() { const statistics& s = g_stats; std::vector body; add_stat_atomic(body, "objects", s.objects); add_stat_atomic(body, "total_objects", s.total_objects); add_stat_atomic(body, "used_memory", s.used_memory); add_stat_atomic(body, "conflicts", s.conflicts); add_stat_atomic(body, "gc_count", s.gc_count); add_stat_atomic(body, "last_gc_elapsed", s.last_gc_elapsed); add_stat_atomic(body, "total_gc_elapsed", s.total_gc_elapsed); add_stat_atomic(body, "curr_connections", s.curr_connections); add_stat_atomic(body, "total_connections", s.total_connections); add_stat_op(body, "command:noop", counter::command::Noop); add_stat_op(body, "command:get", counter::command::Get); add_stat_op(body, "command:acquire", counter::command::Acquire); add_stat_op(body, "command:release", counter::command::Release); add_stat_op(body, "command:stats", counter::command::Stats); add_stat_op(body, "command:dump", counter::command::Dump); char header[HEADER_SIZE]; fill_header(header, counter::status::OK, body.size()); cybozu::tcp_socket::iovec iov[] = { {header, HEADER_SIZE}, {body.data(), body.size()}, }; m_socket.sendv(iov, 2, true); } void response::send_error(counter::status status, const char* message, std::size_t length) { char header[HEADER_SIZE]; fill_header(header, status, length); cybozu::tcp_socket::iovec iov[] = { {header, HEADER_SIZE}, {message, length}, }; m_socket.sendv(iov, 2, true); } }} // namespace yrmcds::counter yrmcds-1.1.5/src/counter/counter.hpp000066400000000000000000000060031262254645600174700ustar00rootroot00000000000000// counter server-side protocol. // (C) 2014 Cybozu. #ifndef YRMCDS_COUNTER_COUNTER_HPP #define YRMCDS_COUNTER_COUNTER_HPP #include #include namespace yrmcds { namespace counter { enum class command: std::uint8_t { Noop = 0x00, Get = 0x01, Acquire = 0x02, Release = 0x03, Stats = 0x10, Dump = 0x11, Unknown, END_OF_COMMAND }; enum class status: std::uint8_t { OK = 0x00, NotFound = 0x01, Invalid = 0x04, ResourceNotAvailable = 0x21, NotAcquired = 0x22, UnknownCommand = 0x81, OutOfMemory = 0x82, }; struct string_slice { const char* p = nullptr; std::size_t len = 0; string_slice() noexcept {} string_slice(const char* p, std::size_t len) noexcept: p(p), len(len) {} }; class request final { public: request(const char* p, std::size_t len) { if( p == nullptr ) throw std::logic_error(" `p` must not be nullptr"); if( len == 0 ) throw std::logic_error(" `len` must not be zero"); parse(p, len); } std::size_t length() const noexcept { return m_request_length; } counter::command command() const noexcept { return m_command; } std::uint8_t flags() const noexcept { return m_flags; } std::uint32_t body_length() const noexcept { return m_body_length; } const char* opaque() const noexcept { return m_opaque; } std::uint32_t resources() const noexcept { return m_resources; } std::uint32_t maximum() const noexcept { return m_maximum; } string_slice name() const noexcept { return m_name; } counter::status status() const noexcept { return m_status; } private: void parse(const char* p, std::size_t len) noexcept; std::size_t m_request_length = 0; counter::command m_command = counter::command::Unknown; std::uint8_t m_flags = 0; std::uint32_t m_body_length = 0; const char* m_opaque = nullptr; std::uint32_t m_resources = 0; std::uint32_t m_maximum = 0; string_slice m_name; counter::status m_status = counter::status::Invalid; }; class response final { public: response(cybozu::tcp_socket& socket, const request& request): m_socket(socket), m_request(request) {} void success(); void error(counter::status status); void get(std::uint32_t consumption); void acquire(std::uint32_t resources); void stats(); void dump(const char* name, std::uint16_t name_len, std::uint32_t consumption, std::uint32_t max_conumption); private: void fill_header(char* header, counter::status status, std::uint32_t body_length); void send_error(counter::status status, const char* message, std::size_t length); cybozu::tcp_socket& m_socket; const request& m_request; }; }} // namespace yrmcds::counter #endif // YRMCDS_COUNTER_COUNTER_HPP yrmcds-1.1.5/src/counter/gc.cpp000066400000000000000000000025671262254645600164100ustar00rootroot00000000000000// (C) 2014 Cybozu. #include "gc.hpp" #include "stats.hpp" namespace yrmcds { namespace counter { void gc_thread::run() { using namespace std::chrono; auto t1 = steady_clock::now(); gc(); g_stats.objects.store(m_objects, std::memory_order_relaxed); g_stats.used_memory.store(m_used_memory, std::memory_order_relaxed); g_stats.conflicts.store(m_conflicts, std::memory_order_relaxed); g_stats.gc_count.fetch_add(1, std::memory_order_relaxed); auto t2 = steady_clock::now(); std::uint64_t us = (std::uint64_t)duration_cast(t2-t1).count(); g_stats.last_gc_elapsed.store(us, std::memory_order_relaxed); g_stats.total_gc_elapsed.fetch_add(us, std::memory_order_relaxed); cybozu::logger::debug() << "Semaphore GC end: elapsed=" << us << "us, survived=" << m_objects; } void gc_thread::gc() { auto pred = [this](const cybozu::hash_key& k, object& obj) -> bool { if( obj.deletable() ) return true; ++m_objects; ++m_objects_in_bucket; if( m_objects_in_bucket == 2 ) ++m_conflicts; m_used_memory += sizeof(cybozu::hash_key) + k.length() + sizeof(object); return false; }; for( auto it = m_hash.begin(); it != m_hash.end(); ++it ) { m_objects_in_bucket = 0; it->gc(pred); } } }} // namespace yrmcds::counter yrmcds-1.1.5/src/counter/gc.hpp000066400000000000000000000016011262254645600164010ustar00rootroot00000000000000// Garbage object collection. // (C) 2014 Cybozu. #ifndef YRMCDS_COUNTER_GC_HPP #define YRMCDS_COUNTER_GC_HPP #include "object.hpp" #include #include namespace yrmcds { namespace counter { class gc_thread final: public cybozu::thread_base { public: explicit gc_thread(cybozu::hash_map& hash): m_hash(hash) {} gc_thread(const gc_thread&) = delete; gc_thread(gc_thread&&) = delete; gc_thread& operator=(const gc_thread&) = delete; gc_thread& operator=(gc_thread&&) = delete; ~gc_thread() { m_thread.join(); } void run(); private: void gc(); cybozu::hash_map& m_hash; std::uint32_t m_objects = 0; std::uint32_t m_conflicts = 0; std::uint64_t m_used_memory = 0; int m_objects_in_bucket = 0; }; }} // namespace yrmcds::counter #endif // YRMCDS_COUNTER_GC_HPP yrmcds-1.1.5/src/counter/handler.cpp000066400000000000000000000045401262254645600174250ustar00rootroot00000000000000// (C) 2014 Cybozu. #include "handler.hpp" #include "../config.hpp" #include "stats.hpp" #include "sockets.hpp" #include namespace { const enum std::memory_order relaxed = std::memory_order_relaxed; } namespace yrmcds { namespace counter { handler::handler(const std::function& finder, cybozu::reactor& reactor) : m_finder(finder), m_reactor(reactor), m_hash(g_config.counter().buckets()) { } bool handler::gc_ready() { std::time_t now = g_current_time.load(relaxed); if( m_gc_thread.get() != nullptr ) { if( ! m_gc_thread->done() ) return false; m_last_gc = now; m_gc_thread = nullptr; // join } unsigned int interval = g_config.counter().stat_interval(); std::time_t boundary = (now / interval) * interval; return m_last_gc < boundary; } void handler::on_master_start() { cybozu::tcp_server_socket::wrapper w = [this](int s, const cybozu::ip_address&) { return make_counter_socket(s); }; std::unique_ptr ss = cybozu::make_server_socket(nullptr, g_config.counter().port(), w); m_reactor.add_resource(std::move(ss), cybozu::reactor::EVENT_IN); } std::unique_ptr handler::make_counter_socket(int s) { unsigned int mc = g_config.counter().max_connections(); if( mc != 0 && g_stats.curr_connections.load(relaxed) >= mc ) return nullptr; return std::unique_ptr( new counter_socket(s, m_finder, m_hash) ); } void handler::on_master_interval() { if( gc_ready() ) { m_gc_thread = std::unique_ptr(new gc_thread(m_hash)); m_gc_thread->start(); } } void handler::on_master_end() { m_gc_thread = nullptr; // join } void handler::dump_stats() { std::uint64_t ops = 0; for( auto& v: g_stats.ops ) { ops += v.load(relaxed); } using logger = cybozu::logger; logger::info() << "counter: " << g_stats.objects.load(relaxed) << " objects, " << g_stats.curr_connections.load(relaxed) << " clients, " << ops << " total ops."; } void handler::clear() { for( auto& bucket: m_hash ) bucket.clear_nolock(); g_stats.total_objects.store(0, relaxed); } }} // namespace yrmcds::counter yrmcds-1.1.5/src/counter/handler.hpp000066400000000000000000000021241262254645600174260ustar00rootroot00000000000000// Semaphore protocol logics and data structures // (C) 2014 Cybozu. #ifndef YRMCDS_COUNTER_HANDLER_HPP #define YRMCDS_COUNTER_HANDLER_HPP #include "../handler.hpp" #include "gc.hpp" #include "object.hpp" #include #include #include #include #include namespace yrmcds { namespace counter { class handler: public protocol_handler { public: handler(const std::function& finder, cybozu::reactor& reactor); virtual void on_master_start() override; virtual void on_master_interval() override; virtual void on_master_end() override; virtual void dump_stats() override; virtual void clear() override; private: bool gc_ready(); std::unique_ptr make_counter_socket(int s); std::function m_finder; cybozu::reactor& m_reactor; cybozu::hash_map m_hash; std::time_t m_last_gc = 0; std::unique_ptr m_gc_thread = nullptr; }; }} // namespace yrmcds::counter #endif // YRMCDS_COUNTER_HANDLER_HPP yrmcds-1.1.5/src/counter/object.hpp000066400000000000000000000042351262254645600172640ustar00rootroot00000000000000// The counter object. // (C) 2014 Cybozu. #ifndef YRMCDS_COUNTER_OBJECT_HPP #define YRMCDS_COUNTER_OBJECT_HPP #include "../config.hpp" #include "../global.hpp" #include "stats.hpp" #include #include namespace yrmcds { namespace counter { class object { public: explicit object(std::uint32_t consumption): m_consumption(consumption), m_max_consumption(consumption), m_last_updated(now()) { g_stats.total_objects.fetch_add(1, std::memory_order_relaxed); } bool acquire(std::uint32_t resources, std::uint32_t maximum) noexcept { if( m_consumption + resources > maximum ) return false; std::time_t t = now(); if( m_last_updated < nearest_boundary(t) ) m_max_consumption = m_consumption + resources; else m_max_consumption = std::max(m_max_consumption, m_consumption + resources); m_last_updated = t; m_consumption += resources; return true; } bool release(std::uint32_t resources) noexcept { if( resources > m_consumption ) return false; std::time_t t = now(); if( m_last_updated < nearest_boundary(t) ) m_max_consumption = m_consumption; m_last_updated = t; m_consumption -= resources; return true; } std::uint32_t consumption() const noexcept { return m_consumption; } std::uint32_t max_consumption() const noexcept { if( m_last_updated < nearest_boundary(now()) ) return m_consumption; return m_max_consumption; } bool deletable() const noexcept { return m_consumption == 0 && m_last_updated < nearest_boundary(now()); } private: static std::time_t now() { return g_current_time.load(std::memory_order_relaxed); } static std::time_t nearest_boundary(std::time_t t) { unsigned int interval = g_config.counter().stat_interval(); return (t / interval) * interval; } std::uint32_t m_consumption; std::uint32_t m_max_consumption; std::time_t m_last_updated; }; }} // namespace yrmcds::counter #endif // YRMCDS_COUNTER_OBJECT_HPP yrmcds-1.1.5/src/counter/sockets.cpp000066400000000000000000000172101262254645600174610ustar00rootroot00000000000000// (C) 2014 Cybozu. #include "sockets.hpp" #include "object.hpp" #include #include #include #include #include namespace yrmcds { namespace counter { counter_socket::counter_socket(int fd, const std::function& finder, cybozu::hash_map& hash) : cybozu::tcp_socket(fd), m_busy(false), m_finder(finder), m_hash(hash), m_pending(0) { g_stats.curr_connections.fetch_add(1); g_stats.total_connections.fetch_add(1); m_recvjob = [this](cybozu::dynbuf& buf) { // load pending data if( ! m_pending.empty() ) { buf.append(m_pending.data(), m_pending.size()); m_pending.reset(); } while( true ) { char* p = buf.prepare(MAX_RECVSIZE); ssize_t n = ::recv(m_fd, p, MAX_RECVSIZE, 0); if( n == -1 ) { if( errno == EAGAIN || errno == EWOULDBLOCK ) break; if( errno == EINTR ) continue; if( errno == ECONNRESET ) { buf.reset(); release_all(); invalidate_and_close(); break; } cybozu::throw_unix_error(errno, "recv"); } if( n == 0 ) { buf.reset(); release_all(); invalidate_and_close(); break; } // if (n != -1) && (n != 0) buf.consume(n); const char* head = buf.data(); std::size_t len = buf.size(); while( len > 0 ) { counter::request parser(head, len); std::size_t c = parser.length(); if( c == 0 ) break; head += c; len -= c; execute(parser); } if( len > MAX_REQUEST_LENGTH ) { cybozu::logger::warning() << "denied too large request of " << len << " bytes."; buf.reset(); release_all(); invalidate_and_close(); break; } buf.erase(head - buf.data()); } // recv returns EAGAIN, or some error happens. if( buf.size() > 0 ) m_pending.append(buf.data(), buf.size()); m_busy.store(false, std::memory_order_release); }; m_sendjob = [this](cybozu::dynbuf& buf) { if( ! write_pending_data() ) invalidate_and_close(); }; } counter_socket::~counter_socket() { // the destructor is the safe place to release remaining resources release_all(); } bool counter_socket::on_readable() { if( m_busy.load(std::memory_order_acquire) ) { m_reactor->add_readable(*this); return true; } // find an idle worker. cybozu::worker* w = m_finder(); if( w == nullptr ) { m_reactor->add_readable(*this); return true; } m_busy.store(true, std::memory_order_release); w->post_job(m_recvjob); return true; } bool counter_socket::on_writable() { cybozu::worker* w = m_finder(); if( w == nullptr ) { // if there is no idle worker, fallback to the default. return cybozu::tcp_socket::on_writable(); } w->post_job(m_sendjob); return true; } void counter_socket::execute(const counter::request& cmd) { counter::response r(*this, cmd); if( cmd.status() != counter::status::OK ) { r.error( cmd.status() ); return; } g_stats.ops[(std::size_t)cmd.command()].fetch_add(1); switch( cmd.command() ) { case counter::command::Noop: r.success(); break; case counter::command::Get: cmd_get(cmd, r); break; case counter::command::Acquire: cmd_acquire(cmd, r); break; case counter::command::Release: cmd_release(cmd, r); break; case counter::command::Stats: r.stats(); break; case counter::command::Dump: cmd_dump(r); break; default: cybozu::logger::info() << "not implemented"; r.error( counter::status::UnknownCommand ); } } void counter_socket::cmd_get(const counter::request& cmd, counter::response& r) { auto h = [this,&cmd,&r](const cybozu::hash_key& k, object& obj) -> bool { r.get(obj.consumption()); return true; }; if( ! m_hash.apply(cybozu::hash_key(cmd.name().p, cmd.name().len), h, nullptr) ) r.error( counter::status::NotFound ); } void counter_socket::cmd_acquire(const counter::request& cmd, counter::response& r) { uint32_t resources = cmd.resources(); uint32_t maximum = cmd.maximum(); auto h = [this,resources,maximum,&r](const cybozu::hash_key& k, object& obj) -> bool { if( ! obj.acquire(resources, maximum) ) { r.error( counter::status::ResourceNotAvailable ); return true; } on_acquire(k, resources); r.acquire(resources); return true; }; auto c = [this,resources,&r](const cybozu::hash_key& k) -> object { on_acquire(k, resources); r.acquire(resources); return object(resources); }; m_hash.apply(cybozu::hash_key(cmd.name().p, cmd.name().len), h, c); } void counter_socket::cmd_release(const counter::request& cmd, counter::response& r) { uint32_t resources = cmd.resources(); auto h = [this,resources,&r](const cybozu::hash_key& k, object& obj) -> bool { if( ! on_release(k, resources) ) { r.error( counter::status::NotAcquired ); return true; } if( ! obj.release(resources) ) { cybozu::dump_stack(); throw std::logic_error(" bug"); } r.success(); return true; }; if( ! m_hash.apply(cybozu::hash_key(cmd.name().p, cmd.name().len), h, nullptr) ) r.error( counter::status::NotFound ); } void counter_socket::cmd_dump(counter::response& r) { auto pred = [this,&r](const cybozu::hash_key& k, object& obj) { r.dump(k.data(), k.length(), obj.consumption(), obj.max_consumption()); }; m_hash.foreach(pred); r.success(); } void counter_socket::on_acquire(const cybozu::hash_key& k, std::uint32_t resources) { auto it = m_acquired_resources.find(&k); if( it != m_acquired_resources.end() ) { it->second += resources; return; } m_acquired_resources.emplace(&k, resources); } bool counter_socket::on_release(const cybozu::hash_key& k, std::uint32_t resources) { auto it = m_acquired_resources.find(&k); if( it == m_acquired_resources.end() ) return false; if( it->second < resources ) return false; if( it->second == resources ) { m_acquired_resources.erase(it); return true; } it->second -= resources; return true; } void counter_socket::release_all() { for( auto& res: m_acquired_resources ) { uint32_t count = res.second; auto h = [count](const cybozu::hash_key&, object& obj) -> bool { if( ! obj.release(count) ) { cybozu::dump_stack(); throw std::logic_error(" release failed"); } return true; }; if( ! m_hash.apply(*res.first, h, nullptr) ) { cybozu::dump_stack(); throw std::logic_error(" not found: " + res.first->str()); } } m_acquired_resources.clear(); } }} // namespace yrmcds::counter yrmcds-1.1.5/src/counter/sockets.hpp000066400000000000000000000043061262254645600174700ustar00rootroot00000000000000// Defines counter sockets for yrmcds. // (C) 2014 Cybozu. #ifndef YRMCDS_COUNTER_SOCKETS_HPP #define YRMCDS_COUNTER_SOCKETS_HPP #include "../constants.hpp" #include "object.hpp" #include "counter.hpp" #include "stats.hpp" #include #include #include #include #include #include #include #include namespace yrmcds { namespace counter { class counter_socket: public cybozu::tcp_socket { public: counter_socket(int fd, const std::function& finder, cybozu::hash_map& hash); virtual ~counter_socket(); void execute(const counter::request& cmd); private: alignas(CACHELINE_SIZE) std::atomic m_busy; const std::function& m_finder; cybozu::hash_map& m_hash; cybozu::dynbuf m_pending; std::function m_recvjob; std::function m_sendjob; std::unordered_map m_acquired_resources; virtual void on_invalidate() override final { g_stats.curr_connections.fetch_sub(1); cybozu::tcp_socket::on_invalidate(); } bool on_readable() override; bool on_writable() override; void cmd_get(const counter::request& cmd, counter::response& r); void cmd_acquire(const counter::request& cmd, counter::response& r); void cmd_release(const counter::request& cmd, counter::response& r); void cmd_dump(counter::response& r); // `on_acquire` augments the number of resources this connection has acquired // recorded in `m_acquired_resources`. void on_acquire(const cybozu::hash_key& k, std::uint32_t resources); // `on_release` reduces the number of resources this connection has acquired // recorded in `m_acquired_resources`. // When this connection has not acquired the specified number of resources, // returns false. Otherwise, returns true. bool on_release(const cybozu::hash_key& k, std::uint32_t resources); void release_all(); }; }} // namespace yrmdcs::counter #endif // YRMCDS_COUNTER_SOCKETS_HPP yrmcds-1.1.5/src/counter/stats.cpp000066400000000000000000000006401262254645600171430ustar00rootroot00000000000000// (C) 2014 Cybozu. #include "stats.hpp" namespace yrmcds { namespace counter { statistics g_stats; void statistics::reset() noexcept { objects = 0; used_memory = 0; conflicts = 0; gc_count = 0; last_gc_elapsed = 0; total_gc_elapsed = 0; total_objects = 0; curr_connections = 0; total_connections = 0; for( auto& v: ops ) v = 0; } }} // namespace yrmcds::counter yrmcds-1.1.5/src/counter/stats.hpp000066400000000000000000000020631262254645600171510ustar00rootroot00000000000000// Atomic counters for statistics. // (C) 2014 Cybozu. #ifndef YRMCDS_COUNTER_STATS_HPP #define YRMCDS_COUNTER_STATS_HPP #include "counter.hpp" #include "../constants.hpp" #include #include namespace yrmcds { namespace counter { struct statistics { statistics() { reset(); } void reset() noexcept; alignas(CACHELINE_SIZE) std::atomic objects; std::atomic used_memory; std::atomic conflicts; std::atomic gc_count; std::atomic last_gc_elapsed; // micro seconds std::atomic total_gc_elapsed; // micro seconds alignas(CACHELINE_SIZE) std::atomic total_objects; alignas(CACHELINE_SIZE) std::atomic curr_connections; std::atomic total_connections; alignas(CACHELINE_SIZE) std::atomic ops[(std::size_t)command::END_OF_COMMAND]; }; extern statistics g_stats; }} // namespace yrmcds::counter #endif // YRMCDS_COUNTER_STATS_HPP yrmcds-1.1.5/src/global.cpp000066400000000000000000000002241262254645600155640ustar00rootroot00000000000000// (C) 2014 Cybozu. #include "global.hpp" namespace yrmcds { std::atomic g_current_time(std::time(nullptr)); } // namespace yrmcds yrmcds-1.1.5/src/global.hpp000066400000000000000000000003731262254645600155760ustar00rootroot00000000000000// Global variables. // (C) 2014 Cybozu. #ifndef YRMCDS_GLOBAL_HPP #define YRMCDS_GLOBAL_HPP #include #include namespace yrmcds { extern std::atomic g_current_time; } // namespace yrmcds #endif // YRMCDS_GLOBAL_HPP yrmcds-1.1.5/src/handler.hpp000066400000000000000000000030201262254645600157430ustar00rootroot00000000000000// Protocol-sepcific logics and data structures. // (C) 2014 Cybozu. #ifndef YRMCDS_HANDLER_HPP #define YRMCDS_HANDLER_HPP #include namespace yrmcds { // An interface for protocol-specific logics. class protocol_handler { public: virtual ~protocol_handler() {} // Called when the server starts. virtual void on_start() {} // Called when the server exits. virtual void on_exit() {} // Called when the server enters the master mode. virtual void on_master_start() {} // Called when the intervals of the reactor loop. virtual void on_master_interval() {} // Called when the server leaves the master mode. virtual void on_master_end() {} // Called when the server enters the slave mode. // // If this function succeeded, returns `true`. // Otherwise, returns `false`. virtual bool on_slave_start() { return true; } // Called when the intervals of the reactor loop. virtual void on_slave_interval() {} // Called when the server leaves the slave mode. virtual void on_slave_end() {} // Called to dump the statistics. // // Implementations should use `cybozu::logger::info()` to emit stats. virtual void dump_stats() = 0; // Called when the server discards all stored data. virtual void clear() {} // If this protocol handler is ready for the reactor GC, // returns true. Otherwise, return false. virtual bool reactor_gc_ready() const { return true; } }; } // namespace yrmcds #endif // YRMCDS_HANDLER_HPP yrmcds-1.1.5/src/main.cpp000066400000000000000000000116131262254645600152540ustar00rootroot00000000000000// The entry point of yrmcdsd. // (C) 2013 Cybozu. #include "config.hpp" #include "constants.hpp" #include "server.hpp" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #define QUOTE(str) #str #define EXPAND_AND_QUOTE(str) QUOTE(str) namespace { #include "../COPYING.hpp" void print_help() { std::cout << "Usage: yrmcdsd [-v] [-h] [-f FILE]" << std::endl; } void print_version() { std::cout << yrmcds::VERSION << std::endl << std::endl << COPYING; } void seed_siphash() { union { char key[16]; std::uint64_t ikey[2]; } k; std::random_device rd; std::uniform_int_distribution dis; k.ikey[0] = dis(rd); k.ikey[1] = dis(rd); cybozu::siphash24_seed(k.key); } bool load_config(const std::vector& args) { auto it = std::find(args.begin(), args.end(), "-f"); using yrmcds::g_config; if( it != args.end() ) { ++it; if( it == args.end() ) { std::cerr << "missing filename after -f" << std::endl; return false; } g_config.load(*it); } else { std::string config_path = EXPAND_AND_QUOTE(DEFAULT_CONFIG); struct stat st; if( cybozu::get_stat(config_path, st) ) g_config.load( config_path ); } if( g_config.secure_erase() && (g_config.max_data_size() > g_config.heap_data_limit()) ) { std::cerr << "WARNING: " << std::endl << " \"secure_erase\" is enabled but your data may be written out to disk." << std::endl << " Consider setting \"max_data_size\" equal to \"heap_data_limit\"." << std::endl; } return true; } } // anonymous namespace int main(int argc, char** argv) { std::vector args; for( int i = 1; i < argc; ++i ) args.push_back(argv[i]); if( std::find(args.begin(), args.end(), "-h") != args.end() ) { print_help(); return 0; } if( std::find(args.begin(), args.end(), "-v") != args.end() ) { print_version(); return 0; } using cybozu::logger; seed_siphash(); try { if( ! load_config(args) ) return 1; if( yrmcds::g_config.lock_memory() ) { if( ::mlockall( MCL_CURRENT | MCL_FUTURE ) == -1 ) cybozu::throw_unix_error(errno, "mlockall"); } // first, set group id (as root) const std::string& g = yrmcds::g_config.group(); if( ! g.empty() ) { if( getuid() != 0 ) { std::cerr << "Ignored group configuration." << std::endl; } else { struct group* grp = getgrnam(g.c_str()); if( grp == nullptr ) { std::cerr << "No such group: " << g << std::endl; return 1; } if( setgid(grp->gr_gid) == -1 ) { std::cerr << "Failed to set group id!" << std::endl; return 1; } } } const std::string& u = yrmcds::g_config.user(); if( ! u.empty() ) { if( getuid() != 0 ) { std::cerr << "Ignored user configuration." << std::endl; } else { struct passwd* p = getpwnam(u.c_str()); if( p == nullptr ) { std::cerr << "No such user: " << u << std::endl; return 1; } if( setuid(p->pw_uid) == -1 ) { std::cerr << "Failed to set user id!" << std::endl; return 1; } } } // make it core dumpable if( prctl(PR_SET_DUMPABLE, 1) == -1 ) { std::cerr << "WARNING: failed to enable core dump!" << std::endl; } // setup logger logger::set_threshold(yrmcds::g_config.threshold()); if( ! yrmcds::g_config.logfile().empty() ) logger::instance().open(yrmcds::g_config.logfile()); yrmcds::server().serve(); } catch( const std::system_error& e ) { cybozu::demangler t( typeid(e).name() ); logger::error() << "[" << t.name() << "] (" << e.code() << ") " << e.what(); std::cerr << "Exception [" << t.name() << "] (" << e.code() << ") " << e.what() << std::endl; return 1; } catch( const std::exception& e ) { cybozu::demangler t( typeid(e).name() ); logger::error() << "[" << t.name() << "] " << e.what(); std::cerr << "Exception [" << t.name() << "] " << e.what() << std::endl; return 1; } return 0; } yrmcds-1.1.5/src/memcache/000077500000000000000000000000001262254645600153645ustar00rootroot00000000000000yrmcds-1.1.5/src/memcache/gc.cpp000066400000000000000000000123051262254645600164620ustar00rootroot00000000000000// (C) 2013 Cybozu. #include "../config.hpp" #include "gc.hpp" #include "replication.hpp" #include "stats.hpp" #include #include namespace yrmcds { namespace memcache { void gc_thread::run() { if( ! cybozu::has_ip_address(g_config.vip()) ) { cybozu::logger::error() << "VIP has been lost. Exiting quickly..."; std::quick_exit(2); } using namespace std::chrono; auto t1 = steady_clock::now(); gc(); if( ! m_new_slaves.empty() ) cybozu::logger::info() << "Initial replication completed for " << m_new_slaves.size() << " new slave(s)."; g_stats.objects.store(m_objects, std::memory_order_relaxed); g_stats.objects_under_1k.store(m_objects_under_1k, std::memory_order_relaxed); g_stats.objects_under_4k.store(m_objects_under_4k, std::memory_order_relaxed); g_stats.objects_under_16k.store(m_objects_under_16k, std::memory_order_relaxed); g_stats.objects_under_64k.store(m_objects_under_64k, std::memory_order_relaxed); g_stats.objects_under_256k.store(m_objects_under_256k, std::memory_order_relaxed); g_stats.objects_under_1m.store(m_objects_under_1m, std::memory_order_relaxed); g_stats.objects_under_4m.store(m_objects_under_4m, std::memory_order_relaxed); g_stats.objects_huge.store(m_objects_huge, std::memory_order_relaxed); g_stats.used_memory.store(m_used_memory, std::memory_order_relaxed); g_stats.conflicts.store(m_conflicts, std::memory_order_relaxed); g_stats.gc_count.fetch_add(1, std::memory_order_relaxed); g_stats.oldest_age.store(m_oldest_age, std::memory_order_relaxed); g_stats.largest_object_size.store(m_largest_object_size, std::memory_order_relaxed); g_stats.last_expirations.store(m_last_expirations, std::memory_order_relaxed); g_stats.last_evictions.store(m_last_evictions, std::memory_order_relaxed); g_stats.total_evictions.fetch_add(m_last_evictions, std::memory_order_relaxed); auto t2 = steady_clock::now(); std::uint64_t us = static_cast( duration_cast(t2-t1).count() ); g_stats.last_gc_elapsed.store(us, std::memory_order_relaxed); g_stats.total_gc_elapsed.fetch_add(us, std::memory_order_relaxed); cybozu::logger::debug() << "GC end: elapsed=" << us << "us, expired=" << m_last_expirations << ", evicted=" << m_last_evictions << ", survived=" << m_objects; } void gc_thread::gc() { std::time_t t = g_stats.flush_time.load(); bool flush = (t != 0) && (std::time(nullptr) >= t); unsigned int evict_age = 0; if( (! flush) && (g_stats.used_memory.load(std::memory_order_relaxed) > g_config.memory_limit()) ) { unsigned int oldest_age = g_stats.oldest_age.load(std::memory_order_relaxed); unsigned int one_hour = 3600U / g_config.gc_interval() + 1; if( oldest_age < (one_hour * 2) ) { evict_age = std::max(1U, oldest_age / 2); } else { evict_age = oldest_age - one_hour; } cybozu::logger::warning() << "Evicting object of " << evict_age << " gc old"; } auto pred = [this,flush,evict_age](const cybozu::hash_key& k, object& obj) ->bool { if( flush && (! obj.locked()) ) { if( ! m_slaves.empty() ) repl_delete(m_slaves, k); return true; } if( evict_age > 0 && obj.age() >= evict_age && (! obj.locked()) ) { ++ m_last_evictions; if( ! m_slaves.empty() ) repl_delete(m_slaves, k); return true; } if( obj.expired() ) { ++ m_last_expirations; if( ! m_slaves.empty() ) repl_delete(m_slaves, k); return true; } obj.survive(m_flushers); if( ++m_objects_in_bucket == 2 ) ++ m_conflicts; ++ m_objects; std::size_t size = obj.size(); if( size < 1024 ) { ++ m_objects_under_1k; } else if( size < 4096 ) { ++ m_objects_under_4k; } else if( size < (16 <<10) ) { ++ m_objects_under_16k; } else if( size < (64 <<10) ) { ++ m_objects_under_64k; } else if( size < (256 <<10) ) { ++ m_objects_under_256k; } else if( size < (1 <<20) ) { ++ m_objects_under_1m; } else if( size < (4 <<20) ) { ++ m_objects_under_4m; } else { ++ m_objects_huge; } m_used_memory += sizeof(cybozu::hash_key) + k.length() + sizeof(object); if( g_config.heap_data_limit() >= obj.size() ) m_used_memory += obj.size(); m_oldest_age = std::max(m_oldest_age, obj.age()); m_largest_object_size = std::max(m_largest_object_size, obj.size()); if( ! m_new_slaves.empty() ) repl_object(m_new_slaves, k, obj, false); return false; }; for( auto it = m_hash.begin(); it != m_hash.end(); ++it ) { m_objects_in_bucket = 0; it->gc(pred); m_flushers.clear(); } if( flush ) g_stats.flush_time.store(0); } }} // namespace yrmcds::memcache yrmcds-1.1.5/src/memcache/gc.hpp000066400000000000000000000037731262254645600165000ustar00rootroot00000000000000// Garbage object collection. // (C) 2013 Cybozu. #ifndef YRMCDS_MEMCACHE_GC_HPP #define YRMCDS_MEMCACHE_GC_HPP #include "object.hpp" #include "sockets.hpp" #include "stats.hpp" #include #include #include #include #include #include #include #include namespace yrmcds { namespace memcache { class gc_thread final: public cybozu::thread_base { public: gc_thread(cybozu::hash_map& m, const std::vector& slaves, const std::vector& new_slaves): m_hash(m), m_slaves(slaves), m_new_slaves(new_slaves) { for( repl_socket* s: new_slaves ) { if( std::find(slaves.begin(), slaves.end(), s) == slaves.end() ) m_slaves.push_back(s); } m_flushers.reserve(10); } gc_thread(const gc_thread&) = delete; gc_thread& operator=(const gc_thread&) = delete; gc_thread(gc_thread&&) = delete; gc_thread& operator=(gc_thread&&) = delete; ~gc_thread() { m_thread.join(); } void run(); private: void gc(); cybozu::hash_map& m_hash; std::vector m_slaves; std::vector m_new_slaves; std::uint32_t m_objects = 0; std::uint32_t m_objects_under_1k = 0; std::uint32_t m_objects_under_4k = 0; std::uint32_t m_objects_under_16k = 0; std::uint32_t m_objects_under_64k = 0; std::uint32_t m_objects_under_256k = 0; std::uint32_t m_objects_under_1m = 0; std::uint32_t m_objects_under_4m = 0; std::uint32_t m_objects_huge = 0; std::size_t m_used_memory = 0; std::uint32_t m_conflicts = 0; std::uint32_t m_oldest_age = 0; std::size_t m_largest_object_size = 0; std::uint32_t m_last_expirations = 0; std::uint32_t m_last_evictions = 0; int m_objects_in_bucket = 0; std::vector m_flushers; }; }} // namespace yrmcds::memcache #endif // YRMCDS_MEMCACHE_GC_HPP yrmcds-1.1.5/src/memcache/handler.cpp000066400000000000000000000141131262254645600175050ustar00rootroot00000000000000// (C) 2014 Cybozu. #include "handler.hpp" #include "../constants.hpp" #include "../global.hpp" #include #include namespace { const enum std::memory_order relaxed = std::memory_order_relaxed; } namespace yrmcds { namespace memcache { handler::handler(const std::function& finder, cybozu::reactor& reactor, syncer& sync) : m_finder(finder), m_reactor(reactor), m_syncer(sync), m_hash(g_config.buckets()) { m_slaves.reserve(MAX_SLAVES); m_new_slaves.reserve(MAX_SLAVES); } bool handler::gc_ready(std::time_t now) { if( m_gc_thread.get() != nullptr ) { if( ! m_gc_thread->done() ) return false; m_last_gc = now; m_gc_thread = nullptr; } std::time_t t = g_stats.flush_time.load(relaxed); if( t != 0 && now >= t ) return true; // run GC immediately if the heap is over used. if( g_stats.used_memory.load(relaxed) > g_config.memory_limit() ) return true; // Run GC when there are new slaves. // In case there are unstable slaves that try to connect to // the master too frequently, the number of consecutive GCs is limited. if( ! m_new_slaves.empty() && m_consecutive_gcs < MAX_CONSECUTIVE_GCS ) { ++m_consecutive_gcs; return true; } if( now > (m_last_gc + g_config.gc_interval()) ) { m_consecutive_gcs = 0; return true; } return false; } bool handler::reactor_gc_ready() const { if( m_gc_thread.get() != nullptr ) return false; return m_new_slaves.empty(); } void handler::on_start() { using cybozu::make_server_socket; cybozu::tcp_server_socket::wrapper w = [this](int s, const cybozu::ip_address&) { return make_memcache_socket(s); }; m_reactor.add_resource(make_server_socket(NULL, g_config.port(), w), cybozu::reactor::EVENT_IN); } void handler::on_master_start() { m_is_slave = false; cybozu::tcp_server_socket::wrapper w = [this](int s, const cybozu::ip_address&) { return make_repl_socket(s); }; m_reactor.add_resource(make_server_socket(NULL, g_config.repl_port(), w), cybozu::reactor::EVENT_IN); } void handler::on_master_interval() { for( auto it = m_slaves.begin(); it != m_slaves.end(); ) { repl_socket* slave = *it; if( ! slave->valid() ) { it = m_slaves.erase(it); continue; } if( slave->timed_out() ) { std::string addr = "unknown address"; try { addr = cybozu::get_peer_ip_address(slave->fileno()).str(); } catch (...) { // ignore errors } cybozu::logger::info() << "No heartbeats from a slave (" << addr << "). Close the replication socket."; // close the socket and release resources if( ! slave->invalidate() ) m_reactor.remove_resource(*slave); it = m_slaves.erase(it); continue; } ++it; } if( gc_ready(g_current_time.load(relaxed)) ) { m_gc_thread = std::unique_ptr( new gc_thread(m_hash, m_slaves, m_new_slaves)); m_new_slaves.clear(); m_gc_thread->start(); } } void handler::on_master_end() { if( m_gc_thread.get() != nullptr ) m_gc_thread = nullptr; // join } bool handler::on_slave_start() { int fd = cybozu::tcp_connect(g_config.vip().str().c_str(), g_config.repl_port()); if( fd == -1 ) { m_reactor.run_once(); return false; } m_repl_client_socket = new repl_client_socket(fd, m_hash); m_reactor.add_resource(std::unique_ptr(m_repl_client_socket), cybozu::reactor::EVENT_IN|cybozu::reactor::EVENT_OUT ); return true; } void handler::on_slave_interval() { // ping to the master char c = '\0'; m_repl_client_socket->send(&c, sizeof(c), true); } void handler::on_slave_end() { if( m_repl_client_socket->valid() ) m_reactor.remove_resource(*m_repl_client_socket); } void handler::dump_stats() { using logger = cybozu::logger; if( m_is_slave ) { logger::info() << "memcache replication stats: " << g_stats.repl_created << " created, " << g_stats.repl_updated << " updated, " << g_stats.repl_removed << " removed."; return; } // master std::uint64_t ops = 0; for( auto& v: g_stats.text_ops ) { ops += v.load(relaxed); } for( auto& v: g_stats.bin_ops ) { ops += v.load(relaxed); } logger::info() << "memcache master: " << m_slaves.size() << " slaves, " << g_stats.objects.load(relaxed) << " objects, " << g_stats.curr_connections.load(relaxed) << " clients, " << ops << " total ops."; } void handler::clear() { for( auto& bucket: m_hash ) bucket.clear_nolock(); g_stats.total_objects.store(0, relaxed); g_stats.repl_created = 0; g_stats.repl_updated = 0; g_stats.repl_removed = 0; } std::unique_ptr handler::make_memcache_socket(int s) { if( m_is_slave ) return nullptr; unsigned int mc = g_config.max_connections(); if( mc != 0 && (g_stats.curr_connections.load(relaxed) >= mc) ) return nullptr; return std::unique_ptr( new memcache_socket(s, m_finder, m_hash, m_slaves) ); } std::unique_ptr handler::make_repl_socket(int s) { if( m_slaves.size() == MAX_SLAVES ) return nullptr; std::unique_ptr t( new repl_socket(s, g_config.repl_bufsize(), m_finder) ); repl_socket* pt = t.get(); m_slaves.push_back(pt); m_syncer.add_request( std::unique_ptr( new sync_request([this,pt]{ m_new_slaves.push_back(pt); }) )); return std::move(t); } }} // namespace yrmcds::memcache yrmcds-1.1.5/src/memcache/handler.hpp000066400000000000000000000032251262254645600175140ustar00rootroot00000000000000// Memcache protocol logics and data structures // (C) 2014 Cybozu. #ifndef YRMCDS_MEMCACHE_HANDLER_HPP #define YRMCDS_MEMCACHE_HANDLER_HPP #include "../handler.hpp" #include "../sync.hpp" #include "gc.hpp" #include "object.hpp" #include "sockets.hpp" #include #include #include #include namespace yrmcds { namespace memcache { class handler: public protocol_handler { public: handler(const std::function& finder, cybozu::reactor& reactor, syncer& syncer); virtual void on_start() override; virtual void on_master_start() override; virtual void on_master_interval() override; virtual void on_master_end() override; virtual bool on_slave_start() override; virtual void on_slave_end() override; virtual void on_slave_interval() override; virtual void dump_stats() override; virtual void clear() override; virtual bool reactor_gc_ready() const override; private: bool gc_ready(std::time_t now); std::unique_ptr make_memcache_socket(int s); std::unique_ptr make_repl_socket(int s); std::function m_finder; cybozu::reactor& m_reactor; syncer& m_syncer; bool m_is_slave = true; cybozu::hash_map m_hash; std::time_t m_last_gc = 0; std::unique_ptr m_gc_thread = nullptr; int m_consecutive_gcs = 0; std::vector m_slaves; std::vector m_new_slaves; repl_client_socket* m_repl_client_socket = nullptr; }; }} // namespace yrmcds::memcache #endif // YRMCDS_MEMCACHE_HANDLER_HPP yrmcds-1.1.5/src/memcache/memcache.cpp000066400000000000000000001236421262254645600176420ustar00rootroot00000000000000// (C) 2013-2015 Cybozu. #include "../config.hpp" #include "../global.hpp" #include "memcache.hpp" #include "stats.hpp" #include #include #include #include #include #include #include #include #include #include #include namespace { using namespace yrmcds; const std::memory_order relaxed = std::memory_order_relaxed; const char CRLF[] = "\x0d\x0a"; const char CR = '\x0d'; const char LF = '\x0a'; const char SP = '\x20'; const char VALUE[] = "VALUE "; const char STATUS_NOT_FOUND[] = "Not found"; const char STATUS_EXISTS[] = "Exists"; const char STATUS_TOO_LARGE[] = "Too large value"; const char STATUS_INVALID[] = "Invalid request"; const char STATUS_NOT_STORED[] = "Not stored"; const char STATUS_NON_NUMERIC[] = "Non-numeric value"; const char STATUS_LOCKED[] = "Locked"; const char STATUS_NOT_LOCKED[] = "Not locked"; const char STATUS_UNKNOWN[] = "Unknown command"; const char STATUS_OOM[] = "Out of memory"; const std::time_t EXPTIME_THRESHOLD = 60*60*24*30; const std::size_t BINARY_HEADER_SIZE = 24; inline const char* cfind(const char* p, char c, std::size_t len) { return (const char*)std::memchr(p, c, len); } template inline UInt to_uint(const char* p, bool& result) { result = false; char* end; unsigned long long i = strtoull(p, &end, 10); if( i == 0 && p == end ) return 0; if( i == std::numeric_limits::max() && errno == ERANGE ) return 0; char c = *end; if( c != CR && c != LF && c != SP ) return 0; if( i > std::numeric_limits::max() ) return 0; result = true; return static_cast(i); } inline std::time_t binary_exptime(const char* p) noexcept { std::uint32_t t; cybozu::ntoh(p, t); if( t == 0 ) return 0; if( t == 0xffffffffUL ) return memcache::binary_request::EXPTIME_NONE; if( t > EXPTIME_THRESHOLD ) return t; return g_current_time.load(relaxed) + t; } } // anonymous namespace namespace yrmcds { namespace memcache { constexpr item text_request::eos; inline void text_request::parse_flushall(const char* b, const char* e) noexcept { m_exptime = g_current_time.load(relaxed); while( *b == SP ) ++b; if( b == e ) { m_valid = true; return; } if( '0' <= *b && *b <= '9' ) { const char* exp_end = cfind(b, SP, e-b); if( exp_end == nullptr ) exp_end = e; bool result; std::time_t t = to_uint(b, result); if( ! result ) return; m_exptime = (t > EXPTIME_THRESHOLD) ? t : (g_current_time.load(relaxed) + t); b = exp_end; } while( *b == SP ) ++b; if( b != e ) { // noreply ? const char* we = cfind(b, SP, e-b); std::size_t noreply_len = (we != nullptr) ? (we - b) : (e - b); if( noreply_len != 7 ) return; if( std::memcmp(b, "noreply", 7) != 0 ) return; m_no_reply = true; b = (we != nullptr) ? we : e; } while( *b == SP ) ++b; if( b != e ) return; // garbage left m_valid = true; } inline void text_request::parse_verbosity(const char* b, const char* e) noexcept { while( *b == SP ) ++b; if( b == e ) return; const char* vb_end = cfind(b, SP, e-b); if( vb_end == nullptr ) vb_end = e; std::size_t len = vb_end - b; if( len == 0 ) return; // invalid if( len == 5 ) { if( std::memcmp("error", b, 5) == 0 ) { m_verbosity = cybozu::severity::error; } else if( std::memcmp("debug", b, 5) == 0 ) { m_verbosity = cybozu::severity::debug; } } if( len == 7 && std::memcmp("warning", b, 7) == 0 ) m_verbosity = cybozu::severity::warning; if( len == 4 && std::memcmp("info", b, 4) == 0 ) m_verbosity = cybozu::severity::info; b = vb_end; while( *b == SP ) ++b; if( b != e ) { // noreply ? const char* we = cfind(b, SP, e-b); std::size_t noreply_len = (we != nullptr) ? (we - b) : (e - b); if( noreply_len != 7 ) return; if( std::memcmp(b, "noreply", 7) != 0 ) return; m_no_reply = true; b = (we != nullptr) ? we : e; } while( *b == SP ) ++b; if( b != e ) return; // garbage left m_valid = true; } inline void text_request::parse_storage(const char* b, const char* e, bool is_cas) noexcept { while( *b == SP ) ++b; if( b == e ) return; const char* key_end = cfind(b, SP, e-b); if( key_end == nullptr ) return; m_key = item(b, key_end-b); b = key_end; while( *b == SP ) ++b; if( b == e ) return; const char* flags_end = cfind(b, SP, e-b); if( flags_end == nullptr ) return; bool result; m_flags = to_uint(b, result); if( ! result ) return; b = flags_end; while( *b == SP ) ++b; if( b == e ) return; const char* exptime_end = cfind(b, SP, e-b); if( exptime_end == nullptr ) return; std::time_t t = to_uint(b, result); if( ! result ) return; if( t != 0 ) m_exptime = (t > EXPTIME_THRESHOLD) ? t : (g_current_time.load(relaxed) + t); b = exptime_end; while( *b == SP ) ++b; if( b == e ) return; std::uint32_t nbytes = to_uint(b, result); if( ! result ) return; const char* bytes_end = cfind(b, SP, e-b); b = (bytes_end == nullptr) ? e : bytes_end; if( is_cas ) { while( *b == SP ) ++b; if( b == e ) return; m_cas_unique = to_uint(b, result); if( ! result ) return; const char* cas_end = cfind(b, SP, e-b); b = (cas_end == nullptr) ? e : cas_end; } while( *b == SP ) ++b; if( b != e ) { // noreply ? const char* we = cfind(b, SP, e-b); std::size_t noreply_len = (we != nullptr) ? (we - b) : (e - b); if( noreply_len != 7 ) return; if( std::memcmp(b, "noreply", 7) != 0 ) return; m_no_reply = true; b = (we != nullptr) ? we : e; } while( *b == SP ) ++b; if( b != e ) return; // garbage left // here, parsing was successful. check data length. b += 2; // CRLF m_request_len = (b - m_p) + nbytes + 2; if( m_len < m_request_len ) { m_request_len = 0; return; } // check CRLF following the data if( *(b+nbytes) != CR || *(b+nbytes+1) != LF ) return; // passed all checks. m_data = item(b, nbytes); m_valid = true; } inline void text_request::parse_delete(const char* b, const char* e) noexcept { while( *b == SP ) ++b; if( b == e ) return; const char* key_end = cfind(b, SP, e-b); if( key_end == nullptr ) key_end = e; m_key = item(b, key_end-b); b = key_end; while( *b == SP ) ++b; if( b != e ) { // noreply ? const char* we = cfind(b, SP, e-b); std::size_t noreply_len = (we != nullptr) ? (we - b) : (e - b); if( noreply_len != 7 ) return; if( std::memcmp(b, "noreply", 7) != 0 ) return; m_no_reply = true; b = (we != nullptr) ? we : e; } while( *b == SP ) ++b; if( b != e ) return; // garbage left m_valid = true; } inline void text_request::parse_touch(const char* b, const char* e) noexcept { while( *b == SP ) ++b; if( b == e ) return; const char* key_end = cfind(b, SP, e-b); if( key_end == nullptr ) return; m_key = item(b, key_end-b); b = key_end; while( *b == SP ) ++b; if( b == e ) return; const char* exptime_end = cfind(b, SP, e-b); if( exptime_end == nullptr ) exptime_end = e; bool result; std::time_t t = to_uint(b, result); if( ! result ) return; if( t != 0 ) m_exptime = (t > EXPTIME_THRESHOLD) ? t : (g_current_time.load(relaxed) + t); b = exptime_end; while( *b == SP ) ++b; if( b != e ) { // noreply ? const char* we = cfind(b, SP, e-b); std::size_t noreply_len = (we != nullptr) ? (we - b) : (e - b); if( noreply_len != 7 ) return; if( std::memcmp(b, "noreply", 7) != 0 ) return; m_no_reply = true; b = (we != nullptr) ? we : e; } while( *b == SP ) ++b; if( b != e ) return; // garbage left m_valid = true; } inline void text_request::parse_lock(const char* b, const char* e) noexcept { while( *b == SP ) ++b; if( b == e ) return; const char* key_end = cfind(b, SP, e-b); if( key_end == nullptr ) key_end = e; m_key = item(b, key_end-b); b = key_end; while( *b == SP ) ++b; if( b != e ) return; // garbage left m_valid = true; } inline void text_request::parse_unlock(const char* b, const char* e) noexcept { while( *b == SP ) ++b; if( b == e ) return; const char* key_end = cfind(b, SP, e-b); if( key_end == nullptr ) key_end = e; m_key = item(b, key_end-b); b = key_end; while( *b == SP ) ++b; if( b != e ) return; // garbage left m_valid = true; } inline void text_request::parse_stats(const char* b, const char* e) noexcept { while( *b == SP ) ++b; if( b == e ) { m_valid = true; return; } const char* value_end = cfind(b, SP, e-b); if( value_end == nullptr ) value_end = e; std::size_t value_len = value_end - b; if( value_len == 3 && std::memcmp(b, "ops", 3) == 0 ) { m_stats = stats_t::OPS; m_valid = true; return; } if( value_len == 5 ) { if( std::memcmp(b, "items", 5) == 0 ) { m_stats = stats_t::ITEMS; m_valid = true; return; } if( std::memcmp(b, "sizes", 5) == 0 ) { m_stats = stats_t::SIZES; m_valid = true; return; } } if( value_len == 8 && std::memcmp(b, "settings", 8) == 0 ) { m_stats = stats_t::SETTINGS; m_valid = true; return; } } inline void text_request::parse_incdec(const char* b, const char* e) noexcept { while( *b == SP ) ++b; if( b == e ) return; const char* key_end = cfind(b, SP, e-b); if( key_end == nullptr ) return; m_key = item(b, key_end-b); b = key_end; while( *b == SP ) ++b; if( b == e ) return; const char* value_end = cfind(b, SP, e-b); if( value_end == nullptr ) value_end = e; bool result; m_value = to_uint(b, result); if( ! result ) return; b = value_end; while( *b == SP ) ++b; if( b != e ) { // noreply ? const char* we = cfind(b, SP, e-b); std::size_t noreply_len = (we != nullptr) ? (we - b) : (e - b); if( noreply_len != 7 ) return; if( std::memcmp(b, "noreply", 7) != 0 ) return; m_no_reply = true; b = (we != nullptr) ? we : e; } while( *b == SP ) ++b; if( b != e ) return; // garbage left m_valid = true; } inline void text_request::parse_get(const char* b, const char* e) noexcept { while( *b == SP ) ++b; if( b == e ) return; const char* key_end = cfind(b, SP, e-b); if( key_end == nullptr ) key_end = e; m_key = item(b, key_end-b); m_valid = true; } inline void text_request::parse_keys(const char* b, const char* e) noexcept { m_valid = true; while( *b == SP ) ++b; if( b == e ) return; const char* key_end = cfind(b, SP, e-b); if( key_end == nullptr ) key_end = e; m_key = item(b, key_end-b); } void text_request::parse() noexcept { const char* eol = cfind(m_p, LF, m_len); if( eol == nullptr ) return; // incomplete m_request_len = eol - m_p + 1; if( m_request_len == 1 ) return; // invalid if( *(--eol) != CR ) return; const char* cmd_start = m_p; while( *cmd_start == SP ) ++cmd_start; const char* cmd_end = cfind(cmd_start, SP, (eol - cmd_start)); if( cmd_end == nullptr ) cmd_end = eol; std::size_t cmd_len = (cmd_end - cmd_start); if( cmd_len == 10 ) { if( std::memcmp(cmd_start, "unlock_all", 10) == 0 ) { m_command = text_command::UNLOCK_ALL; m_valid = true; return; } } if( cmd_len == 9 ) { if( std::memcmp(cmd_start, "flush_all", 9) == 0 ) { m_command = text_command::FLUSH_ALL; parse_flushall(cmd_end, eol); return; } if( std::memcmp(cmd_start, "verbosity", 9) == 0 ) { m_command = text_command::VERBOSITY; parse_verbosity(cmd_end, eol); return; } } if( cmd_len == 7 ) { if( std::memcmp(cmd_start, "version", 7) == 0 ) { m_command = text_command::VERSION; m_valid = true; return; } if( std::memcmp(cmd_start, "prepend", 7) == 0 ) { m_command = text_command::PREPEND; parse_storage(cmd_end, eol, false); return; } if( std::memcmp(cmd_start, "replace", 7) == 0 ) { m_command = text_command::REPLACE; parse_storage(cmd_end, eol, false); return; } } if( cmd_len == 6 ) { if( std::memcmp(cmd_start, "append", 6) == 0 ) { m_command = text_command::APPEND; parse_storage(cmd_end, eol, false); return; } if( std::memcmp(cmd_start, "delete", 6) == 0 ) { m_command = text_command::DELETE; parse_delete(cmd_end, eol); return; } if( std::memcmp(cmd_start, "unlock", 6) == 0 ) { m_command = text_command::UNLOCK; parse_unlock(cmd_end, eol); return; } } if( cmd_len == 5 ) { if( std::memcmp(cmd_start, "touch", 5) == 0 ) { m_command = text_command::TOUCH; parse_touch(cmd_end, eol); return; } if( std::memcmp(cmd_start, "slabs", 5) == 0 ) { m_command = text_command::SLABS; m_valid = true; // ignore. return; } if( std::memcmp(cmd_start, "stats", 5) == 0 ) { m_command = text_command::STATS; parse_stats(cmd_end, eol); return; } } if( cmd_len == 4 ) { if( std::memcmp(cmd_start, "gets", 4) == 0 ) { m_command = text_command::GETS; parse_get(cmd_end, eol); return; } if( std::memcmp(cmd_start, "incr", 4) == 0 ) { m_command = text_command::INCR; parse_incdec(cmd_end, eol); return; } if( std::memcmp(cmd_start, "decr", 4) == 0 ) { m_command = text_command::DECR; parse_incdec(cmd_end, eol); return; } if( std::memcmp(cmd_start, "lock", 4) == 0 ) { m_command = text_command::LOCK; parse_lock(cmd_end, eol); return; } if( std::memcmp(cmd_start, "quit", 4) == 0 ) { m_command = text_command::QUIT; m_valid = true; return; } if( std::memcmp(cmd_start, "keys", 4) == 0 ) { m_command = text_command::KEYS; parse_keys(cmd_end, eol); return; } } if( cmd_len == 3 ) { if( std::memcmp(cmd_start, "set", 3) == 0 ) { m_command = text_command::SET; parse_storage(cmd_end, eol, false); return; } if( std::memcmp(cmd_start, "add", 3) == 0 ) { m_command = text_command::ADD; parse_storage(cmd_end, eol, false); return; } if( std::memcmp(cmd_start, "get", 3) == 0 ) { m_command = text_command::GET; parse_get(cmd_end, eol); return; } if( std::memcmp(cmd_start, "cas", 3) == 0 ) { m_command = text_command::CAS; parse_storage(cmd_end, eol, true); return; } } } void text_response::value(const cybozu::hash_key& key, std::uint32_t flags, const cybozu::dynbuf& data) { if( key.length() > MAX_KEY_LENGTH ) throw std::logic_error("MAX_KEY_LENGTH over bug"); m_iov[0] = {VALUE, sizeof(VALUE) - 1}; m_iov[1] = {key.data(), key.length()}; char buf[MAX_KEY_LENGTH + 100]; static_assert( sizeof(unsigned int) >= sizeof(std::uint32_t), "unsigned int is smaller than std::uint32_t" ); int length = snprintf(buf, sizeof(buf), " %u %llu\x0d\x0a", (unsigned int)flags, (long long unsigned int)data.size()); m_iov[2] = {buf, (std::size_t)length}; m_iov[3] = {data.data(), data.size()}; m_iov[4] = {CRLF, sizeof(CRLF) - 1}; m_socket.sendv(m_iov, 5, false); } void text_response::value(const cybozu::hash_key& key, std::uint32_t flags, const cybozu::dynbuf& data, std::uint64_t cas) { if( key.length() > MAX_KEY_LENGTH ) throw std::logic_error("MAX_KEY_LENGTH over bug"); m_iov[0] = {VALUE, sizeof(VALUE) - 1}; m_iov[1] = {key.data(), key.length()}; char buf[MAX_KEY_LENGTH + 100]; static_assert( sizeof(unsigned int) >= sizeof(std::uint32_t), "unsigned int is smaller than std::uint32_t" ); int length = snprintf(buf, sizeof(buf), " %u %llu %llu\x0d\x0a", (unsigned int)flags, (long long unsigned int)data.size(), (long long unsigned int)cas); m_iov[2] = {buf, (std::size_t)length}; m_iov[3] = {data.data(), data.size()}; m_iov[4] = {CRLF, sizeof(CRLF) - 1}; m_socket.sendv(m_iov, 5, false); } void text_response::value(const cybozu::hash_key& key) { if( key.length() > MAX_KEY_LENGTH ) throw std::logic_error("MAX_KEY_LENGTH over bug"); m_iov[0] = {VALUE, sizeof(VALUE) - 1}; m_iov[1] = {key.data(), key.length()}; m_iov[2] = {CRLF, sizeof(CRLF) - 1}; m_socket.sendv(m_iov, 3, false); } void text_response::stats_settings() { std::ostringstream os; os << "STAT maxbytes " << g_config.memory_limit() << CRLF; os << "STAT tcpport " << g_config.port() << CRLF; os << "STAT replport " << g_config.repl_port() << CRLF; os << "STAT virtual_ip " << g_config.vip().str() << CRLF; os << "STAT evictions on" << CRLF; os << "STAT cas_enabled on" << CRLF; os << "STAT locking on" << CRLF; os << "STAT secure_erase " << (g_config.secure_erase() ? "on" : "off") << CRLF; os << "STAT lock_memory " << (g_config.lock_memory() ? "on" : "off") << CRLF; os << "STAT tmp_dir " << g_config.tempdir() << CRLF; os << "STAT buckets " << g_config.buckets() << CRLF; os << "STAT item_size_max " << g_config.max_data_size() << CRLF; os << "STAT num_threads " << g_config.workers() << CRLF; os << "STAT gc_interval " << g_config.gc_interval() << CRLF; std::string s = os.str(); m_socket.send(s.data(), s.size()); } void text_response::stats_items() { std::ostringstream os; os << "STAT items:1:number " << g_stats.objects.load(relaxed) << CRLF; os << "STAT items:1:age " << g_stats.oldest_age.load(relaxed) << CRLF; os << "STAT items:1:evicted " << g_stats.total_evictions.load(relaxed) << CRLF; os << "STAT items:1:conflicts " << g_stats.conflicts.load(relaxed) << CRLF; os << "STAT items:1:largest " << g_stats.largest_object_size.load(relaxed) << CRLF; std::string s = os.str(); m_socket.send(s.data(), s.size()); } void text_response::stats_sizes() { std::ostringstream os; os << "STAT 1024 " << g_stats.objects_under_1k.load(relaxed) << CRLF; os << "STAT 4096 " << g_stats.objects_under_4k.load(relaxed) << CRLF; os << "STAT 16384 " << g_stats.objects_under_16k.load(relaxed) << CRLF; os << "STAT 65536 " << g_stats.objects_under_64k.load(relaxed) << CRLF; os << "STAT 262144 " << g_stats.objects_under_256k.load(relaxed) << CRLF; os << "STAT 1048576 " << g_stats.objects_under_1m.load(relaxed) << CRLF; os << "STAT 4194304 " << g_stats.objects_under_4m.load(relaxed) << CRLF; os << "STAT huge " << g_stats.objects_huge.load(relaxed) << CRLF; std::string s = os.str(); m_socket.send(s.data(), s.size()); } void text_response::stats_ops() { std::ostringstream os; #define SEND_TEXT_OPS(n,i) \ os << "STAT text:" n " " \ << g_stats.text_ops[(std::size_t)text_command::i].load(relaxed) << CRLF; SEND_TEXT_OPS("set", SET); SEND_TEXT_OPS("add", ADD); SEND_TEXT_OPS("replace", REPLACE); SEND_TEXT_OPS("append", APPEND); SEND_TEXT_OPS("prepend", PREPEND); SEND_TEXT_OPS("cas", CAS); SEND_TEXT_OPS("get", GET); SEND_TEXT_OPS("gets", GETS); SEND_TEXT_OPS("delete", DELETE); SEND_TEXT_OPS("incr", INCR); SEND_TEXT_OPS("decr", DECR); SEND_TEXT_OPS("touch", TOUCH); SEND_TEXT_OPS("lock", LOCK); SEND_TEXT_OPS("unlock", UNLOCK); SEND_TEXT_OPS("unlock_all", UNLOCK_ALL); SEND_TEXT_OPS("slabs", SLABS); SEND_TEXT_OPS("stats", STATS); SEND_TEXT_OPS("flush_all", FLUSH_ALL); SEND_TEXT_OPS("version", VERSION); SEND_TEXT_OPS("verbosity", VERBOSITY); SEND_TEXT_OPS("quit", QUIT); SEND_TEXT_OPS("keys", KEYS); #undef SEND_TEXT_OPS #define SEND_BINARY_OPS(n) \ os << "STAT binary:" #n " " \ << g_stats.bin_ops[(std::size_t)binary_command::n].load(relaxed) << CRLF; SEND_BINARY_OPS(Get); SEND_BINARY_OPS(Set); SEND_BINARY_OPS(Add); SEND_BINARY_OPS(Replace); SEND_BINARY_OPS(Delete); SEND_BINARY_OPS(Increment); SEND_BINARY_OPS(Decrement); SEND_BINARY_OPS(Quit); SEND_BINARY_OPS(Flush); SEND_BINARY_OPS(GetQ); SEND_BINARY_OPS(Noop); SEND_BINARY_OPS(Version); SEND_BINARY_OPS(GetK); SEND_BINARY_OPS(GetKQ); SEND_BINARY_OPS(Append); SEND_BINARY_OPS(Prepend); SEND_BINARY_OPS(Stat); SEND_BINARY_OPS(SetQ); SEND_BINARY_OPS(AddQ); SEND_BINARY_OPS(ReplaceQ); SEND_BINARY_OPS(DeleteQ); SEND_BINARY_OPS(IncrementQ); SEND_BINARY_OPS(DecrementQ); SEND_BINARY_OPS(QuitQ); SEND_BINARY_OPS(FlushQ); SEND_BINARY_OPS(AppendQ); SEND_BINARY_OPS(PrependQ); SEND_BINARY_OPS(Touch); SEND_BINARY_OPS(GaT); SEND_BINARY_OPS(GaTQ); SEND_BINARY_OPS(GaTK); SEND_BINARY_OPS(GaTKQ); SEND_BINARY_OPS(Lock); SEND_BINARY_OPS(LockQ); SEND_BINARY_OPS(Unlock); SEND_BINARY_OPS(UnlockQ); SEND_BINARY_OPS(UnlockAll); SEND_BINARY_OPS(UnlockAllQ); SEND_BINARY_OPS(LaG); SEND_BINARY_OPS(LaGQ); SEND_BINARY_OPS(LaGK); SEND_BINARY_OPS(LaGKQ); SEND_BINARY_OPS(RaU); SEND_BINARY_OPS(RaUQ); SEND_BINARY_OPS(Keys); #undef SEND_BINARY_OPS std::string s = os.str(); m_socket.send(s.data(), s.size()); } void text_response::stats_general(std::size_t n_slaves) { struct rusage ru; if( ::getrusage(RUSAGE_SELF, &ru) == -1 ) cybozu::throw_unix_error(errno, "getrusage"); std::ostringstream os; os << std::setfill('0'); os << "STAT pid " << ::getpid() << CRLF; os << "STAT time " << g_current_time.load(relaxed) << CRLF; os << "STAT version " << VERSION << CRLF; os << "STAT pointer_size " << sizeof(void*)*8 << CRLF; os << "STAT rusage_user " << ru.ru_utime.tv_sec << '.' << std::setw(6) << ru.ru_utime.tv_usec << CRLF; os << "STAT rusage_system " << ru.ru_stime.tv_sec << '.' << std::setw(6) << ru.ru_stime.tv_usec << CRLF; os << "STAT curr_connections " << g_stats.curr_connections.load(relaxed) << CRLF; os << "STAT total_connections " << g_stats.total_connections.load(relaxed) << CRLF; os << "STAT curr_items " << g_stats.objects.load(relaxed) << CRLF; os << "STAT total_items " << g_stats.total_objects.load(relaxed) << CRLF; os << "STAT get_hits " << g_stats.get_hits.load(relaxed) << CRLF; os << "STAT get_misses " << g_stats.get_misses.load(relaxed) << CRLF; os << "STAT cas_hits " << g_stats.cas_hits.load(relaxed) << CRLF; os << "STAT cas_misses " << g_stats.cas_misses.load(relaxed) << CRLF; os << "STAT cas_badval " << g_stats.cas_badval.load(relaxed) << CRLF; os << "STAT bytes " << g_stats.used_memory.load(relaxed) << CRLF; os << "STAT limit_maxbytes " << g_config.memory_limit() << CRLF; os << "STAT threads " << g_config.workers() << CRLF; os << "STAT gc_count " << g_stats.gc_count.load(relaxed) << CRLF; os << "STAT slaves " << n_slaves << CRLF; os << "STAT last_expirations " << g_stats.last_expirations.load(relaxed) << CRLF; os << "STAT last_evictions " << g_stats.last_evictions.load(relaxed) << CRLF; os << "STAT evictions " << g_stats.total_evictions.load(relaxed) << CRLF; os << "STAT last_gc_elapsed " << g_stats.last_gc_elapsed.load(relaxed) << CRLF; os << "STAT total_gc_elapsed " << g_stats.total_gc_elapsed.load(relaxed) << CRLF; std::string s = os.str(); m_socket.send(s.data(), s.size()); } void text_response::version() { m_iov[0] = {TEXT_VERSION, sizeof(TEXT_VERSION) -1}; m_iov[1] = {VERSION, sizeof(VERSION) - 1}; m_iov[2] = {CRLF, sizeof(CRLF) - 1}; m_socket.sendv(m_iov, 3, true); } /* * Binary protocol. */ void binary_request::parse() noexcept { if( m_len < BINARY_HEADER_SIZE ) return; // incomplete std::uint32_t total_len; cybozu::ntoh(m_p + 8, total_len); if( m_len < (BINARY_HEADER_SIZE + total_len) ) return; // incomplete m_request_len = BINARY_HEADER_SIZE + total_len; // Opcode parsing m_command = (binary_command)*(const unsigned char*)(m_p+1); m_quiet = ( m_command == binary_command::GetQ || m_command == binary_command::GetKQ || (binary_command::SetQ <= m_command && m_command <= binary_command::PrependQ) || m_command == binary_command::GaTQ || m_command == binary_command::GaTKQ || (binary_command::LockQ <= m_command && m_command <= binary_command::RaUQ && ((unsigned char)m_command) & 1) ); std::uint16_t key_len; cybozu::ntoh(m_p + 2, key_len); if( key_len > MAX_KEY_LENGTH ) return; // invalid std::uint8_t extras_len = *(const unsigned char*)(m_p + 4); if( total_len < (key_len + extras_len) ) return; // invalid if( key_len > 0 ) m_key = item(m_p + (BINARY_HEADER_SIZE + extras_len), key_len); cybozu::ntoh(m_p + 16, m_cas_unique); std::size_t data_len = total_len - key_len - extras_len; if( data_len > MAX_REQUEST_LENGTH ) { m_status = binary_status::TooLargeValue; return; } if( data_len > 0 ) m_data = item(m_p + (BINARY_HEADER_SIZE + extras_len + key_len), data_len); const char* const p_extra = m_p + BINARY_HEADER_SIZE; switch( m_command ) { case binary_command::Get: case binary_command::GetQ: case binary_command::GetK: case binary_command::GetKQ: case binary_command::GaT: case binary_command::GaTQ: case binary_command::GaTK: case binary_command::GaTKQ: case binary_command::LaG: case binary_command::LaGQ: case binary_command::LaGK: case binary_command::LaGKQ: if( extras_len != 0 && extras_len != 4 ) return; // invalid if( key_len == 0 || data_len > 0 ) return; // invalid if( extras_len == 4 ) { m_exptime = binary_exptime(p_extra); } else { m_exptime = EXPTIME_NONE; } break; case binary_command::Set: case binary_command::SetQ: case binary_command::Add: case binary_command::AddQ: case binary_command::Replace: case binary_command::ReplaceQ: case binary_command::RaU: case binary_command::RaUQ: if( extras_len != 8 || key_len == 0 || data_len == 0 ) return; // invalid cybozu::ntoh(p_extra, m_flags); m_exptime = binary_exptime(p_extra + 4); break; case binary_command::Delete: case binary_command::DeleteQ: if( extras_len != 0 || key_len == 0 || data_len > 0 ) return; // invalid break; case binary_command::Increment: case binary_command::IncrementQ: case binary_command::Decrement: case binary_command::DecrementQ: if( extras_len != 20 || key_len == 0 || data_len > 0 ) return; // invalid cybozu::ntoh(p_extra, m_value); cybozu::ntoh(p_extra + sizeof(std::uint64_t), m_initial); m_exptime = binary_exptime(p_extra + sizeof(std::uint64_t)*2); break; case binary_command::Touch: if( key_len == 0 || extras_len != 4 || data_len != 0 ) return; // invalid m_exptime = binary_exptime(p_extra); break; case binary_command::Flush: case binary_command::FlushQ: if( extras_len != 0 && extras_len != 4 ) return; // invalid if( key_len != 0 || data_len > 0 ) return; // invalid if( extras_len == 4 ) { // yrmcds extension m_exptime = binary_exptime(p_extra); } else { m_exptime = g_current_time.load(relaxed); } break; case binary_command::Append: case binary_command::AppendQ: case binary_command::Prepend: case binary_command::PrependQ: if( extras_len != 0 || key_len == 0 || data_len == 0 ) return; // invalid break; case binary_command::Lock: case binary_command::LockQ: case binary_command::Unlock: case binary_command::UnlockQ: if( extras_len != 0 || key_len == 0 || data_len > 0 ) return; // invalid break; case binary_command::Stat: if( extras_len != 0 || data_len != 0 ) return; // invalid if( key_len == 8 && std::memcmp(p_extra, "settings", 8) == 0 ) { m_stats = stats_t::SETTINGS; } else if( key_len == 5 ) { if( std::memcmp(p_extra, "items", 5) == 0 ) { m_stats = stats_t::ITEMS; } else if( std::memcmp(p_extra, "sizes", 5) == 0 ) { m_stats = stats_t::SIZES; } } else if( key_len == 3 && std::memcmp(p_extra, "ops", 3) == 0 ) { m_stats = stats_t::OPS; } break; case binary_command::Keys: if( extras_len != 0 || data_len > 0 ) return; // invalid break; case binary_command::Quit: case binary_command::QuitQ: case binary_command::Version: case binary_command::Noop: case binary_command::UnlockAll: case binary_command::UnlockAllQ: // no validation. // this is intentional violation against the protocol definition. break; default: m_command = binary_command::Unknown; m_status = binary_status::UnknownCommand; return; } m_status = binary_status::OK; } inline void binary_response::send_error(binary_status status, const char* msg, std::size_t len) { if( len > 100 ) throw std::logic_error(" Too long message"); char header[BINARY_HEADER_SIZE + 100]; fill_header(header, 0, 0, len, 0, status); std::memcpy(&header[BINARY_HEADER_SIZE], msg, len); m_socket.send(header, BINARY_HEADER_SIZE + len, true); } void binary_response::error(binary_status status) { switch( status ) { case binary_status::NotFound: send_error(status, STATUS_NOT_FOUND, sizeof(STATUS_NOT_FOUND) - 1); break; case binary_status::Exists: send_error(status, STATUS_EXISTS, sizeof(STATUS_EXISTS) - 1); break; case binary_status::TooLargeValue: send_error(status, STATUS_TOO_LARGE, sizeof(STATUS_TOO_LARGE) - 1); break; case binary_status::Invalid: send_error(status, STATUS_INVALID, sizeof(STATUS_INVALID) - 1); break; case binary_status::NotStored: send_error(status, STATUS_NOT_STORED, sizeof(STATUS_NOT_STORED) - 1); break; case binary_status::NonNumeric: send_error(status, STATUS_NON_NUMERIC, sizeof(STATUS_NON_NUMERIC) - 1); break; case binary_status::Locked: send_error(status, STATUS_LOCKED, sizeof(STATUS_LOCKED) - 1); break; case binary_status::NotLocked: send_error(status, STATUS_NOT_LOCKED, sizeof(STATUS_NOT_LOCKED) - 1); break; case binary_status::UnknownCommand: send_error(status, STATUS_UNKNOWN, sizeof(STATUS_UNKNOWN) - 1); break; case binary_status::OutOfMemory: send_error(status, STATUS_OOM, sizeof(STATUS_OOM) - 1); break; default: throw std::logic_error(" bug"); } } void binary_response::success() { char header[BINARY_HEADER_SIZE]; fill_header(header, 0, 0, 0, 0); m_socket.send(header, BINARY_HEADER_SIZE, true); } void binary_response::get(std::uint32_t flags, const cybozu::dynbuf& data, std::uint64_t cas, bool flush, const char* key, std::size_t key_len) { char header[BINARY_HEADER_SIZE]; fill_header(header, key_len, sizeof(flags), data.size(), cas); char b_flags[sizeof(flags)]; cybozu::hton(flags, b_flags); m_iov[0] = {header, BINARY_HEADER_SIZE}; m_iov[1] = {b_flags, sizeof(b_flags)}; if( key == nullptr ) { m_iov[2] = {data.data(), data.size()}; m_socket.sendv(m_iov, 3, flush); } else { m_iov[2] = {key, key_len}; m_iov[3] = {data.data(), data.size()}; m_socket.sendv(m_iov, 4, flush); } } void binary_response::key(const char* key, std::size_t key_len) { char header[BINARY_HEADER_SIZE]; fill_header(header, key_len, 0, 0, 0); m_iov[0] = {header, BINARY_HEADER_SIZE}; m_iov[1] = {key, key_len}; m_socket.sendv(m_iov, 2, false); } void binary_response::set(std::uint64_t cas) { char header[BINARY_HEADER_SIZE]; fill_header(header, 0, 0, 0, cas); m_socket.send(header, BINARY_HEADER_SIZE, true); } void binary_response::incdec(std::uint64_t value, std::uint64_t cas) { char header[BINARY_HEADER_SIZE + sizeof(value)]; fill_header(header, 0, 0, sizeof(value), cas); cybozu::hton(value, &header[BINARY_HEADER_SIZE]); m_socket.send(header, sizeof(header), true); } void binary_response::quit() { char header[BINARY_HEADER_SIZE]; fill_header(header, 0, 0, 0, 0); m_socket.send_close(header, sizeof(header)); } inline void binary_response::send_stat(const std::string& key, const std::string& value) { char header[BINARY_HEADER_SIZE]; fill_header(header, key.size(), 0, value.size(), 0); m_iov[0] = {header, sizeof(header)}; m_iov[1] = {key.data(), key.size()}; m_iov[2] = {value.data(), value.size()}; m_socket.sendv(m_iov, 3, false); } void binary_response::stats_settings() { send_stat("maxbytes", std::to_string(g_config.memory_limit())); send_stat("tcpport", std::to_string(g_config.port())); send_stat("replport", std::to_string(g_config.repl_port())); send_stat("virtual_ip", g_config.vip().str()); send_stat("evictions", "on"); send_stat("cas_enabled", "on"); send_stat("locking", "on"); send_stat("secure_erase", g_config.secure_erase() ? "on" : "off"); send_stat("lock_memory", g_config.lock_memory() ? "on" : "off"); send_stat("tmp_dir", g_config.tempdir()); send_stat("buckets", std::to_string(g_config.buckets())); send_stat("item_size_max", std::to_string(g_config.max_data_size())); send_stat("num_threads", std::to_string(g_config.workers())); send_stat("gc_interval", std::to_string(g_config.gc_interval())); success(); } void binary_response::stats_items() { send_stat("items:1:number", std::to_string(g_stats.objects.load(relaxed))); send_stat("items:1:age", std::to_string(g_stats.oldest_age.load(relaxed))); send_stat("items:1:evicted", std::to_string(g_stats.total_evictions.load(relaxed))); send_stat("items:1:conflicts", std::to_string(g_stats.conflicts.load(relaxed))); send_stat("items:1:largest", std::to_string(g_stats.largest_object_size.load(relaxed))); success(); } void binary_response::stats_sizes() { send_stat("1024", std::to_string(g_stats.objects_under_1k.load(relaxed))); send_stat("4096", std::to_string(g_stats.objects_under_4k.load(relaxed))); send_stat("16384", std::to_string(g_stats.objects_under_16k.load(relaxed))); send_stat("65536", std::to_string(g_stats.objects_under_64k.load(relaxed))); send_stat("262144", std::to_string(g_stats.objects_under_256k.load(relaxed))); send_stat("1048576", std::to_string(g_stats.objects_under_1m.load(relaxed))); send_stat("4194304", std::to_string(g_stats.objects_under_4m.load(relaxed))); send_stat("huge", std::to_string(g_stats.objects_huge.load(relaxed))); success(); } void binary_response::stats_ops() { #define SEND_TEXT_OPS(n,i) \ send_stat("text:" n, \ std::to_string(g_stats.text_ops[(std::size_t)text_command::i].load(relaxed))) SEND_TEXT_OPS("set", SET); SEND_TEXT_OPS("add", ADD); SEND_TEXT_OPS("replace", REPLACE); SEND_TEXT_OPS("append", APPEND); SEND_TEXT_OPS("prepend", PREPEND); SEND_TEXT_OPS("cas", CAS); SEND_TEXT_OPS("get", GET); SEND_TEXT_OPS("gets", GETS); SEND_TEXT_OPS("delete", DELETE); SEND_TEXT_OPS("incr", INCR); SEND_TEXT_OPS("decr", DECR); SEND_TEXT_OPS("touch", TOUCH); SEND_TEXT_OPS("lock", LOCK); SEND_TEXT_OPS("unlock", UNLOCK); SEND_TEXT_OPS("unlock_all", UNLOCK_ALL); SEND_TEXT_OPS("slabs", SLABS); SEND_TEXT_OPS("stats", STATS); SEND_TEXT_OPS("flush_all", FLUSH_ALL); SEND_TEXT_OPS("version", VERSION); SEND_TEXT_OPS("verbosity", VERBOSITY); SEND_TEXT_OPS("quit", QUIT); SEND_TEXT_OPS("keys", KEYS); #undef SEND_TEXT_OPS #define SEND_BINARY_OPS(n) \ send_stat("binary:" #n, \ std::to_string(g_stats.bin_ops[(std::size_t)binary_command::n].load(relaxed))) SEND_BINARY_OPS(Get); SEND_BINARY_OPS(Set); SEND_BINARY_OPS(Add); SEND_BINARY_OPS(Replace); SEND_BINARY_OPS(Delete); SEND_BINARY_OPS(Increment); SEND_BINARY_OPS(Decrement); SEND_BINARY_OPS(Quit); SEND_BINARY_OPS(Flush); SEND_BINARY_OPS(GetQ); SEND_BINARY_OPS(Noop); SEND_BINARY_OPS(Version); SEND_BINARY_OPS(GetK); SEND_BINARY_OPS(GetKQ); SEND_BINARY_OPS(Append); SEND_BINARY_OPS(Prepend); SEND_BINARY_OPS(Stat); SEND_BINARY_OPS(SetQ); SEND_BINARY_OPS(AddQ); SEND_BINARY_OPS(ReplaceQ); SEND_BINARY_OPS(DeleteQ); SEND_BINARY_OPS(IncrementQ); SEND_BINARY_OPS(DecrementQ); SEND_BINARY_OPS(QuitQ); SEND_BINARY_OPS(FlushQ); SEND_BINARY_OPS(AppendQ); SEND_BINARY_OPS(PrependQ); SEND_BINARY_OPS(Touch); SEND_BINARY_OPS(GaT); SEND_BINARY_OPS(GaTQ); SEND_BINARY_OPS(GaTK); SEND_BINARY_OPS(GaTKQ); SEND_BINARY_OPS(Lock); SEND_BINARY_OPS(LockQ); SEND_BINARY_OPS(Unlock); SEND_BINARY_OPS(UnlockQ); SEND_BINARY_OPS(UnlockAll); SEND_BINARY_OPS(UnlockAllQ); SEND_BINARY_OPS(LaG); SEND_BINARY_OPS(LaGQ); SEND_BINARY_OPS(LaGK); SEND_BINARY_OPS(LaGKQ); SEND_BINARY_OPS(RaU); SEND_BINARY_OPS(RaUQ); SEND_BINARY_OPS(Keys); #undef SEND_BINARY_OPS success(); } void binary_response::stats_general(std::size_t n_slaves) { struct rusage ru; if( ::getrusage(RUSAGE_SELF, &ru) == -1 ) cybozu::throw_unix_error(errno, "getrusage"); std::ostringstream os_utime, os_stime; os_utime << ru.ru_utime.tv_sec << '.' << std::setfill('0') << std::setw(6) << ru.ru_utime.tv_usec; os_stime << ru.ru_stime.tv_sec << '.' << std::setfill('0') << std::setw(6) << ru.ru_stime.tv_usec; send_stat("pid", std::to_string(::getpid())); send_stat("time", std::to_string(g_current_time.load(relaxed))); send_stat("version", VERSION); send_stat("pointer_size", std::to_string(sizeof(void*)*8)); send_stat("rusage_user", os_utime.str()); send_stat("rusage_system", os_stime.str()); send_stat("curr_connections", std::to_string(g_stats.curr_connections.load(relaxed))); send_stat("total_connections", std::to_string(g_stats.total_connections.load(relaxed))); send_stat("curr_items", std::to_string(g_stats.objects.load(relaxed))); send_stat("total_items", std::to_string(g_stats.total_objects.load(relaxed))); send_stat("get_hits", std::to_string(g_stats.get_hits.load(relaxed))); send_stat("get_misses", std::to_string(g_stats.get_misses.load(relaxed))); send_stat("cas_hits", std::to_string(g_stats.cas_hits.load(relaxed))); send_stat("cas_misses", std::to_string(g_stats.cas_misses.load(relaxed))); send_stat("cas_badval", std::to_string(g_stats.cas_badval.load(relaxed))); send_stat("bytes", std::to_string(g_stats.used_memory.load(relaxed))); send_stat("limit_maxbytes", std::to_string(g_config.memory_limit())); send_stat("threads", std::to_string(g_config.workers())); send_stat("gc_count", std::to_string(g_stats.gc_count.load(relaxed))); send_stat("slaves", std::to_string(n_slaves)); send_stat("last_expirations", std::to_string(g_stats.last_expirations.load(relaxed))); send_stat("last_evictions", std::to_string(g_stats.last_evictions.load(relaxed))); send_stat("evictions", std::to_string(g_stats.total_evictions.load(relaxed))); send_stat("last_gc_elapsed", std::to_string(g_stats.last_gc_elapsed.load(relaxed))); send_stat("total_gc_elapsed", std::to_string(g_stats.total_gc_elapsed.load(relaxed))); success(); } void binary_response::version() { char header[BINARY_HEADER_SIZE + sizeof(VERSION) - 1]; fill_header(header, 0, 0, sizeof(VERSION) - 1, 0); std::memcpy(&header[BINARY_HEADER_SIZE], VERSION, sizeof(VERSION) - 1); m_socket.send(header, sizeof(header), true); } }} // namespace yrmcds::memcache yrmcds-1.1.5/src/memcache/memcache.hpp000066400000000000000000000304761262254645600176510ustar00rootroot00000000000000// memcached server-side protocol. // (C) 2013-2015 Cybozu. #ifndef YRMCDS_MEMCACHE_MEMCACHE_HPP #define YRMCDS_MEMCACHE_MEMCACHE_HPP #include "../constants.hpp" #include #include #include #include #include #include #include #include #include #include namespace yrmcds { namespace memcache { inline bool is_binary_request(const char* p) { return *p == '\x80'; } // Possible stats categories. enum class stats_t { GENERAL, SETTINGS, ITEMS, SIZES, OPS }; using item = std::tuple; // Memcache text commands. enum class text_command { UNKNOWN, SET, ADD, REPLACE, APPEND, PREPEND, CAS, GET, GETS, DELETE, INCR, DECR, TOUCH, LOCK, UNLOCK, UNLOCK_ALL, SLABS, STATS, FLUSH_ALL, VERSION, VERBOSITY, QUIT, KEYS, END_OF_COMMAND // must be defined the last }; // Text request parser. class text_request final { public: text_request(const char* p, std::size_t len): m_p(p), m_len(len) { if( len == 0 || p == nullptr ) throw std::logic_error(" bad ctor arguments"); parse(); } // End of the key stream. static constexpr item eos{}; // Return length of the request. // // Return length of the request. // If the request is incomplete, zero is returned. std::size_t length() const noexcept { return m_request_len; } // Return the command type. text_command command() const noexcept { return m_command; } // Return `true` if the request can be parsed successfully. bool valid() const noexcept { return m_valid; } // `true` if the command has "noreply" option. bool no_reply() const noexcept { return m_no_reply; } // Return `key` sent with various commands. item key() const noexcept { return m_key; } // Return `flags` sent with storage commands. std::uint32_t flags() const noexcept { return m_flags; } // Return `exptime` sent with storage commands and TOUCH and FLUSH_ALL. std::time_t exptime() const noexcept { return m_exptime; } // Return `cas unique` sent with CAS command. std::uint64_t cas_unique() const noexcept { return m_cas_unique; } // Return data block sent with storage commands. item data() const noexcept { return m_data; } // Return `item` for the first key sent with GET or GETS. item first_key() const noexcept { return m_key; } // Return `item` next to the `prev` key in GET or GETS commands. // If the item == , no more key is available. static item next_key(const item& prev) noexcept { const char* b; std::size_t len; std::tie(b, len) = prev; b += len; while( *b == '\x20' ) ++b; const char* e = (const char*)::rawmemchr(b, '\x0d'); if( b == e ) return eos; const char* key_end = (const char*)std::memchr(b, '\x20', e-b); if( key_end == nullptr ) key_end = e; return item(b, key_end-b); } // Return an unsigned 64bit integer value sent with INCR or DECR. std::uint64_t value() const noexcept { return m_value; } // Return the log verbosity sent with VERBOSITY. cybozu::severity verbosity() const noexcept { return m_verbosity; } // Return memcache stats category sent with STATS. stats_t stats() const noexcept { return m_stats; } private: const char* const m_p; const std::size_t m_len; std::size_t m_request_len = 0; text_command m_command = text_command::UNKNOWN; bool m_valid = false; bool m_no_reply = false; item m_key; std::uint32_t m_flags = 0; std::time_t m_exptime = 0; std::uint64_t m_cas_unique = 0; item m_data; std::uint64_t m_value = 0; cybozu::severity m_verbosity = cybozu::severity::info; stats_t m_stats = stats_t::GENERAL; void parse() noexcept; void parse_flushall(const char* b, const char* e) noexcept; void parse_verbosity(const char* b, const char* e) noexcept; void parse_storage(const char* b, const char* e, bool is_cas) noexcept; void parse_delete(const char* b, const char* e) noexcept; void parse_touch(const char* b, const char* e) noexcept; void parse_lock(const char* b, const char* e) noexcept; void parse_unlock(const char* b, const char* e) noexcept; void parse_stats(const char* b, const char* e) noexcept; void parse_incdec(const char* b, const char* e) noexcept; void parse_get(const char* b, const char* e) noexcept; void parse_keys(const char* b, const char* e) noexcept; }; const char TEXT_ERROR[] = "ERROR\x0d\x0a"; const char TEXT_OK[] = "OK\x0d\x0a"; const char TEXT_STORED[] = "STORED\x0d\x0a"; const char TEXT_NOT_STORED[] = "NOT_STORED\x0d\x0a"; const char TEXT_EXISTS[] = "EXISTS\x0d\x0a"; const char TEXT_NOT_FOUND[] = "NOT_FOUND\x0d\x0a"; const char TEXT_TOUCHED[] = "TOUCHED\x0d\x0a"; const char TEXT_DELETED[] = "DELETED\x0d\x0a"; const char TEXT_END[] = "END\x0d\x0a"; const char TEXT_LOCKED[] = "LOCKED\x0d\x0a"; const char TEXT_VERSION[] = "VERSION "; // Text response sender. class text_response final { public: text_response(cybozu::tcp_socket& sock): m_socket(sock) {} void error() { m_socket.send(TEXT_ERROR, sizeof(TEXT_ERROR) - 1, true); } void ok() { m_socket.send(TEXT_OK, sizeof(TEXT_OK) - 1, true); } void end() { m_socket.send(TEXT_END, sizeof(TEXT_END) - 1, true); } void stored() { m_socket.send(TEXT_STORED, sizeof(TEXT_STORED) - 1, true); } void not_stored() { m_socket.send(TEXT_NOT_STORED, sizeof(TEXT_NOT_STORED) - 1, true); } void exists() { m_socket.send(TEXT_EXISTS, sizeof(TEXT_EXISTS) - 1, true); } void touched() { m_socket.send(TEXT_TOUCHED, sizeof(TEXT_TOUCHED) - 1, true); } void deleted() { m_socket.send(TEXT_DELETED, sizeof(TEXT_DELETED) - 1, true); } void not_found() { m_socket.send(TEXT_NOT_FOUND, sizeof(TEXT_NOT_FOUND) - 1, true); } void locked() { m_socket.send(TEXT_LOCKED, sizeof(TEXT_LOCKED) - 1, true); } void value(const cybozu::hash_key& key, std::uint32_t flags, const cybozu::dynbuf& data); void value(const cybozu::hash_key& key, std::uint32_t flags, const cybozu::dynbuf& data, std::uint64_t cas); void value(const cybozu::hash_key& key); void send(const char* p, std::size_t len, bool flush) { m_socket.send(p, len, flush); } void stats_settings(); void stats_items(); void stats_sizes(); void stats_ops(); void stats_general(std::size_t n_slaves); void version(); private: cybozu::tcp_socket& m_socket; cybozu::tcp_socket::iovec m_iov[cybozu::tcp_socket::MAX_IOVCNT]; }; // Memcache binary commands. enum class binary_command: unsigned char { Get = '\x00', Set = '\x01', Add = '\x02', Replace = '\x03', Delete = '\x04', Increment = '\x05', Decrement = '\x06', Quit = '\x07', Flush = '\x08', GetQ = '\x09', Noop = '\x0a', Version = '\x0b', GetK = '\x0c', GetKQ = '\x0d', Append = '\x0e', Prepend = '\x0f', Stat = '\x10', SetQ = '\x11', AddQ = '\x12', ReplaceQ = '\x13', DeleteQ = '\x14', IncrementQ = '\x15', DecrementQ = '\x16', QuitQ = '\x17', FlushQ = '\x18', AppendQ = '\x19', PrependQ = '\x1a', Touch = '\x1c', GaT = '\x1d', GaTQ = '\x1e', GaTK = '\x23', GaTKQ = '\x24', Lock = '\x40', LockQ = '\x41', Unlock = '\x42', UnlockQ = '\x43', UnlockAll = '\x44', UnlockAllQ = '\x45', LaG = '\x46', LaGQ = '\x47', LaGK = '\x48', LaGKQ = '\x49', RaU = '\x4a', RaUQ = '\x4b', Keys = '\x50', Unknown, END_OF_COMMAND // must be defined the last }; // Binary protocol response status enum class binary_status: std::uint16_t { OK = 0x0000, NotFound = 0x0001, Exists = 0x0002, TooLargeValue = 0x0003, Invalid = 0x0004, NotStored = 0x0005, NonNumeric = 0x0006, Locked = 0x0010, NotLocked = 0x0011, UnknownCommand = 0x0081, OutOfMemory = 0x0082 }; // Binary request parser class binary_request final { public: binary_request(const char* p, std::size_t len): m_p(p), m_len(len) { if( len == 0 || p == nullptr ) throw std::logic_error(" bad ctor arguments"); parse(); } // Return length of the request. // // Return length of the request. // If the request is incomplete, zero is returned. std::size_t length() const noexcept { return m_request_len; } // Response status, if determined by the request. binary_status status() const noexcept { return m_status; } // Return the command type. binary_command command() const noexcept { return m_command; } // The command is quiet or not. bool quiet() const noexcept { return m_quiet; } // Return `key`. item key() const noexcept { return m_key; } // Return `opaque` sent with the request. const char* opaque() const noexcept { return m_p + 12; } // Return `cas unique` sent with CAS command. std::uint64_t cas_unique() const noexcept { return m_cas_unique; } // Return `flags` sent with storage commands. std::uint32_t flags() const noexcept { return m_flags; } // Special expiration time for increment/decrement. static const std::time_t EXPTIME_NONE = (std::time_t)-1; // Return `exptime`. std::time_t exptime() const noexcept { return m_exptime; } // Return data block sent with storage commands. item data() const noexcept { return m_data; } // Return an unsigned 64bit integer value for increment or decrement. std::uint64_t value() const noexcept { return m_value; } // Return an unsigned 64bit integer value to initialize an object. std::uint64_t initial() const noexcept { return m_initial; } // Return memcache stats category sent with STATS. stats_t stats() const noexcept { return m_stats; } private: const char* const m_p; const std::size_t m_len; std::size_t m_request_len = 0; binary_status m_status = binary_status::Invalid; binary_command m_command; bool m_quiet; item m_key; std::uint64_t m_cas_unique; std::uint32_t m_flags = 0; std::time_t m_exptime = 0; item m_data; std::uint64_t m_value = 0; std::uint64_t m_initial = 0; stats_t m_stats = stats_t::GENERAL; void parse() noexcept; }; // Binary response sender. class binary_response final { public: binary_response(cybozu::tcp_socket& sock, const binary_request& req): m_socket(sock), m_request(req) {} void error(binary_status status); void success(); void get(std::uint32_t flags, const cybozu::dynbuf& data, std::uint64_t cas, bool flush, const char* key = nullptr, std::size_t key_len = 0); void key(const char* key, std::size_t key_len); void set(std::uint64_t cas); void incdec(std::uint64_t value, std::uint64_t cas); void quit(); void stats_settings(); void stats_items(); void stats_sizes(); void stats_ops(); void stats_general(std::size_t n_slaves); void version(); private: cybozu::tcp_socket& m_socket; const binary_request& m_request; cybozu::tcp_socket::iovec m_iov[cybozu::tcp_socket::MAX_IOVCNT]; void send_error(binary_status status, const char* msg, std::size_t len); void send_stat(const std::string& key, const std::string& value); void fill_header(char* buf, std::uint16_t key_len, std::uint8_t extras_len, std::uint32_t data_len, std::uint64_t cas, binary_status status = binary_status::OK) const noexcept { buf[0] = '\x81'; buf[1] = (char)m_request.command(); cybozu::hton(key_len, buf+2); buf[4] = (char)extras_len; buf[5] = '\x00'; cybozu::hton((std::uint16_t)status, buf+6); std::uint32_t total_len = key_len + extras_len + data_len; cybozu::hton(total_len, buf+8); std::memcpy(buf+12, m_request.opaque(), 4); cybozu::hton(cas, buf+16); } }; }} // namespace yrmcds::memcache #endif // YRMCDS_MEMCACHE_MEMCACHE_HPP yrmcds-1.1.5/src/memcache/object.cpp000066400000000000000000000117211262254645600173400ustar00rootroot00000000000000// (C) 2013 Cybozu. #include "../config.hpp" #include "object.hpp" #include "stats.hpp" #include #include #include #include #include #include namespace { std::uint64_t to_uint64(const char* p, std::size_t len) { unsigned long long ull = std::stoull( std::string(p, len) ); if( ull > std::numeric_limits::max() ) throw std::out_of_range("out of range for uint64"); return static_cast(ull); } } // anonymous namespace namespace yrmcds { namespace memcache { thread_local int g_context = -1; file_flusher::~file_flusher() { if( m_fd == -1 ) return; if( ::fdatasync(m_fd) == 0 ) ::posix_fadvise(m_fd, 0, 0, POSIX_FADV_DONTNEED); ::close(m_fd); } object::object(const char* p, std::size_t len, std::uint32_t flags_, std::time_t exptime) : m_length(len), m_data(0, g_config.secure_erase()), m_file(nullptr), m_flags(flags_), m_exptime(exptime) { if( len > g_config.heap_data_limit() ) { m_file = std::unique_ptr(new tempfile); m_file->write(p, len); } else { if( len > 0 ) m_data.append(p, len); } g_stats.total_objects.fetch_add(1, std::memory_order_relaxed); } object::object(std::uint64_t initial, std::time_t exptime) : m_length(0), m_data(24, g_config.secure_erase()), m_file(nullptr), m_flags(0), m_exptime(exptime) { char s_value[24]; // uint64 can be as large as 20 byte decimal string. m_length = ::snprintf(s_value, sizeof(s_value), "%llu", (unsigned long long)initial); m_data.append(s_value, m_length); g_stats.total_objects.fetch_add(1, std::memory_order_relaxed); } void object::set(const char* p, std::size_t len, std::uint32_t flags_, std::time_t exptime) { m_flags = flags_; m_exptime = exptime; ++ m_cas; m_gc_old = 0; m_data.reset(); if( len > g_config.heap_data_limit() ) { if( m_file.get() == nullptr ) { m_file = std::unique_ptr(new tempfile); } else { m_file->clear(); } m_file->write(p, len); } else { m_file = nullptr; if( len > 0 ) m_data.append(p, len); } m_length = len; } void object::append(const char* p, std::size_t len) { ++ m_cas; m_gc_old = 0; if( len == 0 ) return; std::size_t new_size = m_length + len; if( new_size > g_config.heap_data_limit() ) { if( m_file.get() == nullptr ) { m_file = std::unique_ptr(new tempfile); if( m_length > 0 ) m_file->write(m_data.data(), m_length); m_data.reset(); m_file->write(p, len); } else { m_file->write(p, len); } } else { m_data.append(p, len); } m_length = new_size; } void object::prepend(const char* p, std::size_t len) { ++ m_cas; m_gc_old = 0; if( len == 0 ) return; std::size_t new_size = m_length + len; if( new_size > g_config.heap_data_limit() ) { if( m_file.get() == nullptr ) { m_file = std::unique_ptr(new tempfile); m_file->write(p, len); if( m_length > 0 ) m_file->write(m_data.data(), m_length); m_data.reset(); } else { cybozu::dynbuf buf(new_size); buf.append(p, len); m_file->read_contents(buf); m_file->clear(); m_file->write(buf.data(), new_size); } } else { cybozu::dynbuf buf(new_size); buf.append(p, len); buf.append(m_data.data(), m_length); m_data.swap(buf); } m_length = new_size; } std::uint64_t object::incr(std::uint64_t n) { if( m_file.get() != nullptr ) throw not_a_number{}; std::uint64_t u64_value; try { u64_value = to_uint64(m_data.data(), m_length); } catch( const std::logic_error& ) { throw not_a_number{}; } u64_value += n; char s_value[24]; // uint64 can be as large as 20 byte decimal string. m_length = ::snprintf(s_value, sizeof(s_value), "%llu", (unsigned long long)u64_value); m_data.reset(); m_data.append(s_value, m_length); ++ m_cas; m_gc_old = 0; return u64_value; } std::uint64_t object::decr(std::uint64_t n) { if( m_file.get() != nullptr ) throw not_a_number{}; std::uint64_t u64_value; try { u64_value = to_uint64(m_data.data(), m_length); } catch( const std::logic_error& ) { throw not_a_number{}; } u64_value = (u64_value < n) ? 0 : (u64_value - n); char s_value[24]; // uint64 can be as large as 20 byte decimal string. m_length = ::snprintf(s_value, sizeof(s_value), "%llu", (unsigned long long)u64_value); m_data.reset(); m_data.append(s_value, m_length); ++ m_cas; m_gc_old = 0; return u64_value; } }} // namespace yrmcds::memcache yrmcds-1.1.5/src/memcache/object.hpp000066400000000000000000000117121262254645600173450ustar00rootroot00000000000000// The cache object. // (C) 2013 Cybozu. #ifndef YRMCDS_MEMCACHE_OBJECT_HPP #define YRMCDS_MEMCACHE_OBJECT_HPP #include "stats.hpp" #include "../global.hpp" #include "../tempfile.hpp" #include #include #include #include #include #include #include #include #include #include #include namespace yrmcds { namespace memcache { // Context for object locking. // // The context is in fact the file descriptor of a client connection. extern thread_local int g_context; // Purge temporary file contents from the page cache at dtor. class file_flusher final { public: explicit file_flusher(int fd): m_fd(fd) {} file_flusher(file_flusher&& rhs) { std::swap(m_fd, rhs.m_fd); } file_flusher(const file_flusher&) = delete; file_flusher& operator=(file_flusher&&) = delete; file_flusher& operator=(const file_flusher&) = delete; ~file_flusher(); private: int m_fd = -1; }; // Object in the hash table. // // This class represents an object in the hash table. // Large objects are stored in temporary files. class object final { public: object(const char* p, std::size_t len, std::uint32_t flags_, std::time_t exptime); object(std::uint64_t initial, std::time_t exptime); object(const object&) = delete; object(object&& rhs) noexcept: m_length(rhs.m_length), m_data(std::move(rhs.m_data)), m_file(std::move(rhs.m_file)), m_flags(rhs.m_flags), m_exptime(rhs.m_exptime), m_cas(rhs.m_cas) {} object& operator=(const object&) = delete; object& operator=(object&&) = delete; // Exception thrown by or . struct not_a_number: public std::runtime_error { not_a_number(): std::runtime_error("") {} }; void set(const char* p, std::size_t len, std::uint32_t flags_, std::time_t exptime); void append(const char* p, std::size_t len); void prepend(const char* p, std::size_t len); std::uint64_t incr(std::uint64_t n); std::uint64_t decr(std::uint64_t n); void touch(std::time_t exptime) { m_exptime = exptime; m_gc_old = 0; } const cybozu::dynbuf& data(cybozu::dynbuf& buf) const { m_gc_old = 0; if( m_file.get() == nullptr ) return m_data; buf.reset(); m_file->read_contents(buf); return buf; } std::size_t size() const noexcept { if( m_file.get() == nullptr ) return m_data.size(); return m_file->length(); } std::uint32_t flags() const noexcept { return m_flags; } std::uint64_t cas_unique() const noexcept { return m_cas; } std::uint32_t exptime() const noexcept { return (std::uint32_t)m_exptime; } bool expired() const noexcept { if( locked() ) return false; std::time_t t = g_stats.flush_time.load(std::memory_order_relaxed); std::time_t now = g_current_time.load(std::memory_order_relaxed); if( t != 0 && t <= now ) return true; if( m_exptime == 0 ) return false; return m_exptime <= now; } unsigned int age() const noexcept { return m_gc_old; } void survive(std::vector& flushers) const { ++ m_gc_old; if( m_gc_old != FLUSH_AGE || m_file.get() == nullptr ) return; int new_fd = ::dup(m_file->fileno()); if( new_fd == -1 ) { cybozu::logger::warning() << "Failed to dup a file descriptor"; return; } flushers.emplace_back(new_fd); } void lock() { if( locked() ) throw std::logic_error("object::lock bug"); m_lock = g_context; } void unlock(bool force = false) { if( ! force && ! locked_by_self() ) { cybozu::dump_stack(); std::ostringstream os; os << "object::unlock bug (m_lock=" << m_lock << ", g_context=" << g_context << ", unlocked thread=" << m_unlocker << ", this thread=" << std::this_thread::get_id(); throw std::logic_error(os.str()); } m_unlocker = std::this_thread::get_id(); m_lock = -1; } // Return `true` if this object is locked. bool locked() const noexcept { return m_lock != -1; } // Return `true` if this object is locked by the current context. bool locked_by_self() const noexcept { return m_lock == g_context; } // Return `true` if this object is locked by another context. bool locked_by_other() const noexcept { return locked() && (! locked_by_self()); } private: std::size_t m_length; cybozu::dynbuf m_data; std::unique_ptr m_file; std::uint32_t m_flags; std::time_t m_exptime; std::uint64_t m_cas = 1; mutable unsigned int m_gc_old = 0; int m_lock = -1; std::thread::id m_unlocker; }; }} // namespace yrmcds::memcache #endif // YRMCDS_MEMCACHE_OBJECT_HPP yrmcds-1.1.5/src/memcache/replication.cpp000066400000000000000000000102041262254645600203760ustar00rootroot00000000000000// (C) 2013-2014 Cybozu. #include "memcache.hpp" #include "replication.hpp" #include "stats.hpp" #include #include #include #include #include #include namespace { using namespace yrmcds::memcache; const int BINARY_HEADER_SIZE = 24; inline void fill_header(char* buf, std::uint16_t key_len, std::uint8_t extras_len, std::uint32_t data_len, binary_command cmd) noexcept { std::memset(buf, 0, BINARY_HEADER_SIZE); buf[0] = '\x80'; buf[1] = (char)cmd; cybozu::hton(key_len, buf+2); buf[4] = (char)extras_len; std::uint32_t total_len = key_len + extras_len + data_len; cybozu::hton(total_len, buf+8); } } // anonymous namespace namespace yrmcds { namespace memcache { void repl_object(const std::vector& slaves, const cybozu::hash_key& key, const object& obj, bool flush) { char header[BINARY_HEADER_SIZE]; fill_header(header, key.length(), 8, obj.size(), binary_command::SetQ); char extras[8]; cybozu::hton(obj.flags(), extras); cybozu::hton(obj.exptime(), &extras[4]); cybozu::dynbuf buf(0); const cybozu::dynbuf& data = obj.data(buf); cybozu::tcp_socket::iovec iov[4] = { {header, sizeof(header)}, {extras, sizeof(extras)}, {key.data(), key.length()}, {data.data(), data.size()} }; for( cybozu::tcp_socket* s: slaves ) s->sendv(iov, 4, flush); } void repl_delete(const std::vector& slaves, const cybozu::hash_key& key) { char header[BINARY_HEADER_SIZE]; fill_header(header, key.length(), 0, 0, binary_command::DeleteQ); cybozu::tcp_socket::iovec iov[2] = { {header, sizeof(header)}, {key.data(), key.length()} }; for( cybozu::tcp_socket* s: slaves ) s->sendv(iov, 2, false); } std::size_t repl_recv(const char* p, std::size_t len, cybozu::hash_map& hash) { namespace mc = yrmcds::memcache; std::size_t consumed = 0; while( len > 0 ) { if( ! mc::is_binary_request(p) ) throw std::runtime_error("Invalid replication data"); mc::binary_request parser(p, len); std::size_t n = parser.length(); if( n == 0 ) break; p += n; len -= n; consumed += n; const char* key_data; std::size_t key_len; cybozu::hash_map::handler h = nullptr; cybozu::hash_map::creator c = nullptr; switch( parser.command() ) { case mc::binary_command::SetQ: h = [&parser](const cybozu::hash_key&, object& obj) -> bool { const char* p2; std::size_t len2; std::tie(p2, len2) = parser.data(); ++ g_stats.repl_updated; obj.set(p2, len2, parser.flags(), parser.exptime()); return true; }; c = [&parser](const cybozu::hash_key&) -> object { const char* p2; std::size_t len2; std::tie(p2, len2) = parser.data(); ++ g_stats.repl_created; return object(p2, len2, parser.flags(), parser.exptime()); }; std::tie(key_data, key_len) = parser.key(); cybozu::logger::debug() << "repl: set " << std::string(key_data, key_len); hash.apply_nolock(cybozu::hash_key(key_data, key_len), h, c); break; case mc::binary_command::DeleteQ: std::tie(key_data, key_len) = parser.key(); cybozu::logger::debug() << "repl: remove " << std::string(key_data, key_len); ++ g_stats.repl_removed; hash.remove_nolock(cybozu::hash_key(key_data, key_len), nullptr); break; default: cybozu::logger::error() << "Unknown replication command" << std::hex << (unsigned int)parser.command(); } } return consumed; } }} // namespace yrmcds::memcache yrmcds-1.1.5/src/memcache/replication.hpp000066400000000000000000000013161262254645600204070ustar00rootroot00000000000000// Replication protocol. // (C) 2013 Cybozu. #ifndef YRMCDS_MEMCACHE_REPLICATION_HPP #define YRMCDS_MEMCACHE_REPLICATION_HPP #include "object.hpp" #include "sockets.hpp" #include #include namespace yrmcds { namespace memcache { void repl_object(const std::vector& slaves, const cybozu::hash_key& key, const object& obj, bool flush = true); void repl_delete(const std::vector& slaves, const cybozu::hash_key& key); std::size_t repl_recv(const char* p, std::size_t len, cybozu::hash_map& hash); }} // namespace yrmcds::memcache #endif // YRMCDS_MEMCACHE_REPLICATION_HPP yrmcds-1.1.5/src/memcache/sockets.cpp000066400000000000000000001037241262254645600175520ustar00rootroot00000000000000// (C) 2013 Cybozu. #include "replication.hpp" #include "sockets.hpp" #include #include #include #include #include namespace mc = yrmcds::memcache; using mc::text_command; using mc::binary_command; using mc::binary_status; namespace { const char NON_NUMERIC[] = "CLIENT_ERROR cannot increment or decrement non-numeric value\x0d\x0a"; const char NOT_LOCKED[] = "CLIENT_ERROR object is not locked or not found\x0d\x0a"; const std::memory_order relaxed = std::memory_order_relaxed; } // anonymous namespace namespace yrmcds { namespace memcache { using hash_map = cybozu::hash_map; memcache_socket::memcache_socket(int fd, const std::function& finder, cybozu::hash_map& hash, const std::vector& slaves) : cybozu::tcp_socket(fd), m_busy(false), m_finder(finder), m_hash(hash), m_pending(0), m_slaves_origin(slaves) { m_slaves.reserve(MAX_SLAVES); g_stats.curr_connections.fetch_add(1, relaxed); g_stats.total_connections.fetch_add(1, relaxed); m_recvjob = [this](cybozu::dynbuf& buf) { // set lock context for objects. g_context = m_fd; // load pending data if( ! m_pending.empty() ) { buf.append(m_pending.data(), m_pending.size()); m_pending.reset(); } while( true ) { char* p = buf.prepare(MAX_RECVSIZE); ssize_t n = ::recv(m_fd, p, MAX_RECVSIZE, 0); if( n == -1 ) { if( errno == EAGAIN || errno == EWOULDBLOCK ) break; if( errno == EINTR ) continue; if( errno == ECONNRESET ) { buf.reset(); unlock_all(); invalidate_and_close(); break; } cybozu::throw_unix_error(errno, "recv"); } if( n == 0 ) { buf.reset(); unlock_all(); invalidate_and_close(); break; } // if (n != -1) && (n != 0) buf.consume(n); const char* head = buf.data(); std::size_t len = buf.size(); while( len > 0 ) { if( mc::is_binary_request(head) ) { mc::binary_request parser(head, len); std::size_t c = parser.length(); if( c == 0 ) break; head += c; len -= c; cmd_bin(parser); } else { mc::text_request parser(head, len); std::size_t c = parser.length(); if( c == 0 ) break; head += c; len -= c; cmd_text(parser); } } if( len > MAX_REQUEST_LENGTH ) { cybozu::logger::warning() << "denied too large request of " << len << " bytes."; buf.reset(); unlock_all(); invalidate_and_close(); break; } buf.erase(head - buf.data()); } // recv returns EAGAIN, or some error happens. if( buf.size() > 0 ) m_pending.append(buf.data(), buf.size()); m_busy.store(false, std::memory_order_release); }; m_sendjob = [this](cybozu::dynbuf&) { if( ! write_pending_data() ) invalidate_and_close(); }; } memcache_socket::~memcache_socket() { // the destructor is the safe place to release remaining locks. for( auto& ref: m_locks ) { m_hash.apply(ref.get(), [](const cybozu::hash_key&, object& obj) -> bool { obj.unlock(true); return true; }, nullptr); } } bool memcache_socket::on_readable() { if( m_busy.load(std::memory_order_acquire) ) { m_reactor->add_readable(*this); return true; } // find an idle worker. cybozu::worker* w = m_finder(); if( w == nullptr ) { m_reactor->add_readable(*this); return true; } // copy the current list of slaves m_slaves = m_slaves_origin; m_busy.store(true, std::memory_order_release); w->post_job(m_recvjob); return true; } bool memcache_socket::on_writable() { cybozu::worker* w = m_finder(); if( w == nullptr ) { // if there is no idle worker, fallback to the default. return cybozu::tcp_socket::on_writable(); } w->post_job(m_sendjob); return true; } void memcache_socket::cmd_bin(const memcache::binary_request& cmd) { mc::binary_response r(*this, cmd); if( cmd.status() != binary_status::OK ) { r.error( cmd.status() ); return; } g_stats.bin_ops[(std::size_t)cmd.command()].fetch_add(1, relaxed); const char* p; std::size_t len; hash_map::handler h = nullptr; hash_map::creator c = nullptr; std::function repl = nullptr; std::function pred; std::function foreach_pred; switch( cmd.command() ) { case binary_command::Get: case binary_command::GetQ: case binary_command::GetK: case binary_command::GetKQ: case binary_command::GaT: case binary_command::GaTQ: case binary_command::GaTK: case binary_command::GaTKQ: case binary_command::LaG: case binary_command::LaGQ: case binary_command::LaGK: case binary_command::LaGKQ: h = [this,&cmd,&r](const cybozu::hash_key& k, object& obj) -> bool { if( (binary_command::LaG <= cmd.command()) && (cmd.command() <= binary_command::LaGKQ) ) { if( obj.locked() ) { r.error( binary_status::Locked ); return true; } obj.lock(); add_lock(k); } if( obj.expired() ) return false; if( cmd.exptime() != mc::binary_request::EXPTIME_NONE ) obj.touch( cmd.exptime() ); cybozu::dynbuf buf(0); const cybozu::dynbuf& data = obj.data(buf); if( cmd.command() == binary_command::Get || cmd.command() == binary_command::GetQ || cmd.command() == binary_command::GaT || cmd.command() == binary_command::GaTQ || cmd.command() == binary_command::LaG || cmd.command() == binary_command::LaGQ ) { r.get(obj.flags(), data, obj.cas_unique(), ! cmd.quiet()); } else { r.get(obj.flags(), data, obj.cas_unique(), ! cmd.quiet(), k.data(), k.length()); } return true; }; std::tie(p, len) = cmd.key(); if( ! m_hash.apply(cybozu::hash_key(p, len), h, c) ) { g_stats.get_misses.fetch_add(1, relaxed); if( ! cmd.quiet() || cmd.command() == binary_command::LaGQ ) r.error( binary_status::NotFound ); } else { g_stats.get_hits.fetch_add(1, relaxed); } break; case binary_command::Set: case binary_command::SetQ: case binary_command::Add: case binary_command::AddQ: case binary_command::Replace: case binary_command::ReplaceQ: std::tie(p, len) = cmd.key(); if( len > MAX_KEY_LENGTH ) { r.error( binary_status::Invalid ); return; } if( std::get<1>(cmd.data()) > g_config.max_data_size() ) { r.error( binary_status::TooLargeValue ); return; } h = [this,&cmd,&r](const cybozu::hash_key& k, object& obj) -> bool { if( obj.locked_by_other() ) { r.error( binary_status::Locked ); return true; } if( obj.expired() ) { if( cmd.cas_unique() != 0 || cmd.command() == binary_command::Replace || cmd.command() == binary_command::ReplaceQ ) { if( cmd.cas_unique() != 0 ) g_stats.cas_misses.fetch_add(1, relaxed); r.error( binary_status::NotFound ); return true; } } else if( cmd.command() == binary_command::Add || cmd.command() == binary_command::AddQ ) { r.error( binary_status::Exists ); return true; } if( cmd.cas_unique() != 0 && cmd.cas_unique() != obj.cas_unique() ) { g_stats.cas_badval.fetch_add(1, relaxed); r.error( binary_status::Exists ); return true; } const char* p2; std::size_t len2; std::tie(p2, len2) = cmd.data(); obj.set(p2, len2, cmd.flags(), cmd.exptime()); if( ! cmd.quiet() ) r.set( obj.cas_unique() ); if( cmd.cas_unique() != 0 ) g_stats.cas_hits.fetch_add(1, relaxed); if( ! m_slaves.empty() ) repl_object(m_slaves, k, obj); return true; }; if( cmd.command() != binary_command::Replace && cmd.command() != binary_command::ReplaceQ && cmd.cas_unique() == 0 ) { c = [this,&cmd,&r](const cybozu::hash_key& k) -> object { const char* p2; std::size_t len2; std::tie(p2, len2) = cmd.data(); object o(p2, len2, cmd.flags(), cmd.exptime()); if( ! cmd.quiet() ) r.set( o.cas_unique() ); if( ! m_slaves.empty() ) repl_object(m_slaves, k, o); return std::move(o); }; } if( ! m_hash.apply(cybozu::hash_key(p, len), h, c) ) { if( cmd.cas_unique() != 0 ) g_stats.cas_misses.fetch_add(1, relaxed); r.error( binary_status::NotFound ); } break; case binary_command::RaU: case binary_command::RaUQ: std::tie(p, len) = cmd.key(); if( len > MAX_KEY_LENGTH ) { r.error( binary_status::Invalid ); return; } if( std::get<1>(cmd.data()) > g_config.max_data_size() ) { r.error( binary_status::TooLargeValue ); return; } h = [this,&cmd,&r](const cybozu::hash_key& k, object& obj) -> bool { if( ! obj.locked_by_self() ) { r.error( binary_status::NotLocked ); return true; } if( cmd.cas_unique() != 0 && cmd.cas_unique() != obj.cas_unique() ) { r.error( binary_status::Exists ); return true; } const char* p2; std::size_t len2; std::tie(p2, len2) = cmd.data(); obj.set(p2, len2, cmd.flags(), cmd.exptime()); obj.unlock(); remove_lock(k); if( ! cmd.quiet() ) r.set( obj.cas_unique() ); if( ! m_slaves.empty() ) repl_object(m_slaves, k, obj); return true; }; if( ! m_hash.apply(cybozu::hash_key(p, len), h, c) ) r.error( binary_status::NotFound ); break; case binary_command::Append: case binary_command::AppendQ: std::tie(p, len) = cmd.key(); h = [this,&cmd,&r](const cybozu::hash_key& k, object& obj) -> bool { if( obj.locked_by_other() ) { r.error( binary_status::Locked ); return true; } if( obj.expired() ) return false; if( cmd.cas_unique() != 0 && cmd.cas_unique() != obj.cas_unique() ) { r.error( binary_status::Exists ); return true; } const char* p2; std::size_t len2; std::tie(p2, len2) = cmd.data(); obj.append(p2, len2); if( ! cmd.quiet() ) r.set( obj.cas_unique() ); if( ! m_slaves.empty() ) repl_object(m_slaves, k, obj); return true; }; if( ! m_hash.apply(cybozu::hash_key(p, len), h, c) ) r.error( binary_status::NotFound ); break; case binary_command::Prepend: case binary_command::PrependQ: std::tie(p, len) = cmd.key(); h = [this,&cmd,&r](const cybozu::hash_key& k, object& obj) -> bool { if( obj.locked_by_other() ) { r.error( binary_status::Locked ); return true; } if( obj.expired() ) return false; if( cmd.cas_unique() != 0 && cmd.cas_unique() != obj.cas_unique() ) { r.error( binary_status::Exists ); return true; } const char* p2; std::size_t len2; std::tie(p2, len2) = cmd.data(); obj.prepend(p2, len2); if( ! cmd.quiet() ) r.set( obj.cas_unique() ); if( ! m_slaves.empty() ) repl_object(m_slaves, k, obj); return true; }; if( ! m_hash.apply(cybozu::hash_key(p, len), h, c) ) r.error( binary_status::NotFound ); break; case binary_command::Delete: case binary_command::DeleteQ: std::tie(p, len) = cmd.key(); pred = [this,&cmd,&r](const cybozu::hash_key& k, object& obj) -> bool { if( obj.locked_by_other() ) { r.error( binary_status::Locked ); return false; } if( cmd.cas_unique() != 0 && cmd.cas_unique() != obj.cas_unique() ) { r.error( binary_status::Exists ); return false; } if( obj.locked_by_self() ) remove_lock(k); if( ! cmd.quiet() ) r.success(); if( ! m_slaves.empty() ) repl_delete(m_slaves, k); return true; }; if( ! m_hash.remove_if(cybozu::hash_key(p, len), pred) && ! cmd.quiet() ) r.error( binary_status::NotFound ); break; case binary_command::Increment: case binary_command::IncrementQ: std::tie(p, len) = cmd.key(); h = [this,&cmd,&r](const cybozu::hash_key& k, object& obj) -> bool { if( obj.locked_by_other() ) { r.error( binary_status::Locked ); return true; } if( obj.expired() ) return false; if( cmd.cas_unique() != 0 && cmd.cas_unique() != obj.cas_unique() ) { r.error( binary_status::Exists ); return true; } try { std::uint64_t n = obj.incr( cmd.value() ); if( ! cmd.quiet() ) r.incdec( n, obj.cas_unique() ); if( ! m_slaves.empty() ) repl_object(m_slaves, k, obj); } catch( const object::not_a_number& ) { r.error( binary_status::NonNumeric ); } return true; }; if( cmd.exptime() != mc::binary_request::EXPTIME_NONE ) { c = [this,&cmd,&r](const cybozu::hash_key& k) -> object { object o(cmd.initial(), cmd.exptime()); if( ! cmd.quiet() ) r.incdec( cmd.initial(), o.cas_unique() ); if( ! m_slaves.empty() ) repl_object(m_slaves, k, o); return std::move(o); }; } if( ! m_hash.apply(cybozu::hash_key(p, len), h, c) ) r.error( binary_status::NotFound ); break; case binary_command::Decrement: case binary_command::DecrementQ: std::tie(p, len) = cmd.key(); h = [this,&cmd,&r](const cybozu::hash_key& k, object& obj) -> bool { if( obj.locked_by_other() ) { r.error( binary_status::Locked ); return true; } if( obj.expired() ) return false; if( cmd.cas_unique() != 0 && cmd.cas_unique() != obj.cas_unique() ) { r.error( binary_status::Exists ); return true; } try { std::uint64_t n = obj.decr( cmd.value() ); if( ! cmd.quiet() ) r.incdec( n, obj.cas_unique() ); if( ! m_slaves.empty() ) repl_object(m_slaves, k, obj); } catch( const object::not_a_number& ) { r.error( binary_status::NonNumeric ); } return true; }; if( cmd.exptime() != mc::binary_request::EXPTIME_NONE ) { c = [this,&cmd,&r](const cybozu::hash_key& k) -> object { object o(cmd.initial(), cmd.exptime()); if( ! cmd.quiet() ) r.incdec( cmd.initial(), o.cas_unique() ); if( ! m_slaves.empty() ) repl_object(m_slaves, k, o); return std::move(o); }; } if( ! m_hash.apply(cybozu::hash_key(p, len), h, c) ) r.error( binary_status::NotFound ); break; case binary_command::Touch: std::tie(p, len) = cmd.key(); h = [this,&cmd,&r](const cybozu::hash_key& k, object& obj) -> bool { if( obj.expired() ) return false; obj.touch( cmd.exptime() ); if( ! m_slaves.empty() ) repl_object(m_slaves, k, obj, false); r.set( obj.cas_unique() ); return true; }; if( ! m_hash.apply(cybozu::hash_key(p, len), h, c) ) r.error( binary_status::NotFound ); break; case binary_command::Lock: case binary_command::LockQ: h = [this,&cmd,&r](const cybozu::hash_key& k, object& obj) -> bool { if( obj.expired() ) return false; if( obj.locked() ) { r.error( binary_status::Locked ); return true; } obj.lock(); add_lock(k); if( ! cmd.quiet() ) r.success(); return true; }; std::tie(p, len) = cmd.key(); if( ! m_hash.apply(cybozu::hash_key(p, len), h, c) ) r.error( binary_status::NotFound ); break; case binary_command::Unlock: case binary_command::UnlockQ: h = [this,&cmd,&r](const cybozu::hash_key& k, object& obj) -> bool { if( ! obj.locked_by_self() ) { r.error( binary_status::NotLocked ); return true; } obj.unlock(); remove_lock(k); if( ! cmd.quiet() ) r.success(); return true; }; std::tie(p, len) = cmd.key(); if( ! m_hash.apply(cybozu::hash_key(p, len), h, c) ) r.error( binary_status::NotFound ); break; case binary_command::UnlockAll: case binary_command::UnlockAllQ: unlock_all(); if( ! cmd.quiet() ) r.success(); break; case binary_command::Quit: case binary_command::QuitQ: unlock_all(); if( cmd.quiet() ) { invalidate_and_close(); } else { r.quit(); } break; case binary_command::Flush: case binary_command::FlushQ: g_stats.flush_time.store( cmd.exptime() ); if( ! cmd.quiet() ) r.success(); break; case binary_command::Noop: r.success(); break; case binary_command::Version: r.version(); break; case binary_command::Stat: switch( cmd.stats() ) { case mc::stats_t::SETTINGS: r.stats_settings(); break; case mc::stats_t::ITEMS: r.stats_items(); break; case mc::stats_t::SIZES: r.stats_sizes(); break; case mc::stats_t::OPS: r.stats_ops(); break; default: r.stats_general(m_slaves.size()); } break; case binary_command::Keys: std::tie(p, len) = cmd.key(); foreach_pred = [p,len,&r](const cybozu::hash_key& k, object&) { if( (len == 0) || k.has_prefix(p, len) ) r.key(k.data(), k.length()); }; m_hash.foreach(foreach_pred); r.success(); break; default: cybozu::logger::info() << "not implemented"; r.error( binary_status::UnknownCommand ); } } void memcache_socket::cmd_text(const memcache::text_request& cmd) { mc::text_response r(*this); if( ! cmd.valid() ) { r.error(); return; } g_stats.text_ops[(std::size_t)cmd.command()].fetch_add(1, relaxed); const char* p; std::size_t len; hash_map::handler h = nullptr; hash_map::creator c = nullptr; std::function pred; std::function foreach_pred; switch( cmd.command() ) { case text_command::SET: case text_command::ADD: case text_command::REPLACE: std::tie(p, len) = cmd.key(); if( len > MAX_KEY_LENGTH ) { r.error(); return; } if( std::get<1>(cmd.data()) > g_config.max_data_size() ) { r.error(); return; } h = [this,&cmd,&r](const cybozu::hash_key& k, object& obj) -> bool { if( obj.locked_by_other() ) { if( ! cmd.no_reply() ) r.locked(); return true; } if( obj.expired() ) { if( cmd.command() == text_command::REPLACE ) return false; } else if( cmd.command() == text_command::ADD ) { return false; } const char* p2; std::size_t len2; std::tie(p2, len2) = cmd.data(); obj.set(p2, len2, cmd.flags(), cmd.exptime()); if( ! cmd.no_reply() ) r.stored(); if( ! m_slaves.empty() ) repl_object(m_slaves, k, obj); return true; }; if( cmd.command() != text_command::REPLACE ) { c = [this,&cmd,&r](const cybozu::hash_key& k) -> object { const char* p2; std::size_t len2; std::tie(p2, len2) = cmd.data(); object o(p2, len2, cmd.flags(), cmd.exptime()); if( ! cmd.no_reply() ) r.stored(); if( ! m_slaves.empty() ) repl_object(m_slaves, k, o); return std::move(o); }; } if( ! m_hash.apply(cybozu::hash_key(p, len), h, c) && ! cmd.no_reply() ) r.not_stored(); break; case text_command::APPEND: std::tie(p, len) = cmd.key(); h = [this,&cmd,&r](const cybozu::hash_key& k, object& obj) -> bool { if( obj.locked_by_other() ) { if( ! cmd.no_reply() ) r.locked(); return true; } if( obj.expired() ) return false; const char* p2; std::size_t len2; std::tie(p2, len2) = cmd.data(); obj.append(p2, len2); if( ! cmd.no_reply() ) r.stored(); if( ! m_slaves.empty() ) repl_object(m_slaves, k, obj); return true; }; if( ! m_hash.apply(cybozu::hash_key(p, len), h, c) && ! cmd.no_reply() ) r.not_stored(); break; case text_command::PREPEND: std::tie(p, len) = cmd.key(); h = [this,&cmd,&r](const cybozu::hash_key& k, object& obj) -> bool { if( obj.locked_by_other() ) { if( ! cmd.no_reply() ) r.locked(); return true; } if( obj.expired() ) return false; const char* p2; std::size_t len2; std::tie(p2, len2) = cmd.data(); obj.prepend(p2, len2); if( ! cmd.no_reply() ) r.stored(); if( ! m_slaves.empty() ) repl_object(m_slaves, k, obj); return true; }; if( ! m_hash.apply(cybozu::hash_key(p, len), h, c) && ! cmd.no_reply() ) r.not_stored(); break; case text_command::CAS: std::tie(p, len) = cmd.key(); h = [this,&cmd,&r](const cybozu::hash_key& k, object& obj) -> bool { if( obj.locked_by_other() ) { if( ! cmd.no_reply() ) r.locked(); return true; } if( obj.expired() ) return false; if( obj.cas_unique() != cmd.cas_unique() ) { if( ! cmd.no_reply() ) r.exists(); g_stats.cas_badval.fetch_add(1, relaxed); return true; } const char* p2; std::size_t len2; std::tie(p2, len2) = cmd.data(); obj.set(p2, len2, cmd.flags(), cmd.exptime()); if( ! cmd.no_reply() ) r.stored(); if( ! m_slaves.empty() ) repl_object(m_slaves, k, obj); g_stats.cas_hits.fetch_add(1, relaxed); return true; }; if( ! m_hash.apply(cybozu::hash_key(p, len), h, c) ) { g_stats.cas_misses.fetch_add(1, relaxed); if( ! cmd.no_reply() ) r.not_found(); } break; case text_command::INCR: std::tie(p, len) = cmd.key(); h = [this,&cmd,&r](const cybozu::hash_key& k, object& obj) -> bool { if( obj.locked_by_other() ) { if( ! cmd.no_reply() ) r.locked(); return true; } if( obj.expired() ) return false; try { std::uint64_t n = obj.incr( cmd.value() ); if( ! cmd.no_reply() ) { char buf[24]; int numlen = snprintf(buf, sizeof(buf), "%llu\x0d\x0a", (unsigned long long)n); r.send(buf, numlen, true); } if( ! m_slaves.empty() ) repl_object(m_slaves, k, obj); } catch( const object::not_a_number& ) { if( ! cmd.no_reply() ) r.send(NON_NUMERIC, sizeof(NON_NUMERIC) - 1, true); } return true; }; if( ! m_hash.apply(cybozu::hash_key(p, len), h, c) && ! cmd.no_reply() ) r.not_found(); break; case text_command::DECR: std::tie(p, len) = cmd.key(); h = [this,&cmd,&r](const cybozu::hash_key& k, object& obj) -> bool { if( obj.locked_by_other() ) { if( ! cmd.no_reply() ) r.locked(); return true; } if( obj.expired() ) return false; try { std::uint64_t n = obj.decr( cmd.value() ); if( ! cmd.no_reply() ) { char buf[24]; int numlen = snprintf(buf, sizeof(buf), "%llu\x0d\x0a", (unsigned long long)n); r.send(buf, numlen, true); } if( ! m_slaves.empty() ) repl_object(m_slaves, k, obj); } catch( const object::not_a_number& ) { if( ! cmd.no_reply() ) r.send(NON_NUMERIC, sizeof(NON_NUMERIC) - 1, true); } return true; }; if( ! m_hash.apply(cybozu::hash_key(p, len), h, c) && ! cmd.no_reply() ) r.not_found(); break; case text_command::TOUCH: std::tie(p, len) = cmd.key(); h = [this,&cmd](const cybozu::hash_key& k, object& obj) -> bool { if( obj.expired() ) return false; obj.touch( cmd.exptime() ); if( ! m_slaves.empty() ) repl_object(m_slaves, k, obj, false); return true; }; if( m_hash.apply(cybozu::hash_key(p, len), h, c) ) { if( ! cmd.no_reply() ) r.touched(); } else { if( ! cmd.no_reply() ) r.not_found(); } break; case text_command::DELETE: std::tie(p, len) = cmd.key(); pred = [this,&cmd,&r](const cybozu::hash_key& k, object& obj) -> bool { if( obj.locked_by_other() ) { if( ! cmd.no_reply() ) r.locked(); return false; } if( obj.locked_by_self() ) remove_lock(k); if( ! cmd.no_reply() ) r.deleted(); if( ! m_slaves.empty() ) repl_delete(m_slaves, k); return true; }; if( ! m_hash.remove_if(cybozu::hash_key(p, len), pred) && ! cmd.no_reply() ) r.not_found(); break; case text_command::LOCK: h = [this,&cmd,&r](const cybozu::hash_key& k, object& obj) -> bool { if( obj.expired() ) return false; if( obj.locked() ) { r.locked(); return true; } obj.lock(); add_lock(k); r.ok(); return true; }; std::tie(p, len) = cmd.key(); if( ! m_hash.apply(cybozu::hash_key(p, len), h, c) ) r.not_found(); break; case text_command::UNLOCK: h = [this,&cmd,&r](const cybozu::hash_key& k, object& obj) -> bool { if( ! obj.locked_by_self() ) return false; obj.unlock(); remove_lock(k); r.ok(); return true; }; std::tie(p, len) = cmd.key(); if( ! m_hash.apply(cybozu::hash_key(p, len), h, c) ) r.send(NOT_LOCKED, sizeof(NOT_LOCKED) - 1, true); break; case text_command::UNLOCK_ALL: unlock_all(); r.ok(); break; case text_command::GET: case text_command::GETS: h = [&cmd,&r](const cybozu::hash_key& k, object& obj) -> bool { if( obj.expired() ) return false; cybozu::dynbuf buf(0); const cybozu::dynbuf& data = obj.data(buf); if( cmd.command() == text_command::GETS ) { r.value(k, obj.flags(), data, obj.cas_unique()); } else { r.value(k, obj.flags(), data); } return true; }; for( mc::item it = cmd.first_key(); it != mc::text_request::eos; it = cmd.next_key(it) ) { std::tie(p, len) = it; if( m_hash.apply(cybozu::hash_key(p, len), h, nullptr) ) { g_stats.get_hits.fetch_add(1, relaxed); } else { g_stats.get_misses.fetch_add(1, relaxed); } } r.end(); break; case text_command::KEYS: std::tie(p, len) = cmd.key(); foreach_pred = [p,len,&r](const cybozu::hash_key& k, object&) { if( (len == 0) || k.has_prefix(p, len) ) r.value(k); }; m_hash.foreach(foreach_pred); r.end(); break; case text_command::SLABS: r.ok(); break; case text_command::STATS: switch( cmd.stats() ) { case mc::stats_t::SETTINGS: r.stats_settings(); break; case mc::stats_t::ITEMS: r.stats_items(); break; case mc::stats_t::SIZES: r.stats_sizes(); break; case mc::stats_t::OPS: r.stats_ops(); break; default: r.stats_general(m_slaves.size()); } r.end(); break; case text_command::FLUSH_ALL: g_stats.flush_time.store( cmd.exptime() ); if( ! cmd.no_reply() ) r.ok(); break; case text_command::VERSION: r.version(); break; case text_command::VERBOSITY: cybozu::logger::set_threshold(cmd.verbosity()); if( ! cmd.no_reply() ) r.ok(); break; case text_command::QUIT: unlock_all(); invalidate_and_close(); break; default: cybozu::logger::info() << "not implemented"; r.error(); } } bool repl_socket::on_readable() { // recv and drop. while( true ) { ssize_t n = ::recv(m_fd, &m_recvbuf[0], MAX_RECVSIZE, 0); if( n == -1 ) { if( errno == EAGAIN || errno == EWOULDBLOCK ) break; if( errno == EINTR ) continue; if( errno == ECONNRESET ) return invalidate(); cybozu::throw_unix_error(errno, "recv"); } if( n == 0 ) return invalidate(); m_last_heartbeat = g_current_time.load(relaxed); } return true; } bool repl_socket::on_writable() { cybozu::worker* w = m_finder(); if( w == nullptr ) { // if there is no idle worker, fallback to the default. return cybozu::tcp_socket::on_writable(); } w->post_job(m_sendjob); return true; } bool repl_client_socket::on_readable() { while( true ) { char* p = m_recvbuf.prepare(MAX_RECVSIZE); ssize_t n = ::recv(m_fd, p, MAX_RECVSIZE, 0); if( n == -1 ) { if( errno == EAGAIN || errno == EWOULDBLOCK ) break; if( errno == EINTR ) continue; if( errno == ECONNRESET ) { m_reactor->quit(); return invalidate(); } cybozu::throw_unix_error(errno, "recv"); } if( n == 0 ) { m_reactor->quit(); return invalidate(); } m_recvbuf.consume(n); std::size_t c = repl_recv(m_recvbuf.data(), m_recvbuf.size(), m_hash); m_recvbuf.erase(c); } return true; } }} // namespace yrmcds::memcache yrmcds-1.1.5/src/memcache/sockets.hpp000066400000000000000000000107341262254645600175550ustar00rootroot00000000000000// Defines sockets for yrmcds. // (C) 2013 Cybozu. #ifndef YRMCDS_MEMCACHE_SOCKETS_HPP #define YRMCDS_MEMCACHE_SOCKETS_HPP #include "../constants.hpp" #include "memcache.hpp" #include "object.hpp" #include "stats.hpp" #include #include #include #include #include #include #include namespace yrmcds { namespace memcache { class repl_socket; class memcache_socket: public cybozu::tcp_socket { public: memcache_socket(int fd, const std::function& finder, cybozu::hash_map& hash, const std::vector& slaves); virtual ~memcache_socket(); void add_lock(const cybozu::hash_key& k) { // m_locks are accessed by at most one worker thread, hence // there is no need to guard it with a mutex. m_locks.emplace_back(k); } void remove_lock(const cybozu::hash_key& k) { for( auto it = m_locks.begin(); it != m_locks.end(); ++it ) { if( it->get() == k ) { m_locks.erase(it); return; } } cybozu::dump_stack(); throw std::logic_error(" bug"); } void unlock_all() { for( auto& ref: m_locks ) { m_hash.apply(ref.get(), [](const cybozu::hash_key&, object& obj) -> bool { obj.unlock(false); return true; }, nullptr); } m_locks.clear(); } // Process a binary request command. // @cmd A binary request. void cmd_bin(const memcache::binary_request& cmd); // Process a test request command. // @cmd A binary request. void cmd_text(const memcache::text_request& cmd); private: alignas(CACHELINE_SIZE) std::atomic m_busy; const std::function& m_finder; cybozu::hash_map& m_hash; cybozu::dynbuf m_pending; const std::vector& m_slaves_origin; std::vector m_slaves; cybozu::worker::job m_recvjob; cybozu::worker::job m_sendjob; std::vector> m_locks; virtual void on_invalidate() override final { // In order to avoid races and deadlocks, remaining locks // are not released here. They are released in the destructor // where no other threads have access to this object. g_stats.curr_connections.fetch_sub(1); cybozu::tcp_socket::on_invalidate(); } virtual bool on_readable() override final; virtual bool on_writable() override final; }; class object; class repl_socket: public cybozu::tcp_socket { public: repl_socket(int fd, unsigned int bufcnt, const std::function& finder) : cybozu::tcp_socket(fd, bufcnt), m_finder(finder), m_recvbuf(MAX_RECVSIZE), m_last_heartbeat(g_current_time.load(std::memory_order_relaxed)) { m_sendjob = [this](cybozu::dynbuf&) { if( ! write_pending_data() ) invalidate_and_close(); }; } bool timed_out() const { std::time_t now = g_current_time.load(std::memory_order_relaxed); return m_last_heartbeat + g_config.slave_timeout() <= now; } virtual void on_buffer_full() override { cybozu::logger::warning() << "Replication buffer is full. " << "Please enlarge \"repl_buffer_size\" to avoid the hang up of yrmcds."; } private: const std::function& m_finder; std::vector m_recvbuf; cybozu::worker::job m_sendjob; std::time_t m_last_heartbeat; virtual bool on_readable() override final; virtual bool on_writable() override final; }; class repl_client_socket: public cybozu::tcp_socket { public: repl_client_socket(int fd, cybozu::hash_map& m): cybozu::tcp_socket(fd), m_hash(m), m_recvbuf(30 << 20) {} private: cybozu::hash_map& m_hash; cybozu::dynbuf m_recvbuf; virtual bool on_readable() override final; virtual bool on_hangup() override final { m_reactor->quit(); return invalidate(); } virtual bool on_error() override final { m_reactor->quit(); return invalidate(); } }; }} // namespace yrmcds::memcache #endif // YRMCDS_MEMCACHE_SOCKETS_HPP yrmcds-1.1.5/src/memcache/stats.cpp000066400000000000000000000022301262254645600172230ustar00rootroot00000000000000// (C) 2013-2014 Cybozu. #include "stats.hpp" namespace yrmcds { namespace memcache { statistics g_stats; void statistics::reset() noexcept { objects = 0; objects_under_1k = 0; objects_under_4k = 0; objects_under_16k = 0; objects_under_64k = 0; objects_under_256k = 0; objects_under_1m = 0; objects_under_4m = 0; objects_huge = 0; /* bucket statistics. Updated at every GC. */ used_memory = 0; conflicts = 0; /* GC statistics. Updated at every GC, of course. */ gc_count = 0; oldest_age = 0; largest_object_size = 0; last_expirations = 0; last_evictions = 0; total_evictions = 0; last_gc_elapsed = 0; total_gc_elapsed = 0; /* Replication statistics - non atomic. */ repl_created = 0; repl_updated = 0; repl_removed = 0; /* Realtime staticstics. */ total_objects = 0; flush_time = 0; curr_connections = 0; total_connections = 0; for( auto& v: text_ops ) v = 0; for( auto& v: bin_ops ) v = 0; get_hits = 0; get_misses = 0; cas_hits = 0; cas_misses = 0; cas_badval = 0; } }} // namespace yrmcds::memcache yrmcds-1.1.5/src/memcache/stats.hpp000066400000000000000000000047371262254645600172460ustar00rootroot00000000000000// Atomic counters for statistics. // (C) 2013-2014 Cybozu. #ifndef YRMCDS_MEMCACHE_STATS_HPP #define YRMCDS_MEMCACHE_STATS_HPP #include "memcache.hpp" #include #include #include #include namespace yrmcds { namespace memcache { // statistics counters. struct statistics { statistics() { reset(); } // Reset all counters. void reset() noexcept; /* object counters. Updated at every GC. */ alignas(CACHELINE_SIZE) std::atomic objects; std::atomic objects_under_1k; std::atomic objects_under_4k; std::atomic objects_under_16k; std::atomic objects_under_64k; std::atomic objects_under_256k; std::atomic objects_under_1m; std::atomic objects_under_4m; std::atomic objects_huge; /* bucket statistics. Updated at every GC. */ std::atomic used_memory; std::atomic conflicts; /* GC statistics. Updated at every GC, of course. */ std::atomic gc_count; std::atomic oldest_age; std::atomic largest_object_size; std::atomic last_expirations; std::atomic last_evictions; std::atomic total_evictions; std::atomic last_gc_elapsed; // micro seconds std::atomic total_gc_elapsed; // micro seconds /* Replication statistics - non atomic. */ std::uint64_t repl_created; std::uint64_t repl_updated; std::uint64_t repl_removed; /* Realtime staticstics. */ alignas(CACHELINE_SIZE) std::atomic total_objects; alignas(CACHELINE_SIZE) std::atomic flush_time; // abused by "flush_all" alignas(CACHELINE_SIZE) std::atomic curr_connections; std::atomic total_connections; alignas(CACHELINE_SIZE) std::atomic text_ops[(std::size_t)memcache::text_command::END_OF_COMMAND]; std::atomic bin_ops[(std::size_t)memcache::binary_command::END_OF_COMMAND]; std::atomic get_hits; std::atomic get_misses; std::atomic cas_hits; std::atomic cas_misses; std::atomic cas_badval; }; extern statistics g_stats; }} // namespace yrmcds::memcache #endif // YRMCDS_MEMCACHE_STATS_HPP yrmcds-1.1.5/src/server.cpp000066400000000000000000000121531262254645600156360ustar00rootroot00000000000000// (C) 2013-2014 Cybozu. #include "constants.hpp" #include "memcache/handler.hpp" #include "counter/handler.hpp" #include "server.hpp" #include "global.hpp" #include #include #include #include #include #include #include namespace yrmcds { server::server(): m_syncer(m_workers) { auto finder = [this]() ->cybozu::worker* { std::size_t n_workers = m_workers.size(); for( std::size_t i = 0; i < n_workers; ++i ) { cybozu::worker* pw = m_workers[m_worker_index].get(); m_worker_index = (m_worker_index + 1) % n_workers; if( ! pw->is_running() ) return pw; } return nullptr; }; m_handlers.emplace_back(new memcache::handler(finder, m_reactor, m_syncer)); if( g_config.counter().enable() ) m_handlers.emplace_back(new counter::handler(finder, m_reactor)); } inline bool server::reactor_gc_ready() { if( ! m_reactor.has_garbage() ) return false; if( ! m_syncer.empty() ) return false; auto ready = [](const std::unique_ptr& p) { return p->reactor_gc_ready(); }; return std::all_of(m_handlers.cbegin(), m_handlers.cend(), ready); } void server::serve() { auto res = cybozu::signal_setup({SIGHUP, SIGQUIT, SIGTERM, SIGINT, SIGUSR1}); res->set_handler([this](const struct signalfd_siginfo& si, cybozu::reactor& r) { switch(si.ssi_signo) { case SIGUSR1: for( auto& handler: m_handlers ) handler->dump_stats(); break; case SIGHUP: cybozu::logger::instance().reopen(); cybozu::logger::info() << "got SIGHUP."; break; case SIGQUIT: case SIGTERM: case SIGINT: m_signaled = true; r.quit(); break; } }); m_reactor.add_resource(std::move(res), cybozu::reactor::EVENT_IN); for( auto& handler: m_handlers ) handler->on_start(); if( is_master() ) goto MASTER_ENTRY; while( true ) { for( auto& handler: m_handlers ) handler->clear(); serve_slave(); if( m_signaled ) return; // disconnected from the master for( int i = 0; i < MASTER_CHECKS; ++i ) { if( is_master() ) goto MASTER_ENTRY; std::this_thread::sleep_for( std::chrono::milliseconds(100) ); } } MASTER_ENTRY: serve_master(); std::quick_exit(0); } void server::serve_slave() { for( auto it1 = m_handlers.begin(); it1 != m_handlers.end(); ++it1 ) { if( ! (*it1)->on_slave_start() ) { // failed to start. stop already started handlers. for( auto it2 = m_handlers.begin(); it2 != it1; ++it2 ) (*it2)->on_slave_end(); return; } } cybozu::logger::info() << "Slave start"; m_reactor.run([this](cybozu::reactor& r) { if( is_master() ) { r.quit(); return; } std::time_t now = std::time(nullptr); g_current_time.store(now, std::memory_order_relaxed); for( auto& handler: m_handlers ) handler->on_slave_interval(); r.fix_garbage(); r.gc(); }); for( auto& handler: m_handlers ) handler->on_slave_end(); cybozu::logger::info() << "Slave end"; } void server::serve_master() { cybozu::logger::info() << "Entering master mode"; for( auto& handler: m_handlers ) handler->on_master_start(); auto callback = [this](cybozu::reactor&) { std::time_t now = std::time(nullptr); g_current_time.store(now, std::memory_order_relaxed); if( ! m_syncer.empty() ) m_syncer.check(); for( auto& handler: m_handlers ) handler->on_master_interval(); if( reactor_gc_ready() ) { m_reactor.fix_garbage(); m_syncer.add_request( std::unique_ptr( new sync_request([this]{ m_reactor.gc(); }) )); } }; for( unsigned int i = 0; i < g_config.workers(); ++i ) m_workers.emplace_back(new cybozu::worker(WORKER_BUFSIZE)); for( auto& w: m_workers ) w->start(); auto stop = [this] { m_reactor.invalidate(); for( auto& w: m_workers ) w->stop(); for( auto& handler: m_handlers ) handler->on_master_end(); }; try { cybozu::logger::info() << "Reactor thread id=" << std::this_thread::get_id(); m_reactor.run(callback); cybozu::logger::info() << "Exiting"; } catch( ... ) { stop(); throw; } stop(); } } // namespace yrmcds yrmcds-1.1.5/src/server.hpp000066400000000000000000000016331262254645600156440ustar00rootroot00000000000000// Logics and data structures for the main thread. // (C) 2013-2014 Cybozu. #ifndef YRMCDS_SERVER_HPP #define YRMCDS_SERVER_HPP #include "config.hpp" #include "handler.hpp" #include "sync.hpp" #include #include #include #include #include #include namespace yrmcds { // The yrmcds server class server { public: server(); static bool is_master() { return cybozu::has_ip_address( g_config.vip() ); } void serve(); private: bool m_signaled = false; cybozu::reactor m_reactor; std::vector> m_workers; int m_worker_index = 0; syncer m_syncer; std::vector> m_handlers; bool reactor_gc_ready(); void serve_slave(); void serve_master(); }; } // namespace yrmcds #endif // YRMCDS_SERVER_HPP yrmcds-1.1.5/src/sync.hpp000066400000000000000000000035771262254645600153230ustar00rootroot00000000000000// Worker synchronizer. // (C) 2013 Cybozu. #ifndef YRMCDS_SYNC_HPP #define YRMCDS_SYNC_HPP #include "constants.hpp" #include #include #include #include #include namespace yrmcds { class sync_request final { public: explicit sync_request(std::function callback): m_callback(callback) {} void set(int n) { m_flags[n] = true; } bool check() { if( ! m_flags.all() ) return false; m_callback(); return true; } private: std::function m_callback; std::bitset m_flags; }; class syncer { public: explicit syncer(const std::vector>& workers): m_workers(workers) {} bool empty() const noexcept { return m_requests.empty(); } void add_request(std::unique_ptr req) { const std::size_t n = m_workers.size(); for( std::size_t i = 0; i < n; ++i ) { if( ! m_workers[i]->is_running() ) req->set(i); } for( std::size_t i = n; i < MAX_WORKERS; ++i ) req->set(i); if( ! req->check() ) m_requests.emplace_back( std::move(req) ); } void check() { const std::size_t n = m_workers.size(); for( std::size_t i = 0; i < n; ++i ) { if( ! m_workers[i]->is_running() ) { for( auto& req: m_requests ) req->set(i); } } for( auto it = m_requests.begin(); it != m_requests.end(); ) { if( (*it)->check() ) { it = m_requests.erase(it); } else { ++it; } } } private: const std::vector>& m_workers; std::vector> m_requests; }; } // namespace yrmcds #endif // YRMCDS_SYNC_HPP yrmcds-1.1.5/src/tempfile.cpp000066400000000000000000000021441262254645600161340ustar00rootroot00000000000000// (C) 2013 Cybozu. #include "tempfile.hpp" #include #include #include namespace yrmcds { void tempfile::write(const char* p, std::size_t len) { while( len != 0 ) { ssize_t n = ::write(m_fd, p, len); if( n == -1 ) cybozu::throw_unix_error(errno, "write"); p += n; len -= n; m_length += n; } } void tempfile::clear() { if( lseek(m_fd, 0, SEEK_SET) == (off_t)-1 ) cybozu::throw_unix_error(errno, "lseek"); if( ftruncate(m_fd, 0) == -1 ) cybozu::throw_unix_error(errno, "ftruncate"); m_length = 0; } void tempfile::read_contents(cybozu::dynbuf& buf) const { char* p = buf.prepare(m_length); std::size_t to_read = m_length; while( to_read != 0 ) { ssize_t n = pread(m_fd, p, to_read, m_length - to_read); if( n == -1 ) cybozu::throw_unix_error(errno, "pread"); if( n == 0 ) throw std::runtime_error(" unexpected EOF."); p += n; to_read -= n; buf.consume(n); } } } // namespace yrmcds yrmcds-1.1.5/src/tempfile.hpp000066400000000000000000000032111262254645600161350ustar00rootroot00000000000000// Temporary file to store object data. // (C) 2013 Cybozu. #ifndef YRMCDS_TEMPFILE_HPP #define YRMCDS_TEMPFILE_HPP #include "config.hpp" #include #include #include #include #include #include #include #include namespace yrmcds { // A helper function for . inline int mkstemp_wrap(const std::string& tempdir) { std::string tmpl = tempdir + "/XXXXXX\0"; int fd = ::mkostemp(&(tmpl[0]), O_CLOEXEC); if( fd == -1 ) cybozu::throw_unix_error(errno, "mkstemp"); ::unlink(tmpl.data()); return fd; } // A temporary file abstraction. class tempfile { public: tempfile(): m_fd(mkstemp_wrap(g_config.tempdir())) {} tempfile(const tempfile&) = delete; tempfile& operator=(const tempfile&) = delete; tempfile(tempfile&& rhs) = delete; tempfile& operator=(tempfile&&) = delete; ~tempfile() { ::close(m_fd); } // Append data to the end of the current contents. // @p Pointer to the data. // @len Length of the data void write(const char* p, std::size_t len); // Clear the current file contents. void clear(); // Return the open file descriptor. int fileno() const { return m_fd; } // Read the file contents. // @buf Storage for the file contents. // // Read the file contents and append them to `buf`. void read_contents(cybozu::dynbuf& buf) const; std::size_t length() const noexcept { return m_length; } private: const int m_fd; std::size_t m_length = 0; }; } // namespace yrmcds #endif // YRMCDS_TEMPFILE_HPP yrmcds-1.1.5/test/000077500000000000000000000000001262254645600140125ustar00rootroot00000000000000yrmcds-1.1.5/test/bitfield.cpp000066400000000000000000000003321262254645600162760ustar00rootroot00000000000000#include int main(int argc, char** argv) { struct { unsigned int n : 1; } t; t.n = 0; for( int i = 0; i < 10; ++i ) { std::cout << t.n++ << std::endl; } return 0; } yrmcds-1.1.5/test/capture-this.cpp000066400000000000000000000003051262254645600171240ustar00rootroot00000000000000#include struct A { void foo() { std::cout << "Hello" << std::endl; } void bar() { [this]() { foo(); } (); } }; int main() { A a; a.bar(); return 0; } yrmcds-1.1.5/test/chrono.cpp000066400000000000000000000022141262254645600160050ustar00rootroot00000000000000#include #include #include #include typedef std::chrono::microseconds us_t; template inline us_t::rep to_us(T start, T end) { return std::chrono::duration_cast(end-start).count(); } template inline void print_ratio() { cybozu::demangler t(typeid(T).name()); std::cout << t.name() << " ratio = " << T::period::num << "/" << T::period::den << " second." << std::endl; } int main() { auto t1 = std::chrono::steady_clock::now(); print_ratio(); print_ratio(); print_ratio(); std::cout << "std::chrono::high_resolution_clock is " << (std::chrono::high_resolution_clock::is_steady ? "" : "not ") << "steady." << std::endl; static_assert(std::chrono::steady_clock::is_steady, "std::chrono::steady_clock is not steady!"); auto t2 = std::chrono::steady_clock::now(); std::cout << "t2 - t1 = " << to_us(t1, t2) << "us" << std::endl; return 0; } yrmcds-1.1.5/test/clear_memory.cpp000066400000000000000000000003371262254645600171770ustar00rootroot00000000000000#include #include AUTOTEST(clear_memory) { char hoge[] = "abcdef"; cybozu::clear_memory(hoge, sizeof(hoge)); for( char c : hoge ) { cybozu_assert(c == '\0'); } } yrmcds-1.1.5/test/config.cpp000066400000000000000000000024001262254645600157570ustar00rootroot00000000000000#include "../src/config.hpp" #include const std::string TEST_CONF = "test/test.conf"; AUTOTEST(config) { using yrmcds::g_config; g_config.load(TEST_CONF); cybozu_assert(g_config.port() == 1121); cybozu_assert(g_config.repl_port() == 1122); cybozu_assert(g_config.max_connections() == 10000); cybozu_assert(g_config.user() == "nobody"); cybozu_assert(g_config.group() == "nogroup"); cybozu_assert(g_config.memory_limit() == (1024 << 20)); cybozu_assert(g_config.repl_bufsize() == 100); cybozu_assert(g_config.secure_erase() == true); cybozu_assert(g_config.lock_memory() == true); cybozu_assert(g_config.threshold() == cybozu::severity::warning); cybozu_assert(g_config.max_data_size() == (5 << 20)); cybozu_assert(g_config.heap_data_limit() == (16 << 10)); cybozu_assert(g_config.workers() == 10); cybozu_assert(g_config.gc_interval() == 20); cybozu_assert(g_config.slave_timeout() == 15); cybozu_assert(g_config.counter().enable() == true); cybozu_assert(g_config.counter().port() == 11216); cybozu_assert(g_config.counter().max_connections() == 100); cybozu_assert(g_config.counter().buckets() == 1000001); cybozu_assert(g_config.counter().stat_interval() == 12345); } yrmcds-1.1.5/test/config_parser.cpp000066400000000000000000000010411262254645600173330ustar00rootroot00000000000000#include #include int main(int argc, char** argv) { cybozu::config_parser cp; cp.set("aaa", "3"); std::cout << cp.get_as_int("aaa") << std::endl; cp.set("aaa", "5"); std::cout << cp.get_as_int("aaa") << std::endl; cp.set("bbb", "false"); std::cout << cp.get_as_bool("bbb") << std::endl; if( argc == 2 ) { cp.load(argv[1]); std::cout << cp.get_as_int("memory_limit") << std::endl; std::cout << cp.get("temp_dir") << std::endl; } return 0; } yrmcds-1.1.5/test/connect.cpp000066400000000000000000000010051262254645600161430ustar00rootroot00000000000000#include #include int main(int argc, char** argv) { if( argc != 3 ) { std::cout << "Usage: connect.exe HOST PORT" << std::endl; return 0; } const char* node = argv[1]; std::uint16_t port = static_cast( std::stoi(argv[2]) ); int s = cybozu::tcp_connect(node, port, 5); if( s == -1 ) { std::cout << "failed." << std::endl; } else { std::cout << "connected." << std::endl; close(s); } return 0; } yrmcds-1.1.5/test/counter.cpp000066400000000000000000000165261262254645600162070ustar00rootroot00000000000000#define TEST_DISABLE_AUTO_RUN #include "../src/counter/counter.hpp" #include "counter_client.hpp" #include #include #include #include #include #include #include #include #include #include #include #include #include using namespace counter_client; const char* g_server = nullptr; uint16_t g_port = 11215; int connect_server() { int s = cybozu::tcp_connect(g_server, g_port); if( s == -1 ) return -1; ::fcntl(s, F_SETFL, ::fcntl(s, F_GETFL, 0) & ~O_NONBLOCK); struct timeval tv; tv.tv_sec = 0; tv.tv_usec = 300000; ::setsockopt(s, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv)); int ok = 1; ::setsockopt(s, IPPROTO_TCP, TCP_NODELAY, &ok, sizeof(ok)); return s; } #define ASSERT_RESPONSE(c, r, s, cmd, st) do { \ cybozu_assert( c.recv(r) ); \ cybozu_assert( r.opaque() == s ); \ cybozu_assert( r.command() == yrmcds::counter::command::cmd ); \ cybozu_assert( r.status() == yrmcds::counter::status::st ); \ } while( false ) AUTOTEST(noop) { client c(connect_server()); response r; serial_t s; // send noop 3 times for( int i = 0; i < 3; ++i ) { s = c.noop(); ASSERT_RESPONSE(c, r, s, Noop, OK); } } AUTOTEST(one_client) { client c(connect_server()); response r; serial_t s; s = c.acquire("hoge", 2, 10); ASSERT_RESPONSE(c, r, s, Acquire, OK); cybozu_assert(r.resources() == 2); s = c.get("hoge"); ASSERT_RESPONSE(c, r, s, Get, OK); cybozu_assert(r.consumption() == 2); s = c.acquire("hoge", 3, 10); ASSERT_RESPONSE(c, r, s, Acquire, OK); cybozu_assert(r.resources() == 3); s = c.get("hoge"); ASSERT_RESPONSE(c, r, s, Get, OK); cybozu_assert(r.consumption() == 5); s = c.acquire("hoge", 5, 10); ASSERT_RESPONSE(c, r, s, Acquire, OK); cybozu_assert(r.resources() == 5); s = c.get("hoge"); ASSERT_RESPONSE(c, r, s, Get, OK); cybozu_assert(r.consumption() == 10); s = c.acquire("hoge", 1, 10); ASSERT_RESPONSE(c, r, s, Acquire, ResourceNotAvailable); s = c.release("hoge", 3); ASSERT_RESPONSE(c, r, s, Release, OK); s = c.acquire("hoge", 2, 10); ASSERT_RESPONSE(c, r, s, Acquire, OK); cybozu_assert(r.resources() == 2); } AUTOTEST(multi_clients) { client c1(connect_server()); response r1; serial_t s1; { client c2(connect_server()); response r2; serial_t s2; s1 = c1.acquire("fuga", 1, 10); ASSERT_RESPONSE(c1, r1, s1, Acquire, OK); s2 = c2.acquire("fuga", 9, 10); ASSERT_RESPONSE(c2, r2, s2, Acquire, OK); s1 = c1.acquire("fuga", 1, 10); ASSERT_RESPONSE(c1, r1, s1, Acquire, ResourceNotAvailable); s2 = c2.release("fuga", 1); ASSERT_RESPONSE(c2, r2, s2, Release, OK); s1 = c1.acquire("fuga", 1, 10); ASSERT_RESPONSE(c1, r1, s1, Acquire, OK); s1 = c1.acquire("fuga", 1, 10); ASSERT_RESPONSE(c1, r1, s1, Acquire, ResourceNotAvailable); s2 = c2.acquire("fuga", 1, 10); ASSERT_RESPONSE(c2, r2, s2, Acquire, ResourceNotAvailable); // 8 resources released due to disconnection of c2. } ::timespec ts = {0, 100 * 1000 * 1000}; nanosleep(&ts, nullptr); // wait 100ms s1 = c1.acquire("fuga", 8, 10); ASSERT_RESPONSE(c1, r1, s1, Acquire, OK); s1 = c1.acquire("fuga", 1, 10); ASSERT_RESPONSE(c1, r1, s1, Acquire, ResourceNotAvailable); } AUTOTEST(over_releasing) { client c(connect_server()); response r; serial_t s; s = c.acquire("foo", 1, 10); ASSERT_RESPONSE(c, r, s, Acquire, OK); s = c.release("foo", 2); ASSERT_RESPONSE(c, r, s, Release, NotAcquired); } AUTOTEST(multi_names) { client c(connect_server()); response r; serial_t s; s = c.acquire("a", 1, 10); ASSERT_RESPONSE(c, r, s, Acquire, OK); s = c.acquire("b", 2, 10); ASSERT_RESPONSE(c, r, s, Acquire, OK); s = c.acquire("c", 4, 10); ASSERT_RESPONSE(c, r, s, Acquire, OK); s = c.get("a"); ASSERT_RESPONSE(c, r, s, Get, OK); cybozu_assert(r.consumption() == 1); s = c.get("b"); ASSERT_RESPONSE(c, r, s, Get, OK); cybozu_assert(r.consumption() == 2); s = c.get("c"); ASSERT_RESPONSE(c, r, s, Get, OK); cybozu_assert(r.consumption() == 4); s = c.release("a", 1); ASSERT_RESPONSE(c, r, s, Release, OK); s = c.release("b", 1); ASSERT_RESPONSE(c, r, s, Release, OK); s = c.release("c", 2); ASSERT_RESPONSE(c, r, s, Release, OK); s = c.get("a"); ASSERT_RESPONSE(c, r, s, Get, OK); cybozu_assert(r.consumption() == 0); s = c.get("b"); ASSERT_RESPONSE(c, r, s, Get, OK); cybozu_assert(r.consumption() == 1); s = c.get("c"); ASSERT_RESPONSE(c, r, s, Get, OK); cybozu_assert(r.consumption() == 2); } AUTOTEST(dump) { client c(connect_server()); response r; serial_t s; s = c.acquire("dump:aaa", 5, 10); ASSERT_RESPONSE(c, r, s, Acquire, OK); s = c.acquire("dump:aaa", 3, 10); ASSERT_RESPONSE(c, r, s, Acquire, OK); s = c.release("dump:aaa", 1); ASSERT_RESPONSE(c, r, s, Release, OK); s = c.acquire("dump:bbb", 100, 100); ASSERT_RESPONSE(c, r, s, Acquire, OK); s = c.release("dump:bbb", 100); ASSERT_RESPONSE(c, r, s, Release, OK); s = c.dump(); int count = 0; for(;;) { cybozu_assert( c.recv(r) ); cybozu_assert( r.status() == yrmcds::counter::status::OK ); if( r.status() != yrmcds::counter::status::OK ) break; if( r.body_length() == 0 ) break; if( r.name() == "dump:aaa" ) { cybozu_assert( r.consumption() == 7 ); cybozu_assert( r.max_consumption() == 8 ); ++count; } else if( r.name() == "dump:bbb" ) { cybozu_assert( r.consumption() == 0 ); cybozu_assert( r.max_consumption() == 100 ); ++count; } } cybozu_assert( count == 2 ); } void print_usage() { std::cout << "Usage: counter.exe [SERVER [PORT]]\n" "Environment options:\n" " YRMCDS_SERVER : the name of a yrmcds server.\n" " used only when `SERVER` is unspecified.\n" " YRMCDS_PORT : the port number of a yrmcds server.\n" " used only when `PORT` is unspecified.\n" << std::flush; } // main bool optparse(int argc, char** argv) { if( argc > 3 ) { print_usage(); return false; } g_server = getenv("YRMCDS_SERVER"); const char* env_port = getenv("YRMCDS_PORT"); if( env_port != nullptr ) g_port = std::stoi(env_port); if( argc >= 2 ) g_server = argv[1]; if( argc >= 3 ) g_port = std::stoi(argv[2]); if( g_server == nullptr ) { std::cout << "No server specified." << std::endl; print_usage(); return false; } if( g_port <= 0 || g_port > 65535 ) { std::cout << "Invalid port number: " << argv[2] << std::endl; return false; } int s = connect_server(); if( s == -1 ) { std::cout << "Failed to connect to " << g_server << ":" << g_port << std::endl; return false; } ::close(s); return true; } TEST_MAIN(optparse); yrmcds-1.1.5/test/counter_client.hpp000066400000000000000000000167751262254645600175600ustar00rootroot00000000000000#ifndef YRMCDS_TEST_COUNTER_CLIENT_HPP #define YRMCDS_TEST_COUNTER_CLIENT_HPP #include "../src/counter/counter.hpp" #include #include #include #include #include #include #include #include #include #include #include #include #include namespace counter_client { typedef std::uint32_t serial_t; using yrmcds::counter::string_slice; const std::size_t HEADER_SIZE = 12; struct statistic { std::string name; std::string value; }; class response { public: std::size_t parse(const std::vector buf) { if( buf.size() < HEADER_SIZE ) return 0; std::memcpy(m_header, buf.data(), HEADER_SIZE); if( buf.size() < HEADER_SIZE + body_length() ) return 0; m_body.assign(buf.begin() + HEADER_SIZE, buf.begin() + HEADER_SIZE + body_length()); return HEADER_SIZE + m_body.size(); } std::uint8_t magic() const noexcept { return (uint8_t)m_header[0]; } yrmcds::counter::command command() const noexcept { return (yrmcds::counter::command)m_header[1]; } yrmcds::counter::status status() const noexcept { return (yrmcds::counter::status)m_header[2]; } std::uint32_t body_length() const { uint32_t r; cybozu::ntoh(m_header + 4, r); return r; } const std::string name() const { if( command() == yrmcds::counter::command::Dump ) { uint16_t name_len; cybozu::ntoh(m_body.data() + 8, name_len); return std::string(m_body.data() + 10, name_len); } cybozu_assert(false); return ""; } std::uint32_t opaque() const { uint32_t r; cybozu::ntoh(m_header + 8, r); return r; } std::uint32_t resources() const { cybozu_assert(m_body.size() >= 4); uint32_t r; cybozu::ntoh(m_body.data(), r); return r; } std::uint32_t consumption() const { return resources(); } std::uint32_t max_consumption() const { cybozu_assert(m_body.size() >= 8); uint32_t r; cybozu::ntoh(m_body.data() + 4, r); return r; } std::string message() const { return std::string(m_body.data(), m_body.size()); } std::vector stats() const { std::vector s; const char* end = m_body.data() + m_body.size(); for( const char* p = m_body.data(); p < end; ) { cybozu_assert(p + 4 <= end); std::uint16_t name_len, value_len; cybozu::ntoh(p, name_len); cybozu::ntoh(p + 2, value_len); cybozu_assert(p + 4 + name_len + value_len <= end); s.push_back({ std::string(p + 4, name_len), std::string(p + 4 + name_len, value_len) }); p += 4 + name_len + value_len; } return std::move(s); } private: char m_header[HEADER_SIZE] = {}; std::vector m_body; }; class client { public: client(int fd): m_socket(fd), m_buffer(), m_recv_buffer(1<<16) { cybozu_assert( m_socket != -1 ); } ~client() { ::close(m_socket); } bool recv(response& r) { while( true ) { std::size_t len = r.parse(m_buffer); if( len > 0 ) { m_buffer.erase(m_buffer.begin(), m_buffer.begin() + len); return true; } ssize_t n = ::recv(m_socket, m_recv_buffer.data(), m_recv_buffer.size(), 0); cybozu_assert( n != -1 ); if( n == -1 || n == 0 ) { auto ecnd = std::system_category().default_error_condition(errno); std::cerr << " (" << ecnd.value() << ") " << ecnd.message() << std::endl; return false; } m_buffer.insert(m_buffer.end(), m_recv_buffer.begin(), m_recv_buffer.begin() + n); } } serial_t noop() { char header[HEADER_SIZE]; fill_header(header, 0x00, 0, m_serial); ssize_t n = ::send(m_socket, header, HEADER_SIZE, 0); cybozu_assert( n == ssize_t(HEADER_SIZE) ); return m_serial++; } serial_t get(const std::string& name) { char header[HEADER_SIZE]; fill_header(header, 0x01, 2 + name.size(), m_serial); char body[2]; cybozu::hton((uint16_t)name.size(), body); m_send_buffer.resize(HEADER_SIZE + sizeof(body) + name.size()); std::memcpy(m_send_buffer.data(), header, HEADER_SIZE); std::memcpy(m_send_buffer.data() + HEADER_SIZE, body, sizeof(body)); std::memcpy(m_send_buffer.data() + HEADER_SIZE + sizeof(body), name.data(), name.size()); ssize_t n = ::send(m_socket, m_send_buffer.data(), m_send_buffer.size(), 0); cybozu_assert( n == (ssize_t)m_send_buffer.size() ); return m_serial++; } serial_t acquire(const std::string& name, std::uint32_t resources, std::uint32_t initial) { char header[HEADER_SIZE]; fill_header(header, 0x02, 10 + name.size(), m_serial); char body[10]; cybozu::hton(resources, body); cybozu::hton(initial, body + 4); cybozu::hton((uint16_t)name.size(), body + 8); m_send_buffer.resize(HEADER_SIZE + sizeof(body) + name.size()); std::memcpy(m_send_buffer.data(), header, HEADER_SIZE); std::memcpy(m_send_buffer.data() + HEADER_SIZE, body, sizeof(body)); std::memcpy(m_send_buffer.data() + HEADER_SIZE + sizeof(body), name.data(), name.size()); ssize_t n = ::send(m_socket, m_send_buffer.data(), m_send_buffer.size(), 0); cybozu_assert( n == (ssize_t)m_send_buffer.size() ); return m_serial++; } serial_t release(const std::string& name, std::uint32_t resources) { char header[HEADER_SIZE]; fill_header(header, 0x03, 6 + name.size(), m_serial); char body[6]; cybozu::hton(resources, body); cybozu::hton((uint16_t)name.size(), body + 4); m_send_buffer.resize(HEADER_SIZE + sizeof(body) + name.size()); std::memcpy(m_send_buffer.data(), header, HEADER_SIZE); std::memcpy(m_send_buffer.data() + HEADER_SIZE, body, sizeof(body)); std::memcpy(m_send_buffer.data() + HEADER_SIZE + sizeof(body), name.data(), name.size()); ssize_t n = ::send(m_socket, m_send_buffer.data(), m_send_buffer.size(), 0); cybozu_assert( n == (ssize_t)m_send_buffer.size() ); return m_serial++; } serial_t stats() { char header[HEADER_SIZE]; fill_header(header, 0x10, 0, m_serial); ssize_t n = ::send(m_socket, header, HEADER_SIZE, 0); cybozu_assert( n == ssize_t(HEADER_SIZE) ); return m_serial++; } serial_t dump() { char header[HEADER_SIZE]; fill_header(header, 0x11, 0, m_serial); ssize_t n = ::send(m_socket, header, HEADER_SIZE, 0); cybozu_assert( n == ssize_t(HEADER_SIZE) ); return m_serial++; } private: void fill_header(char* header, uint8_t opcode, uint32_t body_length, uint32_t opaque) { header[0] = 0x90; header[1] = opcode; header[2] = 0; header[3] = 0; cybozu::hton(body_length, header + 4); cybozu::hton(opaque, header + 8); } int m_socket; serial_t m_serial; std::vector m_buffer; std::vector m_recv_buffer; std::vector m_send_buffer; }; } // namespace counter_client #endif // YRMCDS_TEST_COUNTER_CLIENT_HPP yrmcds-1.1.5/test/counter_interactive.cpp000066400000000000000000000106611262254645600205760ustar00rootroot00000000000000#define TEST_DISABLE_AUTO_RUN #include "../src/counter/counter.hpp" #include "counter_client.hpp" using namespace counter_client; const char* g_server; uint16_t g_port; int connect_server() { int s = cybozu::tcp_connect(g_server, g_port); if( s == -1 ) return -1; ::fcntl(s, F_SETFL, ::fcntl(s, F_GETFL, 0) & ~O_NONBLOCK); struct timeval tv; tv.tv_sec = 0; tv.tv_usec = 300000; ::setsockopt(s, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv)); int ok = 1; ::setsockopt(s, IPPROTO_TCP, TCP_NODELAY, &ok, sizeof(ok)); return s; } #define ASSERT_RESPONSE(c, r, s, cmd) do { \ cybozu_assert( c.recv(r) ); \ cybozu_assert( r.opaque() == s ); \ cybozu_assert( r.command() == yrmcds::counter::command::cmd ); \ } while( false ) AUTOTEST(interactive) { client c(connect_server()); response r; serial_t s; while( ! std::cin.eof() ) { std::cout << "Command: " << std::flush; std::string cmd; std::cin >> cmd; if( cmd == "noop" ) { s = c.noop(); ASSERT_RESPONSE(c, r, s, Noop); if( r.status() == yrmcds::counter::status::OK ) std::cout << "OK" << std::endl; else std::cout << r.message() << std::endl; } else if( cmd == "get" ) { std::string name; std::cin >> name; s = c.get(name); ASSERT_RESPONSE(c, r, s, Get); if( r.status() == yrmcds::counter::status::OK ) std::cout << r.consumption() << std::endl; else std::cout << r.message() << std::endl; } else if( cmd == "acquire" ) { std::string name; std::cin >> name; std::uint32_t resources, maximum; std::cin >> resources >> maximum; s = c.acquire(name, resources, maximum); ASSERT_RESPONSE(c, r, s, Acquire); if( r.status() == yrmcds::counter::status::OK ) std::cout << r.resources() << std::endl; else std::cout << r.message() << std::endl; } else if( cmd == "release" ) { std::string name; std::cin >> name; std::uint32_t resources; std::cin >> resources; s = c.release(name, resources); ASSERT_RESPONSE(c, r, s, Release); if( r.status() == yrmcds::counter::status::OK ) std::cout << "OK" << std::endl; else std::cout << r.message() << std::endl; } else if( cmd == "stats" ) { s = c.stats(); ASSERT_RESPONSE(c, r, s, Stats); if( r.status() == yrmcds::counter::status::OK ) { for( auto& stat: r.stats() ) std::cout << stat.name << ": " << stat.value << std::endl; std::cout << "END" << std::endl; } else { std::cout << r.message() << std::endl; } } else if( cmd == "dump" ) { s = c.dump(); for(;;) { ASSERT_RESPONSE(c, r, s, Dump); if( r.status() == yrmcds::counter::status::OK ) { if( r.body_length() == 0 ) { std::cout << "END" << std::endl; break; } std::cout << r.name() << ": " << r.consumption() << ", " << r.max_consumption() << std::endl; } else { std::cout << r.message() << std::endl; } } } if( std::cin.fail() ) { std::cin.clear(); std::cin.ignore(); } } } // main bool optparse(int argc, char** argv) { if( argc != 2 && argc != 3 ) { std::cout << "Usage: counter_interactive.exe SERVER [PORT]" << std::endl; return false; } g_server = argv[1]; g_port = 11215; if( argc == 3 ) { int n = std::stoi(argv[2]); if( n <= 0 || n > 65535 ) { std::cout << "Invalid port number: " << argv[2] << std::endl; return false; } g_port = n; } int s = connect_server(); if( s == -1 ) { std::cout << "Failed to connect to " << g_server << ":" << g_port << std::endl; return false; } ::close(s); return true; } TEST_MAIN(optparse); yrmcds-1.1.5/test/counter_request.cpp000066400000000000000000000214011262254645600177430ustar00rootroot00000000000000#include "../src/counter/counter.hpp" #include #include using namespace yrmcds::counter; #define REQ(n, data) \ const char i##n[] = data; \ request r##n(i##n, sizeof(i##n) - 1); bool operator==(const string_slice& lhs, const char* rhs) { if (lhs.len != std::strlen(rhs)) return false; return std::memcmp(lhs.p, rhs, lhs.len) == 0; } AUTOTEST(noop) { // incomplete request REQ(1, "\x90\x00\x00\x00"); cybozu_assert( r1.length() == 0 ); // valid request REQ(2, "\x90\x00\x00\x00" "\x00\x00\x00\x00" "\x01\x02\x03\x04"); cybozu_assert( r2.length() == 12 ); cybozu_assert( r2.status() == status::OK ); cybozu_assert( r2.command() == command::Noop ); cybozu_assert( std::memcmp(r2.opaque(), "\01\02\03\04", 4) == 0 ); // valid request + incomplete request REQ(3, "\x90\x00\x00\x00" "\x00\x00\x00\x00" "\x01\x02\x03\x04" "\x90\x01\x00\x00"); cybozu_assert( r3.length() == 12 ); cybozu_assert( r3.status() == status::OK ); cybozu_assert( r3.command() == command::Noop ); cybozu_assert( std::memcmp(r3.opaque(), "\01\02\03\04", 4) == 0 ); } AUTOTEST(get) { // valid request REQ(1, "\x90\x01\x00\x00" // magic, opcode, flags, reserved "\x00\x00\x00\x07" // body length "\x01\x02\x03\x04" // opaque "\x00\x05" // name length "Hello") // name cybozu_assert( r1.length() == 12 + 7 ); cybozu_assert( r1.status() == status::OK ); cybozu_assert( r1.command() == command::Get ); cybozu_assert( r1.name() == "Hello" ); // invalid request (name length == 0) REQ(2, "\x90\x01\x00\x00" // magic, opcode, flags, reserved "\x00\x00\x00\x02" // body length "\x01\x02\x03\x04" // opaque "\x00\x00") // name length cybozu_assert( r2.length() == 12 + 2 ); cybozu_assert( r2.status() == status::Invalid ); cybozu_assert( r2.command() == command::Get ); // invalid request (body length < 2 + name length) REQ(3, "\x90\x01\x00\x00" // magic, opcode, flags, reserved "\x00\x00\x00\x06" // body length "\x01\x02\x03\x04" // opaque "\x00\x05" // name length "Hello") // name cybozu_assert( r3.length() == 12 + 6 ); cybozu_assert( r3.status() == status::Invalid ); cybozu_assert( r3.command() == command::Get ); // valid request (name includes 0x00) REQ(4, "\x90\x01\x00\x00" // magic, opcode, flags, reserved "\x00\x00\x00\x07" // body length "\x01\x02\x03\x04" // opaque "\x00\x05" // name length "He\x00lo") // name cybozu_assert( r4.length() == 12 + 7 ); cybozu_assert( r4.status() == status::OK ); cybozu_assert( r4.command() == command::Get ); cybozu_assert( r4.name().len == 5 ); cybozu_assert( std::memcmp(r4.name().p, "He\x00lo", 5) == 0 ); } AUTOTEST(acquire) { // valid request REQ(1, "\x90\x02\x00\x00" // magic, opcode, flags, reserved "\x00\x00\x00\x0b" // body length "\x01\x02\x03\x04" // opaque "\x00\x00\x00\x02" // resources "\x00\x00\x00\x0a" // initial resources "\x00\x01" // name length "x") // name cybozu_assert( r1.length() == 12 + 11 ); cybozu_assert( r1.status() == status::OK ); cybozu_assert( r1.command() == command::Acquire ); cybozu_assert( r1.resources() == 2 ); cybozu_assert( r1.maximum() == 10 ); cybozu_assert( r1.name() == "x" ); // valid request (initial - resources == 0) REQ(2, "\x90\x02\x00\x00" // magic, opcode, flags, reserved "\x00\x00\x00\x0b" // body length "\x01\x02\x03\x04" // opaque "\x00\x00\x00\x0a" // resources "\x00\x00\x00\x0a" // initial resources "\x00\x01" // name length "x") // name cybozu_assert( r2.length() == 12 + 11 ); cybozu_assert( r2.status() == status::OK ); cybozu_assert( r2.command() == command::Acquire ); cybozu_assert( r2.resources() == 10 ); cybozu_assert( r2.maximum() == 10 ); cybozu_assert( r2.name() == "x" ); // invalid request (resources == 0) REQ(3, "\x90\x02\x00\x00" // magic, opcode, flags, reserved "\x00\x00\x00\x0b" // body length "\x01\x02\x03\x04" // opaque "\x00\x00\x00\x00" // resources "\x00\x00\x00\x0a" // initial resources "\x00\x01" // name length "x") // name cybozu_assert( r3.length() == 12 + 11 ); cybozu_assert( r3.status() == status::Invalid ); cybozu_assert( r3.command() == command::Acquire ); // invalid request (initial - resources < 0) REQ(4, "\x90\x02\x00\x00" // magic, opcode, flags, reserved "\x00\x00\x00\x0b" // body length "\x01\x02\x03\x04" // opaque "\x00\x00\x00\x02" // resources "\x00\x00\x00\x01" // initial resources "\x00\x01" // name length "x") // name cybozu_assert( r4.length() == 12 + 11 ); cybozu_assert( r4.status() == status::Invalid ); cybozu_assert( r4.command() == command::Acquire ); // invalid request (name length == 0) REQ(5, "\x90\x02\x00\x00" // magic, opcode, flags, reserved "\x00\x00\x00\x0a" // body length "\x01\x02\x03\x04" // opaque "\x00\x00\x00\x02" // resources "\x00\x00\x00\x0a" // initial resources "\x00\x00") // name length cybozu_assert( r5.length() == 12 + 10 ); cybozu_assert( r5.status() == status::Invalid ); cybozu_assert( r5.command() == command::Acquire ); // invalid request (body length < 10 + name length) REQ(6, "\x90\x02\x00\x00" // magic, opcode, flags, reserved "\x00\x00\x00\x0b" // body length "\x01\x02\x03\x04" // opaque "\x00\x00\x00\x02" // resources "\x00\x00\x00\x0a" // initial resources "\x00\x02" // name length "xy") // name cybozu_assert( r6.length() == 12 + 11 ); cybozu_assert( r6.status() == status::Invalid ); cybozu_assert( r6.command() == command::Acquire ); } AUTOTEST(release) { // valid request REQ(1, "\x90\x03\x00\x00" // magic, opcode, flags, reserved "\x00\x00\x00\x07" // body length "\x01\x02\x03\x04" // opaque "\x00\x00\x00\x02" // resources "\x00\x01" // name length "x") // name cybozu_assert( r1.length() == 12 + 7 ); cybozu_assert( r1.status() == status::OK ); cybozu_assert( r1.command() == command::Release ); cybozu_assert( r1.resources() == 2 ); cybozu_assert( r1.name() == "x" ); // valid request (resources == 0) REQ(2, "\x90\x03\x00\x00" // magic, opcode, flags, reserved "\x00\x00\x00\x07" // body length "\x01\x02\x03\x04" // opaque "\x00\x00\x00\x00" // resources "\x00\x01" // name length "x") // name cybozu_assert( r2.length() == 12 + 7 ); cybozu_assert( r2.status() == status::OK ); cybozu_assert( r2.command() == command::Release ); cybozu_assert( r2.resources() == 0 ); cybozu_assert( r2.name() == "x" ); // invalid request (name length == 0) REQ(3, "\x90\x03\x00\x00" // magic, opcode, flags, reserved "\x00\x00\x00\x06" // body length "\x01\x02\x03\x04" // opaque "\x00\x00\x00\x02" // resources "\x00\x00") // name length cybozu_assert( r3.length() == 12 + 6 ); cybozu_assert( r3.status() == status::Invalid ); cybozu_assert( r3.command() == command::Release ); // invalid request (body length < 6 + name length) REQ(4, "\x90\x03\x00\x00" // magic, opcode, flags, reserved "\x00\x00\x00\x06" // body length "\x01\x02\x03\x04" // opaque "\x00\x00\x00\x02" // resources "\x00\x01" // name length "x") // name cybozu_assert( r4.length() == 12 + 6 ); cybozu_assert( r4.status() == status::Invalid ); cybozu_assert( r4.command() == command::Release ); } AUTOTEST(stats) { REQ(1, "\x90\x10\x00\x00" "\x00\x00\x00\x00" "\x01\x02\x03\x04"); cybozu_assert( r1.length() == 12 ); cybozu_assert( r1.status() == status::OK ); cybozu_assert( r1.command() == command::Stats ); } AUTOTEST(dump) { REQ(1, "\x90\x11\x00\x00" "\x00\x00\x00\x00" "\x01\x02\x03\x04"); cybozu_assert( r1.length() == 12 ); cybozu_assert( r1.status() == status::OK ); cybozu_assert( r1.command() == command::Dump ); } yrmcds-1.1.5/test/cpucount.cpp000066400000000000000000000002071262254645600163550ustar00rootroot00000000000000#include #include int main() { std::cout << std::thread::hardware_concurrency() << std::endl; return 0; } yrmcds-1.1.5/test/detach.cpp000066400000000000000000000003011262254645600157400ustar00rootroot00000000000000#include #include void f() { char c; std::cin >> c; } int main() { std::thread t(f); t.detach(); std::cout << "detached" << std::endl; return 0; } yrmcds-1.1.5/test/dynbuf.cpp000066400000000000000000000023511262254645600160060ustar00rootroot00000000000000#include #include #include AUTOTEST(dynbuf0) { cybozu::dynbuf b(0); cybozu_assert( b.empty() ); b.append("abc", 3); cybozu_assert( b.size() == 3 ); cybozu_assert( ! b.empty() ); b.append("def", 4); cybozu_assert( b.size() == 7 ); cybozu_assert( std::memcmp(b.data(), "abcdef", 7) == 0 ); b.reset(); cybozu_assert( b.size() == 0 ); char* p = b.prepare(7); std::memcpy(p, "aaa", 4); b.consume(4); cybozu_assert( b.size() == 4 ); cybozu_assert( std::memcmp(b.data(), "aaa", 4) == 0 ); b.append("1111112345678", 13); b.erase(10); cybozu_assert( b.size() == 7 ); cybozu_assert( std::memcmp(b.data(), "2345678", 7) == 0 ); } AUTOTEST(dynbuf5) { cybozu::dynbuf b(0); cybozu_assert( b.empty() ); b.append("abc", 3); b.append("defghi", 6); cybozu_assert( b.size() == 9 ); cybozu_assert( std::memcmp(b.data(), "abcdefghi", 9) == 0 ); b.erase(6); cybozu_assert( b.size() == 3 ); cybozu_assert( std::memcmp(b.data(), "ghi", 3) == 0 ); b.append("0123456789", 10); b.erase(1); cybozu_assert( b.size() == 12 ); cybozu_assert( std::memcmp(b.data(), "hi0123456789", 12) == 0 ); b.reset(); } yrmcds-1.1.5/test/echo_server.cpp000066400000000000000000000043551262254645600170310ustar00rootroot00000000000000#include #include #include #include #include #include #include #include #include #include #include class echo_socket: public cybozu::tcp_socket { public: echo_socket(int fd): tcp_socket(fd, 1), m_buffer(1 <<20) {} static bool on_accept(int s) { std::cerr << "accepted." << std::endl; return true; } private: std::vector m_buffer; void echo_back(std::size_t len) { send(m_buffer.data(), len, false); send(m_buffer.data(), len, true); } virtual bool on_readable() override final { std::size_t received = 0; while( true ) { if( m_buffer.size() == received ) { std::cerr << "Too large data received." << std::endl; received = 0; } ssize_t n = ::recv(m_fd, &m_buffer[received], m_buffer.size() - received, 0); if( n == -1 ) { if( errno == EINTR ) continue; if( errno == EAGAIN || errno == EWOULDBLOCK ) { echo_back(received); return true; } auto ecnd = std::system_category().default_error_condition(errno); cybozu::logger::error() << ": (" << ecnd.value() << ") " << ecnd.message(); return false; } if( n == 0 ) return invalidate(); received += n; } } }; int main(int argc, char** argv) { if( argc == 1 ) { std::cout << "Usage: echo_server.exe PORT" << std::endl; return 0; } uint16_t port = static_cast(std::stoi(argv[1])); cybozu::reactor r; cybozu::tcp_server_socket::wrapper w = [](int s, const cybozu::ip_address&) { return std::unique_ptr( new echo_socket(s) ); }; r.add_resource( cybozu::make_server_socket(NULL, port, w), cybozu::reactor::EVENT_IN ); r.run([](cybozu::reactor& r){ std::cerr << "."; }); return 0; } yrmcds-1.1.5/test/endian.cpp000066400000000000000000000012411262254645600157520ustar00rootroot00000000000000#include #include #include int main(int argc, char** argv) { #if __BYTE_ORDER == __LITTLE_ENDIAN std::cout << "little endian" << std::endl; #elif __BYTE_ORDER == __BIG_ENDIAN std::cout << "big endian" << std::endl; #endif std::uint64_t n = 0x1234567887654321ULL; std::cout << std::hex << "n = 0x" << n << std::endl; std::cout << "htobe64(n) = 0x" << htobe64(n) << std::endl; std::uint64_t t; cybozu::hton(n, (char*)&t); std::cout << "hton(n, &t) = 0x" << t << std::endl; std::uint64_t n2; cybozu::ntoh((char*)&t, n2); std::cout << "ntoh(t, n2) = 0x" << n2 << std::endl; return 0; } yrmcds-1.1.5/test/epoll.cpp000066400000000000000000000064511262254645600156370ustar00rootroot00000000000000// test if epoll can receive hangup at clients. #include #include #include #include #include #include #include #include #include #include #include int create_socket(const struct addrinfo& ai) { int s = socket(ai.ai_family, ai.ai_socktype, ai.ai_protocol); if( s == -1 ) throw std::system_error(errno, std::system_category(), "socket"); int enable = 1; if( setsockopt(s, SOL_SOCKET, SO_REUSEADDR, &enable, sizeof(enable)) == -1 ) throw std::system_error(errno, std::system_category(), "setsocketopt"); if( bind(s, ai.ai_addr, ai.ai_addrlen) == -1 ) throw std::system_error(errno, std::system_category(), "bind"); if( listen(s, 10) == -1 ) throw std::system_error(errno, std::system_category(), "listen"); return s; } int main(int argc, char** argv) { if( argc != 2 ) { std::cout << "Usage: epoll PORT" << std::endl; return 0; } struct addrinfo hint; std::memset(&hint, 0, sizeof(hint)); hint.ai_family = AF_UNSPEC; hint.ai_socktype = SOCK_STREAM; hint.ai_protocol = 0; hint.ai_flags = AI_PASSIVE; struct addrinfo *res; int e = getaddrinfo(NULL, argv[1], &hint, &res); if( e == EAI_SYSTEM ) { throw std::system_error(errno, std::system_category(), "getaddrinfo"); } else if( e != 0 ) { throw std::runtime_error(std::string("getaddrinfo: ") + gai_strerror(e)); } const struct addrinfo* p; int s; for( p = res; p != nullptr; p = p->ai_next ) if( p->ai_family == AF_INET6 ) break; if( p != nullptr ) { s = create_socket(*p); } else { for( p = res; p != nullptr; p = p->ai_next ) if( p->ai_family == AF_INET ) break; if( p != nullptr ) { s = create_socket(*p); } else { throw std::runtime_error("No address to bind."); } } struct sockaddr addr; socklen_t slen = sizeof(addr); int clientfd = accept4(s, &addr, &slen, SOCK_NONBLOCK); if( clientfd == -1 ) throw std::system_error(errno, std::system_category(), "accept4"); close(s); int efd = epoll_create(10); if( efd == -1 ) throw std::system_error(errno, std::system_category(), "epoll_create"); struct epoll_event ev; ev.events = EPOLLIN|EPOLLET; ev.data.fd = clientfd; if( epoll_ctl(efd, EPOLL_CTL_ADD, clientfd, &ev) == -1 ) throw std::system_error(errno, std::system_category(), "epoll_ctl"); std::vector buf(1 << 20); struct epoll_event events[10]; while( true ) { int n = epoll_wait(efd, events, 10, -1); for(int i = 0; i < n; ++i ) { if( events[i].events & EPOLLIN ) { std::cout << "EPOLLIN" << std::endl; while( true ) { if( recv(events[i].data.fd, buf.data(), buf.size(), 0) == -1 ) break; } } if( events[i].events & EPOLLHUP ) { std::cout << "EPOLLHUP" << std::endl; break; } if( events[i].events & EPOLLERR ) { std::cout << "EPOLLHUP" << std::endl; } } } return 0; } yrmcds-1.1.5/test/eventfd.cpp000066400000000000000000000023111262254645600161460ustar00rootroot00000000000000#include #include #include #include #include #include #include #include void write_event(int efd, std::uint64_t ev) { ssize_t n = ::write(efd, &ev, sizeof(ev)); if( n != sizeof(ev) ) throw std::system_error(errno, std::system_category(), "write"); std::cerr << "write event of " << n << " bytes." << std::endl; } std::uint64_t read_event(int efd) { std::uint64_t i; ssize_t n = ::read(efd, &i, sizeof(i)); if( n != sizeof(i) ) throw std::system_error(errno, std::system_category(), "read"); return i; } void foo(int efd) { std::cout << "got event: " << read_event(efd) << std::endl; // see what will be returned, 10 or 16? Both can be seen. std::cout << "got event: " << read_event(efd) << std::endl; } int main(int argc, char** argv) { int efd = eventfd(0, EFD_CLOEXEC); if( efd == -1 ) throw std::system_error(errno, std::system_category(), "eventfd"); write_event(efd, 2); std::thread t(foo, efd); std::this_thread::sleep_for(std::chrono::milliseconds(200)); write_event(efd, 10); write_event(efd, 6); t.join(); return 0; } yrmcds-1.1.5/test/filerecv.cpp000066400000000000000000000071731262254645600163250ustar00rootroot00000000000000#include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include std::string g_prefix; int g_counter = 0; const std::size_t MAX_RECV = (2 << 20); // 2 MiB class filerecv_socket: public cybozu::tcp_socket { public: filerecv_socket(int fd): cybozu::tcp_socket(fd), m_file( ::creat((g_prefix+std::to_string(++g_counter)).c_str(), 0644) ), m_buffer(1 << 20) { if( m_fd == -1 ) cybozu::throw_unix_error(errno, "creat"); } ~filerecv_socket() { ::close(m_file); } private: const int m_file; std::size_t m_written = 0; std::vector m_buffer; virtual bool on_readable() override final { std::size_t received = 0; while( true ) { ssize_t n = ::recv(m_fd, m_buffer.data(), m_buffer.size(), 0); if( n == -1 ) { if( errno == EAGAIN || errno == EWOULDBLOCK ) break; cybozu::throw_unix_error(errno, "recv"); } if( n == 0 ) { std::cout << "received and stored " << m_written << " bytes." << std::endl; return invalidate(); } received += n; m_written += n; while( n > 0 ) { ssize_t nw = ::write(m_file, m_buffer.data(), n); if( nw == -1 ) cybozu::throw_unix_error(errno, "write"); n -= nw; } if( received > MAX_RECV ) { m_reactor->add_readable(*this); return true; } } return true; } }; int main(int argc, char** argv) { if( argc != 2 ) { std::cout << "Usage: filerecv.exe PREFIX" << std::endl; return 0; } g_prefix = argv[1]; cybozu::logger::set_threshold(cybozu::severity::debug); cybozu::reactor r; auto sigres = cybozu::signal_setup({SIGHUP, SIGQUIT, SIGTERM, SIGINT}); sigres->set_handler( [](const struct signalfd_siginfo& si, cybozu::reactor& r) { switch(si.ssi_signo) { case SIGHUP: std::cerr << "got SIGHUP." << std::endl; break; case SIGQUIT: std::cerr << "got SIGQUIT." << std::endl; break; case SIGTERM: std::cerr << "got SIGTERM." << std::endl; break; case SIGINT: std::cerr << "got SIGINT." << std::endl; r.quit(); break; } } ); r.add_resource( std::move(sigres), cybozu::reactor::EVENT_IN ); cybozu::tcp_server_socket::wrapper w = [](int s, const cybozu::ip_address&) { return std::unique_ptr( new filerecv_socket(s) ); }; r.add_resource( cybozu::make_server_socket(NULL, 11111, w), cybozu::reactor::EVENT_IN ); r.run([](cybozu::reactor& r){ r.fix_garbage(); r.gc();}); return 0; } yrmcds-1.1.5/test/filesend.cpp000066400000000000000000000054101262254645600163070ustar00rootroot00000000000000#include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include //const std::size_t CHUNK_SIZE = (2 << 20); // 2 MiB const std::size_t CHUNK_SIZE = (512 << 10); // 512 KiB class filesend_socket: public cybozu::tcp_socket { public: filesend_socket(int fd): cybozu::tcp_socket(fd, 2) {} private: virtual bool on_readable() override final { return true; } }; void filesend(const char* path, cybozu::tcp_socket& s) { int fd = open(path, O_RDONLY); if( fd == -1 ) return; struct stat st; if( fstat(fd, &st) == -1 ) { std::cerr << "fstat failed." << std::endl; close(fd); cybozu::throw_unix_error(errno, "fstat"); } off_t length = st.st_size; std::cout << "sending " << length << " bytes." << std::endl; std::vector buffer(CHUNK_SIZE); char* const p_buf = buffer.data(); while( length ) { off_t to_read = std::min(length, (off_t)CHUNK_SIZE); ssize_t n = read(fd, p_buf, to_read); if( n == -1 ) { std::cerr << "read failed." << std::endl; close(fd); cybozu::throw_unix_error(errno, "read"); } length -= n; if( length == 0 ) { std::cerr << "closing." << std::endl; s.send_close(p_buf, n); } else { s.send(p_buf, n); } } close(fd); } int main(int argc, char** argv) { if( argc != 3 ) { std::cout << "Usage: filesend.exe FILENAME THREADS" << std::endl; return 0; } cybozu::logger::set_threshold(cybozu::severity::debug); const char* path = argv[1]; int n_threads = std::stoi(argv[2]); if( n_threads < 0 ) { std::cout << "Invalid number of threads." << std::endl; return 1; } cybozu::reactor r; std::vector threads; for( int i = 0; i < n_threads; ++i ) { int c = cybozu::tcp_connect(NULL, 11111); if( c == -1 ) { std::cerr << "failed to connect to localhost:11111" << std::endl; return 2; } std::unique_ptr s( new filesend_socket(c) ); threads.emplace_back(filesend, path, std::ref(*s)); r.add_resource( std::move(s), cybozu::reactor::EVENT_OUT ); } r.run([&threads](cybozu::reactor& r) { r.fix_garbage(); r.gc(); if( r.size() == 0 ) { for( auto& t: threads ) t.join(); r.quit(); } }); return 0; } yrmcds-1.1.5/test/hash_map.cpp000066400000000000000000000035361262254645600163050ustar00rootroot00000000000000#include #include #include using hash_map = cybozu::hash_map; bool updater(const cybozu::hash_key& k, std::string& s) { std::cerr << "Updating " << k.str() << std::endl; s.append("+"); return true; } bool expired(const cybozu::hash_key&, std::string&) { return false; } bool reader(const cybozu::hash_key&, std::string& s) { std::cout << s << std::endl; return true; } std::string creator(const cybozu::hash_key& k) { return std::string("hoge"); } const char key1[] = "abc"; const cybozu::hash_key hkey1(key1, sizeof(key1)); const char key2[] = "def"; const cybozu::hash_key hkey2(key2, sizeof(key2)); int main(int argc, char** argv) { hash_map m(8); std::cout << "bucket count = " << m.bucket_count() << std::endl; std::cerr << std::boolalpha; std::cerr << m.apply(hkey1, updater, nullptr) << std::endl; std::cerr << m.apply(hkey1, nullptr, creator) << std::endl; std::cerr << m.remove(hkey1, nullptr) << std::endl; std::cerr << m.remove(hkey1, nullptr) << std::endl; std::cerr << m.apply(hkey1, nullptr, creator) << std::endl; std::cerr << m.apply(hkey1, nullptr, creator) << std::endl; std::cerr << m.apply(hkey1, updater, creator) << std::endl; std::cerr << m.apply(hkey1, reader, nullptr) << std::endl; std::cerr << m.apply(hkey2, nullptr, creator) << std::endl; std::cerr << m.apply(hkey2, expired, nullptr) << std::endl; std::cerr << m.apply(hkey2, expired, creator) << std::endl; for( hash_map::bucket& b: m ) { b.gc([](const cybozu::hash_key& key, std::string& s) { std::cout << "collecting " << s << std::endl; return s.size() == 4; }); } std::cerr << m.apply(hkey1, reader, nullptr) << std::endl; std::cerr << m.apply(hkey2, reader, nullptr) << std::endl; return 0; } yrmcds-1.1.5/test/hide_api.cpp000066400000000000000000000006451262254645600162650ustar00rootroot00000000000000#include class A { public: A() {} virtual ~A() {} virtual int f() = 0; }; class B: public A { public: B(): A() {} virtual int g() { return 100; } private: virtual int f() { return 3; } }; int main() { A* a = new B; std::cout << a->f() << std::endl; delete a; // This causes compilation error, as expected! // //B b; //std::cout << b->f() << std::endl; } yrmcds-1.1.5/test/init.cpp000066400000000000000000000003231262254645600154570ustar00rootroot00000000000000#include struct A { A() { std::cout << "A::ctor" << std::endl; } void f() { std::cout << "A::f" << std::endl; } }; int main() { A a{}; a.f(); return 0; } yrmcds-1.1.5/test/ip_address.cpp000066400000000000000000000032171262254645600166360ustar00rootroot00000000000000#include #include AUTOTEST(ipv4) { cybozu_test_no_exception( cybozu::ip_address("123.123.123.0") ); cybozu_test_exception( cybozu::ip_address("123.456.789.0"), cybozu::ip_address::bad_address ); cybozu_test_exception( cybozu::ip_address("localhost"), cybozu::ip_address::bad_address ); cybozu::ip_address ipv4("123.123.123.0"); cybozu_assert( ipv4.is_v4() ); cybozu_assert( ipv4.str() == "123.123.123.0" ); } AUTOTEST(ipv6) { cybozu_test_no_exception( cybozu::ip_address("::1") ); // cybozu_test_no_exception( // cybozu::ip_address("fe80::dead:beaf%lo") // ); cybozu_test_exception( cybozu::ip_address("fg::1"), cybozu::ip_address::bad_address ); cybozu_test_exception( cybozu::ip_address("::1%lo"), cybozu::ip_address::bad_address ); cybozu::ip_address ipv6("fd00:1234::dead:beaf"); cybozu_assert( ipv6.is_v6() ); cybozu_assert( ipv6.str() == "fd00:1234::dead:beaf" ); } AUTOTEST(compare) { using cybozu::ip_address; cybozu_assert( ip_address("127.0.0.1") == ip_address("127.0.0.1") ); cybozu_assert( ip_address("127.0.0.1") != ip_address("162.193.0.1") ); cybozu_assert( ip_address("127.0.0.1") != ip_address("::1") ); cybozu_assert( ip_address("::1") == ip_address("::1") ); // cybozu_assert( ip_address("fe80::1%lo") == ip_address("fe80::1%lo") ); // cybozu_assert( ip_address("fe80::1%lo") != ip_address("fe80::1%eth0") ); } AUTOTEST(has_ip_address) { for( int i = 0; i < 1000; ++i ) cybozu::has_ip_address(cybozu::ip_address("11.11.11.11")); } yrmcds-1.1.5/test/lambda.cpp000066400000000000000000000003251262254645600157360ustar00rootroot00000000000000#include int main(int argc, char** argv) { int x = 3; auto foo = [=]() mutable { ++x; return x * x; }; std::cout << foo() << std::endl; std::cout << foo() << std::endl; return 0; } yrmcds-1.1.5/test/logger.cpp000066400000000000000000000005761262254645600160050ustar00rootroot00000000000000#include /* To see the optimization effects, * 1. run gdbtui, * 2. enter "layout asm", * 3. "b main", "run", then "stepi". */ int main(int argc, char** argv) { if( argc > 1 ) cybozu::logger::instance().open(argv[1]); cybozu::logger::debug() << "hoge, " << 3.145 << ", " << (10LL<<40); cybozu::logger::info() << "fuga"; return 0; } yrmcds-1.1.5/test/lz4.cpp000066400000000000000000000035551262254645600152370ustar00rootroot00000000000000// #include "../lz4/lz4.h" #include #include #include #include #include #include #include #include #include const std::size_t S = 1 << 20; void fill_seq(std::vector& buf) { unsigned char c = 0; char* p = buf.data(); for( std::size_t i = 0; i < S; ++i ) { *p = c++; } } void fill_random(std::vector& buf) { int fd = open("/dev/urandom", O_RDONLY); if( fd == -1 ) { std::cerr << "failed to open /dev/urandom" << std::endl; std::exit(1); } std::size_t to_read = S; char* p = buf.data(); while( to_read != 0 ) { ssize_t n = read(fd, p, to_read); if( n == -1 ) { std::cerr << "failed to read from /dev/urandom" << std::endl; std::exit(1); } p += n; to_read -= n; } close(fd); } int main(int argc, char** argv) { // std::vector buf1(S); // fill_seq(buf1); // //fill_random(buf1); // std::vector buf2(S); // std::vector cbuf(LZ4_COMPRESSBOUND(S)); // std::cout << "cbuf size = " << cbuf.size() << std::endl; // int csize = LZ4_compress(buf1.data(), cbuf.data(), S); // if( csize == 0 ) { // std::cerr << "compression failed." << std::endl; // return 1; // } // std::cout << "compressed: " << csize << " bytes." << std::endl; // // beware that the third parameter is the *uncompressed* size. // int csize2 = LZ4_decompress_fast(cbuf.data(), buf2.data(), S); // if( csize != csize2 ) { // std::cerr << "wtf!?" << std::endl; // return 1; // } // if( std::memcmp(buf1.data(), buf2.data(), S) != 0 ) { // std::cerr << "verify failed." << std::endl; // return 1; // } // std::cout << "verify ok." << std::endl; return 0; } yrmcds-1.1.5/test/macro_stringify.cpp000066400000000000000000000002631262254645600177160ustar00rootroot00000000000000#include #define QUOTE(str) #str #define EXPAND_AND_QUOTE(str) QUOTE(str) int main(int argc, char** argv) { puts(EXPAND_AND_QUOTE(DEFAULT_CONFIG)); return 0; } yrmcds-1.1.5/test/memcache_binary.cpp000066400000000000000000000700251262254645600176300ustar00rootroot00000000000000#include "../src/memcache/memcache.hpp" #include #include #include using namespace yrmcds::memcache; #define REQ(n, data) \ const char i##n[] = data; \ binary_request r##n(i##n, sizeof(i##n) - 1); #define ITEMCMP(t, s) \ cybozu_assert( std::get<1>(t) == (sizeof(s) - 1) ); \ cybozu_assert( std::memcmp(std::get<0>(t), s, sizeof(s) - 1) == 0 ); AUTOTEST(get) { REQ(1, "\x80\x00\x00\x00"); cybozu_assert( r1.length() == 0 ); REQ(2, "\x80\x00\x00\x05\x00\x00\x00\x00" "\x00\x00\x00\x05" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x01" // CAS "Hello" // key ); cybozu_assert( r2.length() == (24 + 5) ); cybozu_assert( r2.status() == binary_status::OK ); cybozu_assert( r2.command() == binary_command::Get ); cybozu_assert( ! r2.quiet() ); cybozu_assert( r2.key() == item(i2 + 24, 5) ); cybozu_assert( r2.exptime() == binary_request::EXPTIME_NONE ); ITEMCMP(r2.key(), "Hello"); cybozu_assert( std::memcmp(r2.opaque(), "\x12\x34\x56\x78", 4) == 0 ); cybozu_assert( r2.cas_unique() == 1 ); REQ(3, "\x80\x00\x00\x05\x00\x00\x00\x00" "\x00\x00\x00\x05" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x01" // CAS "Hell" // key ); cybozu_assert( r3.length() == 0 ); REQ(4, "\x80\x00\x00\x06\x00\x00\x00\x00" "\x00\x00\x00\x05" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x01" // CAS "Hello " // key ); cybozu_assert( r4.length() == (24 + 5) ); cybozu_assert( r4.status() == binary_status::Invalid ); REQ(5, "\x80\x00\x00\x05\x04\x00\x00\x00" "\x00\x00\x00\x09" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x01" // CAS "\x00\x00\x01\x00" // extra (exptime) "Hello " // key ); cybozu_assert( r5.length() == (24 + 9) ); cybozu_assert( r5.status() == binary_status::OK ); cybozu_assert( r5.exptime() != 0 && r5.exptime() != binary_request::EXPTIME_NONE ); const char data6[] = "\x80\x00\x00\x05\x04\x00\x00\x00" "\x00\x00\x00\x09" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x01" // CAS "\x00\x00\x01\x00" // extra (exptime) "Hello" // key "\x80\x09\x00\x05\x00\x00\x00\x00" "\x00\x00\x00\x05" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x01" // CAS "Hello "; // key binary_request r6(data6, sizeof(data6) - 1); binary_request r6_2(data6 + r6.length(), sizeof(data6) - r6.length() - 1); cybozu_assert( r6_2.status() == binary_status::OK ); cybozu_assert( r6_2.command() == binary_command::GetQ ); cybozu_assert( r6_2.quiet() ); REQ(7, "\x80\x81\x00\x00\x00\x00\x00\x00" "\x00\x00\x00\x00" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x00" // CAS ); cybozu_assert( r7.length() == 24 ); cybozu_assert( r7.status() == binary_status::UnknownCommand ); REQ(8, "\x80\x0c\x00\x05\x04\x00\x00\x00" "\x00\x00\x00\x09" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x01" // CAS "\x00\x00\x01\x00" // extra (exptime) "Hello " // key ); cybozu_assert( r8.length() != 0 ); cybozu_assert( r8.status() == binary_status::OK ); cybozu_assert( r8.command() == binary_command::GetK ); cybozu_assert( ! r8.quiet() ); REQ(9, "\x80\x0d\x00\x05\x04\x00\x00\x00" "\x00\x00\x00\x09" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x01" // CAS "\x00\x00\x01\x00" // extra (exptime) "Hello " // key ); cybozu_assert( r9.length() != 0 ); cybozu_assert( r9.status() == binary_status::OK ); cybozu_assert( r9.command() == binary_command::GetKQ ); cybozu_assert( r9.quiet() ); REQ(10, "\x80\x1d\x00\x05\x04\x00\x00\x00" "\x00\x00\x00\x09" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x01" // CAS "\x00\x00\x01\x00" // extra (exptime) "Hello" // key ); cybozu_assert( r10.length() != 0 ); cybozu_assert( r10.status() == binary_status::OK ); cybozu_assert( r10.command() == binary_command::GaT ); cybozu_assert( ! r10.quiet() ); cybozu_assert( r10.exptime() != binary_request::EXPTIME_NONE ); REQ(11, "\x80\x1e\x00\x05\x04\x00\x00\x00" "\x00\x00\x00\x09" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x01" // CAS "\x00\x00\x01\x00" // extra (exptime) "Hello" // key ); cybozu_assert( r11.length() != 0 ); cybozu_assert( r11.status() == binary_status::OK ); cybozu_assert( r11.command() == binary_command::GaTQ ); cybozu_assert( r11.quiet() ); REQ(12, "\x80\x23\x00\x05\x04\x00\x00\x00" "\x00\x00\x00\x09" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x01" // CAS "\x00\x00\x01\x00" // extra (exptime) "Hello" // key ); cybozu_assert( r12.length() != 0 ); cybozu_assert( r12.status() == binary_status::OK ); cybozu_assert( r12.command() == binary_command::GaTK ); cybozu_assert( ! r12.quiet() ); REQ(13, "\x80\x24\x00\x05\x04\x00\x00\x00" "\x00\x00\x00\x09" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x01" // CAS "\x00\x00\x01\x00" // extra (exptime) "Hello" // key ); cybozu_assert( r13.length() != 0 ); cybozu_assert( r13.status() == binary_status::OK ); cybozu_assert( r13.command() == binary_command::GaTKQ ); cybozu_assert( r13.quiet() ); REQ(14, "\x80\x46\x00\x05\x04\x00\x00\x00" "\x00\x00\x00\x09" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x01" // CAS "\x00\x00\x01\x00" // extra (exptime) "Hello" // key ); cybozu_assert( r14.length() == 24 + 9 ); cybozu_assert( r14.status() == binary_status::OK ); cybozu_assert( r14.command() == binary_command::LaG ); cybozu_assert( r14.cas_unique() == 1UL ); cybozu_assert( ! r14.quiet() ); cybozu_assert( r14.exptime() != binary_request::EXPTIME_NONE ); REQ(15, "\x80\x47\x00\x05\x00\x00\x00\x00" "\x00\x00\x00\x05" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x01" // CAS "Hello" // key ); cybozu_assert( r15.length() != 0 ); cybozu_assert( r15.status() == binary_status::OK ); cybozu_assert( r15.command() == binary_command::LaGQ ); cybozu_assert( r15.quiet() ); cybozu_assert( r15.exptime() == binary_request::EXPTIME_NONE ); REQ(16, "\x80\x48\x00\x05\x04\x00\x00\x00" "\x00\x00\x00\x09" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x01" // CAS "\x00\x00\x01\x00" // extra (exptime) "Hello" // key ); cybozu_assert( r16.length() != 0 ); cybozu_assert( r16.status() == binary_status::OK ); cybozu_assert( r16.command() == binary_command::LaGK ); cybozu_assert( ! r16.quiet() ); REQ(17, "\x80\x49\x00\x05\x04\x00\x00\x00" "\x00\x00\x00\x09" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x01" // CAS "\x00\x00\x01\x00" // extra (exptime) "Hello" // key ); cybozu_assert( r17.length() != 0 ); cybozu_assert( r17.status() == binary_status::OK ); cybozu_assert( r17.command() == binary_command::LaGKQ ); cybozu_assert( r17.quiet() ); } AUTOTEST(set_add_replace) { REQ(1, "\x80\x01\x00\x05\x08\x00\x00\x00" "\x00\x00\x00\x12" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x10" // CAS "\x00\x00\x00\x20" // flags "\x11\x11\x22\x33" // extra (exptime) "Hello" // key "World" // body ); cybozu_assert( r1.length() == (24 + 18) ); cybozu_assert( r1.status() == binary_status::OK ); cybozu_assert( r1.command() == binary_command::Set ); cybozu_assert( ! r1.quiet() ); cybozu_assert( r1.cas_unique() == 0x10 ); cybozu_assert( r1.flags() == 0x20 ); cybozu_assert( r1.exptime() == 0x11112233UL ); ITEMCMP(r1.key(), "Hello"); ITEMCMP(r1.data(), "World"); REQ(2, "\x80\x01\x00\x00\x08\x00\x00\x00" "\x00\x00\x00\x0d" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x10" // CAS "\x00\x00\x00\x20" // flags "\x00\x00\x00\x00" // extra (exptime) "World" // body ); cybozu_assert( r2.length() == (24 + 13) ); cybozu_assert( r2.status() == binary_status::Invalid ); cybozu_assert( r2.exptime() == 0 ); REQ(3, "\x80\x02\x00\x06\x08\x00\x00\x00" "\x00\x00\x00\x13" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x10" // CAS "\x00\x00\x00\x20" // flags "\x11\x11\x22\x33" // extra (exptime) "Hello " // key "World" // body "\x80 " ); cybozu_assert( r3.length() == (24 + 19) ); cybozu_assert( r3.status() == binary_status::OK ); cybozu_assert( r3.command() == binary_command::Add ); cybozu_assert( ! r3.quiet() ); REQ(4, "\x80\x03\x00\x06\x08\x00\x00\x00" "\x00\x00\x00\x13" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x10" // CAS "\x00\x00\x00\x20" // flags "\x11\x11\x22\x33" // extra (exptime) "Hello " // key "World" // body "\x80 " ); cybozu_assert( r4.length() == (24 + 19) ); cybozu_assert( r4.status() == binary_status::OK ); cybozu_assert( r4.command() == binary_command::Replace ); cybozu_assert( ! r4.quiet() ); REQ(5, "\x80\x11\x00\x06\x08\x00\x00\x00" "\x00\x00\x00\x13" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x10" // CAS "\x00\x00\x00\x20" // flags "\x11\x11\x22\x33" // extra (exptime) "Hello " // key "World" // body ); cybozu_assert( r5.length() != 0 ); cybozu_assert( r5.status() == binary_status::OK ); cybozu_assert( r5.command() == binary_command::SetQ ); cybozu_assert( r5.quiet() ); REQ(6, "\x80\x12\x00\x06\x08\x00\x00\x00" "\x00\x00\x00\x13" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x10" // CAS "\x00\x00\x00\x20" // flags "\x11\x11\x22\x33" // extra (exptime) "Hello " // key "World" // body ); cybozu_assert( r6.length() != 0 ); cybozu_assert( r6.status() == binary_status::OK ); cybozu_assert( r6.command() == binary_command::AddQ ); cybozu_assert( r6.quiet() ); REQ(7, "\x80\x13\x00\x06\x08\x00\x00\x00" "\x00\x00\x00\x13" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x10" // CAS "\x00\x00\x00\x20" // flags "\x11\x11\x22\x33" // extra (exptime) "Hello " // key "World" // body ); cybozu_assert( r7.length() != 0 ); cybozu_assert( r7.status() == binary_status::OK ); cybozu_assert( r7.command() == binary_command::ReplaceQ ); cybozu_assert( r7.quiet() ); REQ(8, "\x80\x02\x00\x06\x04\x00\x00\x00" "\x00\x00\x00\x0f" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x10" // CAS "\x00\x00\x00\x20" // flags "Hello " // key "World" // body "\x80 " ); cybozu_assert( r8.length() == (24 + 15) ); cybozu_assert( r8.status() == binary_status::Invalid ); REQ(9, "\x80\x13\x00\x00\x08\x00\x00\x00" "\x00\x00\x00\x0d" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x10" // CAS "\x00\x00\x00\x20" // flags "\x11\x11\x22\x33" // extra (exptime) "World" // body ); cybozu_assert( r9.length() != 0 ); cybozu_assert( r9.status() == binary_status::Invalid ); REQ(10, "\x80\x4a\x00\x06\x08\x00\x00\x00" "\x00\x00\x00\x13" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x10" // CAS "\x00\x00\x00\x20" // flags "\x11\x11\x22\x33" // extra (exptime) "Hello " // key "World" // body "\x80 " ); cybozu_assert( r10.length() == (24 + 19) ); cybozu_assert( r10.status() == binary_status::OK ); cybozu_assert( r10.command() == binary_command::RaU ); cybozu_assert( ! r10.quiet() ); ITEMCMP(r10.key(), "Hello "); ITEMCMP(r10.data(), "World"); REQ(11, "\x80\x4b\x00\x06\x08\x00\x00\x00" "\x00\x00\x00\x13" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x10" // CAS "\x00\x00\x00\x20" // flags "\x11\x11\x22\x33" // extra (exptime) "Hello " // key "World" // body "\x80 " ); cybozu_assert( r11.length() != 0 ); cybozu_assert( r11.status() == binary_status::OK ); cybozu_assert( r11.command() == binary_command::RaUQ ); cybozu_assert( r11.quiet() ); } AUTOTEST(delete) { REQ(1, "\x80\x04\x00\x05\x00\x00\x00\x00" "\x00\x00\x00\x05" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x10" // CAS "Hello" // key ); cybozu_assert( r1.length() != 0 ); cybozu_assert( r1.status() == binary_status::OK ); cybozu_assert( r1.command() == binary_command::Delete ); cybozu_assert( ! r1.quiet() ); ITEMCMP(r1.key(), "Hello"); REQ(2, "\x80\x14\x00\x05\x00\x00\x00\x00" "\x00\x00\x00\x05" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x10" // CAS "Hello" // key ); cybozu_assert( r2.length() != 0 ); cybozu_assert( r2.status() == binary_status::OK ); cybozu_assert( r2.command() == binary_command::DeleteQ ); cybozu_assert( r2.quiet() ); REQ(3, "\x80\x14\x00\x00\x00\x00\x00\x00" "\x00\x00\x00\x00" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x10" // CAS ); cybozu_assert( r3.length() != 0 ); cybozu_assert( r3.status() == binary_status::Invalid ); } AUTOTEST(inc_dec) { REQ(1, "\x80\x05\x00\x03\x14\x00\x00\x00" "\x00\x00\x00\x17" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x10" // CAS "\x00\x00\x00\x00\x00\x00\x00\x20" // value "\x00\x00\x00\x00\x00\x00\x00\x01" // inital "\xff\xff\xff\xff" // extra (exptime) "abc" // key ); cybozu_assert( r1.length() == (24 + 23) ); cybozu_assert( r1.status() == binary_status::OK ); cybozu_assert( r1.command() == binary_command::Increment ); cybozu_assert( ! r1.quiet() ); cybozu_assert( r1.value() == 0x20 ); cybozu_assert( r1.initial() == 0x01 ); cybozu_assert( r1.exptime() == binary_request::EXPTIME_NONE ); REQ(2, "\x80\x06\x00\x03\x14\x00\x00\x00" "\x00\x00\x00\x17" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x10" // CAS "\x00\x00\x00\x00\x00\x00\x00\x20" // value "\x00\x00\x00\x00\x00\x00\x00\x01" // inital "\x00\x00\x00\x00" // extra (exptime) "abc" // key ); cybozu_assert( r2.length() == (24 + 23) ); cybozu_assert( r2.status() == binary_status::OK ); cybozu_assert( r2.command() == binary_command::Decrement ); cybozu_assert( ! r2.quiet() ); cybozu_assert( r2.exptime() == 0 ); REQ(3, "\x80\x06\x00\x03\x0c\x00\x00\x00" "\x00\x00\x00\x0f" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x10" // CAS "\x00\x00\x00\x00\x00\x00\x00\x20" // value "\x00\x00\x00\x00" // extra (exptime) "abc" // key ); cybozu_assert( r3.length() != 0 ); cybozu_assert( r3.status() == binary_status::Invalid ); REQ(4, "\x80\x15\x00\x03\x14\x00\x00\x00" "\x00\x00\x00\x17" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x10" // CAS "\x00\x00\x00\x00\x00\x00\x00\x20" // value "\x00\x00\x00\x00\x00\x00\x00\x01" // inital "\x00\x00\x00\x00" // extra (exptime) "abc" // key ); cybozu_assert( r4.length() != 0 ); cybozu_assert( r4.status() == binary_status::OK ); cybozu_assert( r4.command() == binary_command::IncrementQ ); cybozu_assert( r4.quiet() ); REQ(5, "\x80\x16\x00\x03\x14\x00\x00\x00" "\x00\x00\x00\x17" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x10" // CAS "\x00\x00\x00\x00\x00\x00\x00\x20" // value "\x00\x00\x00\x00\x00\x00\x00\x01" // inital "\x00\x00\x00\x00" // extra (exptime) "abc" // key ); cybozu_assert( r5.length() != 0 ); cybozu_assert( r5.status() == binary_status::OK ); cybozu_assert( r5.command() == binary_command::DecrementQ ); cybozu_assert( r5.quiet() ); } AUTOTEST(flush) { REQ(1, "\x80\x08\x00\x00\x00\x00\x00\x00" "\x00\x00\x00\x00" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x10" // CAS ); cybozu_assert( r1.length() != 0 ); cybozu_assert( r1.status() == binary_status::OK ); cybozu_assert( r1.command() == binary_command::Flush ); cybozu_assert( ! r1.quiet() ); cybozu_assert( r1.exptime() != 0 || r1.exptime() != binary_request::EXPTIME_NONE ); REQ(2, "\x80\x08\x00\x00\x04\x00\x00\x00" "\x00\x00\x00\x04" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x10" // CAS "\x11\x22\x33\x44" // extra (exptime) ); cybozu_assert( r2.length() != 0 ); cybozu_assert( r2.status() == binary_status::OK ); cybozu_assert( r2.command() == binary_command::Flush ); cybozu_assert( r2.exptime() == 0x11223344UL ); REQ(3, "\x80\x18\x00\x00\x00\x00\x00\x00" "\x00\x00\x00\x00" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x10" // CAS ); cybozu_assert( r3.length() != 0 ); cybozu_assert( r3.status() == binary_status::OK ); cybozu_assert( r3.command() == binary_command::FlushQ ); cybozu_assert( r3.quiet() ); } AUTOTEST(append_prepend) { REQ(1, "\x80\x0e\x00\x05\x00\x00\x00\x00" "\x00\x00\x00\x06" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x10" // CAS "Hello" // key "!" // body ); cybozu_assert( r1.length() != 0 ); cybozu_assert( r1.status() == binary_status::OK ); cybozu_assert( r1.command() == binary_command::Append ); cybozu_assert( ! r1.quiet() ); ITEMCMP(r1.key(), "Hello"); ITEMCMP(r1.data(), "!"); REQ(2, "\x80\x0f\x00\x05\x00\x00\x00\x00" "\x00\x00\x00\x08" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x10" // CAS "Hello" // key "!!!" // body ); cybozu_assert( r2.length() != 0 ); cybozu_assert( r2.status() == binary_status::OK ); cybozu_assert( r2.command() == binary_command::Prepend ); cybozu_assert( ! r2.quiet() ); ITEMCMP(r2.key(), "Hello"); ITEMCMP(r2.data(), "!!!"); REQ(3, "\x80\x19\x00\x05\x00\x00\x00\x00" "\x00\x00\x00\x08" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x10" // CAS "Hello" // key "!!!" // body ); cybozu_assert( r3.length() != 0 ); cybozu_assert( r3.status() == binary_status::OK ); cybozu_assert( r3.command() == binary_command::AppendQ ); cybozu_assert( r3.quiet() ); REQ(4, "\x80\x1a\x00\x05\x00\x00\x00\x00" "\x00\x00\x00\x08" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x10" // CAS "Hello" // key "!!!" // body ); cybozu_assert( r4.length() != 0 ); cybozu_assert( r4.status() == binary_status::OK ); cybozu_assert( r4.command() == binary_command::PrependQ ); cybozu_assert( r4.quiet() ); } AUTOTEST(lock) { REQ(1, "\x80\x40\x00\x05\x00\x00\x00\x00" "\x00\x00\x00\x05" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x00" // CAS "Hello" // key ); cybozu_assert( r1.length() == 24 + 5 ); cybozu_assert( r1.status() == binary_status::OK ); cybozu_assert( r1.command() == binary_command::Lock ); cybozu_assert( ! r1.quiet() ); ITEMCMP(r1.key(), "Hello"); REQ(2, "\x80\x40\x00\x00\x00\x00\x00\x00" "\x00\x00\x00\x00" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x00" // CAS ); cybozu_assert( r2.length() == 24 ); cybozu_assert( r2.status() == binary_status::Invalid ); REQ(3, "\x80\x41\x00\x05\x00\x00\x00\x00" "\x00\x00\x00\x05" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x00" // CAS "Hello" // key ); cybozu_assert( r3.length() != 0 ); cybozu_assert( r3.status() == binary_status::OK ); cybozu_assert( r3.command() == binary_command::LockQ ); cybozu_assert( r3.quiet() ); } AUTOTEST(unlock) { REQ(1, "\x80\x42\x00\x05\x00\x00\x00\x00" "\x00\x00\x00\x05" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x00" // CAS "Hello" // key ); cybozu_assert( r1.length() == 24 + 5 ); cybozu_assert( r1.status() == binary_status::OK ); cybozu_assert( r1.command() == binary_command::Unlock ); cybozu_assert( ! r1.quiet() ); ITEMCMP(r1.key(), "Hello"); REQ(2, "\x80\x42\x00\x00\x00\x00\x00\x00" "\x00\x00\x00\x00" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x00" // CAS ); cybozu_assert( r2.length() == 24 ); cybozu_assert( r2.status() == binary_status::Invalid ); REQ(3, "\x80\x43\x00\x05\x00\x00\x00\x00" "\x00\x00\x00\x05" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x00" // CAS "Hello" // key ); cybozu_assert( r3.length() != 0 ); cybozu_assert( r3.status() == binary_status::OK ); cybozu_assert( r3.command() == binary_command::UnlockQ ); cybozu_assert( r3.quiet() ); } AUTOTEST(stat) { REQ(1, "\x80\x10\x00\x03\x00\x00\x00\x00" "\x00\x00\x00\x03" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x10" // CAS "foo" // key ); cybozu_assert( r1.length() != 0 ); cybozu_assert( r1.status() == binary_status::OK ); cybozu_assert( r1.command() == binary_command::Stat ); cybozu_assert( ! r1.quiet() ); ITEMCMP(r1.key(), "foo"); cybozu_assert( r1.stats() == stats_t::GENERAL ); REQ(2, "\x80\x10\x00\x00\x00\x00\x00\x00" "\x00\x00\x00\x00" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x10" // CAS ); cybozu_assert( r2.length() != 0 ); cybozu_assert( r2.status() == binary_status::OK ); cybozu_assert( r2.stats() == stats_t::GENERAL ); REQ(3, "\x80\x10\x00\x05\x00\x00\x00\x00" "\x00\x00\x00\x05" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x10" // CAS "items" // key ); cybozu_assert( r3.length() != 0 ); cybozu_assert( r3.status() == binary_status::OK ); cybozu_assert( r3.stats() == stats_t::ITEMS ); REQ(4, "\x80\x10\x00\x05\x00\x00\x00\x00" "\x00\x00\x00\x05" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x10" // CAS "sizes" // key ); cybozu_assert( r4.length() != 0 ); cybozu_assert( r4.status() == binary_status::OK ); cybozu_assert( r4.stats() == stats_t::SIZES ); REQ(5, "\x80\x10\x00\x08\x00\x00\x00\x00" "\x00\x00\x00\x08" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x10" // CAS "settings" // key ); cybozu_assert( r5.length() != 0 ); cybozu_assert( r5.status() == binary_status::OK ); cybozu_assert( r5.stats() == stats_t::SETTINGS ); } AUTOTEST(touch) { REQ(1, "\x80\x1c\x00\x05\x04\x00\x00\x00" "\x00\x00\x00\x09" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x01" // CAS "\x00\x00\x01\x00" // extra (exptime) "Hello " // key ); cybozu_assert( r1.length() == (24 + 9) ); cybozu_assert( r1.status() == binary_status::OK ); cybozu_assert( r1.command() == binary_command::Touch ); cybozu_assert( r1.exptime() != 0 && r1.exptime() != binary_request::EXPTIME_NONE ); REQ(2, "\x80\x1c\x00\x05\x00\x00\x00\x00" "\x00\x00\x00\x05" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x01" // CAS "Hello " // key ); cybozu_assert( r2.length() != 0 ); cybozu_assert( r2.status() == binary_status::Invalid ); } AUTOTEST(keys) { REQ(1, "\x80\x50\x00\x00\x00\x00\x00\x00" "\x00\x00\x00\x00" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x00" // CAS ); cybozu_assert( r1.length() != 0 ); cybozu_assert( r1.status() == binary_status::OK ); cybozu_assert( r1.command() == binary_command::Keys ); cybozu_assert( ! r1.quiet() ); cybozu_assert( std::get<1>(r1.key()) == 0 ); REQ(2, "\x80\x50\x00\x03\x00\x00\x00\x00" "\x00\x00\x00\x03" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x00" // CAS "abc" // key ); cybozu_assert( r2.length() == (24 + 3) ); cybozu_assert( r2.status() == binary_status::OK ); cybozu_assert( r2.command() == binary_command::Keys ); ITEMCMP(r2.key(), "abc"); } AUTOTEST(misc) { REQ(1, "\x80\x07\x00\x00\x00\x00\x00\x00" "\x00\x00\x00\x00" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x10" // CAS ); cybozu_assert( r1.length() != 0 ); cybozu_assert( r1.status() == binary_status::OK ); cybozu_assert( r1.command() == binary_command::Quit ); cybozu_assert( ! r1.quiet() ); REQ(2, "\x80\x17\x00\x00\x00\x00\x00\x00" "\x00\x00\x00\x00" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x10" // CAS ); cybozu_assert( r2.length() != 0 ); cybozu_assert( r2.status() == binary_status::OK ); cybozu_assert( r2.command() == binary_command::QuitQ ); cybozu_assert( r2.quiet() ); REQ(3, "\x80\x0a\x00\x00\x00\x00\x00\x00" "\x00\x00\x00\x00" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x10" // CAS ); cybozu_assert( r3.length() != 0 ); cybozu_assert( r3.status() == binary_status::OK ); cybozu_assert( r3.command() == binary_command::Noop ); REQ(4, "\x80\x0b\x00\x00\x00\x00\x00\x00" "\x00\x00\x00\x00" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x10" // CAS ); cybozu_assert( r4.length() != 0 ); cybozu_assert( r4.status() == binary_status::OK ); cybozu_assert( r4.command() == binary_command::Version ); REQ(5, "\x80\x44\x00\x00\x00\x00\x00\x00" "\x00\x00\x00\x00" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x10" // CAS ); cybozu_assert( r5.length() != 0 ); cybozu_assert( r5.status() == binary_status::OK ); cybozu_assert( r5.command() == binary_command::UnlockAll ); cybozu_assert( ! r5.quiet() ); REQ(6, "\x80\x45\x00\x00\x00\x00\x00\x00" "\x00\x00\x00\x00" // total body "\x12\x34\x56\x78" // opaque "\x00\x00\x00\x00\x00\x00\x00\x10" // CAS ); cybozu_assert( r6.length() != 0 ); cybozu_assert( r6.status() == binary_status::OK ); cybozu_assert( r6.command() == binary_command::UnlockAllQ ); cybozu_assert( r6.quiet() ); } yrmcds-1.1.5/test/memcache_text.cpp000066400000000000000000000333651262254645600173360ustar00rootroot00000000000000#include "../src/memcache/memcache.hpp" #include #include #include #include #include #include #include #include #include using namespace yrmcds; using namespace yrmcds::memcache; const char CR = '\x0d'; const char NL = '\x0a'; const char SP = '\x20'; template inline UInt to_uint(const char* p, bool& result) { result = false; char* end; unsigned long long i = strtoull(p, &end, 10); if( i == 0 && p == end ) return 0; if( i == std::numeric_limits::max() && errno == ERANGE ) return 0; char c = *end; if( c != CR && c != NL && c != SP ) return 0; if( i > std::numeric_limits::max() ) return 0; result = true; return static_cast(i); } AUTOTEST(to_uint) { bool r; to_uint("\n", r); cybozu_assert( ! r ); to_uint("4294967295", r); cybozu_assert( ! r ); to_uint("4294967295 ", r); cybozu_assert( r ); to_uint("4294967295\r", r); cybozu_assert( r ); to_uint("4294967295\n", r); cybozu_assert( r ); to_uint("4294967296\n", r); cybozu_assert( ! r ); to_uint("4294967296\n", r); cybozu_assert( r ); cybozu_assert( to_uint("4294967296\n", r) == 4294967296ULL ); to_uint("-1\n", r); cybozu_assert( ! r ); to_uint(" 429496729\n", r); cybozu_assert( r ); to_uint(" 429496729\n", r); cybozu_assert( r ); to_uint(" 429496729\n", r); cybozu_assert( r ); cybozu_assert( to_uint("18446744073709551615 ", r) == 18446744073709551615ULL ); to_uint("18446744073709551616 ", r); cybozu_assert( ! r ); } #define MEMCACHE_TEST(n, d) \ char data##n[] = d; \ text_request t##n(data##n, sizeof(data##n) - 1); #define VERIFY(n, k, f) \ cybozu_assert( std::memcmp(std::get<0>(t##n.f()), k, sizeof(k)-1) == 0 ) AUTOTEST(empty) { MEMCACHE_TEST(1, "\r\n"); cybozu_assert( t1.length() == 2 ); cybozu_assert( ! t1.valid() ); MEMCACHE_TEST(2, " \n"); cybozu_assert( t2.length() == 2 ); cybozu_assert( ! t2.valid() ); } AUTOTEST(verbosity) { MEMCACHE_TEST(1, " verbosity"); cybozu_assert( t1.length() == 0 ); MEMCACHE_TEST(2, " verbosity\naaa"); cybozu_assert( t2.length() == 11 ); cybozu_assert( ! t2.valid() ); MEMCACHE_TEST(3, " verbosity \r\n"); cybozu_assert( t3.command() == text_command::VERBOSITY ); cybozu_assert( t3.length() != 0 ); cybozu_assert( ! t3.valid() ); cybozu_assert( ! t3.no_reply() ); MEMCACHE_TEST(4, " verbosity warning \r\naaa"); cybozu_assert( t4.valid() ); cybozu_assert( t4.verbosity() == cybozu::severity::warning ); cybozu_assert( ! t4.no_reply() ); MEMCACHE_TEST(5, " verbosity warning \r\nverbosity error\r\n"); cybozu_assert( t5.valid() ); cybozu_assert( t5.verbosity() == cybozu::severity::warning ); text_request t5_2(&data5[t5.length()], sizeof(data5) - 1 - t5.length()); cybozu_assert( t5_2.valid() ); cybozu_assert( t5_2.verbosity() == cybozu::severity::error ); MEMCACHE_TEST(6, " verbosity 1\r\n"); cybozu_assert( t6.valid() ); MEMCACHE_TEST(7, " verbosity error noreply\r\n"); cybozu_assert( t7.valid() ); cybozu_assert( t7.no_reply() ); } AUTOTEST(set) { MEMCACHE_TEST(1, "set \r\n "); cybozu_assert( t1.length() == 6 ); cybozu_assert( t1.command() == text_command::SET ); MEMCACHE_TEST(2, "set abcdef \r\n"); cybozu_assert( std::get<1>(t2.key()) == 6 ); VERIFY(2, "abcdef", key); cybozu_assert( ! t2.valid() ); MEMCACHE_TEST(3, "set abcdef 65536 \r\n"); cybozu_assert( t3.flags() == 65536 ); MEMCACHE_TEST(4, "set abcdef 0 \r\n"); cybozu_assert( t4.flags() == 0 ); MEMCACHE_TEST(5, "set abcdef 4294967295 \r\n"); cybozu_assert( t5.flags() == 4294967295ULL ); MEMCACHE_TEST(6, "set abcdef 4294967296 \r\n"); cybozu_assert( t6.flags() != 4294967296ULL ); MEMCACHE_TEST(7, "set abcdef 3 0 \r\n"); cybozu_assert( t7.flags() == 3 ); cybozu_assert( t7.exptime() == 0 ); cybozu_assert( ! t7.valid() ); MEMCACHE_TEST(8, "set abcdef 3 100 \r\n"); cybozu_assert( t8.exptime() < std::time(nullptr) + 101 ); cybozu_assert( t8.exptime() > 100 ); MEMCACHE_TEST(9, "set abcdef 3 3000000 \r\n"); cybozu_assert( t9.exptime() == 3000000 ); MEMCACHE_TEST(10, "set aaa 100 0 5\r\nabcde\r\nset aaa"); cybozu_assert( t10.valid() ); cybozu_assert( std::get<1>(t10.data()) == 5 ); VERIFY(10, "abcde", data); MEMCACHE_TEST(11, "set aaa 100 0 10\r\nabcdefghija\r\nset aaa"); cybozu_assert( ! t11.valid() ); MEMCACHE_TEST(12, "set aaa 100 0 10\r\nabcdefghij\r"); cybozu_assert( t12.length() == 0 ); MEMCACHE_TEST(13, "set aaa 100 0 10\r\nabcdefghij"); cybozu_assert( t13.length() == 0 ); MEMCACHE_TEST(14, "set aaa 100 0 10 \r\nabcdefghij\r\naaa"); cybozu_assert( t14.valid() ); cybozu_assert( ! t14.no_reply() ); MEMCACHE_TEST(15, "set aaa 100 0 10 noreply \r\nabcdefghij\r\naaa"); cybozu_assert( t15.valid() ); cybozu_assert( t15.no_reply() ); MEMCACHE_TEST(16, "set aaa 100 0 10 norepry\r\nabcdefghij\r\naaa"); cybozu_assert( ! t16.valid() ); MEMCACHE_TEST(17, "set aaa 100 0 10 noreply aaa \r\nabcdefghij\r\naaa"); cybozu_assert( ! t17.valid() ); } AUTOTEST(add) { MEMCACHE_TEST(1, " add aaa 100 0 10 noreply\r\nabcdefghij\r\n"); cybozu_assert( t1.valid() ); cybozu_assert( t1.command() == text_command::ADD ); cybozu_assert( t1.no_reply() ); } AUTOTEST(replace) { MEMCACHE_TEST(1, " replace aaa 100 0 10 noreply \r\nabcdefghij\r\n"); cybozu_assert( t1.valid() ); cybozu_assert( t1.command() == text_command::REPLACE ); cybozu_assert( t1.no_reply() ); } AUTOTEST(append) { MEMCACHE_TEST(1, " append aaa 100 0 10 noreply \r\nabcdefghij\r\n"); cybozu_assert( t1.valid() ); cybozu_assert( t1.command() == text_command::APPEND ); cybozu_assert( t1.no_reply() ); } AUTOTEST(prepend) { MEMCACHE_TEST(1, " prepend aaa 100 0 10 noreply \r\nabcdefghij\r\n"); cybozu_assert( t1.valid() ); cybozu_assert( t1.command() == text_command::PREPEND ); cybozu_assert( t1.no_reply() ); } AUTOTEST(cas) { MEMCACHE_TEST(1, " cas aaa 100 0 10 noreply \r\nabcdefghij\r\n"); cybozu_assert( ! t1.valid() ); cybozu_assert( t1.command() == text_command::CAS ); MEMCACHE_TEST(2, " cas aaa 100 0 10 3 noreply \r\nabcdefghij\r\n"); cybozu_assert( t2.valid() ); cybozu_assert( t2.cas_unique() == 3 ); cybozu_assert( t2.no_reply() ); MEMCACHE_TEST(3, " cas aaa 100 0 10 4\r\nabcdefghij\r\n"); cybozu_assert( t3.valid() ); cybozu_assert( t3.cas_unique() == 4 ); cybozu_assert( ! t3.no_reply() ); } AUTOTEST(delete) { MEMCACHE_TEST(1, "delete aaa noreply \r\nabc"); cybozu_assert( t1.valid() ); cybozu_assert( t1.command() == text_command::DELETE ); cybozu_assert( std::memcmp(std::get<0>(t1.key()), "aaa", 3) == 0 ); cybozu_assert( std::get<1>(t1.key()) == 3 ); cybozu_assert( t1.no_reply() ); MEMCACHE_TEST(2, "delete aaa garbage\r\nabc"); cybozu_assert( ! t2.valid() ); MEMCACHE_TEST(3, "delete aaagarbage\r\nabc"); cybozu_assert( t3.valid() ); cybozu_assert( std::get<1>(t3.key()) == 10 ); cybozu_assert( ! t3.no_reply() ); } AUTOTEST(touch) { MEMCACHE_TEST(1, "touch\r\nabc"); cybozu_assert( ! t1.valid() ); cybozu_assert( t1.command() == text_command::TOUCH ); MEMCACHE_TEST(2, "touch abcd \r\nabc"); cybozu_assert( ! t2.valid() ); cybozu_assert( std::get<1>(t2.key()) == 4 ); MEMCACHE_TEST(3, "touch abcd qqq\r\nabc"); cybozu_assert( ! t3.valid() ); MEMCACHE_TEST(4, "touch abcd 0 \r\nabc"); cybozu_assert( t4.valid() ); cybozu_assert( ! t4.no_reply() ); cybozu_assert( t4.exptime() == 0 ); MEMCACHE_TEST(5, "touch abcd 100 noreply\r\nabc"); cybozu_assert( t5.valid() ); cybozu_assert( t5.no_reply() ); MEMCACHE_TEST(6, "touch abcd 100 noreply 38\r\nabc"); cybozu_assert( ! t6.valid() ); cybozu_assert( t6.length() == 29 ); } AUTOTEST(stats) { MEMCACHE_TEST(1, "stats\r\nabc"); cybozu_assert( t1.valid() ); cybozu_assert( t1.command() == text_command::STATS ); cybozu_assert( t1.stats() == stats_t::GENERAL ); MEMCACHE_TEST(2, "stats hoge \r\nabc"); cybozu_assert( ! t2.valid() ); MEMCACHE_TEST(3, "stats items \r\nabc"); cybozu_assert( t3.valid() ); cybozu_assert( t3.stats() == stats_t::ITEMS ); MEMCACHE_TEST(4, "stats items allow garbage\r\nabc"); cybozu_assert( t4.valid() ); cybozu_assert( t4.stats() == stats_t::ITEMS ); MEMCACHE_TEST(5, "stats settings\r\nabc"); cybozu_assert( t5.valid() ); cybozu_assert( t5.stats() == stats_t::SETTINGS ); MEMCACHE_TEST(6, "stats sizes\r\nabc"); cybozu_assert( t6.valid() ); cybozu_assert( t6.stats() == stats_t::SIZES ); } AUTOTEST(incr) { MEMCACHE_TEST(1, "incr\r\nabc"); cybozu_assert( ! t1.valid() ); cybozu_assert( t1.command() == text_command::INCR ); MEMCACHE_TEST(2, "incr aaaaaa \r\nabc"); cybozu_assert( ! t2.valid() ); cybozu_assert( std::get<1>(t2.key()) == 6 ); MEMCACHE_TEST(3, "incr a 429496729500 \r\nabc"); cybozu_assert( t3.valid() ); cybozu_assert( t3.value() == 429496729500ULL ); cybozu_assert( ! t3.no_reply() ); MEMCACHE_TEST(4, "incr a noreply \r\nabc"); cybozu_assert( ! t4.valid() ); MEMCACHE_TEST(5, "incr a 3 garbagenowallowed\r\nabc"); cybozu_assert( ! t5.valid() ); MEMCACHE_TEST(6, "incr a 3 noreply garbagenowallowed\r\nabc"); cybozu_assert( ! t6.valid() ); MEMCACHE_TEST(7, "incr a 3 noreply \r\nabc"); cybozu_assert( t7.valid() ); cybozu_assert( t7.no_reply() ); } AUTOTEST(decr) { MEMCACHE_TEST(1, "decr tttt 100 noreply \r\nabc"); cybozu_assert( t1.valid() ); cybozu_assert( t1.command() == text_command::DECR ); cybozu_assert( std::get<1>(t1.key()) == 4 ); cybozu_assert( t1.value() == 100 ); cybozu_assert( t1.no_reply() ); } AUTOTEST(lock) { MEMCACHE_TEST(1, "lock hoge \r\n"); cybozu_assert( t1.valid() ); cybozu_assert( t1.command() == text_command::LOCK ); cybozu_assert( std::get<1>(t1.key()) == 4 ); VERIFY(1, "hoge", key); MEMCACHE_TEST(2, "lock hoge leftover\r\n"); cybozu_assert( ! t2.valid() ); cybozu_assert( t2.length() == 20 ); } AUTOTEST(unlock) { MEMCACHE_TEST(1, "unlock hoge \r\n"); cybozu_assert( t1.valid() ); cybozu_assert( t1.command() == text_command::UNLOCK ); cybozu_assert( std::get<1>(t1.key()) == 4 ); VERIFY(1, "hoge", key); MEMCACHE_TEST(2, "unlock hoge leftover\r\n"); cybozu_assert( ! t2.valid() ); cybozu_assert( t2.length() == 22 ); } AUTOTEST(unlockall) { MEMCACHE_TEST(1, "unlock_all \r\n"); cybozu_assert( t1.valid() ); cybozu_assert( t1.command() == text_command::UNLOCK_ALL ); } AUTOTEST(get) { MEMCACHE_TEST(1, "get \r\nabc"); cybozu_assert( ! t1.valid() ); cybozu_assert( t1.length() == 7 ); MEMCACHE_TEST(2, "get 38338 \r\nabc"); cybozu_assert( t2.valid() ); cybozu_assert( t2.command() == text_command::GET ); cybozu_assert( std::get<1>(t2.first_key()) == 5 ); VERIFY(2, "38338", first_key); cybozu_assert( t2.next_key(t2.first_key()) == text_request::eos ); MEMCACHE_TEST(3, "get 38338 4\r\nabc"); cybozu_assert( t3.valid() ); cybozu_assert( std::get<1>(t3.first_key()) == 5 ); VERIFY(3, "38338", first_key); auto next_key = t3.next_key(t3.first_key()); cybozu_assert( next_key != text_request::eos ); cybozu_assert( std::get<1>(next_key) == 1 ); cybozu_assert( *std::get<0>(next_key) == '4' ); next_key = t3.next_key(next_key); cybozu_assert( next_key == text_request::eos ); MEMCACHE_TEST(4, "get 38338 4 \r\nabc"); cybozu_assert( t4.valid() ); next_key = t4.next_key(t4.first_key()); cybozu_assert( std::get<1>(next_key) == 1 ); cybozu_assert( *std::get<0>(next_key) == '4' ); next_key = t4.next_key(next_key); cybozu_assert( next_key == text_request::eos ); } AUTOTEST(gets) { MEMCACHE_TEST(1, "gets 38338 \r\nabc"); cybozu_assert( t1.valid() ); cybozu_assert( t1.command() == text_command::GETS ); } AUTOTEST(flush_all) { MEMCACHE_TEST(1, "flush_all \r\nabc"); cybozu_assert( t1.valid() ); cybozu_assert( t1.command() == text_command::FLUSH_ALL ); MEMCACHE_TEST(2, "flush_all 100 \r\nabc"); cybozu_assert( t2.valid() ); cybozu_assert( ! t2.no_reply() ); cybozu_assert( t2.exptime() != 0 ); cybozu_assert( t2.exptime() < (time(NULL) + 101) ); MEMCACHE_TEST(3, "flush_all 100a \r\nabc"); cybozu_assert( ! t3.valid() ); MEMCACHE_TEST(4, "flush_all 100 noreply \r\nabc"); cybozu_assert( t4.valid() ); cybozu_assert( t4.no_reply() ); } AUTOTEST(keys) { MEMCACHE_TEST(1, "keys\r\n"); cybozu_assert( t1.valid() ); cybozu_assert( t1.command() == text_command::KEYS ); cybozu_assert( std::get<1>(t1.key()) == 0 ); MEMCACHE_TEST(2, "keys abc \r\n"); cybozu_assert( t2.valid() ); cybozu_assert( t2.command() == text_command::KEYS ); cybozu_assert( std::get<1>(t2.key()) == 3 ); VERIFY(2, "abc", key); } AUTOTEST(singles) { MEMCACHE_TEST(1, "slabs 38338 \r\nabc"); cybozu_assert( t1.valid() ); cybozu_assert( t1.command() == text_command::SLABS ); MEMCACHE_TEST(2, "version 38338 \r\nabc"); cybozu_assert( t2.valid() ); cybozu_assert( t2.command() == text_command::VERSION ); MEMCACHE_TEST(3, "quit 38338 \r\nabc"); cybozu_assert( t3.valid() ); cybozu_assert( t3.command() == text_command::QUIT ); } yrmcds-1.1.5/test/move.cpp000066400000000000000000000021511262254645600154630ustar00rootroot00000000000000// move and copy test #include #include #include #include struct A { A() { std::cout << "A ctor." << std::endl; } A(A&& rhs) noexcept { std::cout << "A move." << std::endl; moved = true; } A(const A&) = delete; ~A() { if( ! moved ) std::cout << "A dtor." << std::endl; } private: bool moved = false; }; std::vector v; void f(A a) { v.push_back( std::move(a) ); } void g() { std::unique_ptr pa( new A ); typedef std::unordered_map > map_type; map_type m; m.emplace( 3, std::move(pa) ); std::cout << "get A." << std::endl; auto it = m.find(3); it->second.get(); std::cout << "got A." << std::endl; std::cout << "get A take2." << std::endl; std::unique_ptr& alias = m.at(3); A a( std::move(*alias) ); std::cout << "got A take2." << std::endl; } int main(int argc, char** argv) { A a; f( std::move(a) ); std::vector iv = {1, 2, 3}; std::vector iv2 = iv; for( auto i : iv2 ) { std::cout << i << std::endl; } g(); return 0; } yrmcds-1.1.5/test/mutex.cpp000066400000000000000000000006611262254645600156630ustar00rootroot00000000000000#include #include #include std::mutex lock; int g_counter = 0; void foo() { for (int i = 0; i < 10000; ++i) { std::lock_guard g(lock); ++g_counter; } return; } int main() { std::thread t1(foo); std::thread t2(foo); std::thread t3(foo); foo(); t1.join(); t2.join(); t3.join(); std::cout << g_counter << std::endl; return 0; } yrmcds-1.1.5/test/netiface.cpp000066400000000000000000000013741262254645600163010ustar00rootroot00000000000000#include #include int main(int argc, char** argv) { if( argc < 2 ) { std::cout << "Usage: netiface ADDRESS" << std::endl; return 0; } cybozu::ip_address a(argv[1]); if( a.is_v4() ) { std::cout << "v4!" << std::endl; } else if( a.is_v6() ) { std::cout << "v6!" << std::endl; } cybozu::ip_address b("127.0.0.1"); if( a == b ) { std::cout << a.str() << " == 127.0.0.1" << std::endl; } else { std::cout << a.str() << " != 127.0.0.1" << std::endl; } if( has_ip_address(a) ) { std::cout << a.str() << " is assigned." << std::endl; } else { std::cout << a.str() << " is not assigned." << std::endl; } return 0; } yrmcds-1.1.5/test/object.cpp000066400000000000000000000055301262254645600157670ustar00rootroot00000000000000#include "../src/config.hpp" #include "../src/memcache/object.hpp" #include #include using yrmcds::memcache::object; using cybozu::dynbuf; std::size_t reset_heap_limit() { const char* p = std::getenv("HEAP_LIMIT"); std::size_t n = 0; if( p != nullptr ) n = (std::size_t)std::atoi(p); if( n == 0 ) n = yrmcds::DEFAULT_HEAP_DATA_LIMIT; yrmcds::g_config.set_heap_data_limit(n); return n; } AUTOTEST(set) { reset_heap_limit(); object o1("abcde", 5, 100, 0); cybozu_assert( o1.size() == 5 ); cybozu_assert( o1.flags() == 100 ); dynbuf d1(0); d1.append("abcde", 5); dynbuf d1_(0); std::uint64_t cas = o1.cas_unique(); cybozu_assert( o1.data(d1_) == d1 ); cybozu_assert( o1.cas_unique() == cas ); o1.set("123", 3, 200, 200); cybozu_assert( o1.size() == 3 ); cybozu_assert( o1.flags() == 200 ); d1.reset(); d1.append("123", 3); cybozu_assert( o1.data(d1_) == d1 ); cybozu_assert( o1.cas_unique() != cas ); } AUTOTEST(append_prepend) { reset_heap_limit(); object o1("abcde", 5, 100, 0); std::uint64_t cas = o1.cas_unique(); o1.append("12345", 5); std::uint64_t cas2 = o1.cas_unique(); cybozu_assert( o1.size() == 10 ); cybozu_assert( cas != cas2 ); dynbuf d1(0); d1.append("abcde12345", 10); dynbuf d1_(0); cybozu_assert( o1.data(d1_) == d1 ); o1.prepend("#*A", 3); std::uint64_t cas3 = o1.cas_unique(); cybozu_assert( o1.size() == 13 ); cybozu_assert( cas != cas3 ); cybozu_assert( cas2 != cas3 ); d1.reset(); d1.append("#*Aabcde12345", 13); cybozu_assert( o1.data(d1_) == d1 ); } AUTOTEST(touch) { reset_heap_limit(); object o1("abcde", 5, 100, 0); std::uint64_t cas = o1.cas_unique(); o1.touch(1111); dynbuf d1(0); d1.append("abcde", 5); dynbuf d1_(0); cybozu_assert( o1.size() == 5 ); cybozu_assert( o1.data(d1_) == d1 ); cybozu_assert( o1.cas_unique() == cas ); } AUTOTEST(incr_decr) { if( reset_heap_limit() < 24 ) return; object o1("abcde", 5, 100, 0); cybozu_test_exception( o1.incr(3), object::not_a_number ); object o2("123ab", 5, 100, 0); std::uint64_t cas = o2.cas_unique(); std::uint64_t n = 0; cybozu_test_no_exception( n = o2.incr(3) ); std::uint64_t cas2 = o2.cas_unique(); cybozu_assert( n == 126 ); cybozu_assert( cas != cas2 ); n = o2.decr(3); std::uint64_t cas3 = o2.cas_unique(); cybozu_assert( n == 123 ); cybozu_assert( cas2 != cas3 ); n = o2.decr(200); cybozu_assert( n == 0 ); object o3(" 18446744073709551615", 22, 100, 0); n = o3.decr(1); cybozu_assert( n == 18446744073709551614ULL ); n = o3.incr(2); cybozu_assert( n == 0 ); o3.set(" 111", 4, 0, 0); dynbuf d3(0); d3.append(" 111", 4); dynbuf d3_(0); cybozu_assert( o3.data(d3_) == d3 ); } yrmcds-1.1.5/test/override.cpp000066400000000000000000000002231262254645600163320ustar00rootroot00000000000000struct A { virtual void f() {} virtual ~A() {} }; struct B: public A { void f() override {} }; int main() { B b; return 0; } yrmcds-1.1.5/test/prime.cpp000066400000000000000000000006061262254645600156340ustar00rootroot00000000000000#include #include void show_nearest_prime(unsigned int n) { std::cout << "nearest_prime(" << n << ") = " << cybozu::nearest_prime(n) << std::endl; } int main() { for( int i = 0; i <= 20; ++i ) show_nearest_prime(i); show_nearest_prime(1024); show_nearest_prime(2<<20); show_nearest_prime(2<<30); return 0; } yrmcds-1.1.5/test/protocol_binary.cpp000066400000000000000000001107161262254645600177310ustar00rootroot00000000000000#include "../src/memcache/memcache.hpp" #include #include #define TEST_DISABLE_AUTO_RUN #include #include #include #include #include #include #include #include #include #include #include #include #include using yrmcds::memcache::binary_command; using yrmcds::memcache::binary_status; using yrmcds::memcache::item; const char* g_server = nullptr; std::uint16_t g_port = 11211; typedef char opaque_t[4]; const std::size_t BINARY_HEADER_SIZE = 24; int connect_server() { int s = cybozu::tcp_connect(g_server, g_port); if( s == -1 ) return -1; ::fcntl(s, F_SETFL, ::fcntl(s, F_GETFL, 0) & ~O_NONBLOCK); struct timeval tv; tv.tv_sec = 0; tv.tv_usec = 300000; ::setsockopt(s, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv)); int ok = 1; ::setsockopt(s, IPPROTO_TCP, TCP_NODELAY, &ok, sizeof(ok)); return s; } // compose binary request class request { public: request(binary_command cmd, std::uint16_t key_len, const char* key, char extras_len, const char* extras, std::uint32_t data_len, const char* data, opaque_t* opaque = nullptr, std::uint64_t cas = 0): m_data(BINARY_HEADER_SIZE + key_len + extras_len + data_len), m_p(&m_data[0]) { m_p[0] = '\x80'; m_p[1] = (char)cmd; cybozu::hton(key_len, m_p+2); m_p[4] = extras_len; m_p[5] = 0; m_p[6] = 0; m_p[7] = 0; std::uint32_t total_len = key_len + extras_len + data_len; cybozu::hton(total_len, m_p+8); if( opaque != nullptr ) std::memcpy(m_p+12, opaque, sizeof(opaque_t)); cybozu::hton(cas, m_p+16); char* p = m_p + BINARY_HEADER_SIZE; std::memcpy(p, extras, extras_len); p += extras_len; std::memcpy(p, key, key_len); p += key_len; std::memcpy(p, data, data_len); } const char* data() { return m_p; } std::size_t length() const noexcept { return m_data.size(); } private: std::vector m_data; char* const m_p; }; // parse binary response class response { public: response() {} response(response&&) noexcept = default; // Return length of the response. // // Return length of the response. // If the response is incomplete, zero is returned. std::size_t length() const noexcept { return m_response_len; } // Response status, if determined by the request. binary_status status() const noexcept { return m_status; } // Return `true` if the response status was not OK. bool error() { return m_status != binary_status::OK; } // Return the command type. binary_command command() const noexcept { return m_command; } // Return `key`. item key() const noexcept { return m_key; } // Return `opaque` sent with the request. const char* opaque() const noexcept { return m_p + 12; } // Return `cas unique` sent with CAS command. std::uint64_t cas_unique() const noexcept { return m_cas_unique; } // Return `flags` sent with storage commands. std::uint32_t flags() const noexcept { return m_flags; } // Return data block sent with storage commands. item data() const noexcept { return m_data; } // Return an unsigned 64bit integer value for increment or decrement. std::uint64_t value() const noexcept { return m_value; } bool parse(const char* p, std::size_t len); private: const char* m_p = nullptr; std::size_t m_len = 0; std::size_t m_response_len = 0; binary_status m_status; binary_command m_command; item m_key; std::uint64_t m_cas_unique; std::uint32_t m_flags = 0; item m_data; std::uint64_t m_value = 0; }; bool response::parse(const char* p, std::size_t len) { m_p = p; m_len = len; if( m_len < BINARY_HEADER_SIZE ) return false; cybozu_assert( *m_p == '\x81' ); std::uint32_t total_len; cybozu::ntoh(m_p + 8, total_len); if( m_len < (BINARY_HEADER_SIZE + total_len) ) return false; m_response_len = BINARY_HEADER_SIZE + total_len; m_command = (binary_command)*(unsigned char*)(m_p + 1); std::uint16_t key_len; cybozu::ntoh(m_p + 2, key_len); std::uint8_t extras_len = *(unsigned char*)(m_p + 4); cybozu_assert( total_len >= (key_len + extras_len) ); if( key_len > 0 ) { m_key = item(m_p + (BINARY_HEADER_SIZE + extras_len), key_len); } else { m_key = item(nullptr, 0); } std::uint16_t i_status; cybozu::ntoh(m_p + 6, i_status); m_status = (binary_status)i_status; cybozu::ntoh(m_p + 16, m_cas_unique); std::size_t data_len = total_len - key_len - extras_len; if( data_len > 0 ) { m_data = item(m_p + (BINARY_HEADER_SIZE + extras_len + key_len), data_len); } else { m_data = item(nullptr, 0); } if( extras_len > 0 ) { cybozu_assert( extras_len == 4 ); cybozu::ntoh(m_p + BINARY_HEADER_SIZE, m_flags); } if( (m_command == binary_command::Increment || m_command == binary_command::Decrement) && m_status == binary_status::OK ) { cybozu_assert( data_len == 8 ); if( data_len == 8 ) { cybozu::ntoh(m_p + BINARY_HEADER_SIZE + key_len + extras_len, m_value); } } return true; } class client { public: client(): m_socket(connect_server()), m_buffer(1 << 20) { cybozu_assert( m_socket != -1 ); } ~client() { ::close(m_socket); } bool get_response(response& resp) { m_buffer.erase(m_last_response_size); m_last_response_size = 0; if( m_buffer.empty() ) { if( ! recv() ) { cybozu_assert( m_buffer.empty() ); return false; } } while( true ) { if( resp.parse(m_buffer.data(), m_buffer.size()) ) { m_last_response_size = resp.length(); return true; } cybozu_assert( recv() ); } } void noop(opaque_t* opaque) { request req(binary_command::Noop, 0, nullptr, 0, nullptr, 0, nullptr, opaque); send(req.data(), req.length()); } void get(const std::string& key, bool q) { request req(q ? binary_command::GetQ : binary_command::Get, (std::uint16_t)key.size(), key.data(), 0, nullptr, 0, nullptr); send(req.data(), req.length()); } void getk(const std::string& key, bool q) { request req(q ? binary_command::GetKQ : binary_command::GetK, (std::uint16_t)key.size(), key.data(), 0, nullptr, 0, nullptr); send(req.data(), req.length()); } void touch(const std::string& key, std::uint32_t expire) { char extra[4]; cybozu::hton(expire, extra); request req(binary_command::Touch, (std::uint16_t)key.size(), key.data(), sizeof(extra), extra, 0, nullptr); send(req.data(), req.length()); } void get_and_touch(const std::string& key, std::uint32_t expire, bool q) { char extra[4]; cybozu::hton(expire, extra); request req(q ? binary_command::GaTQ : binary_command::GaT, (std::uint16_t)key.size(), key.data(), sizeof(extra), extra, 0, nullptr); send(req.data(), req.length()); } void getk_and_touch(const std::string& key, std::uint32_t expire, bool q) { char extra[4]; cybozu::hton(expire, extra); request req(q ? binary_command::GaTKQ : binary_command::GaTK, (std::uint16_t)key.size(), key.data(), sizeof(extra), extra, 0, nullptr); send(req.data(), req.length()); } void lock_and_get(const std::string& key, bool q) { request req(q ? binary_command::LaGQ : binary_command::LaG, (std::uint16_t)key.size(), key.data(), 0, nullptr, 0, nullptr); send(req.data(), req.length()); } void lock_and_getk(const std::string& key, bool q) { request req(q ? binary_command::LaGKQ : binary_command::LaGK, (std::uint16_t)key.size(), key.data(), 0, nullptr, 0, nullptr); send(req.data(), req.length()); } void set(const std::string& key, const std::string& data, bool q, std::uint32_t flags, std::uint32_t expire, std::uint64_t cas = 0) { char extra[8]; cybozu::hton(flags, extra); cybozu::hton(expire, &extra[4]); request req(q ? binary_command::SetQ : binary_command::Set, (std::uint16_t)key.size(), key.data(), sizeof(extra), extra, (std::uint32_t)data.length(), data.data(), nullptr, cas); send(req.data(), req.length()); } void replace(const std::string& key, const std::string& data, bool q, std::uint32_t flags, std::uint32_t expire, std::uint64_t cas = 0) { char extra[8]; cybozu::hton(flags, extra); cybozu::hton(expire, &extra[4]); request req(q ? binary_command::ReplaceQ : binary_command::Replace, (std::uint16_t)key.size(), key.data(), sizeof(extra), extra, (std::uint32_t)data.length(), data.data(), nullptr, cas); send(req.data(), req.length()); } void add(const std::string& key, const std::string& data, bool q, std::uint32_t flags, std::uint32_t expire, std::uint64_t cas = 0) { char extra[8]; cybozu::hton(flags, extra); cybozu::hton(expire, &extra[4]); request req(q ? binary_command::AddQ : binary_command::Add, (std::uint16_t)key.size(), key.data(), sizeof(extra), extra, (std::uint32_t)data.length(), data.data(), nullptr, cas); send(req.data(), req.length()); } void replace_and_unlock(const std::string& key, const std::string& data, bool q, std::uint32_t flags, std::uint32_t expire, std::uint64_t cas = 0) { char extra[8]; cybozu::hton(flags, extra); cybozu::hton(expire, &extra[4]); request req(q ? binary_command::RaUQ : binary_command::RaU, (std::uint16_t)key.size(), key.data(), sizeof(extra), extra, (std::uint32_t)data.length(), data.data(), nullptr, cas); send(req.data(), req.length()); } void incr(const std::string& key, std::uint64_t value, bool q, std::uint32_t expire = ~(std::uint32_t)0, std::uint64_t initial = 0, std::uint64_t cas = 0) { char extra[20]; cybozu::hton(value, extra); cybozu::hton(initial, &extra[8]); cybozu::hton(expire, &extra[16]); request req(q ? binary_command::IncrementQ : binary_command::Increment, (std::uint16_t)key.size(), key.data(), sizeof(extra), extra, 0, nullptr, nullptr, cas); send(req.data(), req.length()); } void decr(const std::string& key, std::uint64_t value, bool q, std::uint32_t expire = ~(std::uint32_t)0, std::uint64_t initial = 0, std::uint64_t cas = 0) { char extra[20]; cybozu::hton(value, extra); cybozu::hton(initial, &extra[8]); cybozu::hton(expire, &extra[16]); request req(q ? binary_command::DecrementQ : binary_command::Decrement, (std::uint16_t)key.size(), key.data(), sizeof(extra), extra, 0, nullptr, nullptr, cas); send(req.data(), req.length()); } void append(const std::string& key, const std::string& value, bool q, std::uint64_t cas = 0) { request req(q ? binary_command::AppendQ : binary_command::Append, (std::uint16_t)key.size(), key.data(), 0, nullptr, (std::uint32_t)value.size(), value.data(), nullptr, cas); send(req.data(), req.length()); } void prepend(const std::string& key, const std::string& value, bool q, std::uint64_t cas = 0) { request req(q ? binary_command::PrependQ : binary_command::Prepend, (std::uint16_t)key.size(), key.data(), 0, nullptr, (std::uint32_t)value.size(), value.data(), nullptr, cas); send(req.data(), req.length()); } void remove(const std::string& key, bool q, std::uint64_t cas = 0) { request req(q ? binary_command::DeleteQ : binary_command::Delete, (std::uint16_t)key.size(), key.data(), 0, nullptr, 0, nullptr, nullptr, cas); send(req.data(), req.length()); } void lock(const std::string& key, bool q) { request req(q ? binary_command::LockQ : binary_command::Lock, (std::uint16_t)key.size(), key.data(), 0, nullptr, 0, nullptr); send(req.data(), req.length()); } void unlock(const std::string& key, bool q) { request req(q ? binary_command::UnlockQ : binary_command::Unlock, (std::uint16_t)key.size(), key.data(), 0, nullptr, 0, nullptr); send(req.data(), req.length()); } void unlock_all(bool q) { request req(q ? binary_command::UnlockAllQ : binary_command::UnlockAll, 0, nullptr, 0, nullptr, 0, nullptr); send(req.data(), req.length()); } void flush_all(bool q, std::uint32_t expire = 0) { if( expire == 0 ) { request req(q ? binary_command::FlushQ : binary_command::Flush, 0, nullptr, 0, nullptr, 0, nullptr); send(req.data(), req.length()); return; } char extra[4]; cybozu::hton(expire, extra); request req(q ? binary_command::FlushQ : binary_command::Flush, 0, nullptr, sizeof(extra), extra, 0, nullptr); send(req.data(), req.length()); } void version() { request req(binary_command::Version, 0, nullptr, 0, nullptr, 0, nullptr); send(req.data(), req.length()); } void stat() { request req(binary_command::Stat, 0, nullptr, 0, nullptr, 0, nullptr); send(req.data(), req.length()); } void stat(const std::string& key) { request req(binary_command::Stat, (std::uint16_t)key.size(), key.data(), 0, nullptr, 0, nullptr); send(req.data(), req.length()); } void keys(const std::string& key) { request req(binary_command::Keys, (std::uint16_t)key.size(), key.data(), 0, nullptr, 0, nullptr); send(req.data(), req.length()); } void quit(bool q) { request req(q ? binary_command::QuitQ : binary_command::Quit, 0, nullptr, 0, nullptr, 0, nullptr); send(req.data(), req.length()); } private: const int m_socket; cybozu::dynbuf m_buffer; std::size_t m_last_response_size = 0; void send(const char* p, std::size_t len) { while( len > 0 ) { ssize_t n = ::send(m_socket, p, len, 0); cybozu_assert( n != -1 ); p += n; len -= n; } } bool recv() { char* p = m_buffer.prepare(256 << 10); ssize_t n = ::recv(m_socket, p, 256<<10, 0); cybozu_assert( n != -1 ); if( n == -1 || n == 0 ) return false; m_buffer.consume(n); return true; } }; void print_item(const item& i) { std::cout << std::string(std::get<0>(i), std::get<1>(i)) << std::endl; } bool itemcmp(const item& i, const std::string& s) { if( s.size() != std::get<1>(i) ) return false; return std::memcmp(s.data(), std::get<0>(i), s.size()) == 0; } // tests #define ASSERT_COMMAND(r,cmd) cybozu_assert( r.command() == binary_command::cmd ) #define ASSERT_OK(r) cybozu_assert( r.status() == binary_status::OK ) AUTOTEST(opaque) { client c; response r; c.noop(nullptr); cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, Noop); ASSERT_OK(r); opaque_t zero = {'\0', '\0', '\0', '\0'}; cybozu_assert( std::memcmp(r.opaque(), &zero, sizeof(opaque_t)) == 0 ); opaque_t op1 = {'\x12', '\x23', '\x45', '\x67'}; c.noop(&op1); cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, Noop); ASSERT_OK(r); cybozu_assert( std::memcmp(r.opaque(), &op1, sizeof(opaque_t)) == 0 ); } AUTOTEST(quit) { client c; response r; c.quit(false); cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, Quit); ASSERT_OK( r ); } AUTOTEST(quitq) { client c; response r; c.quit(true); cybozu_assert( ! c.get_response(r) ); } AUTOTEST(version) { client c; response r; c.version(); cybozu_assert( c.get_response(r) ); print_item( r.data() ); } AUTOTEST(set) { client c; response r; c.set("hello", "world", false, 123, 0); cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, Set); ASSERT_OK(r); c.set("hello", "world!", false, 111, 0, r.cas_unique()); cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, Set); ASSERT_OK(r); c.set("hello", "world!!", true, 111, 0, r.cas_unique() + 1); cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, SetQ); cybozu_assert( r.status() == binary_status::Exists ); c.set("not exist", "hoge", true, 111, 0, 100); cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, SetQ); cybozu_assert( r.status() == binary_status::NotFound ); c.set("abc", "def", true, 123, 0); c.noop(nullptr); cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, Noop); c.get("abc", false); cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, Get); ASSERT_OK(r); cybozu_assert( itemcmp(r.data(), "def") ); cybozu_assert( r.flags() == 123 ); c.get("not exist", false); cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, Get); cybozu_assert( r.status() == binary_status::NotFound ); c.get("not exist", true); c.noop(nullptr); cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, Noop); ASSERT_OK(r); } AUTOTEST(expire) { client c; response r; c.set("abc", "def1", true, 123, 1); c.get("abc", false); cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, Get); ASSERT_OK(r); std::this_thread::sleep_for(std::chrono::seconds(3)); c.get("abc", false); cybozu_assert( c.get_response(r) ); cybozu_assert( r.status() == binary_status::NotFound ); } AUTOTEST(touch) { client c; response r; c.remove("abc", true); c.touch("abc", 0); cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, Touch); cybozu_assert( r.status() == binary_status::NotFound ); c.set("abc", "def", true, 10, 2); c.touch("abc", 0); cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, Touch); ASSERT_OK(r); std::uint64_t cas = r.cas_unique(); cybozu_assert( cas != 0 ); std::this_thread::sleep_for(std::chrono::seconds(4)); c.get("abc", false); cybozu_assert( c.get_response(r) ); ASSERT_OK(r); cybozu_assert( r.cas_unique() == cas ); c.touch("abc", 1); cybozu_assert( c.get_response(r) ); ASSERT_OK(r); cybozu_assert( r.cas_unique() == cas ); std::this_thread::sleep_for(std::chrono::seconds(3)); c.get("abc", false); cybozu_assert( c.get_response(r) ); cybozu_assert( r.status() == binary_status::NotFound ); } AUTOTEST(add) { client c; response r; c.set("abc", "def", true, 0, 0); c.add("abc", "123", true, 0, 0); cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, AddQ); cybozu_assert( r.status() == binary_status::Exists ); c.remove("abc", true); c.add("abc", "123", false, 0, 0); cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, Add); ASSERT_OK(r); } AUTOTEST(replace) { client c; response r; c.replace("not exist", "hoge", true, 100, 0); cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, ReplaceQ); cybozu_assert( r.status() == binary_status::NotFound ); c.set("abc", "def", true, 10, 0); c.replace("abc", "123", false, 100, 0); cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, Replace); ASSERT_OK(r); c.get("abc", false); cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, Get); ASSERT_OK(r); cybozu_assert( itemcmp(r.data(), "123") ); cybozu_assert( r.flags() == 100 ); } AUTOTEST(get) { client c; response r; c.remove("unique key", true); c.add("unique key", "value", false, 10, 0); cybozu_assert( c.get_response(r) ); ASSERT_OK(r); c.replace("unique key", "value2", false, 11, 0); cybozu_assert( c.get_response(r) ); ASSERT_OK(r); std::uint64_t cas = r.cas_unique(); c.get("unique key", false); cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, Get); ASSERT_OK(r); cybozu_assert( r.key() == item(nullptr, 0) ); cybozu_assert( itemcmp(r.data(), "value2") ); cybozu_assert( r.flags() == 11 ); cybozu_assert( r.cas_unique() == cas ); c.getk("unique key", false); cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, GetK); ASSERT_OK(r); cybozu_assert( itemcmp(r.key(), "unique key") ); cybozu_assert( itemcmp(r.data(), "value2") ); cybozu_assert( r.flags() == 11 ); cybozu_assert( r.cas_unique() == cas ); } AUTOTEST(get_and_touch) { client c; response r; c.set("abc", "def", true, 10, 2); c.get_and_touch("abc", 0, false); cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, GaT); ASSERT_OK(r); cybozu_assert( std::get<1>(r.key()) == 0 ); cybozu_assert( itemcmp(r.data(), "def") ); std::this_thread::sleep_for(std::chrono::seconds(4)); c.getk_and_touch("abc", 1, false); cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, GaTK); ASSERT_OK(r); cybozu_assert( itemcmp(r.key(), "abc") ); cybozu_assert( itemcmp(r.data(), "def") ); std::this_thread::sleep_for(std::chrono::seconds(3)); c.touch("abc", 0); cybozu_assert( c.get_response(r) ); cybozu_assert( r.status() == binary_status::NotFound ); } AUTOTEST(incr_decr) { client c; response r; c.set("abc", "def", true, 10, 0); c.incr("abc", 10, true); cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, IncrementQ); cybozu_assert( r.status() == binary_status::NonNumeric ); c.remove("abc", true); c.incr("abc", 10, true); cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, IncrementQ); cybozu_assert( r.status() == binary_status::NotFound ); c.incr("abc", 10, false, 0, 12); cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, Increment); ASSERT_OK(r); cybozu_assert( r.value() == 12 ); cybozu_assert( r.flags() == 0 ); std::uint64_t cas = r.cas_unique(); cybozu_assert( cas != 0 ); c.incr("abc", 10, false); cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, Increment); ASSERT_OK(r); cybozu_assert( r.value() == 22 ); cybozu_assert( r.cas_unique() != 0 ); cybozu_assert( cas != r.cas_unique() ); cas = r.cas_unique(); c.decr("abc", 1, false); cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, Decrement); ASSERT_OK(r); cybozu_assert( r.value() == 21 ); cybozu_assert( r.cas_unique() != 0 ); cybozu_assert( cas != r.cas_unique() ); c.decr("abc", 100, false); cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, Decrement); ASSERT_OK(r); cybozu_assert( r.value() == 0 ); c.remove("abc", true); c.decr("abc", 1, true); cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, DecrementQ); cybozu_assert( r.status() == binary_status::NotFound ); c.set("abc", "def", true, 10, 0); c.decr("abc", 1, true); cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, DecrementQ); cybozu_assert( r.status() == binary_status::NonNumeric ); c.remove("abc", true); c.decr("abc", 10, false, 0, 22); cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, Decrement); ASSERT_OK(r); cybozu_assert( r.value() == 22 ); } AUTOTEST(append_prepend) { client c; response r; c.remove("ttt", true); c.append("ttt", "111", true); cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, AppendQ); cybozu_assert( r.status() == binary_status::NotFound ); c.set("ttt", "aaa", true, 0, 0); c.append("ttt", "111", false); cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, Append); ASSERT_OK(r); std::uint64_t cas = r.cas_unique(); cybozu_assert( cas != 0 ); c.get("ttt", false); cybozu_assert( c.get_response(r) ); ASSERT_OK(r); cybozu_assert( itemcmp(r.data(), "aaa111") ); c.prepend("ttt", "222", false); cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, Prepend); ASSERT_OK(r); cybozu_assert( r.cas_unique() != 0 ); cybozu_assert( cas != r.cas_unique() ); c.get("ttt", false); cybozu_assert( c.get_response(r) ); ASSERT_OK(r); cybozu_assert( itemcmp(r.data(), "222aaa111") ); c.remove("ttt", true); c.prepend("ttt", "111", true); cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, PrependQ); cybozu_assert( r.status() == binary_status::NotFound ); } AUTOTEST(delete) { client c; response r; c.set("abc", "def", true, 10, 0); c.remove("abc", false); cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, Delete); ASSERT_OK(r); c.set("abc", "def", false, 10, 0); cybozu_assert( c.get_response(r) ); ASSERT_OK(r); std::uint64_t cas = r.cas_unique(); c.remove("abc", true, cas + 1); cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, DeleteQ); cybozu_assert( r.status() == binary_status::Exists ); c.remove("abc", false, cas); cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, Delete); ASSERT_OK(r); c.remove("abc", false); cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, Delete); cybozu_assert( r.status() == binary_status::NotFound ); c.remove("abc", true); c.noop(nullptr); cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, Noop); } AUTOTEST(lock) { client c; response r; c.set("abc", "10", false, 0, 0); cybozu_assert( c.get_response(r) ); { client c2; c2.lock("abc", false); cybozu_assert( c2.get_response(r) ); ASSERT_COMMAND(r, Lock); ASSERT_OK(r); c.lock("abc", true); cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, LockQ); cybozu_assert( r.status() == binary_status::Locked ); c2.quit(false); cybozu_assert( c2.get_response(r) ); } c.lock("abc", false); cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, Lock); ASSERT_OK(r); c.quit(false); c.get_response(r); } AUTOTEST(unlock) { client c; response r; c.remove("abc", false); cybozu_assert( c.get_response(r) ); ASSERT_OK(r); c.lock("abc", true); cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, LockQ); cybozu_assert( r.status() == binary_status::NotFound ); c.unlock("abc", true); cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, UnlockQ); cybozu_assert( r.status() == binary_status::NotFound ); c.set("abc", "def", true, 10, 0); c.unlock("abc", true); cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, UnlockQ); cybozu_assert( r.status() == binary_status::NotLocked ); c.lock("abc", false); cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, Lock); ASSERT_OK(r); client c2; c2.unlock("abc", true); cybozu_assert( c2.get_response(r) ); ASSERT_COMMAND(r, UnlockQ); cybozu_assert( r.status() == binary_status::NotLocked ); c.unlock("abc", false); cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, Unlock); ASSERT_OK(r); c2.lock("abc", false); cybozu_assert( c2.get_response(r) ); ASSERT_COMMAND(r, Lock); ASSERT_OK(r); c2.quit(false); c2.get_response(r); } AUTOTEST(unlock_all) { client c; response r; c.set("abc", "1", true, 10, 0); c.set("def", "2", true, 10, 0); c.lock("abc", false); cybozu_assert( c.get_response(r) ); ASSERT_OK(r); c.lock("def", false); cybozu_assert( c.get_response(r) ); ASSERT_OK(r); client c2; c2.lock("abc", true); cybozu_assert( c2.get_response(r) ); cybozu_assert( r.status() == binary_status::Locked ); c2.lock("def", true); cybozu_assert( c2.get_response(r) ); cybozu_assert( r.status() == binary_status::Locked ); c.unlock_all(false); cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, UnlockAll); ASSERT_OK(r); c2.lock("abc", false); cybozu_assert( c2.get_response(r) ); ASSERT_OK(r); c2.lock("def", false); cybozu_assert( c2.get_response(r) ); ASSERT_OK(r); c2.quit(false); c2.get_response(r); } AUTOTEST(lock_and_get) { client c; response r; c.remove("abc", true); c.lock_and_get("abc", true); cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, LaGQ); cybozu_assert( r.status() == binary_status::NotFound ); c.set("abc", "def", true, 10, 0); c.lock_and_get("abc", false); cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, LaG); ASSERT_OK(r); cybozu_assert( std::get<1>(r.key()) == 0 ); cybozu_assert( itemcmp(r.data(), "def") ); std::uint64_t cas = r.cas_unique(); c.lock_and_get("abc", true); cybozu_assert( c.get_response(r) ); cybozu_assert( r.status() == binary_status::Locked ); client c2; c2.lock_and_get("abc", true); cybozu_assert( c2.get_response(r) ); cybozu_assert( r.status() == binary_status::Locked ); c2.replace_and_unlock("abc", "ghi", true, 20, 0); cybozu_assert( c2.get_response(r) ); ASSERT_COMMAND(r, RaUQ); cybozu_assert( r.status() == binary_status::NotLocked ); c.replace_and_unlock("abc", "ghi", false, 20, 0); cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, RaU); ASSERT_OK(r); cybozu_assert( r.cas_unique() != cas ); cybozu_assert( r.cas_unique() != 0 ); c2.lock_and_getk("abc", false); cybozu_assert( c2.get_response(r) ); ASSERT_OK(r); cybozu_assert( itemcmp(r.key(), "abc") ); cybozu_assert( itemcmp(r.data(), "ghi") ); c2.quit(false); c2.get_response(r); } AUTOTEST(flush) { client c; response r; c.set("abc", "def", true, 10, 0); c.flush_all(true, 2); c.get("abc", false); cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, Get); ASSERT_OK(r); std::this_thread::sleep_for(std::chrono::seconds(4)); c.get("abc", false); cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, Get); cybozu_assert( r.status() == binary_status::NotFound ); c.set("abc", "234", true, 10, 0); c.flush_all(false); cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, Flush); ASSERT_OK(r); c.get("abc", false); cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, Get); cybozu_assert( r.status() == binary_status::NotFound ); } AUTOTEST(stat_general) { client c; response r; c.stat(); while( true ) { cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, Stat); ASSERT_OK(r); if( std::get<1>(r.key()) == 0 ) break; continue; std::cout << std::string(std::get<0>(r.key()), std::get<1>(r.key())) << ": " << std::string(std::get<0>(r.data()), std::get<1>(r.data())) << std::endl; } } AUTOTEST(stat_settings) { client c; response r; c.stat("settings"); while( true ) { cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, Stat); ASSERT_OK(r); if( std::get<1>(r.key()) == 0 ) break; continue; std::cout << std::string(std::get<0>(r.key()), std::get<1>(r.key())) << ": " << std::string(std::get<0>(r.data()), std::get<1>(r.data())) << std::endl; } } AUTOTEST(stat_sizes) { client c; response r; c.stat("sizes"); while( true ) { cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, Stat); ASSERT_OK(r); if( std::get<1>(r.key()) == 0 ) break; continue; std::cout << std::string(std::get<0>(r.key()), std::get<1>(r.key())) << ": " << std::string(std::get<0>(r.data()), std::get<1>(r.data())) << std::endl; } } AUTOTEST(stat_items) { client c; response r; c.stat("items"); while( true ) { cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, Stat); ASSERT_OK(r); if( std::get<1>(r.key()) == 0 ) break; continue; std::cout << std::string(std::get<0>(r.key()), std::get<1>(r.key())) << ": " << std::string(std::get<0>(r.data()), std::get<1>(r.data())) << std::endl; } } AUTOTEST(stat_ops) { client c; response r; c.stat("ops"); while( true ) { cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, Stat); ASSERT_OK(r); if( std::get<1>(r.key()) == 0 ) break; continue; std::cout << std::string(std::get<0>(r.key()), std::get<1>(r.key())) << ": " << std::string(std::get<0>(r.data()), std::get<1>(r.data())) << std::endl; } } AUTOTEST(keys) { client c; response r; c.flush_all(true, 0); std::this_thread::sleep_for(std::chrono::seconds(2)); c.set("foo", "bar", true, 10, 0); c.set("fo1", "bar", true, 10, 0); c.set("zzzzzzzzz", "bar", true, 10, 0); c.keys(""); int n = 0; while( true ) { cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, Keys); ASSERT_OK(r); if( std::get<1>(r.key()) == 0 ) break; ++n; } cybozu_assert( n == 3 ); n = 0; c.keys("f"); while( true ) { cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, Keys); ASSERT_OK(r); if( std::get<1>(r.key()) == 0 ) break; ++n; } cybozu_assert( n == 2 ); n = 0; c.keys("foo"); while( true ) { cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, Keys); ASSERT_OK(r); if( std::get<1>(r.key()) == 0 ) break; ++n; } cybozu_assert( n == 1 ); n = 0; c.keys("zzzz"); while( true ) { cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, Keys); ASSERT_OK(r); if( std::get<1>(r.key()) == 0 ) break; ++n; } cybozu_assert( n == 1 ); n = 0; c.keys("b"); while( true ) { cybozu_assert( c.get_response(r) ); ASSERT_COMMAND(r, Keys); ASSERT_OK(r); if( std::get<1>(r.key()) == 0 ) break; ++n; } cybozu_assert( n == 0 ); } void print_usage() { std::cout << "Usage: protocol_binary.exe [SERVER [PORT]]\n" "Environment options:\n" " YRMCDS_SERVER : the name of a yrmcds server.\n" " used only when `SERVER` is unspecified.\n" " YRMCDS_PORT : the port number of a yrmcds server.\n" " used only when `PORT` is unspecified.\n" << std::flush; } // main bool optparse(int argc, char** argv) { if( argc > 3 ) { print_usage(); return false; } g_server = getenv("YRMCDS_SERVER"); const char* env_port = getenv("YRMCDS_PORT"); if( env_port != nullptr ) g_port = std::stoi(env_port); if( argc >= 2 ) g_server = argv[1]; if( argc >= 3 ) g_port = std::stoi(argv[2]); if( g_server == nullptr ) { std::cout << "No server specified." << std::endl; print_usage(); return false; } if( g_port <= 0 || g_port > 65535 ) { std::cout << "Invalid port number: " << g_port << std::endl; return false; } int s = connect_server(); if( s == -1 ) { std::cout << "Failed to connect to " << g_server << std::endl; return false; } ::close(s); return true; } TEST_MAIN(optparse); yrmcds-1.1.5/test/range.cpp000066400000000000000000000002551262254645600156140ustar00rootroot00000000000000#include #include int main() { const int a[] = {1, 2, 3, 4, 5}; for( auto i: a ) { std::cout << i << std::endl; } return 0; } yrmcds-1.1.5/test/reactor.cpp000066400000000000000000000034511262254645600161600ustar00rootroot00000000000000#include #include #include #include #include #include #include #include int g_counter = 0; bool got_hangup = false; bool data_read = false; class in_resource: public cybozu::resource { public: in_resource(int fd): cybozu::resource(fd) {} ~in_resource() { std::cout << "stdin_resource dtor." << std::endl; } private: virtual bool on_readable() override final { char buf[512]; buf[sizeof(buf) - 1] = '\0'; ssize_t t = ::read(fileno(), buf, sizeof(buf) - 1); if( t == -1 ) cybozu::throw_unix_error(errno, "read"); std::cout << "aaa: " << t << std::endl; buf[t] = '\0'; std::cout << "read: " << buf << std::endl; data_read = true; return false; } virtual bool on_writable() override final { return true; } virtual bool on_hangup() override final { std::cout << "got hangup!" << std::endl; got_hangup = true; return true; } }; void f(cybozu::reactor& r) { g_counter++; if( g_counter > 2 ) { r.quit(); return; } std::cout << g_counter << std::endl; if( r.fix_garbage() ) r.gc(); } int main(int argc, char** argv) { cybozu::logger::set_threshold(cybozu::severity::debug); int fds[2]; if( ::pipe2(fds, O_NONBLOCK) == -1 ) cybozu::throw_unix_error(errno, "pipe"); if( ::write(fds[1], "Hello!", 6) == -1 ) cybozu::throw_unix_error(errno, "write"); ::close(fds[1]); cybozu::reactor r; std::unique_ptr res(new in_resource(fds[0])); r.add_resource(std::move(res), cybozu::reactor::EVENT_IN); r.run(f); assert(got_hangup); assert(data_read); return 0; } yrmcds-1.1.5/test/reference_wrapper.cpp000066400000000000000000000012271262254645600202160ustar00rootroot00000000000000#include #include #include struct foo { int i; }; int main() { int *p = nullptr; std::reference_wrapper a[3] = {*p, *p, *p}; int i = 3; a[0] = std::ref(i); if( &(a[2].get()) == nullptr ) { std::cout << "null!" << std::endl; } else { std::cout << "wtf" << std::endl; } a[0].get() = 5; std::cout << "i = " << i << std::endl; foo f; f.i = 3; std::vector> v; v.emplace_back(f); std::cout << "f.i = " << v[0].get().i << std::endl; f.i = 5; std::cout << "f.i = " << v[0].get().i << std::endl; return 0; } yrmcds-1.1.5/test/signalfd.cpp000066400000000000000000000064131262254645600163110ustar00rootroot00000000000000#include #include #include #include #include #include #include #include #include #include const int MAX_EVENTS = 10; bool readall(int fd, char* p, std::size_t len) { while( len != 0 ) { ssize_t n = read(fd, p, len); if( n == -1 ) { if( errno == EAGAIN || errno == EWOULDBLOCK ) { return false; } else { throw std::runtime_error("read failed."); } } len -= n; p += n; } return true; } void read_signal(int sfd) { struct signalfd_siginfo si; while( true ) { if( ! readall(sfd, (char*)&si, sizeof(si)) ) return; switch(si.ssi_signo) { case SIGHUP: std::cout << "SIGHUP" << std::endl; break; case SIGQUIT: std::cout << "SIGQUIT" << std::endl; break; case SIGTERM: std::cout << "SIGTERM" << std::endl; break; case SIGCHLD: std::cout << "SIGCHLD" << std::endl; break; default: std::cout << "Unknown signal: " << si.ssi_signo << std::endl; } } } int wait_signal(int sfd) { int efd = epoll_create(10); if( efd == -1 ) { std::cerr << "epoll_create failed." << std::endl; close(efd); return 1; } struct epoll_event ev; ev.events = EPOLLIN|EPOLLET; ev.data.fd = sfd; if( epoll_ctl(efd, EPOLL_CTL_ADD, sfd, &ev) == -1 ) { std::cerr << "epoll_ctl failed." << std::endl; close(efd); return 1; } std::cerr << "pid=" << getpid() << std::endl; std::cerr << "waiting for signals..." << std::endl; std::this_thread::sleep_for(std::chrono::seconds(10)); struct epoll_event events[MAX_EVENTS]; int n = epoll_wait(efd, events, MAX_EVENTS, -1); if( n == -1 ) { std::cerr << "epoll_wait failed." << std::endl; close(efd); return 1; } for( int i = 0; i < n; ++i ) { if( events[i].data.fd != sfd ) { std::cerr << "wtf!?" << std::endl; close(efd); return 1; } read_signal(sfd); } close(efd); return 0; } int main(int argc, char** argv) { if( argc == 1 ) return 0; // block signals and setup signalfd // Threads created by the main thread inherit the signal mask. sigset_t sigs[1]; sigemptyset(sigs); sigaddset(sigs, SIGQUIT); sigaddset(sigs, SIGHUP); sigaddset(sigs, SIGTERM); sigaddset(sigs, SIGCHLD); if( pthread_sigmask(SIG_BLOCK, sigs, NULL) != 0 ) { std::cerr << "pthread_sigmask failed." << std::endl; return 1; } int sfd = signalfd(-1, sigs, SFD_NONBLOCK); if( sfd == -1 ) { std::cerr << "signalfd failed." << std::endl; return 1; } // ignore SIGPIPE // As per man 7 signal, the signal disposition is a per-process attribute. struct sigaction act; std::memset(&act, 0, sizeof(act)); act.sa_handler = SIG_IGN; if( sigaction(SIGPIPE, &act, NULL) == -1 ) { std::cerr << "sigaction failed." << std::endl; return 1; } int ret = wait_signal(sfd); close(sfd); return ret; } yrmcds-1.1.5/test/siphash.cpp000066400000000000000000000036541262254645600161650ustar00rootroot00000000000000#include #include #include const std::uint64_t vectors[64] = { 0x726fdb47dd0e0e31ULL, 0x74f839c593dc67fdULL, 0x0d6c8009d9a94f5aULL, 0x85676696d7fb7e2dULL, 0xcf2794e0277187b7ULL, 0x18765564cd99a68dULL, 0xcbc9466e58fee3ceULL, 0xab0200f58b01d137ULL, 0x93f5f5799a932462ULL, 0x9e0082df0ba9e4b0ULL, 0x7a5dbbc594ddb9f3ULL, 0xf4b32f46226bada7ULL, 0x751e8fbc860ee5fbULL, 0x14ea5627c0843d90ULL, 0xf723ca908e7af2eeULL, 0xa129ca6149be45e5ULL, 0x3f2acc7f57c29bdbULL, 0x699ae9f52cbe4794ULL, 0x4bc1b3f0968dd39cULL, 0xbb6dc91da77961bdULL, 0xbed65cf21aa2ee98ULL, 0xd0f2cbb02e3b67c7ULL, 0x93536795e3a33e88ULL, 0xa80c038ccd5ccec8ULL, 0xb8ad50c6f649af94ULL, 0xbce192de8a85b8eaULL, 0x17d835b85bbb15f3ULL, 0x2f2e6163076bcfadULL, 0xde4daaaca71dc9a5ULL, 0xa6a2506687956571ULL, 0xad87a3535c49ef28ULL, 0x32d892fad841c342ULL, 0x7127512f72f27cceULL, 0xa7f32346f95978e3ULL, 0x12e0b01abb051238ULL, 0x15e034d40fa197aeULL, 0x314dffbe0815a3b4ULL, 0x027990f029623981ULL, 0xcadcd4e59ef40c4dULL, 0x9abfd8766a33735cULL, 0x0e3ea96b5304a7d0ULL, 0xad0c42d6fc585992ULL, 0x187306c89bc215a9ULL, 0xd4a60abcf3792b95ULL, 0xf935451de4f21df2ULL, 0xa9538f0419755787ULL, 0xdb9acddff56ca510ULL, 0xd06c98cd5c0975ebULL, 0xe612a3cb9ecba951ULL, 0xc766e62cfcadaf96ULL, 0xee64435a9752fe72ULL, 0xa192d576b245165aULL, 0x0a8787bf8ecb74b2ULL, 0x81b3e73d20b49b6fULL, 0x7fa8220ba3b2eceaULL, 0x245731c13ca42499ULL, 0xb78dbfaf3a8d83bdULL, 0xea1ad565322a1a0bULL, 0x60e61c23a3795013ULL, 0x6606d7e446282b93ULL, 0x6ca4ecb15c5f91e1ULL, 0x9f626da15c9625f3ULL, 0xe51b38608ef25f57ULL, 0x958a324ceb064572ULL, }; AUTOTEST(siphash) { const char key[16] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0xa, 0xb, 0xc, 0xd, 0xe, 0xf }; cybozu::siphash24_seed(key); char plaintext[64]; for (int i = 0; i < 64; i++) { plaintext[i] = (char)i; cybozu_assert(cybozu::siphash24(plaintext, i) == vectors[i]); } } yrmcds-1.1.5/test/sleep.cpp000066400000000000000000000003321262254645600156240ustar00rootroot00000000000000#include #include #include int main() { std::cerr << "Hello "; std::this_thread::sleep_for(std::chrono::milliseconds(300)); std::cerr << "World!" << std::endl; return 0; } yrmcds-1.1.5/test/spinlock.cpp000066400000000000000000000011011262254645600163310ustar00rootroot00000000000000#include #include #include #include alignas(CACHELINE_SIZE) cybozu::spinlock g_lock; alignas(CACHELINE_SIZE) unsigned int g_counter = 0; void accum(int n) { for(int i = 0; i < n; ++i) { std::lock_guard guard(g_lock); ++g_counter; } } int main() { std::thread t1(accum, 100000); std::thread t2(accum, 100000); std::thread t3(accum, 100000); t1.join(); t2.join(); t3.join(); std::cout << "accumulated value = " << g_counter << std::endl; return 0; } yrmcds-1.1.5/test/system_error.cpp000066400000000000000000000010701262254645600172510ustar00rootroot00000000000000#include #include #include #include #include #include int main(int argc, char** argv) { open("/hoge/fuga/faaa", O_WRONLY|O_CREAT, 0700); try { throw std::system_error(errno, std::system_category(), "open"); } catch( const std::exception& e ) { std::cout << e.what() << std::endl; } auto ec = std::system_category().default_error_condition(3); std::cout << "value=" << ec.value() << ", message=" << ec.message() << std::endl; return 0; } yrmcds-1.1.5/test/tempfile.cpp000066400000000000000000000017631262254645600163320ustar00rootroot00000000000000#include "../src/tempfile.hpp" #include #include #include AUTOTEST(mkstemp_wrap) { yrmcds::mkstemp_wrap("/var/tmp"); } AUTOTEST(tempfile) { yrmcds::tempfile t; t.write("abcde", 5); cybozu::dynbuf buf(10); t.read_contents(buf); cybozu_assert( buf.size() == 5 ); cybozu_assert( std::memcmp(buf.data(), "abcde", 5) == 0 ); buf.reset(); t.read_contents(buf); cybozu_assert( buf.size() == 5 ); cybozu_assert( std::memcmp(buf.data(), "abcde", 5) == 0 ); buf.reset(); t.write("xxx", 3); t.read_contents(buf); cybozu_assert( buf.size() == 8 ); cybozu_assert( std::memcmp(buf.data(), "abcdexxx", 8) == 0 ); buf.reset(); t.clear(); t.read_contents(buf); cybozu_assert( buf.size() == 0 ); buf.append("123", 3); t.write("789", 3); t.read_contents(buf); cybozu_assert( buf.size() == 6 ); cybozu_assert( std::memcmp(buf.data(), "123789", 6) == 0 ); buf.reset(); } yrmcds-1.1.5/test/test.conf000066400000000000000000000010421262254645600156350ustar00rootroot00000000000000# Configuration file for yrmcds virtual_ip = 127.0.0.1 port = 1121 repl_port = 1122 max_connections = 10000 temp_dir = "/var/tmp" user = nobody group = nogroup log.threshold = warning log.file = "/var/log/yrmcds.log" buckets = 1000000 max_data_size = 5M heap_data_limit = 16K memory_limit = 1024M repl_buffer_size= 100 secure_erase = true lock_memory = true workers = 10 gc_interval = 20 slave_timeout = 15 counter.enable = true counter.port = 11216 counter.max_connections = 100 counter.buckets = 1000001 counter.stat_interval = 12345 yrmcds-1.1.5/test/thread.cpp000066400000000000000000000011701262254645600157640ustar00rootroot00000000000000#include #include #include #include #include #include struct ttt: public cybozu::thread_base { ttt(): cybozu::thread_base() {} int i = 3; void run() { std::cout << "i = " << i << std::endl; throw std::runtime_error("hoge"); } void stop() { m_thread.join(); } }; int main() { std::unique_ptr p; std::once_flag f; std::call_once(f, [&]{ p.reset(new int); }); std::cout << "*p=" << *p << std::endl; ttt t; t.start(); t.stop(); return 0; } yrmcds-1.1.5/test/tuple.cpp000066400000000000000000000010431262254645600156450ustar00rootroot00000000000000#include #include #include #include int main(int argc, char** argv) { std::vector> p; void* vv = std::malloc(10); std::cout << vv << std::endl; p.emplace_back(vv, 3); for( auto t: p ) { void* v; std::tie(v, std::ignore) = t; std::cout << v << std::endl; } std::free(vv); auto tt = std::make_tuple(1, "abc"); std::get<0>(tt) = 3; int n; std::tie(n, std::ignore) = tt; std::cout << n << std::endl; return 0; } yrmcds-1.1.5/test/unordered_map.cpp000066400000000000000000000006761262254645600173530ustar00rootroot00000000000000#include #include #include int main(int argc, char** argv) { std::unordered_map m; std::cout << "Default max load factor: " << m.max_load_factor() << std::endl; std::cout << "Default bucket count: " << m.bucket_count() << std::endl; m.max_load_factor(1.0); m.reserve(65536); std::cout << "Bucket count now: " << m.bucket_count() << std::endl; return 0; } yrmcds-1.1.5/test/vector.cpp000066400000000000000000000027401262254645600160230ustar00rootroot00000000000000#include #include #include #include #include #include #if DEBUG_NEW void* operator new(std::size_t n) { std::cerr << "allocating " << n << " bytes." << std::endl; return std::malloc(n); } void operator delete(void* p) { std::cerr << "deallocating" << std::endl; return std::free(p); } #endif int main(int argc, char** argv) { std::cout << "sizeof(std::vector) = " << sizeof(std::vector) << std::endl; std::vector v {'a', 'b', 'c', 'd', 'e', 'f'}; std::cout << "v.size() = " << v.size() << std::endl; auto it = v.begin(); while( it != v.end() ) { if( *it == 'b' || *it == 'c' ) { it = v.erase(it); } else { ++it; } } for( char c: v ) std::cout << c; std::cout << std::endl; const char* hello = "Hello World"; std::vector hv(hello+3, hello+7); for( char c: hv ) std:: cerr << c; std::cerr << std::endl; // without v.erase, the vector size will not be changed. v.erase(std::remove_if(v.begin(), v.end(), [](char c) { return (c&1) == 1; }), v.end()); for( char c: v ) std::cout << c; std::cout << std::endl; std::vector v2; v2.reserve(5); v2.push_back('1'); v2.push_back('2'); std::vector v3 = {'a', 'b', 'c', 'd'}; v2 = v3; std::cout << v2.capacity() << std::endl; return 0; } yrmcds-1.1.5/test/vector_bool.cpp000066400000000000000000000012701262254645600170330ustar00rootroot00000000000000#include #include #include #include void foo() { std::vector vb(30); //std::fill(vb.begin(), vb.end(), true); assert( std::none_of(vb.begin(), vb.end(), [](bool b){return b;}) ); vb[3] = true; assert( ! std::none_of(vb.begin(), vb.end(), [](bool b){return b;}) ); assert( std::any_of(vb.begin(), vb.end(), [](bool b){return b;}) ); assert( ! std::all_of(vb.begin(), vb.end(), [](bool b){return b;}) ); std::fill(vb.begin(), vb.end(), true); assert( std::all_of(vb.begin(), vb.end(), [](bool b){return b;}) ); } int main(int argc, char** argv) { for( int i = 0; i < 10; ++i ) foo(); return 0; }