Lunchbox  1.13.0
Multi-threaded C++ toolbox library for all application developers creating high-performance multi-threaded programs.
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Modules Pages
lunchbox::PersistentMap Class Reference

Unified interface to save key-value pairs in a persistent store. More...

#include <persistentMap.h>

+ Inheritance diagram for lunchbox::PersistentMap:
+ Collaboration diagram for lunchbox::PersistentMap:

Public Member Functions

 PersistentMap (const std::string &uri=std::string())
 Construct a new persistent map. More...
 
 PersistentMap (const URI &uri)
 Construct a persistent map using an URI. More...
 
 ~PersistentMap ()
 Destruct the persistent map. More...
 
template<class V >
bool insert (const std::string &key, const V &value)
 Insert or update a value in the database. More...
 
template<class V >
bool insert (const std::string &key, const std::vector< V > &values)
 Insert or update a vector of values in the database. More...
 
template<class V >
bool insert (const std::string &key, const std::set< V > &values)
 Insert or update a set of values in the database. More...
 
std::string operator[] (const std::string &key) const
 Retrieve a value for a key. More...
 
template<class V >
std::vector< V > getVector (const std::string &key)
 Retrieve a value as a vector for a key. More...
 
template<class V >
std::set< V > getSet (const std::string &key)
 Retrieve a value as a set for a key. More...
 
bool contains (const std::string &key) const
 
template<>
bool _insert (const std::string &k, const std::string &v, const boost::false_type &)
 

Static Public Member Functions

static bool handles (const URI &uri)
 

Detailed Description

Unified interface to save key-value pairs in a persistent store.

Example:

/* Copyright (c) 2014-2015, Stefan.Eilemann@epfl.ch
*
* This library is free software; you can redistribute it and/or modify it under
* the terms of the GNU Lesser General Public License version 2.1 as published
* by the Free Software Foundation.
*
* This library is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
* FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
* details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this library; if not, write to the Free Software Foundation, Inc.,
* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
*/
#define TEST_RUNTIME 240 //seconds
#include <test.h>
#include <lunchbox/clock.h>
#include <lunchbox/os.h>
#include <lunchbox/persistentMap.h>
#include <lunchbox/rng.h>
#ifdef LUNCHBOX_USE_LEVELDB
# include <leveldb/db.h>
#endif
#ifdef LUNCHBOX_USE_SKV
# include <FxLogger/FxLogger.hpp>
#endif
#include <boost/format.hpp>
#include <stdexcept>
const int ints[] = { 17, 53, 42, 65535, 32768 };
const size_t numInts = sizeof( ints ) / sizeof( int );
const int64_t loopTime = 1000;
bool perfTest = false;
template< class T > void insertVector( PersistentMap& map )
{
std::vector< T > vector;
for( size_t i = 0; i < numInts; ++i )
vector.push_back( T( ints[ i ] ));
TEST( map.insert( typeid( vector ).name(), vector ));
}
template< class T > void readVector( const PersistentMap& map )
{
const std::vector< T >& vector =
map.getVector< T >( typeid( vector ).name( ));
TESTINFO( vector.size() == numInts, vector.size() << " != " << numInts );
for( size_t i = 0; i < numInts; ++i )
TEST( vector[ i ] == T( ints[i] ));
}
template< class T > void insertVector( PersistentMap& map, const size_t elems )
{
std::vector< T > vector;
for( size_t i = 0; i < elems; ++i )
vector.push_back( i );
TEST( map.insert( std::string( "bulk" ) + typeid( vector ).name(), vector ));
}
template< class T > void readVector( const PersistentMap& map, const size_t elems )
{
const std::vector< T >& vector =
map.getVector< T >( std::string( "bulk" ) + typeid( vector ).name());
TESTINFO( vector.size() == elems, vector.size() << " != " << elems );
for( size_t i = 0; i < numInts; ++i )
TESTINFO( vector[ i ] == T( i ), vector[ i ] << " != " << i );
}
void read( const PersistentMap& map )
{
const std::set< uint32_t >& bigSet =
map.getSet< uint32_t >( "std::set< uint32_t >" );
TEST( bigSet.size() == 1000 );
for( uint32_t i = 1; i <= 1000; ++i )
TEST( bigSet.find( i ) != bigSet.end( ));
TEST( map[ "foo" ] == "bar" );
TEST( map[ "bar" ].empty( ));
TEST( map.get< bool >( "bValue" ) == true );
TEST( map.get< int >( "iValue" ) == 42 );
readVector< int >( map );
readVector< uint16_t >( map );
const std::set< int >& set = map.getSet< int >( "std::set< int >" );
TESTINFO( set.size() == numInts, set.size() << " != " << numInts );
for( size_t i = 0; i < numInts; ++i )
TESTINFO( set.find( ints[i] ) != set.end(),
ints[i] << " not found in set" );
}
void read( const std::string& uri )
{
PersistentMap map( uri );
read( map );
}
void setup( const std::string& uri )
{
PersistentMap map( uri );
TEST( map.insert( "foo", "bar" ));
TEST( map.contains( "foo" ));
TESTINFO( map[ "foo" ] == "bar",
map[ "foo" ] << " length " << map[ "foo" ].length( ));
TEST( map[ "bar" ].empty( ));
TEST( map.insert( "the quick brown fox", "jumped over something" ));
TESTINFO( map[ "the quick brown fox" ] == "jumped over something",
map[ "the quick brown fox" ] );
TEST( map.insert( "hans", std::string( "dampf" )));
TESTINFO( map[ "hans" ] == "dampf", map[ "hans" ] );
const bool bValue = true;
TEST( map.insert( "bValue", bValue ));
TEST( map.get< bool >( "bValue" ) == bValue );
const int iValue = 42;
TEST( map.insert( "iValue", iValue ));
TEST( map.get< int >( "iValue" ) == iValue );
TEST( map.insert( "coffee", 0xC0FFEE ));
map.setByteswap( true );
TEST( map.get< unsigned >( "coffee" ) == 0xEEFFC000u );
map.setByteswap( false );
TEST( map.get< int >( "coffee" ) == 0xC0FFEE );
insertVector< int >( map );
insertVector< uint16_t >( map );
readVector< int >( map );
readVector< uint16_t >( map );
insertVector< int >( map, LB_128KB );
insertVector< uint16_t >( map, LB_128KB );
map.fetch( std::string( "bulk" ) + typeid( std::vector< int > ).name( ));
map.fetch( std::string( "bulk" ) + typeid( std::vector<uint16_t> ).name( ));
readVector< int >( map, LB_128KB );
readVector< uint16_t >( map, LB_128KB );
std::set< int > set( ints, ints + numInts );
TEST( map.insert( "std::set< int >", set ));
std::set< uint32_t > bigSet;
for( uint32_t i = 1; i <= 1000; ++i )
bigSet.insert( i );
TEST( map.insert( "std::set< uint32_t >", bigSet ));
read( map );
}
void benchmark( const std::string& uri, const uint64_t queueDepth,
const size_t valueSize )
{
static std::string lastURI;
if( uri != lastURI )
{
std::cout << uri << std::endl;
lastURI = uri;
}
PersistentMap map( uri );
map.setQueueDepth( queueDepth );
// Prepare keys and value
keys.resize( queueDepth + 1 );
for( uint64_t i = 0; i <= queueDepth; ++i )
keys[i].assign( reinterpret_cast< char* >( &i ), 8 );
std::string value( valueSize, '*' );
for( size_t i = 0; i < valueSize; ++i )
value[i] = rng.get<char>();
// write performance
uint64_t i = 0;
while( clock.getTime64() < loopTime )
{
map.insert( keys[ i % (queueDepth+1) ], value );
++i;
}
map.flush();
const float writeTime = clock.getTimef() / 1000.f;
const uint64_t wOps = i;
TEST( i > queueDepth );
// read performance
clock.reset();
if( queueDepth == 0 ) // sync read
{
for( i = 0; i < wOps && clock.getTime64() < loopTime; ++i ) // read keys
map[ keys[ i % (queueDepth+1) ]];
}
else // fetch + async read
{
for( i = 0; i < queueDepth; ++i ) // prefetch queueDepth keys
TEST( map.fetch( keys[ i % (queueDepth+1) ], valueSize ) );
for( ; i < wOps && clock.getTime64() < loopTime; ++i ) // read keys
{
map[ keys[ (i - queueDepth) % (queueDepth+1) ] ];
TEST( map.fetch( keys[ i % (queueDepth+1) ], valueSize ));
}
for( uint64_t j = i - queueDepth; j <= i; ++j ) // drain fetched keys
map[ keys[ j % (queueDepth+1) ]];
}
const float readTime = clock.getTimef() / 1000.f;
const size_t rOps = i;
std::cout << boost::format( "%6i, %6i, %9.2f, %9.2f, %9.2f, %9.2f")
// cppcheck-suppress zerodivcond
% queueDepth % valueSize % (rOps/readTime) % (wOps/writeTime)
% (rOps/1024.f/1024.f*valueSize/readTime)
% (wOps/1024.f/1024.f*valueSize/writeTime) << std::endl;
if( !perfTest )
{
// check contents of store (not all to save time on bigger tests)
for( uint64_t j = 0; j < wOps && clock.getTime64() < loopTime; ++j )
{
const std::string& val = map[ keys[ j % (queueDepth+1) ]];
TESTINFO( val.size() == valueSize,
val.size() << " != " << valueSize );
TEST( val == value );
}
}
// try to make sure there's nothing outstanding if we messed up in our test
map.flush();
}
void testGenericFailures()
{
try
{
setup( "foobar://" );
}
catch( const std::runtime_error& )
{
return;
}
TESTINFO( false, "Missing exception" );
}
void testLevelDBFailures()
{
#ifdef LUNCHBOX_USE_LEVELDB
try
{
setup( "leveldb:///doesnotexist/deadbeef/coffee" );
}
catch( const std::runtime_error& )
{
return;
}
TESTINFO( false, "Missing exception" );
#endif
}
int main( int, char* argv[] )
{
perfTest = std::string( argv[0] ).find( "perf-" ) != std::string::npos;
if( perfTest )
std::cout
<< " async, value, reads/s, writes/s, read MB/s, write MB/s"
<< std::endl;
try
{
#ifdef LUNCHBOX_USE_LEVELDB
setup( "" );
setup( "leveldb://" );
setup( "leveldb://persistentMap2.leveldb" );
read( "" );
read( "leveldb://" );
read( "leveldb://persistentMap2.leveldb" );
if( perfTest )
for( size_t i=1; i <= 65536; i = i<<2 )
benchmark( "leveldb://", 0, i );
#endif
#ifdef LUNCHBOX_USE_SKV
FxLogger_Init( argv[0] );
setup( "skv://" );
read( "skv://" );
if( perfTest )
{
benchmark( "skv://", 0, 64 );
for( size_t i=1; i <= 65536; i = i<<1 )
benchmark( "skv://", i, 64 );
for( size_t i=1; i <= 65536; i = i<<2 )
benchmark( "skv://", 65536, i );
}
#endif
}
#ifdef LUNCHBOX_USE_LEVELDB
catch( const leveldb::Status& status )
{
TESTINFO( !"exception", status.ToString( ));
}
#endif
catch( const std::runtime_error& error )
{
#ifdef LUNCHBOX_USE_SKV
if( error.what() !=
std::string( "skv init failed: SKV_ERRNO_CONN_FAILED" ))
#endif
{
TESTINFO( !"exception", error.what( ));
}
}
testGenericFailures();
testLevelDBFailures();
return EXIT_SUCCESS;
}

Definition at line 43 of file persistentMap.h.

Constructor & Destructor Documentation

lunchbox::PersistentMap::PersistentMap ( const std::string &  uri = std::string())

Construct a new persistent map.

Depending on the URI scheme an implementation backend is chosen. If no URI is given, a default one is selected. Available implementations are:

  • leveldb://path (if LUNCHBOX_USE_LEVELDB is defined)
  • skv://path_to_config#pdsname (if LUNCHBOX_USE_SKV is defined)
Parameters
urithe storage backend and destination.
Exceptions
std::runtime_errorif no suitable implementation is found.
std::runtime_errorif opening the leveldb failed.
Version
1.9.2
lunchbox::PersistentMap::PersistentMap ( const URI uri)
explicit

Construct a persistent map using an URI.

See other ctor for details.

Version
1.9.2
lunchbox::PersistentMap::~PersistentMap ( )

Destruct the persistent map.

Version
1.9.2

Member Function Documentation

bool lunchbox::PersistentMap::contains ( const std::string &  key) const
Returns
true if the key exists.
Version
1.9.2
template<class V >
std::set< V > lunchbox::PersistentMap::getSet ( const std::string &  key)
inline

Retrieve a value as a set for a key.

Parameters
keythe key to retreive.
Returns
the values, or an empty set if the key is not available.
Version
1.9.2

Definition at line 194 of file persistentMap.h.

template<class V >
std::vector< V > lunchbox::PersistentMap::getVector ( const std::string &  key)
inline

Retrieve a value as a vector for a key.

Parameters
keythe key to retreive.
Returns
the values, or an empty vector if the key is not available.
Version
1.9.2

Definition at line 186 of file persistentMap.h.

static bool lunchbox::PersistentMap::handles ( const URI uri)
static
Returns
true if an implementation for the given URI is available.
Version
1.9.2
template<class V >
bool lunchbox::PersistentMap::insert ( const std::string &  key,
const V &  value 
)
inline

Insert or update a value in the database.

Parameters
keythe key to store the value.
valuethe value stored at the key.
Returns
true on success, false otherwise
Exceptions
std::runtime_errorif the value is not copyable
Version
1.9.2

Definition at line 85 of file persistentMap.h.

Referenced by insert().

+ Here is the caller graph for this function:

template<class V >
bool lunchbox::PersistentMap::insert ( const std::string &  key,
const std::vector< V > &  values 
)
inline

Insert or update a vector of values in the database.

Parameters
keythe key to store the value.
valuethe values stored at the key.
Returns
true on success, false otherwise
Exceptions
std::runtime_errorif the vector values are not copyable
Version
1.9.2

Definition at line 98 of file persistentMap.h.

template<class V >
bool lunchbox::PersistentMap::insert ( const std::string &  key,
const std::set< V > &  values 
)
inline

Insert or update a set of values in the database.

Parameters
keythe key to store the value.
valuethe values stored at the key.
Returns
true on success, false otherwise
Exceptions
std::runtime_errorif the set values are not copyable
Version
1.9.2

Definition at line 111 of file persistentMap.h.

References insert().

+ Here is the call graph for this function:

std::string lunchbox::PersistentMap::operator[] ( const std::string &  key) const

Retrieve a value for a key.

Parameters
keythe key to retreive.
Returns
the value, or an empty string if the key is not available.
Version
1.9.2

The documentation for this class was generated from the following file: