Start to add LevelDB support

git-svn-id: http://svn.us.apache.org/repos/asf/tuscany@1172029 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
rfeng 2011-09-17 17:44:55 +00:00
commit e2a1e0c4a5
10 changed files with 795 additions and 0 deletions

View file

@ -0,0 +1,49 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
INCLUDES = -I${LEVELDB_INCLUDE}
incl_HEADERS = *.hpp
incldir = $(prefix)/include/components/kvdb
dist_comp_SCRIPTS = leveldb
compdir=$(prefix)/components/kvdb
comp_DATA = leveldb.prefix
leveldb.prefix: $(top_builddir)/config.status
echo ${LEVELDB_PREFIX} >leveldb.prefix
EXTRA_DIST = kvdb.composite kvdb.componentType
comp_LTLIBRARIES = libkvdb.la
noinst_DATA = libkvdb${libsuffix}
libkvdb_la_SOURCES = kvdb.cpp
libkvdb_la_LDFLAGS = -L${LEVELDB_LIB} -R${LEVELDB_LIB} -lleveldb
libkvdb${libsuffix}:
ln -s .libs/libkvdb${libsuffix}
leveldb_test_SOURCES = leveldb-test.cpp
leveldb_test_LDFLAGS = -L${LEVELDB_LIB} -R${LEVELDB_LIB} -lleveldb
client_test_SOURCES = client-test.cpp
client_test_LDFLAGS = -lxml2 -lcurl -lmozjs
dist_noinst_SCRIPTS = kvdb-test server-test
noinst_PROGRAMS = leveldb-test client-test
TESTS = kvdb-test server-test

View file

@ -0,0 +1,139 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
/* $Rev$ $Date$ */
/**
* Test KV (Key-Value) database component.
*/
#include <assert.h>
#include "stream.hpp"
#include "string.hpp"
#include "list.hpp"
#include "value.hpp"
#include "monad.hpp"
#include "perf.hpp"
#include "../../modules/http/http.hpp"
namespace tuscany {
namespace nosqldb {
const string uri("http://localhost:8090/nosqldb");
bool testNoSqlDb() {
http::CURLSession cs("", "", "", "");
const list<value> i = list<value>() + "content" + (list<value>() + "item"
+ (list<value>() + "name" + string("Apple"))
+ (list<value>() + "price" + string("$2.99")));
const list<value> a = list<value>() + (list<value>() + "entry"
+ (list<value>() + "title" + string("item"))
+ (list<value>() + "id" + string("cart-53d67a61-aa5e-4e5e-8401-39edeba8b83b"))
+ i);
const failable<value> id = http::post(a, uri, cs);
assert(hasContent(id));
const string p = path(content(id));
{
const failable<value> val = http::get(uri + p, cs);
assert(hasContent(val));
assert(content(val) == a);
}
const list<value> j = list<value>() + "content" + (list<value>() + "item"
+ (list<value>() + "name" + string("Apple"))
+ (list<value>() + "price" + string("$3.55")));
const list<value> b = list<value>() + (list<value>() + "entry"
+ (list<value>() + "title" + string("item"))
+ (list<value>() + "id" + string("cart-53d67a61-aa5e-4e5e-8401-39edeba8b83b"))
+ j);
{
const failable<value> r = http::put(b, uri + p, cs);
assert(hasContent(r));
assert(content(r) == value(true));
}
{
const failable<value> val = http::get(uri + p, cs);
assert(hasContent(val));
assert(content(val) == b);
}
{
const failable<value> r = http::del(uri + p, cs);
assert(hasContent(r));
assert(content(r) == value(true));
}
{
const failable<value> val = http::get(uri + p, cs);
assert(!hasContent(val));
}
return true;
}
struct getLoop {
const string path;
const value entry;
http::CURLSession cs;
getLoop(const string& path, const value& entry, http::CURLSession cs) : path(path), entry(entry), cs(cs) {
}
const bool operator()() const {
const failable<value> val = http::get(uri + path, cs);
assert(hasContent(val));
assert(content(val) == entry);
return true;
}
};
bool testGetPerf() {
const list<value> i = list<value>() + "content" + (list<value>() + "item"
+ (list<value>() + "name" + string("Apple"))
+ (list<value>() + "price" + string("$4.55")));
const list<value> a = list<value>() + (list<value>() + "entry"
+ (list<value>() + "title" + string("item"))
+ (list<value>() + "id" + string("cart-53d67a61-aa5e-4e5e-8401-39edeba8b83b"))
+ i);
http::CURLSession cs("", "", "", "");
const failable<value> id = http::post(a, uri, cs);
assert(hasContent(id));
const string p = path(content(id));
const lambda<bool()> gl = getLoop(p, a, cs);
cout << "NoSqldb get test " << time(gl, 5, 200) << " ms" << endl;
return true;
}
}
}
int main() {
tuscany::cout << "Testing..." << tuscany::endl;
tuscany::nosqldb::testNoSqlDb();
tuscany::nosqldb::testGetPerf();
tuscany::cout << "OK" << tuscany::endl;
return 0;
}

View file

@ -0,0 +1,29 @@
#!/bin/sh
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
# Setup
mkdir -p tmp
./tinycdb -c -m tmp/test.cdb </dev/null
# Test
./tinycdb-test 2>/dev/null
rc=$?
# Cleanup
exit $rc

View file

@ -0,0 +1,28 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
-->
<componentType xmlns="http://docs.oasis-open.org/ns/opencsa/sca/200912"
xmlns:xsd="http://www.w3.org/2001/XMLSchema"
xmlns:t="http://tuscany.apache.org/xmlns/sca/1.1"
targetNamespace="http://tuscany.apache.org/xmlns/sca/components">
<service name="nvdb"/>
<property name="dbname" type="xsd:string"/>
</componentType>

View file

@ -0,0 +1,32 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
-->
<composite xmlns="http://docs.oasis-open.org/ns/opencsa/sca/200912"
targetNamespace="http://tuscany.apache.org/xmlns/sca/components"
name="nvdb">
<component name="nvdb">
<implementation.cpp path="." library="libnvdb"/>
<property name="dbname">tmp/test.cdb</property>
<service name="nvdb">
<binding.http uri="nvdb"/>
</service>
</component>
</composite>

View file

@ -0,0 +1,123 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
/* $Rev$ $Date$ */
/**
* LevelDB-based database component implementation.
*/
#include "string.hpp"
#include "function.hpp"
#include "list.hpp"
#include "value.hpp"
#include "monad.hpp"
#include "leveldb.hpp"
namespace tuscany {
namespace nvdb {
/**
* Get an item from the database.
*/
const failable<value> get(const list<value>& params, leveldb::LevelDB& cdb) {
return leveldb::get(car(params), cdb);
}
/**
* Post an item to the database.
*/
const failable<value> post(const list<value>& params, leveldb::LevelDB& cdb) {
const value id = append<value>(car(params), mklist(mkuuid()));
const failable<bool> val = leveldb::post(id, cadr(params), cdb);
if (!hasContent(val))
return mkfailure<value>(reason(val));
return id;
}
/**
* Put an item into the database.
*/
const failable<value> put(const list<value>& params, leveldb::LevelDB& cdb) {
const failable<bool> val = leveldb::put(car(params), cadr(params), cdb);
if (!hasContent(val))
return mkfailure<value>(reason(val));
return value(content(val));
}
/**
* Delete an item from the database.
*/
const failable<value> del(const list<value>& params, leveldb::LevelDB& cdb) {
const failable<bool> val = leveldb::del(car(params), cdb);
if (!hasContent(val))
return mkfailure<value>(reason(val));
return value(content(val));
}
/**
* Component implementation lambda function.
*/
class applyNoSqldb {
public:
applyNoSqldb(leveldb::LevelDB& cdb) : cdb(cdb) {
}
const value operator()(const list<value>& params) const {
const value func(car(params));
if (func == "get")
return get(cdr(params), cdb);
if (func == "post")
return post(cdr(params), cdb);
if (func == "put")
return put(cdr(params), cdb);
if (func == "delete")
return del(cdr(params), cdb);
return tuscany::mkfailure<tuscany::value>();
}
private:
leveldb::LevelDB& cdb;
};
/**
* Start the component.
*/
const failable<value> start(unused const list<value>& params) {
// Connect to the configured database and table
const value dbname = ((lambda<value(list<value>)>)car(params))(list<value>());
leveldb::LevelDB& cdb = *(new (gc_new<leveldb::LevelDB>()) leveldb::LevelDB(dbname));
// Return the component implementation lambda function
return value(lambda<value(const list<value>&)>(applyNoSqldb(cdb)));
}
}
}
extern "C" {
const tuscany::value apply(const tuscany::list<tuscany::value>& params) {
const tuscany::value func(car(params));
if (func == "start")
return tuscany::nosqldb::start(cdr(params));
return tuscany::mkfailure<tuscany::value>();
}
}

View file

@ -0,0 +1,24 @@
#!/bin/sh
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
here=`echo "import os; print os.path.realpath('$0')" | python`; here=`dirname $here`
leveldb_prefix=`cat $here/leveldb.prefix`
$leveldb_prefix/bin/cdb $*

View file

@ -0,0 +1,82 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
/* $Rev$ $Date$ */
/**
* Test TinyCDB access functions.
*/
#include <assert.h>
#include "stream.hpp"
#include "string.hpp"
#include "perf.hpp"
#include "tinycdb.hpp"
namespace tuscany {
namespace tinycdb {
bool testTinyCDB() {
TinyCDB cdb("tmp/test.cdb");
const value k = mklist<value>("a");
assert(hasContent(post(k, string("AAA"), cdb)));
assert((get(k, cdb)) == value(string("AAA")));
assert(hasContent(put(k, string("aaa"), cdb)));
assert((get(k, cdb)) == value(string("aaa")));
assert(hasContent(del(k, cdb)));
assert(!hasContent(get(k, cdb)));
return true;
}
struct getLoop {
const value k;
TinyCDB& cdb;
getLoop(const value& k, TinyCDB& cdb) : k(k), cdb(cdb) {
}
const bool operator()() const {
assert((get(k, cdb)) == value(string("CCC")));
return true;
}
};
bool testGetPerf() {
const value k = mklist<value>("c");
TinyCDB cdb("tmp/test.cdb");
assert(hasContent(post(k, string("CCC"), cdb)));
const lambda<bool()> gl = getLoop(k, cdb);
cout << "TinyCDB get test " << time(gl, 5, 100000) << " ms" << endl;
return true;
}
}
}
int main() {
tuscany::cout << "Testing..." << tuscany::endl;
tuscany::tinycdb::testTinyCDB();
tuscany::tinycdb::testGetPerf();
tuscany::cout << "OK" << tuscany::endl;
return 0;
}

View file

@ -0,0 +1,249 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
/* $Rev$ $Date$ */
#ifndef tuscany_leveldb_hpp
#define tuscany_leveldb_hpp
#include <fcntl.h>
#include <unistd.h>
#include <sys/stat.h>
#include <leveldb/db.h>
#include "string.hpp"
#include "list.hpp"
#include "value.hpp"
#include "monad.hpp"
#include "../../modules/scheme/eval.hpp"
namespace tuscany {
namespace leveldb {
/**
* A reallocatable buffer.
*/
class buffer {
public:
operator void*() const throw() {
return buf;
}
operator unsigned char*() const throw() {
return (unsigned char*)buf;
}
operator char*() const throw() {
return (char*)buf;
}
private:
buffer(const unsigned int size, void* buf) : size(size), buf(buf) {
}
unsigned int size;
void* buf;
friend const buffer mkbuffer(const unsigned int sz);
friend const buffer mkbuffer(const buffer& b, const unsigned int newsz);
friend const bool free(const buffer& b);
};
/**
* Make a new buffer.
*/
const buffer mkbuffer(const unsigned int sz) {
return buffer(sz, malloc(sz));
}
/**
* Make a new buffer by reallocating an existing one.
*/
const buffer mkbuffer(const buffer& b, const unsigned int sz) {
if (sz <= b.size)
return b;
return buffer(sz, realloc(b.buf, sz));
}
/**
* Free a buffer.
*/
const bool free(const buffer&b) {
::free(b.buf);
return true;
}
/**
* Represents a LevelDB connection.
*/
class LevelDB {
public:
LevelDB() : owner(false), fd(-1) {
debug("leveldb::leveldb");
st.st_ino = 0;
}
LevelDB(const string& name) : owner(true), name(name), fd(-1) {
debug(name, "leveldb::leveldb::name");
st.st_ino = 0;
}
LevelDB(const LevelDB& c) : owner(false), name(c.name), fd(c.fd) {
debug("leveldb::leveldb::copy");
st.st_ino = c.st.st_ino;
}
~LevelDB() {
debug("leveldb::~leveldb");
if (!owner)
return;
if (fd == -1)
return;
close(fd);
}
private:
bool owner;
string name;
leveldb::DB* db;
struct stat st;
friend const string dbname(const LevelDB& db);
friend const failable<int> dbopen(LevelDB& db);
friend const failable<bool> dbclose(LevelDB& db);
};
/**
* Return the name of the database.
*/
const string dbname(const LevelDB& db) {
return db.name;
}
/**
* Open a database.
*/
const failable<int> dbopen(LevelDB& db) {
// Get database file serial number
struct stat st;
const int s = stat(c_str(db.name), &st);
if (s == -1)
return mkfailure<int>(
string("Couldn't leveldb read database stat ") + db.name);
leveldb::DB* ldb;
leveldb::Options options;
options.create_if_missing = true;
leveldb::Status status = leveldb::DB::Open(options, s, &ldb);
db.db = ldb;
}
/**
* Close a database.
*/
const failable<bool> dbclose(LevelDB& db) {
delete db.db;
db.db = NULL;
return true;
}
const failable<bool> post(const value& key, const value& val, LevelDB& db) {
debug(key, "leveldb::post::key");
debug(val, "leveldb::post::value");
debug(dbname(db), "leveldb::post::dbname");
const string ks(scheme::writeValue(key));
const string vs(scheme::writeValue(val));
put(ks, vs, db);
debug(r, "leveldb::post::result");
return r;
}
const failable<bool> put(const value& key, const value& val, LevelDB& db) {
debug(key, "leveldb::put::key");
debug(val, "leveldb::put::value");
debug(dbname(db), "leveldb::put::dbname");
const string ks(scheme::writeValue(key));
const string vs(scheme::writeValue(val));
debug(r, "leveldb::put::result");
return r;
}
/**
* Get an item from the database.
*/
const failable<value> get(const value& key, LevelDB& db) {
debug(key, "leveldb::get::key");
debug(dbname(db), "leveldb::get::dbname");
const string ks(scheme::writeValue(key));
std::string data;
db.db->Get(leveldb::ReadOptions(), key, &data);
const value val(scheme::readValue(string(data)));
debug(val, "leveldb::get::result");
return val;
}
/**
* Delete an item from the database
*/
struct delUpdate {
const string ks;
delUpdate(const string& ks) : ks(ks) {
}
const failable<bool> operator()(buffer& buf, const unsigned int klen, unused const unsigned int vlen) const {
if (ks == string((char*)buf, klen))
return false;
return true;
}
};
struct delFinish {
delFinish() {
}
const failable<bool> operator()(unused struct db_make& dbm) const {
return true;
}
};
const failable<bool> del(const value& key, LevelDB& db) {
debug(key, "leveldb::delete::key");
debug(dbname(db), "leveldb::delete::dbname");
const string ks(scheme::writeValue(key));
leveldb::Status s = db.db->Delete(leveldb::WriteOptions(), key);
debug(r, "leveldb::delete::result");
return s.ok();
}
}
}
#endif /* tuscany_leveldb_hpp */

View file

@ -0,0 +1,40 @@
#!/bin/sh
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
# Setup
../../modules/http/httpd-conf tmp localhost 8090 ../../modules/http/htdocs
../../modules/server/server-conf tmp
../../modules/server/scheme-conf tmp
cat >>tmp/conf/httpd.conf <<EOF
SCAContribution `pwd`/
SCAComposite nosqldb.composite
EOF
./leveldb -c -m tmp/test.cdb </dev/null
../../modules/http/httpd-start tmp
sleep 2
# Test
./client-test #2>/dev/null
rc=$?
# Cleanup
../../modules/http/httpd-stop tmp
sleep 2
exit $rc