From e2a1e0c4a5c4c3328e2833c176212d8e1406af4c Mon Sep 17 00:00:00 2001 From: rfeng Date: Sat, 17 Sep 2011 17:44:55 +0000 Subject: Start to add LevelDB support git-svn-id: http://svn.us.apache.org/repos/asf/tuscany@1172029 13f79535-47bb-0310-9956-ffa450edef68 --- sca-cpp/trunk/components/kvdb/Makefile.am | 49 +++++ sca-cpp/trunk/components/kvdb/client-test.cpp | 139 +++++++++++++ sca-cpp/trunk/components/kvdb/kvdb-test | 29 +++ sca-cpp/trunk/components/kvdb/kvdb.componentType | 28 +++ sca-cpp/trunk/components/kvdb/kvdb.composite | 32 +++ sca-cpp/trunk/components/kvdb/kvdb.cpp | 123 +++++++++++ sca-cpp/trunk/components/kvdb/leveldb | 24 +++ sca-cpp/trunk/components/kvdb/leveldb-test.cpp | 82 ++++++++ sca-cpp/trunk/components/kvdb/leveldb.hpp | 249 +++++++++++++++++++++++ sca-cpp/trunk/components/kvdb/server-test | 40 ++++ 10 files changed, 795 insertions(+) create mode 100644 sca-cpp/trunk/components/kvdb/Makefile.am create mode 100644 sca-cpp/trunk/components/kvdb/client-test.cpp create mode 100755 sca-cpp/trunk/components/kvdb/kvdb-test create mode 100644 sca-cpp/trunk/components/kvdb/kvdb.componentType create mode 100644 sca-cpp/trunk/components/kvdb/kvdb.composite create mode 100644 sca-cpp/trunk/components/kvdb/kvdb.cpp create mode 100755 sca-cpp/trunk/components/kvdb/leveldb create mode 100644 sca-cpp/trunk/components/kvdb/leveldb-test.cpp create mode 100644 sca-cpp/trunk/components/kvdb/leveldb.hpp create mode 100755 sca-cpp/trunk/components/kvdb/server-test (limited to 'sca-cpp') diff --git a/sca-cpp/trunk/components/kvdb/Makefile.am b/sca-cpp/trunk/components/kvdb/Makefile.am new file mode 100644 index 0000000000..3212f3f5c8 --- /dev/null +++ b/sca-cpp/trunk/components/kvdb/Makefile.am @@ -0,0 +1,49 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +INCLUDES = -I${LEVELDB_INCLUDE} + +incl_HEADERS = *.hpp +incldir = $(prefix)/include/components/kvdb + +dist_comp_SCRIPTS = leveldb +compdir=$(prefix)/components/kvdb + +comp_DATA = leveldb.prefix +leveldb.prefix: $(top_builddir)/config.status + echo ${LEVELDB_PREFIX} >leveldb.prefix + +EXTRA_DIST = kvdb.composite kvdb.componentType + +comp_LTLIBRARIES = libkvdb.la +noinst_DATA = libkvdb${libsuffix} + +libkvdb_la_SOURCES = kvdb.cpp +libkvdb_la_LDFLAGS = -L${LEVELDB_LIB} -R${LEVELDB_LIB} -lleveldb +libkvdb${libsuffix}: + ln -s .libs/libkvdb${libsuffix} + +leveldb_test_SOURCES = leveldb-test.cpp +leveldb_test_LDFLAGS = -L${LEVELDB_LIB} -R${LEVELDB_LIB} -lleveldb + +client_test_SOURCES = client-test.cpp +client_test_LDFLAGS = -lxml2 -lcurl -lmozjs + +dist_noinst_SCRIPTS = kvdb-test server-test +noinst_PROGRAMS = leveldb-test client-test +TESTS = kvdb-test server-test + diff --git a/sca-cpp/trunk/components/kvdb/client-test.cpp b/sca-cpp/trunk/components/kvdb/client-test.cpp new file mode 100644 index 0000000000..fc31a99e69 --- /dev/null +++ b/sca-cpp/trunk/components/kvdb/client-test.cpp @@ -0,0 +1,139 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* $Rev$ $Date$ */ + +/** + * Test KV (Key-Value) database component. + */ + +#include +#include "stream.hpp" +#include "string.hpp" + +#include "list.hpp" +#include "value.hpp" +#include "monad.hpp" +#include "perf.hpp" +#include "../../modules/http/http.hpp" + +namespace tuscany { +namespace nosqldb { + +const string uri("http://localhost:8090/nosqldb"); + +bool testNoSqlDb() { + http::CURLSession cs("", "", "", ""); + + const list i = list() + "content" + (list() + "item" + + (list() + "name" + string("Apple")) + + (list() + "price" + string("$2.99"))); + const list a = list() + (list() + "entry" + + (list() + "title" + string("item")) + + (list() + "id" + string("cart-53d67a61-aa5e-4e5e-8401-39edeba8b83b")) + + i); + + const failable id = http::post(a, uri, cs); + assert(hasContent(id)); + + const string p = path(content(id)); + { + const failable val = http::get(uri + p, cs); + assert(hasContent(val)); + assert(content(val) == a); + } + + const list j = list() + "content" + (list() + "item" + + (list() + "name" + string("Apple")) + + (list() + "price" + string("$3.55"))); + const list b = list() + (list() + "entry" + + (list() + "title" + string("item")) + + (list() + "id" + string("cart-53d67a61-aa5e-4e5e-8401-39edeba8b83b")) + + j); + + { + const failable r = http::put(b, uri + p, cs); + assert(hasContent(r)); + assert(content(r) == value(true)); + } + { + const failable val = http::get(uri + p, cs); + assert(hasContent(val)); + assert(content(val) == b); + } + { + const failable r = http::del(uri + p, cs); + assert(hasContent(r)); + assert(content(r) == value(true)); + } + { + const failable val = http::get(uri + p, cs); + assert(!hasContent(val)); + } + + return true; +} + +struct getLoop { + const string path; + const value entry; + http::CURLSession cs; + getLoop(const string& path, const value& entry, http::CURLSession cs) : path(path), entry(entry), cs(cs) { + } + const bool operator()() const { + const failable val = http::get(uri + path, cs); + assert(hasContent(val)); + assert(content(val) == entry); + return true; + } +}; + +bool testGetPerf() { + const list i = list() + "content" + (list() + "item" + + (list() + "name" + string("Apple")) + + (list() + "price" + string("$4.55"))); + const list a = list() + (list() + "entry" + + (list() + "title" + string("item")) + + (list() + "id" + string("cart-53d67a61-aa5e-4e5e-8401-39edeba8b83b")) + + i); + + http::CURLSession cs("", "", "", ""); + const failable id = http::post(a, uri, cs); + assert(hasContent(id)); + const string p = path(content(id)); + + const lambda gl = getLoop(p, a, cs); + cout << "NoSqldb get test " << time(gl, 5, 200) << " ms" << endl; + + return true; +} + +} +} + +int main() { + tuscany::cout << "Testing..." << tuscany::endl; + + tuscany::nosqldb::testNoSqlDb(); + tuscany::nosqldb::testGetPerf(); + + tuscany::cout << "OK" << tuscany::endl; + + return 0; +} diff --git a/sca-cpp/trunk/components/kvdb/kvdb-test b/sca-cpp/trunk/components/kvdb/kvdb-test new file mode 100755 index 0000000000..3730ed67a4 --- /dev/null +++ b/sca-cpp/trunk/components/kvdb/kvdb-test @@ -0,0 +1,29 @@ +#!/bin/sh + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Setup +mkdir -p tmp +./tinycdb -c -m tmp/test.cdb /dev/null +rc=$? + +# Cleanup +exit $rc diff --git a/sca-cpp/trunk/components/kvdb/kvdb.componentType b/sca-cpp/trunk/components/kvdb/kvdb.componentType new file mode 100644 index 0000000000..1c56ab6807 --- /dev/null +++ b/sca-cpp/trunk/components/kvdb/kvdb.componentType @@ -0,0 +1,28 @@ + + + + + + + + diff --git a/sca-cpp/trunk/components/kvdb/kvdb.composite b/sca-cpp/trunk/components/kvdb/kvdb.composite new file mode 100644 index 0000000000..378f418cfc --- /dev/null +++ b/sca-cpp/trunk/components/kvdb/kvdb.composite @@ -0,0 +1,32 @@ + + + + + + + tmp/test.cdb + + + + + + diff --git a/sca-cpp/trunk/components/kvdb/kvdb.cpp b/sca-cpp/trunk/components/kvdb/kvdb.cpp new file mode 100644 index 0000000000..70ae1eda74 --- /dev/null +++ b/sca-cpp/trunk/components/kvdb/kvdb.cpp @@ -0,0 +1,123 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* $Rev$ $Date$ */ + +/** + * LevelDB-based database component implementation. + */ + +#include "string.hpp" +#include "function.hpp" +#include "list.hpp" +#include "value.hpp" +#include "monad.hpp" +#include "leveldb.hpp" + +namespace tuscany { +namespace nvdb { + +/** + * Get an item from the database. + */ +const failable get(const list& params, leveldb::LevelDB& cdb) { + return leveldb::get(car(params), cdb); +} + +/** + * Post an item to the database. + */ +const failable post(const list& params, leveldb::LevelDB& cdb) { + const value id = append(car(params), mklist(mkuuid())); + const failable val = leveldb::post(id, cadr(params), cdb); + if (!hasContent(val)) + return mkfailure(reason(val)); + return id; +} + +/** + * Put an item into the database. + */ +const failable put(const list& params, leveldb::LevelDB& cdb) { + const failable val = leveldb::put(car(params), cadr(params), cdb); + if (!hasContent(val)) + return mkfailure(reason(val)); + return value(content(val)); +} + +/** + * Delete an item from the database. + */ +const failable del(const list& params, leveldb::LevelDB& cdb) { + const failable val = leveldb::del(car(params), cdb); + if (!hasContent(val)) + return mkfailure(reason(val)); + return value(content(val)); +} + +/** + * Component implementation lambda function. + */ +class applyNoSqldb { +public: + applyNoSqldb(leveldb::LevelDB& cdb) : cdb(cdb) { + } + + const value operator()(const list& params) const { + const value func(car(params)); + if (func == "get") + return get(cdr(params), cdb); + if (func == "post") + return post(cdr(params), cdb); + if (func == "put") + return put(cdr(params), cdb); + if (func == "delete") + return del(cdr(params), cdb); + return tuscany::mkfailure(); + } + +private: + leveldb::LevelDB& cdb; +}; + +/** + * Start the component. + */ +const failable start(unused const list& params) { + // Connect to the configured database and table + const value dbname = ((lambda)>)car(params))(list()); + leveldb::LevelDB& cdb = *(new (gc_new()) leveldb::LevelDB(dbname)); + + // Return the component implementation lambda function + return value(lambda&)>(applyNoSqldb(cdb))); +} + +} +} + +extern "C" { + +const tuscany::value apply(const tuscany::list& params) { + const tuscany::value func(car(params)); + if (func == "start") + return tuscany::nosqldb::start(cdr(params)); + return tuscany::mkfailure(); +} + +} diff --git a/sca-cpp/trunk/components/kvdb/leveldb b/sca-cpp/trunk/components/kvdb/leveldb new file mode 100755 index 0000000000..f9c54465f6 --- /dev/null +++ b/sca-cpp/trunk/components/kvdb/leveldb @@ -0,0 +1,24 @@ +#!/bin/sh + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +here=`echo "import os; print os.path.realpath('$0')" | python`; here=`dirname $here` +leveldb_prefix=`cat $here/leveldb.prefix` + +$leveldb_prefix/bin/cdb $* + diff --git a/sca-cpp/trunk/components/kvdb/leveldb-test.cpp b/sca-cpp/trunk/components/kvdb/leveldb-test.cpp new file mode 100644 index 0000000000..b3b4ea7fd7 --- /dev/null +++ b/sca-cpp/trunk/components/kvdb/leveldb-test.cpp @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* $Rev$ $Date$ */ + +/** + * Test TinyCDB access functions. + */ + +#include +#include "stream.hpp" +#include "string.hpp" +#include "perf.hpp" +#include "tinycdb.hpp" + +namespace tuscany { +namespace tinycdb { + +bool testTinyCDB() { + TinyCDB cdb("tmp/test.cdb"); + const value k = mklist("a"); + + assert(hasContent(post(k, string("AAA"), cdb))); + assert((get(k, cdb)) == value(string("AAA"))); + assert(hasContent(put(k, string("aaa"), cdb))); + assert((get(k, cdb)) == value(string("aaa"))); + assert(hasContent(del(k, cdb))); + assert(!hasContent(get(k, cdb))); + + return true; +} + +struct getLoop { + const value k; + TinyCDB& cdb; + getLoop(const value& k, TinyCDB& cdb) : k(k), cdb(cdb) { + } + const bool operator()() const { + assert((get(k, cdb)) == value(string("CCC"))); + return true; + } +}; + +bool testGetPerf() { + const value k = mklist("c"); + TinyCDB cdb("tmp/test.cdb"); + assert(hasContent(post(k, string("CCC"), cdb))); + + const lambda gl = getLoop(k, cdb); + cout << "TinyCDB get test " << time(gl, 5, 100000) << " ms" << endl; + return true; +} + +} +} + +int main() { + tuscany::cout << "Testing..." << tuscany::endl; + + tuscany::tinycdb::testTinyCDB(); + tuscany::tinycdb::testGetPerf(); + + tuscany::cout << "OK" << tuscany::endl; + + return 0; +} diff --git a/sca-cpp/trunk/components/kvdb/leveldb.hpp b/sca-cpp/trunk/components/kvdb/leveldb.hpp new file mode 100644 index 0000000000..893166c361 --- /dev/null +++ b/sca-cpp/trunk/components/kvdb/leveldb.hpp @@ -0,0 +1,249 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* $Rev$ $Date$ */ + +#ifndef tuscany_leveldb_hpp +#define tuscany_leveldb_hpp + +#include +#include +#include +#include + +#include "string.hpp" +#include "list.hpp" +#include "value.hpp" +#include "monad.hpp" +#include "../../modules/scheme/eval.hpp" + +namespace tuscany { +namespace leveldb { + +/** + * A reallocatable buffer. + */ +class buffer { +public: + operator void*() const throw() { + return buf; + } + + operator unsigned char*() const throw() { + return (unsigned char*)buf; + } + + operator char*() const throw() { + return (char*)buf; + } + +private: + buffer(const unsigned int size, void* buf) : size(size), buf(buf) { + } + + unsigned int size; + void* buf; + + friend const buffer mkbuffer(const unsigned int sz); + friend const buffer mkbuffer(const buffer& b, const unsigned int newsz); + friend const bool free(const buffer& b); +}; + +/** + * Make a new buffer. + */ +const buffer mkbuffer(const unsigned int sz) { + return buffer(sz, malloc(sz)); +} + +/** + * Make a new buffer by reallocating an existing one. + */ +const buffer mkbuffer(const buffer& b, const unsigned int sz) { + if (sz <= b.size) + return b; + return buffer(sz, realloc(b.buf, sz)); +} + +/** + * Free a buffer. + */ +const bool free(const buffer&b) { + ::free(b.buf); + return true; +} + +/** + * Represents a LevelDB connection. + */ +class LevelDB { +public: + LevelDB() : owner(false), fd(-1) { + debug("leveldb::leveldb"); + st.st_ino = 0; + } + + LevelDB(const string& name) : owner(true), name(name), fd(-1) { + debug(name, "leveldb::leveldb::name"); + st.st_ino = 0; + } + + LevelDB(const LevelDB& c) : owner(false), name(c.name), fd(c.fd) { + debug("leveldb::leveldb::copy"); + st.st_ino = c.st.st_ino; + } + + ~LevelDB() { + debug("leveldb::~leveldb"); + if (!owner) + return; + if (fd == -1) + return; + close(fd); + } + +private: + bool owner; + string name; + leveldb::DB* db; + struct stat st; + + friend const string dbname(const LevelDB& db); + friend const failable dbopen(LevelDB& db); + friend const failable dbclose(LevelDB& db); +}; + +/** + * Return the name of the database. + */ +const string dbname(const LevelDB& db) { + return db.name; +} + +/** + * Open a database. + */ +const failable dbopen(LevelDB& db) { + + // Get database file serial number + struct stat st; + const int s = stat(c_str(db.name), &st); + if (s == -1) + return mkfailure( + string("Couldn't leveldb read database stat ") + db.name); + + leveldb::DB* ldb; + leveldb::Options options; + options.create_if_missing = true; + leveldb::Status status = leveldb::DB::Open(options, s, &ldb); + db.db = ldb; +} + +/** + * Close a database. + */ +const failable dbclose(LevelDB& db) { + delete db.db; + db.db = NULL; + return true; +} + + +const failable post(const value& key, const value& val, LevelDB& db) { + debug(key, "leveldb::post::key"); + debug(val, "leveldb::post::value"); + debug(dbname(db), "leveldb::post::dbname"); + + const string ks(scheme::writeValue(key)); + const string vs(scheme::writeValue(val)); + + put(ks, vs, db); + + debug(r, "leveldb::post::result"); + return r; +} + + +const failable put(const value& key, const value& val, LevelDB& db) { + debug(key, "leveldb::put::key"); + debug(val, "leveldb::put::value"); + debug(dbname(db), "leveldb::put::dbname"); + + const string ks(scheme::writeValue(key)); + const string vs(scheme::writeValue(val)); + + debug(r, "leveldb::put::result"); + return r; +} + +/** + * Get an item from the database. + */ +const failable get(const value& key, LevelDB& db) { + debug(key, "leveldb::get::key"); + debug(dbname(db), "leveldb::get::dbname"); + + const string ks(scheme::writeValue(key)); + + std::string data; + db.db->Get(leveldb::ReadOptions(), key, &data); + const value val(scheme::readValue(string(data))); + + debug(val, "leveldb::get::result"); + return val; +} + +/** + * Delete an item from the database + */ +struct delUpdate { + const string ks; + delUpdate(const string& ks) : ks(ks) { + } + const failable operator()(buffer& buf, const unsigned int klen, unused const unsigned int vlen) const { + if (ks == string((char*)buf, klen)) + return false; + return true; + } +}; + +struct delFinish { + delFinish() { + } + const failable operator()(unused struct db_make& dbm) const { + return true; + } +}; + +const failable del(const value& key, LevelDB& db) { + debug(key, "leveldb::delete::key"); + debug(dbname(db), "leveldb::delete::dbname"); + + const string ks(scheme::writeValue(key)); + + leveldb::Status s = db.db->Delete(leveldb::WriteOptions(), key); + + debug(r, "leveldb::delete::result"); + return s.ok(); +} + +} +} + +#endif /* tuscany_leveldb_hpp */ diff --git a/sca-cpp/trunk/components/kvdb/server-test b/sca-cpp/trunk/components/kvdb/server-test new file mode 100755 index 0000000000..1836336c70 --- /dev/null +++ b/sca-cpp/trunk/components/kvdb/server-test @@ -0,0 +1,40 @@ +#!/bin/sh + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Setup +../../modules/http/httpd-conf tmp localhost 8090 ../../modules/http/htdocs +../../modules/server/server-conf tmp +../../modules/server/scheme-conf tmp +cat >>tmp/conf/httpd.conf </dev/null +rc=$? + +# Cleanup +../../modules/http/httpd-stop tmp +sleep 2 +exit $rc -- cgit v1.2.3