From a0f68830211cbea1112794922939f8c5e90d7c4e Mon Sep 17 00:00:00 2001 From: jsdelfino Date: Sun, 17 Apr 2011 22:14:18 +0000 Subject: [PATCH] Fix representation of null values and escape control characters in JSON and HTTP query strings. git-svn-id: http://svn.us.apache.org/repos/asf/tuscany@1094210 13f79535-47bb-0310-9956-ffa450edef68 --- sca-cpp/trunk/kernel/gc.hpp | 29 ++++-- sca-cpp/trunk/kernel/list.hpp | 2 +- sca-cpp/trunk/kernel/value.hpp | 28 ++++-- sca-cpp/trunk/modules/http/http.hpp | 100 +++++++++++++++---- sca-cpp/trunk/modules/http/httpd-conf | 6 +- sca-cpp/trunk/modules/http/httpd.hpp | 20 +++- sca-cpp/trunk/modules/js/eval.hpp | 8 +- sca-cpp/trunk/modules/js/htdocs/jsonutil.js | 2 + sca-cpp/trunk/modules/scheme/environment.hpp | 5 +- sca-cpp/trunk/modules/scheme/eval-test.cpp | 38 ++++--- sca-cpp/trunk/modules/scheme/io.hpp | 43 +++++--- sca-cpp/trunk/modules/wsgi/jsonutil.py | 2 + 12 files changed, 215 insertions(+), 68 deletions(-) diff --git a/sca-cpp/trunk/kernel/gc.hpp b/sca-cpp/trunk/kernel/gc.hpp index 09dcd1e5ac..d843b50ca0 100644 --- a/sca-cpp/trunk/kernel/gc.hpp +++ b/sca-cpp/trunk/kernel/gc.hpp @@ -36,6 +36,23 @@ namespace tuscany { +#ifdef WANT_MAINTAINER_MODE + +/** + * Force a core dump on assertion violation. + */ +bool assertOrFail(const bool expr) { + if (!expr) + *(char*)NULL = '\0'; + return true; +} + +#else + +#define assertOrFail(expr) + +#endif + /** * Pointer to a value. */ @@ -117,7 +134,7 @@ private: apr_pool_t* mkpool() { apr_pool_t* p = NULL; apr_pool_create(&p, NULL); - assert(p != NULL); + assertOrFail(p != NULL); return p; } @@ -164,7 +181,7 @@ apr_pool_t* gc_current_pool() { // Create a parent pool for the current thread apr_pool_create(&apr_pool, NULL); - assert(apr_pool != NULL); + assertOrFail(apr_pool != NULL); gc_pool_stack = apr_pool; return apr_pool; } @@ -196,7 +213,7 @@ public: gc_scoped_pool() : gc_pool(NULL), prev(gc_current_pool()), owner(true) { apr_pool_create(&apr_pool, NULL); - assert(apr_pool != NULL); + assertOrFail(apr_pool != NULL); gc_push_pool(apr_pool); } @@ -230,7 +247,7 @@ template apr_status_t gc_pool_cleanup(void* v) { template T* gc_new(apr_pool_t* p) { void* gc_new_ptr = apr_palloc(p, sizeof(T)); - assert(gc_new_ptr != NULL); + assertOrFail(gc_new_ptr != NULL); apr_pool_cleanup_register(p, gc_new_ptr, gc_pool_cleanup, apr_pool_cleanup_null) ; return static_cast(gc_new_ptr); } @@ -254,7 +271,7 @@ template apr_status_t gc_pool_acleanup(void* v) { template T* gc_anew(apr_pool_t* p, size_t n) { size_t* gc_anew_ptr = static_cast(apr_palloc(p, sizeof(size_t) + sizeof(T[n]))); - assert(gc_anew_ptr != NULL); + assertOrFail(gc_anew_ptr != NULL); *gc_anew_ptr = n; apr_pool_cleanup_register(p, gc_anew_ptr, gc_pool_acleanup, apr_pool_cleanup_null) ; return (T*)(gc_anew_ptr + 1); @@ -273,7 +290,7 @@ template T* gc_anew(size_t n) { */ char* gc_cnew(apr_pool_t* p, size_t n) { char* gc_cnew_ptr = static_cast(apr_palloc(p, n)); - assert(gc_cnew_ptr != NULL); + assertOrFail(gc_cnew_ptr != NULL); return gc_cnew_ptr; } diff --git a/sca-cpp/trunk/kernel/list.hpp b/sca-cpp/trunk/kernel/list.hpp index a8dbcc1b0c..df7bc27c03 100644 --- a/sca-cpp/trunk/kernel/list.hpp +++ b/sca-cpp/trunk/kernel/list.hpp @@ -301,7 +301,7 @@ template const list mklist(const T& a, const T& b, const T& c, co */ template const T car(const list& p) { // Abort if trying to access the car of a nil list - assert(!isNil(p.cdr)); + assertOrFail(!isNil(p.cdr)); return p.car; } diff --git a/sca-cpp/trunk/kernel/value.hpp b/sca-cpp/trunk/kernel/value.hpp index 07be7f5c82..3e0dd0002f 100644 --- a/sca-cpp/trunk/kernel/value.hpp +++ b/sca-cpp/trunk/kernel/value.hpp @@ -95,10 +95,10 @@ class value { public: enum ValueType { - Undefined, Symbol, String, List, Number, Bool, Lambda, Ptr + Nil, Symbol, String, List, Number, Bool, Lambda, Ptr }; - value() : type(value::Undefined) { + value() : type(value::Nil) { debug_inc(countValues); debug_inc(countEValues); debug_watchValue(); @@ -239,8 +239,8 @@ public: if(this == &v) return true; switch(type) { - case value::Undefined: - return v.type == value::Undefined; + case value::Nil: + return v.type == value::Nil; case value::List: return v.type == value::List && lst()() == v.lst()(); case value::Lambda: @@ -447,6 +447,18 @@ const string watchValue(const value& v) { #endif +/** + * Write an escaped string value to a stream. + */ +ostream& escvwrite(const char* s, ostream& out) { + if (*s == '\0') + return out; + if (*s == '\\' || *s == '"') + out << '\\'; + out << *s; + return escvwrite(s + 1, out); +} + /** * Write a value to a stream. */ @@ -459,7 +471,9 @@ ostream& operator<<(ostream& out, const value& v) { case value::Symbol: return out << v.str()(); case value::String: - return out << '\"' << v.str()() << '\"'; + out << '\"'; + escvwrite(c_str(v.str()()), out); + return out << '\"'; case value::Number: return out << v.num()(); case value::Bool: @@ -474,7 +488,7 @@ ostream& operator<<(ostream& out, const value& v) { return out << "gc_ptr::" << p; } default: - return out << "undefined"; + return out << "nil"; } } @@ -489,7 +503,7 @@ const value::ValueType type(const value& v) { * Returns true if a value is nil. */ const bool isNil(const value& v) { - return type(v) == value::Undefined; + return type(v) == value::Nil; } /** diff --git a/sca-cpp/trunk/modules/http/http.hpp b/sca-cpp/trunk/modules/http/http.hpp index 7050021fb9..1a207669c0 100644 --- a/sca-cpp/trunk/modules/http/http.hpp +++ b/sca-cpp/trunk/modules/http/http.hpp @@ -30,6 +30,8 @@ #include #include #include +#include +#include #include #include #include @@ -93,6 +95,7 @@ private: friend CURL* handle(const CURLSession& cs); friend apr_socket_t* sock(const CURLSession& cs); + friend const failable setup(const string& url, const CURLSession& cs); friend const failable connect(const string& url, CURLSession& cs); friend const failable send(const char* c, const size_t l, const CURLSession& cs); friend const failable recv(char* c, const size_t l, const CURLSession& cs); @@ -142,6 +145,45 @@ const string apreason(apr_status_t rc) { return apr_strerror(rc, buf, sizeof(buf)); } +/** + * Escape a URI or a query argument. + */ +const char escape_c2x[] = "0123456789ABCDEF"; + +const string escape(const string& unesc, const char* reserv) { + char* copy = (char*)apr_palloc(gc_current_pool(), 3 * length(unesc) + 3); + const unsigned char* s = (const unsigned char *)c_str(unesc); + unsigned char* d = (unsigned char*)copy; + unsigned c; + while ((c = *s)) { + if (!apr_isalnum(c) && !strchr(reserv, c)) { + *d++ = '%'; + *d++ = escape_c2x[c >> 4]; + *d++ = escape_c2x[c & 0xf]; + } + else { + *d++ = (unsigned char)c; + } + ++s; + } + *d = '\0'; + return copy; +} + +const string escapeURI(const string& uri) { + debug(uri, "http::escapeURI::uri"); + const string e = escape(uri, "?$-_.+!*'(),:@&=/~%"); + debug(e, "http::escapeURI::result"); + return e; +} + +const string escapeArg(const string& arg) { + debug(arg, "http::escapeArg::arg"); + const string e = escape(arg, "-_.~"); + debug(e, "http::escapeArg::result"); + return e; +} + /** * Setup a CURL session */ @@ -177,7 +219,7 @@ const failable setup(const string& url, const CURLSession& cs) { } // Set target URL - curl_easy_setopt(ch, CURLOPT_URL, c_str(url)); + curl_easy_setopt(ch, CURLOPT_URL, c_str(escapeURI(url))); return ch; } @@ -430,6 +472,22 @@ const failable get(const string& url, const CURLSession& cs) { debug(val, "http::get::result"); return val; } + if (contains(ct, "application/x-javascript")) { + // Read a JSON document enclosed in a javascript function call + // Extract the JSON out of the enclosing parenthesis + ostringstream os; + write(ls, os); + const string s = str(os); + const size_t fp = find(s, '('); + const size_t lp = find_last(s, ')'); + const list jls = mklist(substr(s, fp + 1, lp - (fp + 1))); + debug(jls, "http::get::javascript::content"); + + js::JSContext cx; + const value val(json::jsonValues(content(json::readJSON(jls, cx)))); + debug(val, "http::get::result"); + return val; + } if (contains(ct, "text/xml") || contains(ct, "application/xml") || isXML(ls)) { // Read an XML document const value val(elementsToValues(readXML(ls))); @@ -620,21 +678,6 @@ const failable recv(char* c, const size_t l, const CURLSession& cs) { return recv(c, l, cs); } - -/** - * Filter path segment in a list of arguments. - */ -const bool filterPath(const value& arg) { - return isString(arg); -} - -/** - * Filter query string arguments in a list of arguments. - */ -const bool filterQuery(const value& arg) { - return isList(arg); -} - /** * Converts a list of key value pairs to a query string. */ @@ -653,6 +696,28 @@ const string queryString(const list > args) { return str(queryString(args, os)); } +/** + * Filter path segment in a list of arguments. + */ +const bool filterPath(const value& arg) { + return isString(arg); +} + +/** + * Filter query string arguments in a list of arguments. + */ +const bool filterQuery(const value& arg) { + return isList(arg); +} + +/** + * Escape a query string argument. + */ +const value escapeQuery(const value& arg) { + return arg; + //return mklist(car(arg), escapeArg(cadr(arg))); +} + /** * HTTP client proxy function. */ @@ -665,8 +730,7 @@ struct proxy { if (fun == "get") { const list lp = filter(filterPath, cadr(args)); debug(lp, "http::queryString::arg"); - const list lq = filter(filterQuery, cadr(args)); - debug(lq, "http::get::query"); + const list lq = map(escapeQuery, filter(filterQuery, cadr(args))); const value p = path(lp); const value q = queryString(lq); const failable val = get(uri + p + (q != ""? string("?") + q : string("")), cs); diff --git a/sca-cpp/trunk/modules/http/httpd-conf b/sca-cpp/trunk/modules/http/httpd-conf index cfa3952521..7c9190d08c 100755 --- a/sca-cpp/trunk/modules/http/httpd-conf +++ b/sca-cpp/trunk/modules/http/httpd-conf @@ -82,9 +82,9 @@ AddCharset utf-8 .js .css # Configure cache control SetEnvIf Request_URI "^/app.html$" must-revalidate -Header onsuccess merge Cache-Control public -Header onsuccess merge Cache-Control max-age=31536000 -Header onsuccess merge Cache-Control must-revalidate env=must-revalidate +Header onsuccess set Cache-Control "max-age=86400" env=!must-revalidate +Header set Cache-Control "must-revalidate, max-age=0" env=must-revalidate +Header set Expires "Tue, 01 Jan 1980 00:00:00 GMT" env=must-revalidate # Set default document root DocumentRoot $htdocs diff --git a/sca-cpp/trunk/modules/http/httpd.hpp b/sca-cpp/trunk/modules/http/httpd.hpp index 0dd7920f65..86a89d8823 100644 --- a/sca-cpp/trunk/modules/http/httpd.hpp +++ b/sca-cpp/trunk/modules/http/httpd.hpp @@ -251,6 +251,20 @@ const string unescape(const string& uri) { return b; } +/** + * Unescape a list of key of value pairs representing query args. + */ +const list unescapeArg(const list a) { + return mklist(car(a), unescape(cadr(a))); +} + +const list > unescapeArgs(const list > args) { + debug(args, "httpd::unescape::args"); + const list > uargs = map, list>(unescapeArg, args); + debug(uargs, "httpd::unescape::result"); + return uargs; +} + /** * Returns a list of key value pairs from the args in a query string. */ @@ -328,11 +342,11 @@ const failable writeResult(const failable >& ls, const string& const string ob(str(os)); // Make sure browsers come back and check for updated dynamic content - apr_table_setn(r->headers_out, "Expires", "Tue, 01 Jan 1980 00:00:00 GMT"); - apr_table_setn(r->headers_out, "Cache-Control", "must-revalidate"); + // The actual header setup is configured in httpd-conf, based on the must-revalidate env variable + apr_table_set(r->subprocess_env, apr_pstrdup(r->pool, "must-revalidate"), apr_pstrdup(r->pool, "true")); // Compute and return an Etag for the returned content - const string etag(ap_md5(r->pool, (const unsigned char*)c_str(ob))); + const string etag(ap_md5_binary(r->pool, (const unsigned char*)c_str(ob), (int)length(ob))); // Check for an If-None-Match header and just return a 304 not-modified status // if the Etag matches the Etag presented by the client, to save bandwith diff --git a/sca-cpp/trunk/modules/js/eval.hpp b/sca-cpp/trunk/modules/js/eval.hpp index cdb9037bb7..79ae2aec65 100644 --- a/sca-cpp/trunk/modules/js/eval.hpp +++ b/sca-cpp/trunk/modules/js/eval.hpp @@ -177,9 +177,10 @@ const list jsPropertiesToValues(const list& propertiesSoFar, JSObj jsval idv; JS_IdToValue(cx, id, &idv); if(JSVAL_IS_STRING(idv)) { - if (isNil(val) && !isList(val)) - return jsPropertiesToValues(propertiesSoFar, o, i, cx); const string name = JS_GetStringBytes(JSVAL_TO_STRING(idv)); + if (isNil(val) && !isList(val)) + return jsPropertiesToValues(cons (mklist (element, c_str(name), val), propertiesSoFar), o, i, cx); + //return jsPropertiesToValues(propertiesSoFar, o, i, cx); if (substr(name, 0, 1) == atsign) return jsPropertiesToValues(cons(mklist(attribute, c_str(substr(name, 1)), val), propertiesSoFar), o, i, cx); if (isList(val) && !isJSArray(val)) @@ -257,6 +258,9 @@ const jsval valueToJSVal(const value& val, const js::JSContext& cx) { return OBJECT_TO_JSVAL(valuesToJSElements(JS_NewArrayObject(cx, 0, NULL), val, 0, cx)); return OBJECT_TO_JSVAL(valuesToJSProperties(JS_NewObject(cx, NULL, NULL, NULL), val, cx)); } + case value::Nil: { + return JSVAL_NULL; + } default: { return JSVAL_VOID; } diff --git a/sca-cpp/trunk/modules/js/htdocs/jsonutil.js b/sca-cpp/trunk/modules/js/htdocs/jsonutil.js index 47e3ec4130..8aa291bc89 100644 --- a/sca-cpp/trunk/modules/js/htdocs/jsonutil.js +++ b/sca-cpp/trunk/modules/js/htdocs/jsonutil.js @@ -77,6 +77,8 @@ json.jsPropertiesToValues = function(propertiesSoFar, o, i) { * Converts a JSON val to a value. */ json.jsValToValue = function(jsv) { + if (jsv == null) + return null; if (isList(jsv)) return json.jsPropertiesToValues(mklist(), jsv, reverse(range(0, jsv.length))); if (typeof jsv == 'object') diff --git a/sca-cpp/trunk/modules/scheme/environment.hpp b/sca-cpp/trunk/modules/scheme/environment.hpp index bfb415a978..303a37cb3c 100644 --- a/sca-cpp/trunk/modules/scheme/environment.hpp +++ b/sca-cpp/trunk/modules/scheme/environment.hpp @@ -118,8 +118,11 @@ const value definitionVariable(const value& exp) { const value definitionValue(const value& exp) { const list exps(exp); - if(isSymbol(car(cdr(exps)))) + if(isSymbol(car(cdr(exps)))) { + if (isNil(cdr(cdr(exps)))) + return value(); return car(cdr(cdr(exps))); + } const list lexps(car(cdr(exps))); return makeLambda(cdr(lexps), cdr(cdr(exps))); } diff --git a/sca-cpp/trunk/modules/scheme/eval-test.cpp b/sca-cpp/trunk/modules/scheme/eval-test.cpp index 7c4c0c69c4..dd97bc358d 100644 --- a/sca-cpp/trunk/modules/scheme/eval-test.cpp +++ b/sca-cpp/trunk/modules/scheme/eval-test.cpp @@ -72,20 +72,30 @@ bool testRead() { } bool testWrite() { - const list i = list() - + (list() + "item" + "cart-53d67a61-aa5e-4e5e-8401-39edeba8b83b" - + (list() + "item" - + (list() + "name" + "Apple") - + (list() + "price" + "$2.99"))) - + (list() + "item" + "cart-53d67a61-aa5e-4e5e-8401-39edeba8b83c" - + (list() + "item" - + (list() + "name" + "Orange") - + (list() + "price" + "$3.55"))); - const list a = cons("Feed", cons("feed-1234", i)); - ostringstream os; - writeValue(a, os); - istringstream is(str(os)); - assert(readValue(is) == a); + { + const list i = list() + + (list() + "item" + "cart-53d67a61-aa5e-4e5e-8401-39edeba8b83b" + + (list() + "item" + + (list() + "name" + "Apple") + + (list() + "price" + "$2.99"))) + + (list() + "item" + "cart-53d67a61-aa5e-4e5e-8401-39edeba8b83c" + + (list() + "item" + + (list() + "name" + "Orange") + + (list() + "price" + "$3.55"))); + const list a = cons("Feed", cons("feed-1234", i)); + ostringstream os; + writeValue(a, os); + istringstream is(str(os)); + assert(readValue(is) == a); + } + { + const list i = mklist("x", value()); + const list a = mklist(i); + ostringstream os; + writeValue(a, os); + istringstream is(str(os)); + assert(readValue(is) == a); + } return true; } diff --git a/sca-cpp/trunk/modules/scheme/io.hpp b/sca-cpp/trunk/modules/scheme/io.hpp index 6928739d17..8f9d70e7fe 100644 --- a/sca-cpp/trunk/modules/scheme/io.hpp +++ b/sca-cpp/trunk/modules/scheme/io.hpp @@ -88,14 +88,14 @@ const bool isQuote(const value& token) { return token == quoteSymbol; } -const value skipComment(istream& in); +const failable skipComment(istream& in); const value readQuoted(istream& in); const value readIdentifier(const char chr, istream& in); const value readString(istream& in); const value readNumber(const char chr, istream& in); const value readValue(istream& in); -const value readToken(istream& in) { +const failable readToken(istream& in) { const char firstChar = readChar(in); if(isWhitespace(firstChar)) return readToken(in); @@ -114,12 +114,12 @@ const value readToken(istream& in) { if(isDigit(firstChar)) return readNumber(firstChar, in); if(firstChar == -1) - return value(); + return mkfailure(); logStream() << "Illegal lexical syntax '" << firstChar << "'" << endl; return readToken(in); } -const value skipComment(istream& in) { +const failable skipComment(istream& in) { const char nextChar = readChar(in); if (nextChar == '\n') return readToken(in); @@ -131,8 +131,11 @@ const value readQuoted(istream& in) { } const list readList(const list& listSoFar, istream& in) { - const value token = readToken(in); - if(isNil(token) || isRightParenthesis(token)) + const failable ftoken = readToken(in); + if (!hasContent(ftoken)) + return reverse(listSoFar); + const value token = content(ftoken); + if(isRightParenthesis(token)) return reverse(listSoFar); if(isLeftParenthesis(token)) return readList(cons(value(readList(list (), in)), listSoFar), in); @@ -159,14 +162,22 @@ const value readIdentifier(const char chr, istream& in) { return value((bool)false); if (val == "true") return value((bool)true); + if (val == "nil") + return value(); return val; } const list readStringHelper(const list& listSoFar, istream& in) { const char nextChar = readChar(in); - if(nextChar != -1 && nextChar != '"') - return readStringHelper(cons(nextChar, listSoFar), in); - return reverse(listSoFar); + if(nextChar == -1 || nextChar == '"') + return reverse(listSoFar); + if (nextChar == '\\') { + const char escapedChar = readChar(in); + if (escapedChar == -1) + return reverse(listSoFar); + return readStringHelper(cons(escapedChar, listSoFar), in); + } + return readStringHelper(cons(nextChar, listSoFar), in); } const value readString(istream& in) { @@ -185,17 +196,23 @@ const value readNumber(const char chr, istream& in) { } const value readValue(istream& in) { - const value nextToken = readToken(in); + const failable fnextToken = readToken(in); + if (!hasContent(fnextToken)) + return value(); + const value nextToken = content(fnextToken); if(isLeftParenthesis(nextToken)) - return readList(list (), in); + return readList(list(), in); return nextToken; } const value readValue(const string s) { istringstream in(s); - const value nextToken = readToken(in); + const failable fnextToken = readToken(in); + if (!hasContent(fnextToken)) + return value(); + const value nextToken = content(fnextToken); if(isLeftParenthesis(nextToken)) - return readList(list (), in); + return readList(list(), in); return nextToken; } diff --git a/sca-cpp/trunk/modules/wsgi/jsonutil.py b/sca-cpp/trunk/modules/wsgi/jsonutil.py index ad8f5fcc6b..849b44405b 100644 --- a/sca-cpp/trunk/modules/wsgi/jsonutil.py +++ b/sca-cpp/trunk/modules/wsgi/jsonutil.py @@ -57,6 +57,8 @@ def jsPropertiesToValues(propertiesSoFar, o, i): # Converts a JSON val to a value def jsValToValue(jsv): + if jsv is None: + return None if isinstance(jsv, dict): return jsPropertiesToValues((), jsv, tuple(jsv.keys())) if isList(jsv):