Enable log collection using Scribe, and some minor log cleanup.

git-svn-id: http://svn.us.apache.org/repos/asf/tuscany@1208240 13f79535-47bb-0310-9956-ffa450edef68

11 files changed, 143 insertions, 39 deletions

diff --git a/sca-cpp/trunk/.gitignore b/sca-cpp/trunk/.gitignore
index 1dd44f8fd9..d1d2818389 100644
--- a/sca-cpp/trunk/.gitignore
+++ b/sca-cpp/trunk/.gitignore
@@ -136,6 +136,7 @@ js-test
 js-eval
 file-test
 test-start*
+test-stop*
 xml-value
 value-xml
 json-value
diff --git a/sca-cpp/trunk/components/cache/datacache.cpp b/sca-cpp/trunk/components/cache/datacache.cpp
index 502a57671d..994aa4840e 100644
--- a/sca-cpp/trunk/components/cache/datacache.cpp
+++ b/sca-cpp/trunk/components/cache/datacache.cpp
@@ -55,7 +55,7 @@ const failable<value> get(const value& key, const lambda<value(const list<value>
     if (isNil(val2)) {
         ostringstream os;
         os << "Couldn't get cache entry: " << key;
-        return mkfailure<value>(str(os));
+        return mkfailure<value>(str(os), false);
     }
 
     // Update level1 cache
diff --git a/sca-cpp/trunk/components/cache/memcache.hpp b/sca-cpp/trunk/components/cache/memcache.hpp
index 5d23ff7c42..f19f18c4ec 100644
--- a/sca-cpp/trunk/components/cache/memcache.hpp
+++ b/sca-cpp/trunk/components/cache/memcache.hpp
@@ -180,7 +180,7 @@ const failable<value> get(const value& key, const MemCached& cache) {
     if (rc != APR_SUCCESS) {
         ostringstream os;
         os << "Couldn't get memcached entry: " << key;
-        return mkfailure<value>(str(os));
+        return mkfailure<value>(str(os), false);
     }
     const value val(scheme::readValue(string(data, size)));
 
diff --git a/sca-cpp/trunk/kernel/monad.hpp b/sca-cpp/trunk/kernel/monad.hpp
index 34e39c72d9..c21f6409b7 100644
--- a/sca-cpp/trunk/kernel/monad.hpp
+++ b/sca-cpp/trunk/kernel/monad.hpp
@@ -242,7 +242,7 @@ private:
     template<typename A, typename B> friend const bool hasContent(const failable<A, B>& m);
     template<typename A, typename B> friend const A content(const failable<A, B>& m);
     template<typename A, typename B> friend const B reason(const failable<A, B>& m);
-    template<typename A, typename B> friend const failable<A, B> mkfailure(const B& f);
+    template<typename A, typename B> friend const failable<A, B> mkfailure(const B& f, const bool log = true);
     template<typename A> friend const failable<A, string> mkfailure();
 
     bool hasv;
@@ -276,22 +276,26 @@ template<typename V, typename F> const lambda<failable<V, F>(const V)> success()
 /**
  * Returns a failable monad with a failure in it.
  */
-template<typename V, typename F> const failable<V, F> mkfailure(const F& f) {
+template<typename V, typename F> const failable<V, F> mkfailure(const F& f, const bool log = true) {
 #ifdef WANT_MAINTAINER_MODE
-    ostringstream os;
-    os << f;
-    if (length(str(os)) != 0)
-        debug(f, "failable::mkfailure");
+    if (log) {
+        ostringstream os;
+        os << f;
+        if (length(str(os)) != 0)
+            debug(f, "failable::mkfailure");
+    }
 #else
-    ostringstream os;
-    os << f;
-    if (length(str(os)) != 0)
-        cfailure << "failable::mkfailure" << ": " << f << endl;
+    if (log) {
+        ostringstream os;
+        os << f;
+        if (length(str(os)) != 0)
+            cfailure << "failable::mkfailure" << ": " << f << endl;
+    }
 #endif
     return failable<V, F>(false, f);
 }
 
-template<typename V> const failable<V> mkfailure(const char* f) {
+template<typename V> const failable<V> mkfailure(const char* f, const bool log = true) {
     return mkfailure<V, string>(string(f));
 }
 
diff --git a/sca-cpp/trunk/modules/edit/ssl-start b/sca-cpp/trunk/modules/edit/ssl-start
index 5103566a5c..015a5dda66 100755
--- a/sca-cpp/trunk/modules/edit/ssl-start
+++ b/sca-cpp/trunk/modules/edit/ssl-start
@@ -27,6 +27,18 @@ jsprefix=`echo "import os; print os.path.realpath('$here/../js')" | python`
 ../../modules/http/ssl-ca-conf tmp sca-store.com
 ../../modules/http/ssl-cert-conf tmp sca-store.com server
 
+# Configure and start logging
+if [ -x ../../components/log/scribe-cat ]; then
+    ../../components/log/scribed-central-conf tmp
+    ../../components/log/scribed-client-conf tmp localhost
+    ../../components/log/scribed-central-start tmp
+    ../../components/log/scribed-client-start tmp
+fi
+
+# Start memcached
+../../components/cache/memcached-start 11211
+../../components/cache/memcached-start 11212
+
 # Configure server
 ../../modules/http/httpd-conf tmp sca-store.com 8090 htdocs
 ../../modules/http/httpd-event-conf tmp
@@ -41,21 +53,21 @@ jsprefix=`echo "import os; print os.path.realpath('$here/../js')" | python`
 # Configure OAuth authentication
 # Configure your OAuth app keys here
 ../../modules/oauth/oauth-conf tmp
-../../modules/oauth/oauth-memcached-conf tmp sca-store.com 11212
+../../modules/oauth/oauth-memcached-conf tmp localhost 11212
 ../../modules/oauth/oauth2-appkey-conf tmp facebook.com 12345 67890
 
 # Configure OpenID step2 authentication
 ../../modules/openid/openid-conf tmp
 ../../modules/openid/openid-step2-conf tmp
-../../modules/openid/openid-memcached-conf tmp sca-store.com 11212
+../../modules/openid/openid-memcached-conf tmp localhost 11212
 
 # Configure authorized users
 #../../modules/http/group-auth-conf tmp john
 #../../modules/http/group-auth-conf tmp jane
 #../../modules/http/group-auth-conf tmp admin
-# Configure your OpenID and OAuth ids here
-../../modules/http/group-auth-conf tmp https://www.google.com/accounts/o8/id?id=45678
-../../modules/http/group-auth-conf tmp 23456789
+# Configure the email addresses associated with your OpenID and OAuth ids here
+../../modules/http/group-auth-conf tmp john@sca-store.com
+../../modules/http/group-auth-conf tmp jane@sca-store.com
 
 # Configure mod-security
 ../../modules/http/mod-security-conf tmp
@@ -64,8 +76,42 @@ jsprefix=`echo "import os; print os.path.realpath('$here/../js')" | python`
 ../../modules/server/server-conf tmp
 ../../modules/python/python-conf tmp
 
+# Configure server log streaming
+if [ -x ../../components/log/scribe-cat ]; then
+    cat >tmp/conf/log.conf <<EOF
+# Generated by: ssl-start $*
+ErrorLog "|$here/../../components/log/scribe-cat server"
+CustomLog "|$here/../../components/log/scribe-cat server" combined
+
+EOF
+
+    cat >tmp/conf/log-ssl.conf <<EOF
+# Generated by: ssl-start $*
+CustomLog "|$here/../../components/log/scribe-cat server" sslcombined
+
+EOF
+
+    cat >tmp/conf/mod-security-log.conf <<EOF
+# Generated by: ssl-start $*
+SecAuditLog "|$here/../../components/log/scribe-cat secaudit"
+
+EOF
+
+fi
+
+# Configure certificate mime type
+cat >>tmp/conf/svhost-ssl.conf <<EOF
+# Generated by: ssl-start $*
+# Certificate mime type
+<Location /ca.crt>
+ForceType application/x-x509-ca-cert
+</Location>
+
+EOF
+
 # Configure error pages
 cat >>tmp/conf/svhost-ssl.conf <<EOF
+# Generated by: ssl-start $*
 # Error pages
 ErrorDocument 404 /public/notfound/
 ErrorDocument 401 /public/notauth/
@@ -76,7 +122,7 @@ EOF
 
 # Configure SCA contributions
 cat >>tmp/conf/httpd.conf <<EOF
-
+# Generated by: ssl-start $*
 # Configure SCA Composite
 SCAContribution `pwd`/
 SCAComposite edit.composite
@@ -89,7 +135,7 @@ EOF
 
 # Configure resource aliases
 cat >>tmp/conf/httpd.conf <<EOF
-
+# Generated by: ssl-start $*
 Alias /home/home.png $here/htdocs/home/home.png
 Alias /home/home.b64 $here/htdocs/home/home.b64
 
@@ -100,7 +146,7 @@ EOF
 
 # Configure app resource aliases
 cat >>tmp/conf/svhost-ssl.conf <<EOF
-
+# Generated by: ssl-start $*
 <Location /v>
 RewriteEngine on
 
@@ -119,10 +165,6 @@ EOF
 # Create application database directories
 mkdir -p tmp/appdata/filedb
 
-# Start memcached
-../../components/cache/memcached-start 11211
-../../components/cache/memcached-start 11212
-
 # Start server
 ../../modules/http/httpd-start tmp
 
diff --git a/sca-cpp/trunk/modules/edit/start b/sca-cpp/trunk/modules/edit/start
index 361cead2df..c3ef1f7c4d 100755
--- a/sca-cpp/trunk/modules/edit/start
+++ b/sca-cpp/trunk/modules/edit/start
@@ -23,6 +23,18 @@
 here=`echo "import os; print os.path.realpath('$0')" | python`; here=`dirname $here`
 jsprefix=`echo "import os; print os.path.realpath('$here/../js')" | python`
 
+# Configure and start logging
+if [ -x ../../components/log/scribe-cat ]; then
+    ../../components/log/scribed-central-conf tmp
+    ../../components/log/scribed-client-conf tmp localhost
+    ../../components/log/scribed-central-start tmp
+    ../../components/log/scribed-client-start tmp
+fi
+
+# Start memcached
+../../components/cache/memcached-start 11211
+../../components/cache/memcached-start 11212
+
 # Configure server
 ../../modules/http/httpd-conf tmp sca-store.com 8090 htdocs
 ../../modules/http/httpd-event-conf tmp
@@ -31,8 +43,26 @@ jsprefix=`echo "import os; print os.path.realpath('$here/../js')" | python`
 ../server/server-conf tmp
 ../python/python-conf tmp
 
+# Configure server log streaming
+if [ -x ../../components/log/scribe-cat ]; then
+    cat >tmp/conf/log.conf <<EOF
+# Generated by: start $*
+ErrorLog "|$here/../../components/log/scribe-cat server"
+CustomLog "|$here/../../components/log/scribe-cat server" combined
+
+EOF
+
+    cat >tmp/conf/log-ssl.conf <<EOF
+# Generated by: start $*
+CustomLog "|$here/../../components/log/scribe-cat server" sslcombined
+
+EOF
+
+fi
+
 # Configure error pages
 cat >>tmp/conf/svhost.conf <<EOF
+# Generated by: start $*
 # Error pages
 ErrorDocument 404 /public/notfound/
 ErrorDocument 401 /public/notauth/
@@ -43,6 +73,7 @@ EOF
 
 # Configure SCA contributions
 cat >>tmp/conf/httpd.conf <<EOF
+# Generated by: start $*
 # Configure SCA Composite
 SCAContribution `pwd`/
 SCAComposite edit.composite
@@ -55,7 +86,7 @@ EOF
 
 # Configure resource aliases
 cat >>tmp/conf/httpd.conf <<EOF
-
+# Generated by: start $*
 Alias /home/home.png $here/htdocs/home/home.png
 Alias /home/home.b64 $here/htdocs/home/home.b64
 
@@ -66,7 +97,7 @@ EOF
 
 # Configure app resource aliases
 cat >>tmp/conf/svhost.conf <<EOF
-
+# Generated by: start $*
 <Location /v>
 RewriteEngine on
 
@@ -85,10 +116,6 @@ EOF
 # Create application database directories
 mkdir -p tmp/appdata/filedb
 
-# Start memcached
-../../components/cache/memcached-start 11211
-../../components/cache/memcached-start 11212
-
 # Start server
 ../../modules/http/httpd-start tmp
 
diff --git a/sca-cpp/trunk/modules/edit/stop b/sca-cpp/trunk/modules/edit/stop
index 60c25f88f5..8e250de88b 100755
--- a/sca-cpp/trunk/modules/edit/stop
+++ b/sca-cpp/trunk/modules/edit/stop
@@ -18,6 +18,12 @@
 #  under the License.
 
 ../../modules/http/httpd-stop tmp
+
 ../../components/cache/memcached-stop 11211
 ../../components/cache/memcached-stop 11212
 
+if [ -x ../../components/log/scribe-cat ]; then
+    ../../components/log/scribed-client-stop tmp
+    ../../components/log/scribed-central-stop tmp
+fi
+
diff --git a/sca-cpp/trunk/modules/http/httpd-conf b/sca-cpp/trunk/modules/http/httpd-conf
index 9b55270bea..5d4e9f5485 100755
--- a/sca-cpp/trunk/modules/http/httpd-conf
+++ b/sca-cpp/trunk/modules/http/httpd-conf
@@ -82,9 +82,10 @@ HostNameLookups Off
 # status response-size "referrer" "user-agent" "user-track" local-IP
 # virtual-host response-time bytes-received bytes-sent
 LogLevel notice
-ErrorLog $root/logs/error_log
 LogFormat "[%{%a %b %d %H:%M:%S %Y}t] [access] %h %l %u \"%r\" %>s %b \"%{Referer}i\" \"%{User-agent}i\" \"%{cookie}n\" %A %V %D %I %O" combined
-CustomLog $root/logs/access_log combined
+Include conf/log.conf
+
+# Configure tracking
 CookieTracking on
 CookieName TuscanyVisitorId
 CookieStyle Cookie
@@ -93,7 +94,7 @@ CookieExpires 31556926
 # Configure Mime types and default charsets
 TypesConfig $here/conf/mime.types
 AddDefaultCharset utf-8
-AddCharset utf-8 .js .css
+AddCharset utf-8 .html .js .css
 
 # Configure cache control
 SetEnvIf Request_URI "^/app.html$" must-revalidate
@@ -183,6 +184,14 @@ Require all granted
 
 EOF
 
+# Configure logging
+cat >$root/conf/log.conf <<EOF
+# Generated by: httpd-conf $*
+ErrorLog $root/logs/error_log
+CustomLog $root/logs/access_log combined
+
+EOF
+
 # Run with the prefork MPM
 cat >$root/conf/mpm.conf <<EOF
 # Generated by: httpd-conf $*
diff --git a/sca-cpp/trunk/modules/http/httpd-ssl-conf b/sca-cpp/trunk/modules/http/httpd-ssl-conf
index 50da9e6b54..dc3b71bcac 100755
--- a/sca-cpp/trunk/modules/http/httpd-ssl-conf
+++ b/sca-cpp/trunk/modules/http/httpd-ssl-conf
@@ -138,7 +138,7 @@ SSLRequire %{SSL_CIPHER_USEKEYSIZE} >= 128
 # "SSL-client-I-DN" "SSL-client-S-DN" "user-track" local-IP virtual-host
 # response-time bytes-received bytes-sent
 LogFormat "[%{%a %b %d %H:%M:%S %Y}t] [sslaccess] %h %l %u %{SSL_PROTOCOL}x %{SSL_CIPHER}x \"%r\" %>s %b \"%{Referer}i\" \"%{User-agent}i\" \"%{SSL_CLIENT_I_DN}x\" \"%{SSL_CLIENT_S_DN}x\" \"%{cookie}n\" %A %V %D %I %O" sslcombined
-CustomLog $root/logs/ssl_access_log sslcombined
+Include conf/log-ssl.conf
 
 # Enable HTTPS reverse proxy
 ProxyRequests Off
@@ -154,6 +154,14 @@ SSLProxyCheckPeerCN Off
 
 EOF
 
+# Configure logging
+cat >$root/conf/log-ssl.conf <<EOF
+# Generated by: httpd-ssl-conf $*
+CustomLog $root/logs/ssl_access_log sslcombined
+
+EOF
+
+# Configure virtual hosts
 proxycert="server"
 if [ "$proxyconf" != "" ]; then
     proxycert="proxy"
diff --git a/sca-cpp/trunk/modules/http/mod-security-conf b/sca-cpp/trunk/modules/http/mod-security-conf
index a1373906fd..fdc4d8e24d 100755
--- a/sca-cpp/trunk/modules/http/mod-security-conf
+++ b/sca-cpp/trunk/modules/http/mod-security-conf
@@ -46,7 +46,7 @@ cat >$root/conf/mod-security.conf <<EOF
 # Generated by: mod-security-conf $*
 # Enable mod-security rules
 SecRuleEngine On
-SecDefaultAction "phase:2,pass,log"
+SecDefaultAction "phase:2,pass,nolog,auditlog"
 
 #SecDebugLog $root/logs//modsec_debug_log
 #SecDebugLogLevel 3
@@ -100,12 +100,12 @@ SecResponseBodyLimitAction ProcessPartial
 SecTmpDir $root/tmp/
 SecDataDir $root/tmp/
 
-# Enable mod-security audit
+# Enable mod-security audit log
 SecAuditEngine RelevantOnly
 SecAuditLogRelevantStatus "^(?:5|4(?!04))"
 SecAuditLogParts ABIJDEFHKZ
 SecAuditLogType Serial
-SecAuditLog $root/logs/modsec_audit_log
+Include conf/mod-security-log.conf
 
 # Use & as application/x-www-form-urlencoded parameter separator
 SecArgumentSeparator &
@@ -188,3 +188,10 @@ Include ${modsecurity_prefix}/optional_rules/modsecurity_crs_47_skip_outbound_ch
 Include ${modsecurity_prefix}/optional_rules/modsecurity_crs_55_application_defects.conf
 EOF
 
+# Configure audit logging
+cat >$root/conf/mod-security-log.conf <<EOF
+# Generated by: mod-security-conf $*
+SecAuditLog $root/logs/modsec_audit_log
+
+EOF
+
diff --git a/sca-cpp/trunk/modules/server/mod-eval.hpp b/sca-cpp/trunk/modules/server/mod-eval.hpp
index b94636074a..4c305fccbf 100644
--- a/sca-cpp/trunk/modules/server/mod-eval.hpp
+++ b/sca-cpp/trunk/modules/server/mod-eval.hpp
@@ -97,7 +97,7 @@ const bool hasVirtualCompositeConf(const ServerConf& sc) {
 const failable<value> failableResult(const list<value>& v) {
     if (isNil(cdr(v)))
         return car(v);
-    return mkfailure<value>(string(cadr(v)));
+    return mkfailure<value>(string(cadr(v)), false);
 }
 
 /**