Cleanup Scribe log output and add ability to log to a firehose Unix fifo pipe.

git-svn-id: http://svn.us.apache.org/repos/asf/tuscany@1311137 13f79535-47bb-0310-9956-ffa450edef68

14 files changed, 273 insertions, 39 deletions

diff --git a/sca-cpp/trunk/components/cache/memcached-start b/sca-cpp/trunk/components/cache/memcached-start
index 692a066010..5a71e7189f 100755
--- a/sca-cpp/trunk/components/cache/memcached-start
+++ b/sca-cpp/trunk/components/cache/memcached-start
@@ -34,7 +34,7 @@ memcached_prefix=`cat $here/memcached.prefix`
 
 if [ -f "$root/memcached/log.conf" ]; then
     log=`cat $root/memcached/log.conf`
-    v="-vv"
+    v="-v"
 else
     mkdir -p $root/logs
     log="cat >>$root/logs/memcached"
diff --git a/sca-cpp/trunk/components/log/Makefile.am b/sca-cpp/trunk/components/log/Makefile.am
index 5d685babec..0e96be5697 100644
--- a/sca-cpp/trunk/components/log/Makefile.am
+++ b/sca-cpp/trunk/components/log/Makefile.am
@@ -22,7 +22,7 @@ INCLUDES = -I${THRIFT_INCLUDE} -I${FB303_INCLUDE}
 incl_HEADERS = *.hpp
 incldir = $(prefix)/include/components/log
 
-dist_comp_SCRIPTS = scribed-central-conf scribed-client-conf scribed-central-start scribed-central-stop scribed-client-start scribed-client-stop scribe-tail-start scribe-tail-stop
+dist_comp_SCRIPTS = scribed-central-conf scribed-central-firehose-conf scribed-central-mkfirehose scribed-client-conf scribed-central-start scribed-central-stop scribed-client-start scribed-client-stop scribe-tail-start scribe-tail-stop
 compdir=$(prefix)/components/log
 
 comp_DATA = scribe.prefix thrift.prefix
diff --git a/sca-cpp/trunk/components/log/scribe-cat.cpp b/sca-cpp/trunk/components/log/scribe-cat.cpp
index 77011e8b12..7f48ddb59b 100644
--- a/sca-cpp/trunk/components/log/scribe-cat.cpp
+++ b/sca-cpp/trunk/components/log/scribe-cat.cpp
@@ -36,33 +36,35 @@
 namespace tuscany {
 namespace scribecat {
 
-int cat(const string& category, const string& type) {
+int cat(const string& host, const string& category, const string& type) {
     // Connect to Scribe
     scribe::Scribe& sc = *(new (gc_new<scribe::Scribe>()) scribe::Scribe("localhost", 1464));
 
     // Read lines from stdin and log them
-    char buf[8192];
+    char buf[8193];
     for (;;) {
-        const char* s = fgets(buf, 8192, stdin);
+        gc_scoped_pool();
+
+        // Write line prefix
+        ostringstream os;
+        os << "[" << host << "] ";
+        if (length(type) != 0)
+            os << "[" << logTime() << "] [" << type << "] ";
+        const string prefix = str(os);
+        const int pl = length(prefix);
+        strcpy(buf, c_str(prefix));
+
+        // Read log line
+        const char* s = fgets(buf + pl, 8192 - pl, stdin);
         if (s == NULL)
             return 0;
         const size_t l = strlen(s);
         if (l < 2)
             return 0;
-        buf[l - 1] = '\0';
+        buf[pl + l - 1] = '\0';
 
-        // Log each line as is
-        if (length(type) == 0) {
-            const failable<bool> val = scribe::log(buf, category, sc);
-            if (!hasContent(val))
-                return 1;
-            continue;
-        }
-        
-        // Log each line prefixed with time and a type tag
-        ostringstream os;
-        os << "[" << logTime() << "] [" << type << "] " << buf;
-        const failable<bool> val = scribe::log(c_str(str(os)), category, sc);
+        // Log the line
+        const failable<bool> val = scribe::log(buf, category, sc);
         if (!hasContent(val))
             return 1;
     }
@@ -72,6 +74,6 @@ int cat(const string& category, const string& type) {
 }
 
 int main(const int argc, const char** argv) {
-    return tuscany::scribecat::cat(argc < 2? "default" : argv[1], argc < 3? "" : argv[2]);
+    return tuscany::scribecat::cat(argc < 2? "localhost" : argv[1], argc < 3? "default" : argv[2], argc < 4? "" : argv[3]);
 }
 
diff --git a/sca-cpp/trunk/components/log/scribe-tail-start b/sca-cpp/trunk/components/log/scribe-tail-start
index 22f5101053..fc469b5488 100755
--- a/sca-cpp/trunk/components/log/scribe-tail-start
+++ b/sca-cpp/trunk/components/log/scribe-tail-start
@@ -35,10 +35,11 @@ else
         file=$1
     fi
 fi
+host=`hostname`
 
 mkdir -p `dirname $file`
 touch $file
 file=`echo "import os; print os.path.realpath('$file')" | python`
 
-tail -f -n 0 $file | $here/scribe-cat $category $type &
+tail -f -n 0 $file | $here/scribe-cat $host $category $type &
 
diff --git a/sca-cpp/trunk/components/log/scribed-central-conf b/sca-cpp/trunk/components/log/scribed-central-conf
index d8d008623d..066dad4add 100755
--- a/sca-cpp/trunk/components/log/scribed-central-conf
+++ b/sca-cpp/trunk/components/log/scribed-central-conf
@@ -42,7 +42,6 @@ check_interval=3
 <store>
 category=default
 type=buffer
-
 target_write_size=20480
 max_write_interval=1
 buffer_send_rate=2
@@ -50,7 +49,10 @@ retry_interval=30
 retry_interval_range=10
 
 <primary>
+category=default
 type=file
+target_write_size=20480
+max_write_interval=1
 fs_type=std
 file_path=$root/scribe/logs/central
 base_filename=central
@@ -62,7 +64,10 @@ rotate_minute=10
 </primary>
 
 <secondary>
+category=default
 type=file
+target_write_size=20480
+max_write_interval=1
 fs_type=std
 file_path=$root/scribe/logs/central-secondary
 base_filename=central
diff --git a/sca-cpp/trunk/components/log/scribed-central-firehose-conf b/sca-cpp/trunk/components/log/scribed-central-firehose-conf
new file mode 100755
index 0000000000..660cf7820a
--- /dev/null
+++ b/sca-cpp/trunk/components/log/scribed-central-firehose-conf
@@ -0,0 +1,125 @@
+#!/bin/sh
+
+#  Licensed to the Apache Software Foundation (ASF) under one
+#  or more contributor license agreements.  See the NOTICE file
+#  distributed with this work for additional information
+#  regarding copyright ownership.  The ASF licenses this file
+#  to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#  
+#    http://www.apache.org/licenses/LICENSE-2.0
+#    
+#  Unless required by applicable law or agreed to in writing,
+#  software distributed under the License is distributed on an
+#  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+#  KIND, either express or implied.  See the License for the
+#  specific language governing permissions and limitations
+#  under the License.
+
+# Generate a Scribe central conf
+here=`echo "import os; print os.path.realpath('$0')" | python`; here=`dirname $here`
+mkdir -p $1
+root=`echo "import os; print os.path.realpath('$1')" | python`
+
+port=$2
+if [ "$port" = "" ]; then
+    port="1463"
+fi
+
+mkdir -p $root/scribe/conf
+mkdir -p $root/scribe/logs/central
+mkdir -p $root/scribe/logs/central-secondary
+mkdir -p $root/scribe/logs/firehose
+mkdir -p $root/scribe/logs/firehose-secondary
+
+cat >$root/scribe/conf/scribe-central.conf <<EOF
+# Generated by: scribed-central-conf $*
+# Scribe central configuration
+port=$port
+max_msg_per_second=2000000
+check_interval=3
+
+# Log store configuration
+<store>
+category=default
+type=multi
+target_write_size=20480
+max_write_interval=1
+
+<store0>
+category=default
+type=buffer
+target_write_size=20480
+max_write_interval=1
+buffer_send_rate=2
+retry_interval=30
+retry_interval_range=10
+
+<primary>
+category=default
+type=file
+target_write_size=20480
+max_write_interval=1
+fs_type=std
+file_path=$root/scribe/logs/central
+base_filename=central
+max_size=1000000
+add_newlines=1
+rotate_period=daily
+rotate_hour=0
+rotate_minute=10
+</primary>
+
+<secondary>
+category=default
+type=file
+target_write_size=20480
+max_write_interval=1
+fs_type=std
+file_path=$root/scribe/logs/central-secondary
+base_filename=central
+max_size=3000000
+</secondary>
+
+</store0>
+
+<store1>
+category=default
+type=buffer
+target_write_size=20480
+max_write_interval=1
+buffer_send_rate=2
+retry_interval=30
+retry_interval_range=10
+
+<primary>
+category=default
+type=file
+target_write_size=20480
+max_write_interval=1
+fs_type=std
+file_path=$root/scribe/logs/firehose
+base_filename=central
+max_size=1000000
+add_newlines=1
+write_stats=no
+create_symlink=no
+</primary>
+
+<secondary>
+category=default
+type=file
+target_write_size=20480
+max_write_interval=1
+fs_type=std
+file_path=$root/scribe/logs/firehose-secondary
+base_filename=central
+max_size=3000000
+</secondary>
+
+</store1>
+
+</store>
+
+EOF
diff --git a/sca-cpp/trunk/components/log/scribed-central-mkfirehose b/sca-cpp/trunk/components/log/scribed-central-mkfirehose
new file mode 100755
index 0000000000..9b02305d1d
--- /dev/null
+++ b/sca-cpp/trunk/components/log/scribed-central-mkfirehose
@@ -0,0 +1,34 @@
+#!/bin/sh
+
+#  Licensed to the Apache Software Foundation (ASF) under one
+#  or more contributor license agreements.  See the NOTICE file
+#  distributed with this work for additional information
+#  regarding copyright ownership.  The ASF licenses this file
+#  to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#  
+#    http://www.apache.org/licenses/LICENSE-2.0
+#    
+#  Unless required by applicable law or agreed to in writing,
+#  software distributed under the License is distributed on an
+#  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+#  KIND, either express or implied.  See the License for the
+#  specific language governing permissions and limitations
+#  under the License.
+
+# Create a firehose fifo pipe for a log category
+here=`echo "import os; print os.path.realpath('$0')" | python`; here=`dirname $here`
+mkdir -p $1
+root=`echo "import os; print os.path.realpath('$1')" | python`
+
+category=$2
+if [ "$category" = "" ]; then
+    category="default"
+fi
+
+mkdir -p $root/scribe/logs/firehose/$category
+if [ ! -e "$root/scribe/logs/firehose/$category/$category""_00000" ]; then
+    mkfifo "$root/scribe/logs/firehose/$category/$category""_00000"
+fi
+
diff --git a/sca-cpp/trunk/components/sqldb/pgsql-conf b/sca-cpp/trunk/components/sqldb/pgsql-conf
index 21222a1183..58d38412ca 100755
--- a/sca-cpp/trunk/components/sqldb/pgsql-conf
+++ b/sca-cpp/trunk/components/sqldb/pgsql-conf
@@ -61,15 +61,15 @@ cat >>$root/sqldb/data/postgresql.conf <<EOF
 # Generated by: pgsql-conf $*
 
 # Setup logging
-log_min_messages = INFO
-log_min_error_statement = INFO
-log_min_duration_statement = 0
+log_min_messages = NOTICE
+log_min_error_statement = NOTICE
+log_min_duration_statement = -1
 log_checkpoints = on
-log_connections = on
-log_disconnections = on
-log_duration = on
+log_connections = off
+log_disconnections = off
+log_duration = off
 log_lock_waits = on
-log_statement = all
+log_statement = none
 
 # Listen
 listen_addresses = '$ip'
diff --git a/sca-cpp/trunk/components/sqldb/pgsql-standby-conf b/sca-cpp/trunk/components/sqldb/pgsql-standby-conf
index b5dd238282..bc0d4e9981 100755
--- a/sca-cpp/trunk/components/sqldb/pgsql-standby-conf
+++ b/sca-cpp/trunk/components/sqldb/pgsql-standby-conf
@@ -63,7 +63,7 @@ mkdir -p $root/sqldb/archive
 
 # Initialize from a backup of the master
 if [ ! -f $root/sqldb/data/postgresql.conf ]; then
-    (curl -L http://$mhost:$mhttpport/pgsql-backup | tar -C $root/sqldb -xz) 2>&1 | sh $root/sqldb/logger
+    (curl -L -# http://$mhost:$mhttpport/pgsql-backup | tar -C $root/sqldb -xz) 2>&1 | grep -v "100.0%" | sh $root/sqldb/logger
     rm -rf $root/sqldb/data/postmaster.pid $root/sqldb/data/pg_xlog
     mkdir -p $root/sqldb/data/pg_xlog/archive_status
     chmod 700 $root/sqldb/data/pg_xlog/archive_status
@@ -73,7 +73,18 @@ fi
 cp $root/sqldb/data/postgresql-init.conf $root/sqldb/data/postgresql.conf
 cat >>$root/sqldb/data/postgresql.conf <<EOF
 
-# Generated by: standby-conf $*
+# Generated by: pgsql-standby-conf $*
+
+# Setup logging
+log_min_messages = NOTICE
+log_min_error_statement = NOTICE
+log_min_duration_statement = -1
+log_checkpoints = on
+log_connections = off
+log_disconnections = off
+log_duration = off
+log_lock_waits = on
+log_statement = none
 
 # Listen
 listen_addresses = '$ip'
@@ -95,7 +106,7 @@ EOF
 
 # Generate recovery configuration
 cat >$root/sqldb/data/recovery.conf << EOF
-# Generated by: pgsql-slave-conf $*
+# Generated by: pgsql-standby-conf $*
 
 # Start in standby mode
 standby_mode = 'on'
@@ -104,7 +115,7 @@ primary_conninfo = 'host=$mhost port=$mport'
 # Failover
 trigger_file = '$root/sqldb/failover'
 
-restore_command = 'curl http://$mhost:$mhttpport/pgsql-archive/%f -o "%p"'
+restore_command = 'curl -L -# http://$mhost:$mhttpport/pgsql-archive/%f -o "%p" 2>&1 | grep -v "100.0%"'
 
 EOF
 
@@ -119,7 +130,7 @@ chmod 700 $root/sqldb/scripts/backup
 # Configure HTTPD to serve backup and archive files
 if [ -f "$root/conf/httpd.conf" ]; then
     cat >>$root/conf/httpd.conf <<EOF
-# Generated by: pgsql-conf $*
+# Generated by: pgsql-standby-conf $*
 
 # Serve PostgreSQL backup and WAL archive files
 ScriptAlias /pgsql-backup "$root/sqldb/scripts/backup"
diff --git a/sca-cpp/trunk/hosting/server/ssl-start b/sca-cpp/trunk/hosting/server/ssl-start
index 49d2b424b9..bbe6a46ea1 100755
--- a/sca-cpp/trunk/hosting/server/ssl-start
+++ b/sca-cpp/trunk/hosting/server/ssl-start
@@ -22,6 +22,7 @@
 
 here=`echo "import os; print os.path.realpath('$0')" | python`; here=`dirname $here`
 jsprefix=`echo "import os; print os.path.realpath('$here/../../modules/js')" | python`
+host=`hostname`
 
 # Create SSL certificates
 ../../modules/http/ssl-ca-conf tmp www.example.com
@@ -81,20 +82,20 @@ fi
 if [ -x ../../components/log/scribe-cat ]; then
     cat >tmp/conf/log.conf <<EOF
 # Generated by: ssl-start $*
-ErrorLog "|$here/../../components/log/scribe-cat server"
-CustomLog "|$here/../../components/log/scribe-cat server" combined
+ErrorLog "|$here/../../components/log/scribe-cat $host server"
+CustomLog "|$here/../../components/log/scribe-cat $host server" combined
 
 EOF
 
     cat >tmp/conf/log-ssl.conf <<EOF
 # Generated by: ssl-start $*
-CustomLog "|$here/../../components/log/scribe-cat server" sslcombined
+CustomLog "|$here/../../components/log/scribe-cat $host server" sslcombined
 
 EOF
 
 #    cat >tmp/conf/mod-security-log.conf <<EOF
 # Generated by: ssl-start $*
-#SecAuditLog "|$here/../../components/log/scribe-cat secaudit"
+#SecAuditLog "|$here/../../components/log/scribe-cat $host secaudit"
 #
 #EOF
 
diff --git a/sca-cpp/trunk/hosting/server/start b/sca-cpp/trunk/hosting/server/start
index 1737efb7b1..a608ccb2e5 100755
--- a/sca-cpp/trunk/hosting/server/start
+++ b/sca-cpp/trunk/hosting/server/start
@@ -22,6 +22,7 @@
 
 here=`echo "import os; print os.path.realpath('$0')" | python`; here=`dirname $here`
 jsprefix=`echo "import os; print os.path.realpath('$here/../../modules/js')" | python`
+host=`hostname`
 
 # Configure and start logging
 if [ -x ../../components/log/scribe-cat ]; then
@@ -48,8 +49,8 @@ if [ -x ../../components/log/scribe-cat ]; then
     cat >tmp/conf/log.conf <<EOF
 # Generated by: start $*
 LogLevel notice
-ErrorLog "|$here/../../components/log/scribe-cat server"
-CustomLog "|$here/../../components/log/scribe-cat server" combined
+ErrorLog "|$here/../../components/log/scribe-cat $host server"
+CustomLog "|$here/../../components/log/scribe-cat $host server" combined
 
 EOF
 
diff --git a/sca-cpp/trunk/patches/scribe.patch b/sca-cpp/trunk/patches/scribe.patch
new file mode 100644
index 0000000000..6340c4634d
--- /dev/null
+++ b/sca-cpp/trunk/patches/scribe.patch
@@ -0,0 +1,50 @@
+--- src/common.h
++++ src/common.h
+@@ -42,6 +42,8 @@
+ #include <sys/types.h>
+ #include <sys/stat.h>
+ #include <unistd.h>
++#include <signal.h>
++#include <fcntl.h>
+ #include <boost/shared_ptr.hpp>
+ #include <boost/filesystem/operations.hpp>
+ #include <boost/filesystem/convenience.hpp>
+
+--- src/file.cpp
++++ src/file.cpp
+@@ -74,9 +74,21 @@ bool StdFile::openRead() {
+ }
+ 
+ bool StdFile::openWrite() {
++  // if file is a fifo, temporarily open it for read
++  int fd = -1;
++  struct stat st;
++  int s = stat(filename.c_str(), &st);
++  if (s != -1 && S_ISFIFO(st.st_mode))
++    fd = ::open(filename.c_str(), O_RDONLY | O_NONBLOCK);
++
+   // open file for write in append mode
+   ios_base::openmode mode = fstream::out | fstream::app;
+-  return open(mode);
++  bool r = open(mode);
++
++  // close fifo
++  if (fd != -1)
++    ::close(fd);
++  return r;
+ }
+ 
+ bool StdFile::openTruncate() {
+
+--- src/scribe_server.cpp
++++ src/scribe_server.cpp
+@@ -55,6 +55,8 @@ int main(int argc, char **argv) {
+     if (-1 == setrlimit(RLIMIT_NOFILE, &r_fd)) {
+       LOG_OPER("setrlimit error (setting max fd size)");
+     }
++    
++    signal(SIGPIPE, SIG_IGN);
+ 
+     int next_option;
+     const char* const short_options = "hp:c:";
+
diff --git a/sca-cpp/trunk/ubuntu/ubuntu-install b/sca-cpp/trunk/ubuntu/ubuntu-install
index 231e76b433..ce634a1e21 100755
--- a/sca-cpp/trunk/ubuntu/ubuntu-install
+++ b/sca-cpp/trunk/ubuntu/ubuntu-install
@@ -302,6 +302,8 @@ fi
 curl -OL http://github.com/downloads/facebook/scribe/scribe-2.2.tar.gz
 tar xzf scribe-2.2.tar.gz
 cd scribe
+curl -OL http://svn.apache.org/repos/asf/tuscany/sca-cpp/trunk/patches/scribe.patch
+patch -p0 <scribe.patch
 ./bootstrap.sh --prefix=$build/scribe-2.2-bin PY_PREFIX=$build/scribe-2.2-bin --with-thriftpath=$build/thrift-0.2.0-bin --with-fb303path=$build/thrift-0.2.0-bin/contrib/fb303 --disable-static
 make
 make install
diff --git a/sca-cpp/trunk/ubuntu/ubuntu-install-all b/sca-cpp/trunk/ubuntu/ubuntu-install-all
index c87f9a8541..63500e953b 100755
--- a/sca-cpp/trunk/ubuntu/ubuntu-install-all
+++ b/sca-cpp/trunk/ubuntu/ubuntu-install-all
@@ -356,6 +356,8 @@ fi
 curl -OL http://github.com/downloads/facebook/scribe/scribe-2.2.tar.gz
 tar xzf scribe-2.2.tar.gz
 cd scribe
+curl -OL http://svn.apache.org/repos/asf/tuscany/sca-cpp/trunk/patches/scribe.patch
+patch -p0 <scribe.patch
 ./bootstrap.sh --prefix=$build/scribe-2.2-bin PY_PREFIX=$build/scribe-2.2-bin --with-thriftpath=$build/thrift-0.2.0-bin --with-fb303path=$build/thrift-0.2.0-bin/contrib/fb303 --disable-static
 make
 make install