From 10753b34f309db0fbcc484716bafff09faada6fd Mon Sep 17 00:00:00 2001 From: jsdelfino Date: Mon, 9 Apr 2012 06:22:16 +0000 Subject: Cleanup Scribe log output and add ability to log to a firehose Unix fifo pipe. git-svn-id: http://svn.us.apache.org/repos/asf/tuscany@1311137 13f79535-47bb-0310-9956-ffa450edef68 --- sca-cpp/trunk/components/cache/memcached-start | 2 +- sca-cpp/trunk/components/log/Makefile.am | 2 +- sca-cpp/trunk/components/log/scribe-cat.cpp | 36 +++--- sca-cpp/trunk/components/log/scribe-tail-start | 3 +- sca-cpp/trunk/components/log/scribed-central-conf | 7 +- .../components/log/scribed-central-firehose-conf | 125 +++++++++++++++++++++ .../components/log/scribed-central-mkfirehose | 34 ++++++ sca-cpp/trunk/components/sqldb/pgsql-conf | 14 +-- sca-cpp/trunk/components/sqldb/pgsql-standby-conf | 21 +++- sca-cpp/trunk/hosting/server/ssl-start | 9 +- sca-cpp/trunk/hosting/server/start | 5 +- sca-cpp/trunk/patches/scribe.patch | 50 +++++++++ sca-cpp/trunk/ubuntu/ubuntu-install | 2 + sca-cpp/trunk/ubuntu/ubuntu-install-all | 2 + 14 files changed, 273 insertions(+), 39 deletions(-) create mode 100755 sca-cpp/trunk/components/log/scribed-central-firehose-conf create mode 100755 sca-cpp/trunk/components/log/scribed-central-mkfirehose create mode 100644 sca-cpp/trunk/patches/scribe.patch (limited to 'sca-cpp') diff --git a/sca-cpp/trunk/components/cache/memcached-start b/sca-cpp/trunk/components/cache/memcached-start index 692a066010..5a71e7189f 100755 --- a/sca-cpp/trunk/components/cache/memcached-start +++ b/sca-cpp/trunk/components/cache/memcached-start @@ -34,7 +34,7 @@ memcached_prefix=`cat $here/memcached.prefix` if [ -f "$root/memcached/log.conf" ]; then log=`cat $root/memcached/log.conf` - v="-vv" + v="-v" else mkdir -p $root/logs log="cat >>$root/logs/memcached" diff --git a/sca-cpp/trunk/components/log/Makefile.am b/sca-cpp/trunk/components/log/Makefile.am index 5d685babec..0e96be5697 100644 --- a/sca-cpp/trunk/components/log/Makefile.am +++ b/sca-cpp/trunk/components/log/Makefile.am @@ -22,7 +22,7 @@ INCLUDES = -I${THRIFT_INCLUDE} -I${FB303_INCLUDE} incl_HEADERS = *.hpp incldir = $(prefix)/include/components/log -dist_comp_SCRIPTS = scribed-central-conf scribed-client-conf scribed-central-start scribed-central-stop scribed-client-start scribed-client-stop scribe-tail-start scribe-tail-stop +dist_comp_SCRIPTS = scribed-central-conf scribed-central-firehose-conf scribed-central-mkfirehose scribed-client-conf scribed-central-start scribed-central-stop scribed-client-start scribed-client-stop scribe-tail-start scribe-tail-stop compdir=$(prefix)/components/log comp_DATA = scribe.prefix thrift.prefix diff --git a/sca-cpp/trunk/components/log/scribe-cat.cpp b/sca-cpp/trunk/components/log/scribe-cat.cpp index 77011e8b12..7f48ddb59b 100644 --- a/sca-cpp/trunk/components/log/scribe-cat.cpp +++ b/sca-cpp/trunk/components/log/scribe-cat.cpp @@ -36,33 +36,35 @@ namespace tuscany { namespace scribecat { -int cat(const string& category, const string& type) { +int cat(const string& host, const string& category, const string& type) { // Connect to Scribe scribe::Scribe& sc = *(new (gc_new()) scribe::Scribe("localhost", 1464)); // Read lines from stdin and log them - char buf[8192]; + char buf[8193]; for (;;) { - const char* s = fgets(buf, 8192, stdin); + gc_scoped_pool(); + + // Write line prefix + ostringstream os; + os << "[" << host << "] "; + if (length(type) != 0) + os << "[" << logTime() << "] [" << type << "] "; + const string prefix = str(os); + const int pl = length(prefix); + strcpy(buf, c_str(prefix)); + + // Read log line + const char* s = fgets(buf + pl, 8192 - pl, stdin); if (s == NULL) return 0; const size_t l = strlen(s); if (l < 2) return 0; - buf[l - 1] = '\0'; + buf[pl + l - 1] = '\0'; - // Log each line as is - if (length(type) == 0) { - const failable val = scribe::log(buf, category, sc); - if (!hasContent(val)) - return 1; - continue; - } - - // Log each line prefixed with time and a type tag - ostringstream os; - os << "[" << logTime() << "] [" << type << "] " << buf; - const failable val = scribe::log(c_str(str(os)), category, sc); + // Log the line + const failable val = scribe::log(buf, category, sc); if (!hasContent(val)) return 1; } @@ -72,6 +74,6 @@ int cat(const string& category, const string& type) { } int main(const int argc, const char** argv) { - return tuscany::scribecat::cat(argc < 2? "default" : argv[1], argc < 3? "" : argv[2]); + return tuscany::scribecat::cat(argc < 2? "localhost" : argv[1], argc < 3? "default" : argv[2], argc < 4? "" : argv[3]); } diff --git a/sca-cpp/trunk/components/log/scribe-tail-start b/sca-cpp/trunk/components/log/scribe-tail-start index 22f5101053..fc469b5488 100755 --- a/sca-cpp/trunk/components/log/scribe-tail-start +++ b/sca-cpp/trunk/components/log/scribe-tail-start @@ -35,10 +35,11 @@ else file=$1 fi fi +host=`hostname` mkdir -p `dirname $file` touch $file file=`echo "import os; print os.path.realpath('$file')" | python` -tail -f -n 0 $file | $here/scribe-cat $category $type & +tail -f -n 0 $file | $here/scribe-cat $host $category $type & diff --git a/sca-cpp/trunk/components/log/scribed-central-conf b/sca-cpp/trunk/components/log/scribed-central-conf index d8d008623d..066dad4add 100755 --- a/sca-cpp/trunk/components/log/scribed-central-conf +++ b/sca-cpp/trunk/components/log/scribed-central-conf @@ -42,7 +42,6 @@ check_interval=3 category=default type=buffer - target_write_size=20480 max_write_interval=1 buffer_send_rate=2 @@ -50,7 +49,10 @@ retry_interval=30 retry_interval_range=10 +category=default type=file +target_write_size=20480 +max_write_interval=1 fs_type=std file_path=$root/scribe/logs/central base_filename=central @@ -62,7 +64,10 @@ rotate_minute=10 +category=default type=file +target_write_size=20480 +max_write_interval=1 fs_type=std file_path=$root/scribe/logs/central-secondary base_filename=central diff --git a/sca-cpp/trunk/components/log/scribed-central-firehose-conf b/sca-cpp/trunk/components/log/scribed-central-firehose-conf new file mode 100755 index 0000000000..660cf7820a --- /dev/null +++ b/sca-cpp/trunk/components/log/scribed-central-firehose-conf @@ -0,0 +1,125 @@ +#!/bin/sh + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Generate a Scribe central conf +here=`echo "import os; print os.path.realpath('$0')" | python`; here=`dirname $here` +mkdir -p $1 +root=`echo "import os; print os.path.realpath('$1')" | python` + +port=$2 +if [ "$port" = "" ]; then + port="1463" +fi + +mkdir -p $root/scribe/conf +mkdir -p $root/scribe/logs/central +mkdir -p $root/scribe/logs/central-secondary +mkdir -p $root/scribe/logs/firehose +mkdir -p $root/scribe/logs/firehose-secondary + +cat >$root/scribe/conf/scribe-central.conf < +category=default +type=multi +target_write_size=20480 +max_write_interval=1 + + +category=default +type=buffer +target_write_size=20480 +max_write_interval=1 +buffer_send_rate=2 +retry_interval=30 +retry_interval_range=10 + + +category=default +type=file +target_write_size=20480 +max_write_interval=1 +fs_type=std +file_path=$root/scribe/logs/central +base_filename=central +max_size=1000000 +add_newlines=1 +rotate_period=daily +rotate_hour=0 +rotate_minute=10 + + + +category=default +type=file +target_write_size=20480 +max_write_interval=1 +fs_type=std +file_path=$root/scribe/logs/central-secondary +base_filename=central +max_size=3000000 + + + + + +category=default +type=buffer +target_write_size=20480 +max_write_interval=1 +buffer_send_rate=2 +retry_interval=30 +retry_interval_range=10 + + +category=default +type=file +target_write_size=20480 +max_write_interval=1 +fs_type=std +file_path=$root/scribe/logs/firehose +base_filename=central +max_size=1000000 +add_newlines=1 +write_stats=no +create_symlink=no + + + +category=default +type=file +target_write_size=20480 +max_write_interval=1 +fs_type=std +file_path=$root/scribe/logs/firehose-secondary +base_filename=central +max_size=3000000 + + + + + + +EOF diff --git a/sca-cpp/trunk/components/log/scribed-central-mkfirehose b/sca-cpp/trunk/components/log/scribed-central-mkfirehose new file mode 100755 index 0000000000..9b02305d1d --- /dev/null +++ b/sca-cpp/trunk/components/log/scribed-central-mkfirehose @@ -0,0 +1,34 @@ +#!/bin/sh + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Create a firehose fifo pipe for a log category +here=`echo "import os; print os.path.realpath('$0')" | python`; here=`dirname $here` +mkdir -p $1 +root=`echo "import os; print os.path.realpath('$1')" | python` + +category=$2 +if [ "$category" = "" ]; then + category="default" +fi + +mkdir -p $root/scribe/logs/firehose/$category +if [ ! -e "$root/scribe/logs/firehose/$category/$category""_00000" ]; then + mkfifo "$root/scribe/logs/firehose/$category/$category""_00000" +fi + diff --git a/sca-cpp/trunk/components/sqldb/pgsql-conf b/sca-cpp/trunk/components/sqldb/pgsql-conf index 21222a1183..58d38412ca 100755 --- a/sca-cpp/trunk/components/sqldb/pgsql-conf +++ b/sca-cpp/trunk/components/sqldb/pgsql-conf @@ -61,15 +61,15 @@ cat >>$root/sqldb/data/postgresql.conf <&1 | sh $root/sqldb/logger + (curl -L -# http://$mhost:$mhttpport/pgsql-backup | tar -C $root/sqldb -xz) 2>&1 | grep -v "100.0%" | sh $root/sqldb/logger rm -rf $root/sqldb/data/postmaster.pid $root/sqldb/data/pg_xlog mkdir -p $root/sqldb/data/pg_xlog/archive_status chmod 700 $root/sqldb/data/pg_xlog/archive_status @@ -73,7 +73,18 @@ fi cp $root/sqldb/data/postgresql-init.conf $root/sqldb/data/postgresql.conf cat >>$root/sqldb/data/postgresql.conf <$root/sqldb/data/recovery.conf << EOF -# Generated by: pgsql-slave-conf $* +# Generated by: pgsql-standby-conf $* # Start in standby mode standby_mode = 'on' @@ -104,7 +115,7 @@ primary_conninfo = 'host=$mhost port=$mport' # Failover trigger_file = '$root/sqldb/failover' -restore_command = 'curl http://$mhost:$mhttpport/pgsql-archive/%f -o "%p"' +restore_command = 'curl -L -# http://$mhost:$mhttpport/pgsql-archive/%f -o "%p" 2>&1 | grep -v "100.0%"' EOF @@ -119,7 +130,7 @@ chmod 700 $root/sqldb/scripts/backup # Configure HTTPD to serve backup and archive files if [ -f "$root/conf/httpd.conf" ]; then cat >>$root/conf/httpd.conf <tmp/conf/log.conf <tmp/conf/log-ssl.conf <tmp/conf/mod-security-log.conf <tmp/conf/log.conf < + #include + #include ++#include ++#include + #include + #include + #include + +--- src/file.cpp ++++ src/file.cpp +@@ -74,9 +74,21 @@ bool StdFile::openRead() { + } + + bool StdFile::openWrite() { ++ // if file is a fifo, temporarily open it for read ++ int fd = -1; ++ struct stat st; ++ int s = stat(filename.c_str(), &st); ++ if (s != -1 && S_ISFIFO(st.st_mode)) ++ fd = ::open(filename.c_str(), O_RDONLY | O_NONBLOCK); ++ + // open file for write in append mode + ios_base::openmode mode = fstream::out | fstream::app; +- return open(mode); ++ bool r = open(mode); ++ ++ // close fifo ++ if (fd != -1) ++ ::close(fd); ++ return r; + } + + bool StdFile::openTruncate() { + +--- src/scribe_server.cpp ++++ src/scribe_server.cpp +@@ -55,6 +55,8 @@ int main(int argc, char **argv) { + if (-1 == setrlimit(RLIMIT_NOFILE, &r_fd)) { + LOG_OPER("setrlimit error (setting max fd size)"); + } ++ ++ signal(SIGPIPE, SIG_IGN); + + int next_option; + const char* const short_options = "hp:c:"; + diff --git a/sca-cpp/trunk/ubuntu/ubuntu-install b/sca-cpp/trunk/ubuntu/ubuntu-install index 231e76b433..ce634a1e21 100755 --- a/sca-cpp/trunk/ubuntu/ubuntu-install +++ b/sca-cpp/trunk/ubuntu/ubuntu-install @@ -302,6 +302,8 @@ fi curl -OL http://github.com/downloads/facebook/scribe/scribe-2.2.tar.gz tar xzf scribe-2.2.tar.gz cd scribe +curl -OL http://svn.apache.org/repos/asf/tuscany/sca-cpp/trunk/patches/scribe.patch +patch -p0