archive/apache-tuscany
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Cleanup Scribe log output and add ability to log to a firehose Unix fifo pipe.

Browse source 
git-svn-id: http://svn.us.apache.org/repos/asf/tuscany@1311137 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
jsdelfino 2012-04-09 06:22:16 +00:00
commit 10753b34f3
14 changed files with 273 additions and 39 deletions
Download patch file Download diff file Expand all files Collapse all files

2
sca-cpp/trunk/components/cache/memcached-start vendored
View file

@ -34,7 +34,7 @@ memcached_prefix=`cat $here/memcached.prefix`
if [ -f "$root/memcached/log.conf" ]; then
log=`cat $root/memcached/log.conf`
v="-vv"
v="-v"
else
mkdir -p $root/logs
log="cat >>$root/logs/memcached"

2
sca-cpp/trunk/components/log/Makefile.am
View file

@ -22,7 +22,7 @@ INCLUDES = -I${THRIFT_INCLUDE} -I${FB303_INCLUDE}
incl_HEADERS = *.hpp
incldir = $(prefix)/include/components/log
dist_comp_SCRIPTS = scribed-central-conf scribed-client-conf scribed-central-start scribed-central-stop scribed-client-start scribed-client-stop scribe-tail-start scribe-tail-stop
dist_comp_SCRIPTS = scribed-central-conf scribed-central-firehose-conf scribed-central-mkfirehose scribed-client-conf scribed-central-start scribed-central-stop scribed-client-start scribed-client-stop scribe-tail-start scribe-tail-stop
compdir=$(prefix)/components/log
comp_DATA = scribe.prefix thrift.prefix

36
sca-cpp/trunk/components/log/scribe-cat.cpp
View file

@ -36,33 +36,35 @@
namespace tuscany {
namespace scribecat {
int cat(const string& category, const string& type) {
int cat(const string& host, const string& category, const string& type) {
// Connect to Scribe
scribe::Scribe& sc = *(new (gc_new<scribe::Scribe>()) scribe::Scribe("localhost", 1464));
// Read lines from stdin and log them
char buf[8192];
char buf[8193];
for (;;) {
const char* s = fgets(buf, 8192, stdin);
gc_scoped_pool();
// Write line prefix
ostringstream os;
os << "[" << host << "] ";
if (length(type) != 0)
os << "[" << logTime() << "] [" << type << "] ";
const string prefix = str(os);
const int pl = length(prefix);
strcpy(buf, c_str(prefix));
// Read log line
const char* s = fgets(buf + pl, 8192 - pl, stdin);
if (s == NULL)
return 0;
const size_t l = strlen(s);
if (l < 2)
return 0;
buf[l - 1] = '\0';
buf[pl + l - 1] = '\0';
// Log each line as is
if (length(type) == 0) {
const failable<bool> val = scribe::log(buf, category, sc);
if (!hasContent(val))
return 1;
continue;
}
// Log each line prefixed with time and a type tag
ostringstream os;
os << "[" << logTime() << "] [" << type << "] " << buf;
const failable<bool> val = scribe::log(c_str(str(os)), category, sc);
// Log the line
const failable<bool> val = scribe::log(buf, category, sc);
if (!hasContent(val))
return 1;
}
@ -72,6 +74,6 @@ int cat(const string& category, const string& type) {
}
int main(const int argc, const char** argv) {
return tuscany::scribecat::cat(argc < 2? "default" : argv[1], argc < 3? "" : argv[2]);
return tuscany::scribecat::cat(argc < 2? "localhost" : argv[1], argc < 3? "default" : argv[2], argc < 4? "" : argv[3]);
}

3
sca-cpp/trunk/components/log/scribe-tail-start
View file

@ -35,10 +35,11 @@ else
file=$1
fi
fi
host=`hostname`
mkdir -p `dirname $file`
touch $file
file=`echo "import os; print os.path.realpath('$file')" | python`
tail -f -n 0 $file | $here/scribe-cat $category $type &
tail -f -n 0 $file | $here/scribe-cat $host $category $type &

7
sca-cpp/trunk/components/log/scribed-central-conf
View file

@ -42,7 +42,6 @@ check_interval=3
<store>
category=default
type=buffer
target_write_size=20480
max_write_interval=1
buffer_send_rate=2
@ -50,7 +49,10 @@ retry_interval=30
retry_interval_range=10
<primary>
category=default
type=file
target_write_size=20480
max_write_interval=1
fs_type=std
file_path=$root/scribe/logs/central
base_filename=central
@ -62,7 +64,10 @@ rotate_minute=10
</primary>
<secondary>
category=default
type=file
target_write_size=20480
max_write_interval=1
fs_type=std
file_path=$root/scribe/logs/central-secondary
base_filename=central

125
sca-cpp/trunk/components/log/scribed-central-firehose-conf Executable file
View file

@ -0,0 +1,125 @@
#!/bin/sh
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
# Generate a Scribe central conf
here=`echo "import os; print os.path.realpath('$0')" | python`; here=`dirname $here`
mkdir -p $1
root=`echo "import os; print os.path.realpath('$1')" | python`
port=$2
if [ "$port" = "" ]; then
port="1463"
fi
mkdir -p $root/scribe/conf
mkdir -p $root/scribe/logs/central
mkdir -p $root/scribe/logs/central-secondary
mkdir -p $root/scribe/logs/firehose
mkdir -p $root/scribe/logs/firehose-secondary
cat >$root/scribe/conf/scribe-central.conf <<EOF
# Generated by: scribed-central-conf $*
# Scribe central configuration
port=$port
max_msg_per_second=2000000
check_interval=3
# Log store configuration
<store>
category=default
type=multi
target_write_size=20480
max_write_interval=1
<store0>
category=default
type=buffer
target_write_size=20480
max_write_interval=1
buffer_send_rate=2
retry_interval=30
retry_interval_range=10
<primary>
category=default
type=file
target_write_size=20480
max_write_interval=1
fs_type=std
file_path=$root/scribe/logs/central
base_filename=central
max_size=1000000
add_newlines=1
rotate_period=daily
rotate_hour=0
rotate_minute=10
</primary>
<secondary>
category=default
type=file
target_write_size=20480
max_write_interval=1
fs_type=std
file_path=$root/scribe/logs/central-secondary
base_filename=central
max_size=3000000
</secondary>
</store0>
<store1>
category=default
type=buffer
target_write_size=20480
max_write_interval=1
buffer_send_rate=2
retry_interval=30
retry_interval_range=10
<primary>
category=default
type=file
target_write_size=20480
max_write_interval=1
fs_type=std
file_path=$root/scribe/logs/firehose
base_filename=central
max_size=1000000
add_newlines=1
write_stats=no
create_symlink=no
</primary>
<secondary>
category=default
type=file
target_write_size=20480
max_write_interval=1
fs_type=std
file_path=$root/scribe/logs/firehose-secondary
base_filename=central
max_size=3000000
</secondary>
</store1>
</store>
EOF

34
sca-cpp/trunk/components/log/scribed-central-mkfirehose Executable file
View file

@ -0,0 +1,34 @@
#!/bin/sh
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
# Create a firehose fifo pipe for a log category
here=`echo "import os; print os.path.realpath('$0')" | python`; here=`dirname $here`
mkdir -p $1
root=`echo "import os; print os.path.realpath('$1')" | python`
category=$2
if [ "$category" = "" ]; then
category="default"
fi
mkdir -p $root/scribe/logs/firehose/$category
if [ ! -e "$root/scribe/logs/firehose/$category/$category""_00000" ]; then
mkfifo "$root/scribe/logs/firehose/$category/$category""_00000"
fi

14
sca-cpp/trunk/components/sqldb/pgsql-conf
View file

@ -61,15 +61,15 @@ cat >>$root/sqldb/data/postgresql.conf <<EOF
# Generated by: pgsql-conf $*
# Setup logging
log_min_messages = INFO
log_min_error_statement = INFO
log_min_duration_statement = 0
log_min_messages = NOTICE
log_min_error_statement = NOTICE
log_min_duration_statement = -1
log_checkpoints = on
log_connections = on
log_disconnections = on
log_duration = on
log_connections = off
log_disconnections = off
log_duration = off
log_lock_waits = on
log_statement = all
log_statement = none
# Listen
listen_addresses = '$ip'

21
sca-cpp/trunk/components/sqldb/pgsql-standby-conf
View file

@ -63,7 +63,7 @@ mkdir -p $root/sqldb/archive
# Initialize from a backup of the master
if [ ! -f $root/sqldb/data/postgresql.conf ]; then
(curl -L http://$mhost:$mhttpport/pgsql-backup | tar -C $root/sqldb -xz) 2>&1 | sh $root/sqldb/logger
(curl -L -# http://$mhost:$mhttpport/pgsql-backup | tar -C $root/sqldb -xz) 2>&1 | grep -v "100.0%" | sh $root/sqldb/logger
rm -rf $root/sqldb/data/postmaster.pid $root/sqldb/data/pg_xlog
mkdir -p $root/sqldb/data/pg_xlog/archive_status
chmod 700 $root/sqldb/data/pg_xlog/archive_status
@ -73,7 +73,18 @@ fi
cp $root/sqldb/data/postgresql-init.conf $root/sqldb/data/postgresql.conf
cat >>$root/sqldb/data/postgresql.conf <<EOF
# Generated by: standby-conf $*
# Generated by: pgsql-standby-conf $*
# Setup logging
log_min_messages = NOTICE
log_min_error_statement = NOTICE
log_min_duration_statement = -1
log_checkpoints = on
log_connections = off
log_disconnections = off
log_duration = off
log_lock_waits = on
log_statement = none
# Listen
listen_addresses = '$ip'
@ -95,7 +106,7 @@ EOF
# Generate recovery configuration
cat >$root/sqldb/data/recovery.conf << EOF
# Generated by: pgsql-slave-conf $*
# Generated by: pgsql-standby-conf $*
# Start in standby mode
standby_mode = 'on'
@ -104,7 +115,7 @@ primary_conninfo = 'host=$mhost port=$mport'
# Failover
trigger_file = '$root/sqldb/failover'
restore_command = 'curl http://$mhost:$mhttpport/pgsql-archive/%f -o "%p"'
restore_command = 'curl -L -# http://$mhost:$mhttpport/pgsql-archive/%f -o "%p" 2>&1 | grep -v "100.0%"'
EOF
@ -119,7 +130,7 @@ chmod 700 $root/sqldb/scripts/backup
# Configure HTTPD to serve backup and archive files
if [ -f "$root/conf/httpd.conf" ]; then
cat >>$root/conf/httpd.conf <<EOF
# Generated by: pgsql-conf $*
# Generated by: pgsql-standby-conf $*
# Serve PostgreSQL backup and WAL archive files
ScriptAlias /pgsql-backup "$root/sqldb/scripts/backup"

9
sca-cpp/trunk/hosting/server/ssl-start
View file

@ -22,6 +22,7 @@
here=`echo "import os; print os.path.realpath('$0')" | python`; here=`dirname $here`
jsprefix=`echo "import os; print os.path.realpath('$here/../../modules/js')" | python`
host=`hostname`
# Create SSL certificates
../../modules/http/ssl-ca-conf tmp www.example.com
@ -81,20 +82,20 @@ fi
if [ -x ../../components/log/scribe-cat ]; then
cat >tmp/conf/log.conf <<EOF
# Generated by: ssl-start $*
ErrorLog "|$here/../../components/log/scribe-cat server"
CustomLog "|$here/../../components/log/scribe-cat server" combined
ErrorLog "|$here/../../components/log/scribe-cat $host server"
CustomLog "|$here/../../components/log/scribe-cat $host server" combined
EOF
cat >tmp/conf/log-ssl.conf <<EOF
# Generated by: ssl-start $*
CustomLog "|$here/../../components/log/scribe-cat server" sslcombined
CustomLog "|$here/../../components/log/scribe-cat $host server" sslcombined
EOF
# cat >tmp/conf/mod-security-log.conf <<EOF
# Generated by: ssl-start $*
#SecAuditLog "|$here/../../components/log/scribe-cat secaudit"
#SecAuditLog "|$here/../../components/log/scribe-cat $host secaudit"
#
#EOF

5
sca-cpp/trunk/hosting/server/start
View file

@ -22,6 +22,7 @@
here=`echo "import os; print os.path.realpath('$0')" | python`; here=`dirname $here`
jsprefix=`echo "import os; print os.path.realpath('$here/../../modules/js')" | python`
host=`hostname`
# Configure and start logging
if [ -x ../../components/log/scribe-cat ]; then
@ -48,8 +49,8 @@ if [ -x ../../components/log/scribe-cat ]; then
cat >tmp/conf/log.conf <<EOF
# Generated by: start $*
LogLevel notice
ErrorLog "|$here/../../components/log/scribe-cat server"
CustomLog "|$here/../../components/log/scribe-cat server" combined
ErrorLog "|$here/../../components/log/scribe-cat $host server"
CustomLog "|$here/../../components/log/scribe-cat $host server" combined
EOF

50
sca-cpp/trunk/patches/scribe.patch Normal file
View file

@ -0,0 +1,50 @@
--- src/common.h
+++ src/common.h
@@ -42,6 +42,8 @@
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
+#include <signal.h>
+#include <fcntl.h>
#include <boost/shared_ptr.hpp>
#include <boost/filesystem/operations.hpp>
#include <boost/filesystem/convenience.hpp>
--- src/file.cpp
+++ src/file.cpp
@@ -74,9 +74,21 @@ bool StdFile::openRead() {
}
bool StdFile::openWrite() {
+ // if file is a fifo, temporarily open it for read
+ int fd = -1;
+ struct stat st;
+ int s = stat(filename.c_str(), &st);
+ if (s != -1 && S_ISFIFO(st.st_mode))
+ fd = ::open(filename.c_str(), O_RDONLY | O_NONBLOCK);
+
// open file for write in append mode
ios_base::openmode mode = fstream::out | fstream::app;
- return open(mode);
+ bool r = open(mode);
+
+ // close fifo
+ if (fd != -1)
+ ::close(fd);
+ return r;
}
bool StdFile::openTruncate() {
--- src/scribe_server.cpp
+++ src/scribe_server.cpp
@@ -55,6 +55,8 @@ int main(int argc, char **argv) {
if (-1 == setrlimit(RLIMIT_NOFILE, &r_fd)) {
LOG_OPER("setrlimit error (setting max fd size)");
}
+
+ signal(SIGPIPE, SIG_IGN);
int next_option;
const char* const short_options = "hp:c:";

2
sca-cpp/trunk/ubuntu/ubuntu-install
View file

@ -302,6 +302,8 @@ fi
curl -OL http://github.com/downloads/facebook/scribe/scribe-2.2.tar.gz
tar xzf scribe-2.2.tar.gz
cd scribe
curl -OL http://svn.apache.org/repos/asf/tuscany/sca-cpp/trunk/patches/scribe.patch
patch -p0 <scribe.patch
./bootstrap.sh --prefix=$build/scribe-2.2-bin PY_PREFIX=$build/scribe-2.2-bin --with-thriftpath=$build/thrift-0.2.0-bin --with-fb303path=$build/thrift-0.2.0-bin/contrib/fb303 --disable-static
make
make install

2
sca-cpp/trunk/ubuntu/ubuntu-install-all
View file

@ -356,6 +356,8 @@ fi
curl -OL http://github.com/downloads/facebook/scribe/scribe-2.2.tar.gz
tar xzf scribe-2.2.tar.gz
cd scribe
curl -OL http://svn.apache.org/repos/asf/tuscany/sca-cpp/trunk/patches/scribe.patch
patch -p0 <scribe.patch
./bootstrap.sh --prefix=$build/scribe-2.2-bin PY_PREFIX=$build/scribe-2.2-bin --with-thriftpath=$build/thrift-0.2.0-bin --with-fb303path=$build/thrift-0.2.0-bin/contrib/fb303 --disable-static
make
make install