summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rwxr-xr-xsca-cpp/trunk/components/cache/memcached-start2
-rw-r--r--sca-cpp/trunk/components/log/Makefile.am2
-rw-r--r--sca-cpp/trunk/components/log/scribe-cat.cpp36
-rwxr-xr-xsca-cpp/trunk/components/log/scribe-tail-start3
-rwxr-xr-xsca-cpp/trunk/components/log/scribed-central-conf7
-rwxr-xr-xsca-cpp/trunk/components/log/scribed-central-firehose-conf125
-rwxr-xr-xsca-cpp/trunk/components/log/scribed-central-mkfirehose34
-rwxr-xr-xsca-cpp/trunk/components/sqldb/pgsql-conf14
-rwxr-xr-xsca-cpp/trunk/components/sqldb/pgsql-standby-conf21
-rwxr-xr-xsca-cpp/trunk/hosting/server/ssl-start9
-rwxr-xr-xsca-cpp/trunk/hosting/server/start5
-rw-r--r--sca-cpp/trunk/patches/scribe.patch50
-rwxr-xr-xsca-cpp/trunk/ubuntu/ubuntu-install2
-rwxr-xr-xsca-cpp/trunk/ubuntu/ubuntu-install-all2
14 files changed, 273 insertions, 39 deletions
diff --git a/sca-cpp/trunk/components/cache/memcached-start b/sca-cpp/trunk/components/cache/memcached-start
index 692a066010..5a71e7189f 100755
--- a/sca-cpp/trunk/components/cache/memcached-start
+++ b/sca-cpp/trunk/components/cache/memcached-start
@@ -34,7 +34,7 @@ memcached_prefix=`cat $here/memcached.prefix`
if [ -f "$root/memcached/log.conf" ]; then
log=`cat $root/memcached/log.conf`
- v="-vv"
+ v="-v"
else
mkdir -p $root/logs
log="cat >>$root/logs/memcached"
diff --git a/sca-cpp/trunk/components/log/Makefile.am b/sca-cpp/trunk/components/log/Makefile.am
index 5d685babec..0e96be5697 100644
--- a/sca-cpp/trunk/components/log/Makefile.am
+++ b/sca-cpp/trunk/components/log/Makefile.am
@@ -22,7 +22,7 @@ INCLUDES = -I${THRIFT_INCLUDE} -I${FB303_INCLUDE}
incl_HEADERS = *.hpp
incldir = $(prefix)/include/components/log
-dist_comp_SCRIPTS = scribed-central-conf scribed-client-conf scribed-central-start scribed-central-stop scribed-client-start scribed-client-stop scribe-tail-start scribe-tail-stop
+dist_comp_SCRIPTS = scribed-central-conf scribed-central-firehose-conf scribed-central-mkfirehose scribed-client-conf scribed-central-start scribed-central-stop scribed-client-start scribed-client-stop scribe-tail-start scribe-tail-stop
compdir=$(prefix)/components/log
comp_DATA = scribe.prefix thrift.prefix
diff --git a/sca-cpp/trunk/components/log/scribe-cat.cpp b/sca-cpp/trunk/components/log/scribe-cat.cpp
index 77011e8b12..7f48ddb59b 100644
--- a/sca-cpp/trunk/components/log/scribe-cat.cpp
+++ b/sca-cpp/trunk/components/log/scribe-cat.cpp
@@ -36,33 +36,35 @@
namespace tuscany {
namespace scribecat {
-int cat(const string& category, const string& type) {
+int cat(const string& host, const string& category, const string& type) {
// Connect to Scribe
scribe::Scribe& sc = *(new (gc_new<scribe::Scribe>()) scribe::Scribe("localhost", 1464));
// Read lines from stdin and log them
- char buf[8192];
+ char buf[8193];
for (;;) {
- const char* s = fgets(buf, 8192, stdin);
+ gc_scoped_pool();
+
+ // Write line prefix
+ ostringstream os;
+ os << "[" << host << "] ";
+ if (length(type) != 0)
+ os << "[" << logTime() << "] [" << type << "] ";
+ const string prefix = str(os);
+ const int pl = length(prefix);
+ strcpy(buf, c_str(prefix));
+
+ // Read log line
+ const char* s = fgets(buf + pl, 8192 - pl, stdin);
if (s == NULL)
return 0;
const size_t l = strlen(s);
if (l < 2)
return 0;
- buf[l - 1] = '\0';
+ buf[pl + l - 1] = '\0';
- // Log each line as is
- if (length(type) == 0) {
- const failable<bool> val = scribe::log(buf, category, sc);
- if (!hasContent(val))
- return 1;
- continue;
- }
-
- // Log each line prefixed with time and a type tag
- ostringstream os;
- os << "[" << logTime() << "] [" << type << "] " << buf;
- const failable<bool> val = scribe::log(c_str(str(os)), category, sc);
+ // Log the line
+ const failable<bool> val = scribe::log(buf, category, sc);
if (!hasContent(val))
return 1;
}
@@ -72,6 +74,6 @@ int cat(const string& category, const string& type) {
}
int main(const int argc, const char** argv) {
- return tuscany::scribecat::cat(argc < 2? "default" : argv[1], argc < 3? "" : argv[2]);
+ return tuscany::scribecat::cat(argc < 2? "localhost" : argv[1], argc < 3? "default" : argv[2], argc < 4? "" : argv[3]);
}
diff --git a/sca-cpp/trunk/components/log/scribe-tail-start b/sca-cpp/trunk/components/log/scribe-tail-start
index 22f5101053..fc469b5488 100755
--- a/sca-cpp/trunk/components/log/scribe-tail-start
+++ b/sca-cpp/trunk/components/log/scribe-tail-start
@@ -35,10 +35,11 @@ else
file=$1
fi
fi
+host=`hostname`
mkdir -p `dirname $file`
touch $file
file=`echo "import os; print os.path.realpath('$file')" | python`
-tail -f -n 0 $file | $here/scribe-cat $category $type &
+tail -f -n 0 $file | $here/scribe-cat $host $category $type &
diff --git a/sca-cpp/trunk/components/log/scribed-central-conf b/sca-cpp/trunk/components/log/scribed-central-conf
index d8d008623d..066dad4add 100755
--- a/sca-cpp/trunk/components/log/scribed-central-conf
+++ b/sca-cpp/trunk/components/log/scribed-central-conf
@@ -42,7 +42,6 @@ check_interval=3
<store>
category=default
type=buffer
-
target_write_size=20480
max_write_interval=1
buffer_send_rate=2
@@ -50,7 +49,10 @@ retry_interval=30
retry_interval_range=10
<primary>
+category=default
type=file
+target_write_size=20480
+max_write_interval=1
fs_type=std
file_path=$root/scribe/logs/central
base_filename=central
@@ -62,7 +64,10 @@ rotate_minute=10
</primary>
<secondary>
+category=default
type=file
+target_write_size=20480
+max_write_interval=1
fs_type=std
file_path=$root/scribe/logs/central-secondary
base_filename=central
diff --git a/sca-cpp/trunk/components/log/scribed-central-firehose-conf b/sca-cpp/trunk/components/log/scribed-central-firehose-conf
new file mode 100755
index 0000000000..660cf7820a
--- /dev/null
+++ b/sca-cpp/trunk/components/log/scribed-central-firehose-conf
@@ -0,0 +1,125 @@
+#!/bin/sh
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Generate a Scribe central conf
+here=`echo "import os; print os.path.realpath('$0')" | python`; here=`dirname $here`
+mkdir -p $1
+root=`echo "import os; print os.path.realpath('$1')" | python`
+
+port=$2
+if [ "$port" = "" ]; then
+ port="1463"
+fi
+
+mkdir -p $root/scribe/conf
+mkdir -p $root/scribe/logs/central
+mkdir -p $root/scribe/logs/central-secondary
+mkdir -p $root/scribe/logs/firehose
+mkdir -p $root/scribe/logs/firehose-secondary
+
+cat >$root/scribe/conf/scribe-central.conf <<EOF
+# Generated by: scribed-central-conf $*
+# Scribe central configuration
+port=$port
+max_msg_per_second=2000000
+check_interval=3
+
+# Log store configuration
+<store>
+category=default
+type=multi
+target_write_size=20480
+max_write_interval=1
+
+<store0>
+category=default
+type=buffer
+target_write_size=20480
+max_write_interval=1
+buffer_send_rate=2
+retry_interval=30
+retry_interval_range=10
+
+<primary>
+category=default
+type=file
+target_write_size=20480
+max_write_interval=1
+fs_type=std
+file_path=$root/scribe/logs/central
+base_filename=central
+max_size=1000000
+add_newlines=1
+rotate_period=daily
+rotate_hour=0
+rotate_minute=10
+</primary>
+
+<secondary>
+category=default
+type=file
+target_write_size=20480
+max_write_interval=1
+fs_type=std
+file_path=$root/scribe/logs/central-secondary
+base_filename=central
+max_size=3000000
+</secondary>
+
+</store0>
+
+<store1>
+category=default
+type=buffer
+target_write_size=20480
+max_write_interval=1
+buffer_send_rate=2
+retry_interval=30
+retry_interval_range=10
+
+<primary>
+category=default
+type=file
+target_write_size=20480
+max_write_interval=1
+fs_type=std
+file_path=$root/scribe/logs/firehose
+base_filename=central
+max_size=1000000
+add_newlines=1
+write_stats=no
+create_symlink=no
+</primary>
+
+<secondary>
+category=default
+type=file
+target_write_size=20480
+max_write_interval=1
+fs_type=std
+file_path=$root/scribe/logs/firehose-secondary
+base_filename=central
+max_size=3000000
+</secondary>
+
+</store1>
+
+</store>
+
+EOF
diff --git a/sca-cpp/trunk/components/log/scribed-central-mkfirehose b/sca-cpp/trunk/components/log/scribed-central-mkfirehose
new file mode 100755
index 0000000000..9b02305d1d
--- /dev/null
+++ b/sca-cpp/trunk/components/log/scribed-central-mkfirehose
@@ -0,0 +1,34 @@
+#!/bin/sh
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Create a firehose fifo pipe for a log category
+here=`echo "import os; print os.path.realpath('$0')" | python`; here=`dirname $here`
+mkdir -p $1
+root=`echo "import os; print os.path.realpath('$1')" | python`
+
+category=$2
+if [ "$category" = "" ]; then
+ category="default"
+fi
+
+mkdir -p $root/scribe/logs/firehose/$category
+if [ ! -e "$root/scribe/logs/firehose/$category/$category""_00000" ]; then
+ mkfifo "$root/scribe/logs/firehose/$category/$category""_00000"
+fi
+
diff --git a/sca-cpp/trunk/components/sqldb/pgsql-conf b/sca-cpp/trunk/components/sqldb/pgsql-conf
index 21222a1183..58d38412ca 100755
--- a/sca-cpp/trunk/components/sqldb/pgsql-conf
+++ b/sca-cpp/trunk/components/sqldb/pgsql-conf
@@ -61,15 +61,15 @@ cat >>$root/sqldb/data/postgresql.conf <<EOF
# Generated by: pgsql-conf $*
# Setup logging
-log_min_messages = INFO
-log_min_error_statement = INFO
-log_min_duration_statement = 0
+log_min_messages = NOTICE
+log_min_error_statement = NOTICE
+log_min_duration_statement = -1
log_checkpoints = on
-log_connections = on
-log_disconnections = on
-log_duration = on
+log_connections = off
+log_disconnections = off
+log_duration = off
log_lock_waits = on
-log_statement = all
+log_statement = none
# Listen
listen_addresses = '$ip'
diff --git a/sca-cpp/trunk/components/sqldb/pgsql-standby-conf b/sca-cpp/trunk/components/sqldb/pgsql-standby-conf
index b5dd238282..bc0d4e9981 100755
--- a/sca-cpp/trunk/components/sqldb/pgsql-standby-conf
+++ b/sca-cpp/trunk/components/sqldb/pgsql-standby-conf
@@ -63,7 +63,7 @@ mkdir -p $root/sqldb/archive
# Initialize from a backup of the master
if [ ! -f $root/sqldb/data/postgresql.conf ]; then
- (curl -L http://$mhost:$mhttpport/pgsql-backup | tar -C $root/sqldb -xz) 2>&1 | sh $root/sqldb/logger
+ (curl -L -# http://$mhost:$mhttpport/pgsql-backup | tar -C $root/sqldb -xz) 2>&1 | grep -v "100.0%" | sh $root/sqldb/logger
rm -rf $root/sqldb/data/postmaster.pid $root/sqldb/data/pg_xlog
mkdir -p $root/sqldb/data/pg_xlog/archive_status
chmod 700 $root/sqldb/data/pg_xlog/archive_status
@@ -73,7 +73,18 @@ fi
cp $root/sqldb/data/postgresql-init.conf $root/sqldb/data/postgresql.conf
cat >>$root/sqldb/data/postgresql.conf <<EOF
-# Generated by: standby-conf $*
+# Generated by: pgsql-standby-conf $*
+
+# Setup logging
+log_min_messages = NOTICE
+log_min_error_statement = NOTICE
+log_min_duration_statement = -1
+log_checkpoints = on
+log_connections = off
+log_disconnections = off
+log_duration = off
+log_lock_waits = on
+log_statement = none
# Listen
listen_addresses = '$ip'
@@ -95,7 +106,7 @@ EOF
# Generate recovery configuration
cat >$root/sqldb/data/recovery.conf << EOF
-# Generated by: pgsql-slave-conf $*
+# Generated by: pgsql-standby-conf $*
# Start in standby mode
standby_mode = 'on'
@@ -104,7 +115,7 @@ primary_conninfo = 'host=$mhost port=$mport'
# Failover
trigger_file = '$root/sqldb/failover'
-restore_command = 'curl http://$mhost:$mhttpport/pgsql-archive/%f -o "%p"'
+restore_command = 'curl -L -# http://$mhost:$mhttpport/pgsql-archive/%f -o "%p" 2>&1 | grep -v "100.0%"'
EOF
@@ -119,7 +130,7 @@ chmod 700 $root/sqldb/scripts/backup
# Configure HTTPD to serve backup and archive files
if [ -f "$root/conf/httpd.conf" ]; then
cat >>$root/conf/httpd.conf <<EOF
-# Generated by: pgsql-conf $*
+# Generated by: pgsql-standby-conf $*
# Serve PostgreSQL backup and WAL archive files
ScriptAlias /pgsql-backup "$root/sqldb/scripts/backup"
diff --git a/sca-cpp/trunk/hosting/server/ssl-start b/sca-cpp/trunk/hosting/server/ssl-start
index 49d2b424b9..bbe6a46ea1 100755
--- a/sca-cpp/trunk/hosting/server/ssl-start
+++ b/sca-cpp/trunk/hosting/server/ssl-start
@@ -22,6 +22,7 @@
here=`echo "import os; print os.path.realpath('$0')" | python`; here=`dirname $here`
jsprefix=`echo "import os; print os.path.realpath('$here/../../modules/js')" | python`
+host=`hostname`
# Create SSL certificates
../../modules/http/ssl-ca-conf tmp www.example.com
@@ -81,20 +82,20 @@ fi
if [ -x ../../components/log/scribe-cat ]; then
cat >tmp/conf/log.conf <<EOF
# Generated by: ssl-start $*
-ErrorLog "|$here/../../components/log/scribe-cat server"
-CustomLog "|$here/../../components/log/scribe-cat server" combined
+ErrorLog "|$here/../../components/log/scribe-cat $host server"
+CustomLog "|$here/../../components/log/scribe-cat $host server" combined
EOF
cat >tmp/conf/log-ssl.conf <<EOF
# Generated by: ssl-start $*
-CustomLog "|$here/../../components/log/scribe-cat server" sslcombined
+CustomLog "|$here/../../components/log/scribe-cat $host server" sslcombined
EOF
# cat >tmp/conf/mod-security-log.conf <<EOF
# Generated by: ssl-start $*
-#SecAuditLog "|$here/../../components/log/scribe-cat secaudit"
+#SecAuditLog "|$here/../../components/log/scribe-cat $host secaudit"
#
#EOF
diff --git a/sca-cpp/trunk/hosting/server/start b/sca-cpp/trunk/hosting/server/start
index 1737efb7b1..a608ccb2e5 100755
--- a/sca-cpp/trunk/hosting/server/start
+++ b/sca-cpp/trunk/hosting/server/start
@@ -22,6 +22,7 @@
here=`echo "import os; print os.path.realpath('$0')" | python`; here=`dirname $here`
jsprefix=`echo "import os; print os.path.realpath('$here/../../modules/js')" | python`
+host=`hostname`
# Configure and start logging
if [ -x ../../components/log/scribe-cat ]; then
@@ -48,8 +49,8 @@ if [ -x ../../components/log/scribe-cat ]; then
cat >tmp/conf/log.conf <<EOF
# Generated by: start $*
LogLevel notice
-ErrorLog "|$here/../../components/log/scribe-cat server"
-CustomLog "|$here/../../components/log/scribe-cat server" combined
+ErrorLog "|$here/../../components/log/scribe-cat $host server"
+CustomLog "|$here/../../components/log/scribe-cat $host server" combined
EOF
diff --git a/sca-cpp/trunk/patches/scribe.patch b/sca-cpp/trunk/patches/scribe.patch
new file mode 100644
index 0000000000..6340c4634d
--- /dev/null
+++ b/sca-cpp/trunk/patches/scribe.patch
@@ -0,0 +1,50 @@
+--- src/common.h
++++ src/common.h
+@@ -42,6 +42,8 @@
+ #include <sys/types.h>
+ #include <sys/stat.h>
+ #include <unistd.h>
++#include <signal.h>
++#include <fcntl.h>
+ #include <boost/shared_ptr.hpp>
+ #include <boost/filesystem/operations.hpp>
+ #include <boost/filesystem/convenience.hpp>
+
+--- src/file.cpp
++++ src/file.cpp
+@@ -74,9 +74,21 @@ bool StdFile::openRead() {
+ }
+
+ bool StdFile::openWrite() {
++ // if file is a fifo, temporarily open it for read
++ int fd = -1;
++ struct stat st;
++ int s = stat(filename.c_str(), &st);
++ if (s != -1 && S_ISFIFO(st.st_mode))
++ fd = ::open(filename.c_str(), O_RDONLY | O_NONBLOCK);
++
+ // open file for write in append mode
+ ios_base::openmode mode = fstream::out | fstream::app;
+- return open(mode);
++ bool r = open(mode);
++
++ // close fifo
++ if (fd != -1)
++ ::close(fd);
++ return r;
+ }
+
+ bool StdFile::openTruncate() {
+
+--- src/scribe_server.cpp
++++ src/scribe_server.cpp
+@@ -55,6 +55,8 @@ int main(int argc, char **argv) {
+ if (-1 == setrlimit(RLIMIT_NOFILE, &r_fd)) {
+ LOG_OPER("setrlimit error (setting max fd size)");
+ }
++
++ signal(SIGPIPE, SIG_IGN);
+
+ int next_option;
+ const char* const short_options = "hp:c:";
+
diff --git a/sca-cpp/trunk/ubuntu/ubuntu-install b/sca-cpp/trunk/ubuntu/ubuntu-install
index 231e76b433..ce634a1e21 100755
--- a/sca-cpp/trunk/ubuntu/ubuntu-install
+++ b/sca-cpp/trunk/ubuntu/ubuntu-install
@@ -302,6 +302,8 @@ fi
curl -OL http://github.com/downloads/facebook/scribe/scribe-2.2.tar.gz
tar xzf scribe-2.2.tar.gz
cd scribe
+curl -OL http://svn.apache.org/repos/asf/tuscany/sca-cpp/trunk/patches/scribe.patch
+patch -p0 <scribe.patch
./bootstrap.sh --prefix=$build/scribe-2.2-bin PY_PREFIX=$build/scribe-2.2-bin --with-thriftpath=$build/thrift-0.2.0-bin --with-fb303path=$build/thrift-0.2.0-bin/contrib/fb303 --disable-static
make
make install
diff --git a/sca-cpp/trunk/ubuntu/ubuntu-install-all b/sca-cpp/trunk/ubuntu/ubuntu-install-all
index c87f9a8541..63500e953b 100755
--- a/sca-cpp/trunk/ubuntu/ubuntu-install-all
+++ b/sca-cpp/trunk/ubuntu/ubuntu-install-all
@@ -356,6 +356,8 @@ fi
curl -OL http://github.com/downloads/facebook/scribe/scribe-2.2.tar.gz
tar xzf scribe-2.2.tar.gz
cd scribe
+curl -OL http://svn.apache.org/repos/asf/tuscany/sca-cpp/trunk/patches/scribe.patch
+patch -p0 <scribe.patch
./bootstrap.sh --prefix=$build/scribe-2.2-bin PY_PREFIX=$build/scribe-2.2-bin --with-thriftpath=$build/thrift-0.2.0-bin --with-fb303path=$build/thrift-0.2.0-bin/contrib/fb303 --disable-static
make
make install