Added test to measure file size after many inserts and deletes.

git-svn-id: file:///svn/toku/tokudb@16084 c7de825b-a66e-492c-adef-691d508d4ae1

src/tests/filesize.c

@@ -0,0 +1,208 @@
+/* -*- mode: C; c-basic-offset: 4 -*- */
+
+/* Idea: 
+ *  create a dictionary
+ *  repeat:  
+ *    lots of inserts
+ *    checkpoint
+ *    note file size
+ *    lots of deletes
+ *    checkpoint 
+ *    note file size
+ *
+ */
+
+#include "test.h"
+
+#define PATHSIZE 1024
+
+DB_ENV *env;
+DB *db;
+char dbname[] = "foo.db";
+char path[PATHSIZE];
+
+const int envflags = DB_INIT_MPOOL|DB_CREATE|DB_THREAD |DB_INIT_LOCK|DB_PRIVATE;
+
+int ninsert, nread, nread_notfound, nread_failed, ndelete, ndelete_notfound, ndelete_failed;
+
+static void
+close_em (void)
+{
+    int r;
+    r = db->close(db, 0);   CKERR(r);
+    r = env->close(env, 0); CKERR(r);
+}
+
+
+static void
+setup(void)
+{
+    system("rm -rf " ENVDIR);
+    toku_os_mkdir(ENVDIR, S_IRWXU+S_IRWXG+S_IRWXO);
+    int r;
+    r = db_env_create(&env, 0);                                         CKERR(r);
+    r = env->open(env, ENVDIR, envflags, S_IRWXU+S_IRWXG+S_IRWXO);      CKERR(r);
+    r = db_create(&db, env, 0);                                         CKERR(r);
+    r = db->open(db, NULL, dbname, NULL, DB_BTREE, DB_CREATE, 0666);    CKERR(r);
+}
+
+
+static void
+fill_rand(int n, uint64_t * d) {
+    for (int i = 0; i < n; i++){
+	*(d+i) = random64();
+    }
+}
+
+static void
+insert_n (u_int32_t ah) {
+    int datasize = 1500;
+    uint64_t vdata[datasize];
+    fill_rand(datasize, vdata);
+    u_int32_t an = htonl(ah);
+    //    printf("insert an = %0X (ah = %0X)\n", an, ah);
+    DBT key = {.size = 4,             .data=&an };
+    DBT val = {.size = sizeof(vdata), .data=vdata};
+    int r = db->put(db, NULL, &key, &val, DB_YESOVERWRITE);
+    CKERR(r);
+    ninsert++;
+}
+
+static void
+delete_n (u_int32_t ah)
+{
+    u_int32_t an = htonl(ah);
+    //    printf("delete an = %0X (ah = %0X)\n", an, ah);
+    DBT key = {.size = 4,             .data=&an };
+    int r = db->del(db, NULL, &key, DB_DELETE_ANY);
+    if (r == 0)
+	ndelete++;
+    else if (r == DB_NOTFOUND)
+	ndelete_notfound++;
+    else
+	ndelete_failed++;
+#ifdef USE_BDB
+    assert(r==0 || r==DB_NOTFOUND);
+#else
+    CKERR(r);
+#endif
+}
+
+static void
+scan(int n) {
+    DBT k,v;
+    DBC *dbc;
+    int i,r;
+
+    r = db->cursor(db, 0, &dbc, 0);                           
+    CKERR(r);
+    memset(&k, 0, sizeof(k));
+    memset(&v, 0, sizeof(v));
+    r = dbc->c_get(dbc, &k, &v, DB_FIRST);
+    if (r==0) {
+	nread++;
+	//	printf("First read, r = %0X, key = %0X (size=%d)\n", r, *(uint32_t*)k.data, k.size);
+    }
+    else if (r == DB_NOTFOUND)  {
+	nread_notfound++;
+	//	printf("First read failed: %d\n", r);
+    }
+    else
+	nread_failed++;
+    for (i = 1; i<n; i++) {
+	r = dbc->c_get(dbc, &k, &v, DB_NEXT);
+	if (r == 0) {
+	    nread++;
+	    //	    printf("read key = %0X (size=%d)\n", *(uint32_t*)k.data, k.size);
+	}
+	else if (r == DB_NOTFOUND)
+	    nread_notfound++;
+	else
+	    nread_failed++;
+    }
+    r = dbc->c_close(dbc);
+    CKERR(r);
+
+}
+
+static void
+get_file_pathname(void) {
+    DBT dname;
+    DBT iname;
+    dbt_init(&dname, dbname, sizeof(dbname));
+    dbt_init(&iname, NULL, 0);
+    iname.flags |= DB_DBT_MALLOC;
+    int r = env->get_iname(env, &dname, &iname);
+    CKERR(r);
+    sprintf(path, "%s/%s", ENVDIR, (char*)iname.data);
+    printf("path = %s\n", path);
+}
+
+
+static int 
+getsizeM(void) {
+    toku_struct_stat buf;
+    int r = toku_stat(path, &buf);
+    CKERR(r);    
+    int sizeM = (int)buf.st_size >> 20;
+    return sizeM;
+}
+
+static void
+test_filesize (void)
+{
+    int N=1<<16;
+    int r, i, sizeM;
+
+    get_file_pathname();
+    
+    for (int iter = 0; iter < 3; iter++) {
+	int offset = N * iter;
+
+	for (i=0; i<N; i++) {
+	    insert_n(i + offset);
+	}
+	
+	r = env->txn_checkpoint(env, 0, 0, 0);
+	CKERR(r);
+	sizeM = getsizeM();
+	printf("Filesize after iteration %d insertion and checkpoint = %dM\n", iter, sizeM);
+	
+	int preserve = 2;
+	for (i = preserve; i<(N); i++) {  // leave a little at the beginning
+	    delete_n(i + offset);
+	}
+	scan(N);
+
+	r = env->txn_checkpoint(env, 0, 0, 0);
+	CKERR(r);
+	sizeM = getsizeM();
+	printf("Filesize after iteration %d deletion and checkpoint 1 = %dM\n", iter, sizeM);
+
+	insert_n(preserve+offset + 1);
+	r = env->txn_checkpoint(env, 0, 0, 0);
+	CKERR(r);
+	sizeM = getsizeM();
+	printf("Filesize after iteration %d deletion and checkpoint 2 = %dM\n", iter, sizeM);
+
+	insert_n(preserve+offset+2);
+	r = env->txn_checkpoint(env, 0, 0, 0);
+	CKERR(r);
+	sizeM = getsizeM();
+	printf("Filesize after iteration %d deletion and checkpoint 3 = %dM\n", iter, sizeM);
+
+	printf("ninsert = %d\n", ninsert);
+	printf("nread = %d, nread_notfound = %d, nread_failed = %d\n", nread, nread_notfound, nread_failed);
+	printf("ndelete = %d, ndelete_notfound = %d, ndelete_failed = %d\n", ndelete, ndelete_notfound, ndelete_failed);
+    }
+}
+
+int test_main (int argc __attribute__((__unused__)), char *argv[] __attribute__((__unused__))) {
+    parse_args(argc, argv);
+    setup();
+    print_engine_status(env);
+    test_filesize();
+    print_engine_status(env);
+    close_em();
+    return 0;
+}