FT-242 Break up fttypes.h completely.

FT-296 Move bytestring class to utils/

ft/block_table.cc

@@ -97,7 +97,6 @@ PATENT RIGHTS GRANT:
 #include "ft/block_allocator.h"
 #include "ft/block_table.h"
 #include "ft/ft-internal.h"        // ugly but pragmatic, need access to dirty bits while holding translation lock
-#include "ft/fttypes.h"
 // TODO: reorganize this dependency
 #include "ft/ft-ops.h" // for toku_maybe_truncate_file
 #include "ft/rbuf.h"
@@ -925,10 +924,9 @@ translation_deserialize_from_buffer(struct translation *t,    // destination int
     assert(t->smallest_never_used_blocknum.b >= RESERVED_BLOCKNUMS);
     t->blocknum_freelist_head       = rbuf_blocknum(&rt); 
     XMALLOC_N(t->length_of_array, t->block_translation);
-    int64_t i;
-    for (i=0; i < t->length_of_array; i++) {
-        t->block_translation[i].u.diskoff = rbuf_diskoff(&rt);
-        t->block_translation[i].size    = rbuf_diskoff(&rt);
+    for (int64_t i = 0; i < t->length_of_array; i++) {
+        t->block_translation[i].u.diskoff = rbuf_DISKOFF(&rt);
+        t->block_translation[i].size = rbuf_DISKOFF(&rt);
     }
     assert(calculate_size_on_disk(t)                                     == (int64_t)size_on_disk);
     assert(t->block_translation[RESERVED_BLOCKNUM_TRANSLATION].size      == (int64_t)size_on_disk);

ft/block_table.h

@@ -91,11 +91,25 @@ PATENT RIGHTS GRANT:
 #ident "Copyright (c) 2007-2013 Tokutek Inc.  All rights reserved."
 #ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
 
-#include "fttypes.h"
-#include "ft/ft-internal.h"
+#include <db.h>
+
+#include "portability/toku_stdint.h"
+
+struct ft;
 
 typedef struct block_table *BLOCK_TABLE;
 
+typedef struct blocknum_s { int64_t b; } BLOCKNUM;
+static inline BLOCKNUM make_blocknum(int64_t b) {
+    BLOCKNUM result = { .b = b };
+    return result;
+}
+static const BLOCKNUM ROLLBACK_NONE = { .b = 0 };
+
+
+// Offset in a disk. -1 is the 'null' pointer.
+typedef int64_t DISKOFF;
+
 // Needed by tests, ftdump
 struct block_translation_pair {
     union { // If in the freelist, use next_free_blocknum, otherwise diskoff.
@@ -109,8 +123,8 @@ void toku_blocktable_create_new(BLOCK_TABLE *btp);
 int toku_blocktable_create_from_buffer(int fd, BLOCK_TABLE *btp, DISKOFF location_on_disk, DISKOFF size_on_disk, unsigned char *translation_buffer);
 void toku_blocktable_destroy(BLOCK_TABLE *btp);
 
-void toku_ft_lock(FT ft);
-void toku_ft_unlock(FT ft);
+void toku_ft_lock(struct ft *ft);
+void toku_ft_unlock(struct ft *ft);
 
 void toku_block_translation_note_start_checkpoint_unlocked(BLOCK_TABLE bt);
 void toku_block_translation_note_end_checkpoint(BLOCK_TABLE bt, int fd);
@@ -118,38 +132,39 @@ void toku_block_translation_note_skipped_checkpoint(BLOCK_TABLE bt);
 void toku_maybe_truncate_file_on_open(BLOCK_TABLE bt, int fd);
 
 //Blocknums
-void toku_allocate_blocknum(BLOCK_TABLE bt, BLOCKNUM *res, FT ft);
-void toku_allocate_blocknum_unlocked(BLOCK_TABLE bt, BLOCKNUM *res, FT ft);
-void toku_free_blocknum(BLOCK_TABLE bt, BLOCKNUM *b, FT ft, bool for_checkpoint);
+void toku_allocate_blocknum(BLOCK_TABLE bt, BLOCKNUM *res, struct ft *ft);
+void toku_allocate_blocknum_unlocked(BLOCK_TABLE bt, BLOCKNUM *res, struct ft *ft);
+void toku_free_blocknum(BLOCK_TABLE bt, BLOCKNUM *b, struct ft *ft, bool for_checkpoint);
 void toku_verify_blocknum_allocated(BLOCK_TABLE bt, BLOCKNUM b);
 void toku_block_verify_no_data_blocks_except_root(BLOCK_TABLE bt, BLOCKNUM root);
 void toku_free_unused_blocknums(BLOCK_TABLE bt, BLOCKNUM root);
 void toku_block_verify_no_free_blocknums(BLOCK_TABLE bt);
-void toku_realloc_descriptor_on_disk(BLOCK_TABLE bt, DISKOFF size, DISKOFF *offset, FT ft, int fd);
-void toku_realloc_descriptor_on_disk_unlocked(BLOCK_TABLE bt, DISKOFF size, DISKOFF *offset, FT ft);
+void toku_realloc_descriptor_on_disk(BLOCK_TABLE bt, DISKOFF size, DISKOFF *offset, struct ft *ft, int fd);
+void toku_realloc_descriptor_on_disk_unlocked(BLOCK_TABLE bt, DISKOFF size, DISKOFF *offset, struct ft *ft);
 void toku_get_descriptor_offset_size(BLOCK_TABLE bt, DISKOFF *offset, DISKOFF *size);
 
 //Blocks and Blocknums
-void toku_blocknum_realloc_on_disk(BLOCK_TABLE bt, BLOCKNUM b, DISKOFF size, DISKOFF *offset, FT ft, int fd, bool for_checkpoint);
+void toku_blocknum_realloc_on_disk(BLOCK_TABLE bt, BLOCKNUM b, DISKOFF size, DISKOFF *offset, struct ft *ft, int fd, bool for_checkpoint);
 void toku_translate_blocknum_to_offset_size(BLOCK_TABLE bt, BLOCKNUM b, DISKOFF *offset, DISKOFF *size);
 
 //Serialization
 void toku_serialize_translation_to_wbuf(BLOCK_TABLE bt, int fd, struct wbuf *w, int64_t *address, int64_t *size);
-
 void toku_block_table_swap_for_redirect(BLOCK_TABLE old_bt, BLOCK_TABLE new_bt);
 
-
 //DEBUG ONLY (ftdump included), tests included
 void toku_blocknum_dump_translation(BLOCK_TABLE bt, BLOCKNUM b);
 void toku_dump_translation_table_pretty(FILE *f, BLOCK_TABLE bt);
 void toku_dump_translation_table(FILE *f, BLOCK_TABLE bt);
 void toku_block_free(BLOCK_TABLE bt, uint64_t offset);
 typedef int (*BLOCKTABLE_CALLBACK)(BLOCKNUM b, int64_t size, int64_t address, void *extra);
-enum translation_type {TRANSLATION_NONE=0,
+
+enum translation_type {
+    TRANSLATION_NONE = 0,
     TRANSLATION_CURRENT,
     TRANSLATION_INPROGRESS,
     TRANSLATION_CHECKPOINTED,
-                       TRANSLATION_DEBUG};
+    TRANSLATION_DEBUG
+};
 
 int toku_blocktable_iterate(BLOCK_TABLE bt, enum translation_type type, BLOCKTABLE_CALLBACK f, void *extra, bool data_only, bool used_only); 
 void toku_blocktable_internal_fragmentation(BLOCK_TABLE bt, int64_t *total_sizep, int64_t *used_sizep);
@@ -166,8 +181,40 @@ int toku_blocktable_iterate_translation_tables(BLOCK_TABLE, uint64_t, int (*)(ui
 
 //Unmovable reserved first, then reallocable.
 // We reserve one blocknum for the translation table itself.
-enum {RESERVED_BLOCKNUM_NULL       =0,
+enum {
+    RESERVED_BLOCKNUM_NULL = 0,
     RESERVED_BLOCKNUM_TRANSLATION = 1,
     RESERVED_BLOCKNUM_DESCRIPTOR = 2,
-      RESERVED_BLOCKNUMS};
+    RESERVED_BLOCKNUMS
+};
 
+// For serialize / deserialize
+
+#include "ft/wbuf.h"
+
+static inline void wbuf_BLOCKNUM (struct wbuf *w, BLOCKNUM b) {
+    wbuf_ulonglong(w, b.b);
+}
+
+static inline void wbuf_nocrc_BLOCKNUM (struct wbuf *w, BLOCKNUM b) {
+    wbuf_nocrc_ulonglong(w, b.b);
+}
+
+static inline void wbuf_DISKOFF(struct wbuf *wb, DISKOFF off) {
+    wbuf_ulonglong(wb, (uint64_t) off);
+}
+
+#include "ft/rbuf.h"
+
+static inline DISKOFF rbuf_DISKOFF(struct rbuf *rb) {
+    return rbuf_ulonglong(rb);
+}
+
+static inline BLOCKNUM rbuf_blocknum(struct rbuf *rb) {
+    BLOCKNUM result = make_blocknum(rbuf_longlong(rb));
+    return result;
+}
+
+static inline void rbuf_ma_BLOCKNUM(struct rbuf *rb, memarena *UU(ma), BLOCKNUM *blocknum) {
+    *blocknum = rbuf_blocknum(rb);
+}

ft/bndata.cc

@@ -89,8 +89,8 @@ PATENT RIGHTS GRANT:
 #ident "Copyright (c) 2007-2013 Tokutek Inc.  All rights reserved."
 #ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
 
-#include <bndata.h>
-#include <ft-ops.h>
+#include <ft/bndata.h>
+#include <ft/ft-internal.h>
 
 using namespace toku;
 uint32_t bn_data::klpair_disksize(const uint32_t klpair_len, const klpair_struct *klpair) const {
@@ -129,14 +129,14 @@ void bn_data::initialize_from_separate_keys_and_vals(uint32_t num_entries, struc
     uint32_t ndone_before = rb->ndone;
     init_zero();
     invariant(all_keys_same_length);  // Until otherwise supported.
-    bytevec keys_src;
+    const void *keys_src;
     rbuf_literal_bytes(rb, &keys_src, key_data_size);
     //Generate dmt
     this->m_buffer.create_from_sorted_memory_of_fixed_size_elements(
             keys_src, num_entries, key_data_size, fixed_klpair_length);
     toku_mempool_construct(&this->m_buffer_mempool, val_data_size);
 
-    bytevec vals_src;
+    const void *vals_src;
     rbuf_literal_bytes(rb, &vals_src, val_data_size);
 
     if (num_entries > 0) {
@@ -256,7 +256,7 @@ void bn_data::deserialize_from_rbuf(uint32_t num_entries, struct rbuf *rb, uint3
         }
     }
     // Version >= 26 and version 25 deserialization are now identical except that <= 25 might allocate too much memory.
-    bytevec bytes;
+    const void *bytes;
     rbuf_literal_bytes(rb, &bytes, data_size);
     const unsigned char *CAST_FROM_VOIDP(buf, bytes);
     if (data_size == 0) {

ft/cachetable.h

@@ -94,7 +94,9 @@ PATENT RIGHTS GRANT:
 
 #include <fcntl.h>
 
-#include "ft/fttypes.h"
+#include "ft/block_table.h"
+#include "ft/logger.h"
+#include "ft/txn.h"
 #include "util/minicron.h"
 
 // Maintain a cache mapping from cachekeys to values (void*)
@@ -159,7 +161,7 @@ uint32_t toku_get_cleaner_iterations_unlocked (CACHETABLE ct);
 // create and initialize a cache table
 // size_limit is the upper limit on the size of the size of the values in the table
 // pass 0 if you want the default
-int toku_cachetable_create(CACHETABLE *result, long size_limit, LSN initial_lsn, TOKULOGGER);
+int toku_cachetable_create(CACHETABLE *result, long size_limit, LSN initial_lsn, struct tokulogger *logger);
 
 // Create a new cachetable.
 // Effects: a new cachetable is created and initialized.
@@ -184,9 +186,9 @@ int toku_cachefile_of_iname_in_env (CACHETABLE ct, const char *iname_in_env, CAC
 // Return the filename
 char *toku_cachefile_fname_in_cwd (CACHEFILE cf);
 
-void toku_cachetable_begin_checkpoint (CHECKPOINTER cp, TOKULOGGER);
+void toku_cachetable_begin_checkpoint (CHECKPOINTER cp, struct tokulogger *logger);
 
-void toku_cachetable_end_checkpoint(CHECKPOINTER cp, TOKULOGGER logger, 
+void toku_cachetable_end_checkpoint(CHECKPOINTER cp, struct tokulogger *logger, 
                                    void (*testcallback_f)(void*),  void * testextra);
 
 // Shuts down checkpoint thread
@@ -544,15 +546,15 @@ void toku_cachefile_unlink_on_close(CACHEFILE cf);
 bool toku_cachefile_is_unlink_on_close(CACHEFILE cf);
 
 // Return the logger associated with the cachefile
-TOKULOGGER toku_cachefile_logger (CACHEFILE);
+struct tokulogger *toku_cachefile_logger(CACHEFILE cf);
 
 // Return the filenum associated with the cachefile
-FILENUM toku_cachefile_filenum (CACHEFILE);
+FILENUM toku_cachefile_filenum(CACHEFILE cf);
 
 // Effect: Return a 32-bit hash key.  The hash key shall be suitable for using with bitmasking for a table of size power-of-two.
-uint32_t toku_cachetable_hash (CACHEFILE cachefile, CACHEKEY key);
+uint32_t toku_cachetable_hash(CACHEFILE cf, CACHEKEY key);
 
-uint32_t toku_cachefile_fullhash_of_header (CACHEFILE cachefile);
+uint32_t toku_cachefile_fullhash_of_header(CACHEFILE cf);
 
 // debug functions
 

ft/checkpoint.cc

@@ -126,18 +126,18 @@ PATENT RIGHTS GRANT:
  *
  *****/
 
-#include <toku_portability.h>
 #include <time.h>
 
+#include "portability/toku_portability.h"
+#include "portability/toku_atomic.h"
+
+#include "ft/cachetable.h"
 #include "ft/ft.h"
-#include "fttypes.h"
-#include "cachetable.h"
-#include "log-internal.h"
-#include "logger.h"
-#include "checkpoint.h"
-#include <portability/toku_atomic.h>
-#include <util/status.h>
-#include <util/frwlock.h>
+#include "ft/log-internal.h"
+#include "ft/logger.h"
+#include "ft/checkpoint.h"
+#include "util/frwlock.h"
+#include "util/status.h"
 
 ///////////////////////////////////////////////////////////////////////////////////
 // Engine status

ft/checkpoint.h

@@ -160,13 +160,11 @@ typedef enum {SCHEDULED_CHECKPOINT  = 0,   // "normal" checkpoint taken on check
 // Callbacks are called during checkpoint procedure while checkpoint_safe lock is still held.
 // Callbacks are primarily intended for use in testing.
 // caller_id identifies why the checkpoint is being taken.
-int toku_checkpoint(CHECKPOINTER cp, TOKULOGGER logger,
-                    void (*callback_f)(void*),  void * extra,
-                    void (*callback2_f)(void*), void * extra2,
+int toku_checkpoint(CHECKPOINTER cp, struct tokulogger *logger,
+                    void (*callback_f)(void *extra), void *extra,
+                    void (*callback2_f)(void *extra2), void *extra2,
                     checkpoint_caller_t caller_id);
 
-
-
 /******
  * These functions are called from the ydb level.
  * They return status information and have no side effects.

ft/comparator.h

@@ -94,11 +94,13 @@ PATENT RIGHTS GRANT:
 #include <string.h>
 
 #include <ft/ybt.h>
-#include <ft/fttypes.h>
+//#include <ft/fttypes.h>
 #include <portability/memory.h>
 
+typedef int (*ft_compare_func)(DB *db, const DBT *a, const DBT *b);
+
 // TODO: this should really all be encapsulated in ft/comparator.cc
-int toku_builtin_compare_fun(DB *, const DBT *a, const DBT *b) __attribute__((__visibility__("default")));
+int toku_builtin_compare_fun(DB *db, const DBT *a, const DBT *b) __attribute__((__visibility__("default")));
 
 namespace toku {
 

ft/cursor.cc

@@ -226,8 +226,8 @@ static int ft_cursor_compare_set(const ft_search &search, const DBT *x) {
 }
 
 static int
-ft_cursor_current_getf(ITEMLEN keylen,                 bytevec key,
-                        ITEMLEN vallen,                 bytevec val,
+ft_cursor_current_getf(uint32_t keylen,                 const void *key,
+                        uint32_t vallen,                 const void *val,
                         void *v, bool lock_only) {
     struct ft_cursor_search_struct *CAST_FROM_VOIDP(bcss, v);
     int r;
@@ -286,7 +286,7 @@ int toku_ft_cursor_last(FT_CURSOR cursor, FT_GET_CALLBACK_FUNCTION getf, void *g
     return r;
 }
 
-int toku_ft_cursor_check_restricted_range(FT_CURSOR c, bytevec key, ITEMLEN keylen) {
+int toku_ft_cursor_check_restricted_range(FT_CURSOR c, const void *key, uint32_t keylen) {
     if (c->out_of_range_error) {
         FT ft = c->ft_handle->ft;
         DBT found_key;
@@ -367,8 +367,8 @@ int toku_ft_cursor_next(FT_CURSOR cursor, FT_GET_CALLBACK_FUNCTION getf, void *g
     return r;
 }
 
-static int ft_cursor_search_eq_k_x_getf(ITEMLEN keylen, bytevec key,
-                                        ITEMLEN vallen, bytevec val,
+static int ft_cursor_search_eq_k_x_getf(uint32_t keylen, const void *key,
+                                        uint32_t vallen, const void *val,
                                         void *v, bool lock_only) {
     struct ft_cursor_search_struct *CAST_FROM_VOIDP(bcss, v);
     int r;

ft/cursor.h

@@ -220,7 +220,7 @@ bool toku_ft_cursor_uninitialized(FT_CURSOR cursor) __attribute__ ((warn_unused_
 
 void toku_ft_cursor_peek(FT_CURSOR cursor, const DBT **pkey, const DBT **pval);
 
-int toku_ft_cursor_check_restricted_range(FT_CURSOR cursor, bytevec key, ITEMLEN keylen);
+int toku_ft_cursor_check_restricted_range(FT_CURSOR cursor, const void *key, uint32_t keylen);
 
 int toku_ft_cursor_shortcut(FT_CURSOR cursor, int direction, uint32_t index, bn_data *bd,
                             FT_GET_CALLBACK_FUNCTION getf, void *getf_v,

ft/ft-cachetable-wrappers.cc

@@ -90,7 +90,6 @@ PATENT RIGHTS GRANT:
 #ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
 
 #include "ft/block_table.h"
-#include "ft/fttypes.h"
 #include "ft/ft-cachetable-wrappers.h"
 #include "ft/ft-flusher.h"
 #include "ft/ft-internal.h"

ft/ft-flusher-internal.h

@@ -91,8 +91,6 @@ PATENT RIGHTS GRANT:
 #ident "Copyright (c) 2007-2013 Tokutek Inc.  All rights reserved."
 #ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
 
-#include <fttypes.h>
-
 #define flt_flush_before_applying_inbox 1
 #define flt_flush_before_child_pin 2
 #define ft_flush_aflter_child_pin 3

ft/ft-internal.h

@@ -92,8 +92,16 @@ PATENT RIGHTS GRANT:
 #ident "Copyright (c) 2007-2013 Tokutek Inc.  All rights reserved."
 #ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
 
-#include <portability/toku_config.h>
-#include <toku_race_tools.h>
+#include "portability/toku_config.h"
+#include "portability/toku_list.h"
+#include "portability/toku_race_tools.h"
+
+#include "ft/cachetable.h"
+#include "ft/comparator.h"
+#include "ft/ft.h"
+#include "ft/ft-ops.h"
+#include "ft/node.h"
+#include "ft/rollback.h"
 
 // Symbol TOKUDB_REVISION is not defined by fractal-tree makefiles, so
 // BUILD_ID of 1000 indicates development build of main, not a release build.  
@@ -103,19 +111,6 @@ PATENT RIGHTS GRANT:
 #error
 #endif
 
-#include "ft_layout_version.h"
-#include "block_allocator.h"
-#include "cachetable.h"
-#include "toku_list.h"
-#include <util/omt.h>
-#include "leafentry.h"
-#include "compress.h"
-#include <util/omt.h>
-#include "ft/bndata.h"
-#include "ft/comparator.h"
-#include "ft/rollback.h"
-#include "ft/msg_buffer.h"
-
 struct block_table;
 struct ft_search;
 
@@ -200,6 +195,7 @@ struct ft_header {
 
     STAT64INFO_S on_disk_stats;
 };
+typedef struct ft_header *FT_HEADER;
 
 // ft_header is always the current version.
 struct ft {
@@ -267,7 +263,6 @@ struct ft {
     // - if our attempt fails because the key was not in range of the rightmost leaf, we reset the score back to 0
     uint32_t seqinsert_score;
 };
-typedef struct ft *FT;
 
 // Allocate a DB struct off the stack and only set its comparison
 // descriptor. We don't bother setting any other fields because
@@ -312,6 +307,8 @@ bool toku_ftnode_pf_req_callback(void* ftnode_pv, void* read_extraargs);
 int toku_ftnode_pf_callback(void* ftnode_pv, void* UU(disk_data), void* read_extraargs, int fd, PAIR_ATTR* sizep);
 int toku_ftnode_cleaner_callback( void *ftnode_pv, BLOCKNUM blocknum, uint32_t fullhash, void *extraargs);
 
+CACHETABLE_WRITE_CALLBACK get_write_callbacks_for_node(FT ft);
+
 /* serialization code */
 void toku_create_compressed_partition_from_available(FTNODE node, int childnum,
                                                      enum toku_compression_method compression_method,
@@ -497,7 +494,7 @@ void toku_create_new_ftnode(FT_HANDLE ft_handle, FTNODE *result, int height, int
 // toku_testsetup_initialize() must be called before any other test_setup_xxx() functions are called.
 void toku_testsetup_initialize(void);
 int toku_testsetup_leaf(FT_HANDLE ft_h, BLOCKNUM *blocknum, int n_children, char **keys, int *keylens);
-int toku_testsetup_nonleaf (FT_HANDLE ft_h, int height, BLOCKNUM *diskoff, int n_children, BLOCKNUM *children, char **keys, int *keylens);
+int toku_testsetup_nonleaf (FT_HANDLE ft_h, int height, BLOCKNUM *blocknum, int n_children, BLOCKNUM *children, char **keys, int *keylens);
 int toku_testsetup_root(FT_HANDLE ft_h, BLOCKNUM);
 int toku_testsetup_get_sersize(FT_HANDLE ft_h, BLOCKNUM); // Return the size on disk.
 int toku_testsetup_insert_to_leaf (FT_HANDLE ft_h, BLOCKNUM, const char *key, int keylen, const char *val, int vallen);
@@ -685,11 +682,11 @@ int toku_upgrade_msn_from_root_to_header(int fd, FT ft) __attribute__((nonnull))
 // The cursor object will have been updated (so that if result==0 the current value is the value being passed)
 //  (If r!=0 then the cursor won't have been updated.)
 // If r!=0, it's up to the callback function to return that value of r.
-// A 'key' bytevec of NULL means that element is not found (effectively infinity or
+// A 'key' pointer of NULL means that element is not found (effectively infinity or
 // -infinity depending on direction)
 // When lock_only is false, the callback does optional lock tree locking and then processes the key and val.
 // When lock_only is true, the callback only does optional lock tree locking.
-typedef int (*FT_GET_CALLBACK_FUNCTION)(ITEMLEN keylen, bytevec key, ITEMLEN vallen, bytevec val, void *extra, bool lock_only);
+typedef int (*FT_GET_CALLBACK_FUNCTION)(uint32_t keylen, const void *key, uint32_t vallen, const void *val, void *extra, bool lock_only);
 
 typedef bool (*FT_CHECK_INTERRUPT_CALLBACK)(void *extra);
 

ft/ft-node-deserialize.cc

@@ -132,7 +132,7 @@ int
 read_and_check_magic(struct rbuf *rb)
 {
     int r = 0;
-    bytevec magic;
+    const void *magic;
     rbuf_literal_bytes(rb, &magic, 8);
     if (memcmp(magic, "tokuleaf", 8)!=0 &&
         memcmp(magic, "tokunode", 8)!=0) {

ft/ft-ops.cc

@@ -2286,7 +2286,7 @@ static int ft_leaf_get_relative_key_pos(FT ft, FTNODE leaf, const DBT *key, bool
 
 static void ft_insert_directly_into_leaf(FT ft, FTNODE leaf, int target_childnum, DBT *key, DBT *val,
                                          XIDS message_xids, enum ft_msg_type type, txn_gc_info *gc_info);
-static int getf_nothing(ITEMLEN, bytevec, ITEMLEN, bytevec, void *, bool);
+static int getf_nothing(uint32_t, const void *, uint32_t, const void *, void *, bool);
 
 static int ft_maybe_insert_into_rightmost_leaf(FT ft, DBT *key, DBT *val, XIDS message_xids, enum ft_msg_type type,
                                                txn_gc_info *gc_info, bool unique)
@@ -3957,7 +3957,7 @@ try_again:
 
 /* ********************************* delete **************************************/
 static int
-getf_nothing (ITEMLEN UU(keylen), bytevec UU(key), ITEMLEN UU(vallen), bytevec UU(val), void *UU(pair_v), bool UU(lock_only)) {
+getf_nothing (uint32_t UU(keylen), const void *UU(key), uint32_t UU(vallen), const void *UU(val), void *UU(pair_v), bool UU(lock_only)) {
     return 0;
 }
 
@@ -4720,7 +4720,7 @@ int toku_ft_strerror_r(int error, char *buf, size_t buflen)
 }
 
 // when a and b are chars, return a-b is safe here because return type is int.  No over/underflow possible.
-int toku_keycompare (bytevec key1, ITEMLEN key1len, bytevec key2, ITEMLEN key2len) {
+int toku_keycompare (const void *key1, uint32_t key1len, const void *key2, uint32_t key2len) {
     int comparelen = key1len<key2len ? key1len : key2len;
     const unsigned char *k1;
     const unsigned char *k2;

ft/ft-ops.h

@@ -93,13 +93,15 @@ PATENT RIGHTS GRANT:
 
 // This must be first to make the 64-bit file mode work right in Linux
 #define _FILE_OFFSET_BITS 64
-#include "fttypes.h"
-#include "ybt.h"
+
 #include <db.h>
-#include "cachetable.h"
-#include "log.h"
-#include "compress.h"
+
+#include "ft/cachetable.h"
+#include "ft/comparator.h"
 #include "ft/msg.h"
+#include "ft/ybt.h"
+
+typedef struct ft_handle *FT_HANDLE;
 
 int toku_open_ft_handle (const char *fname, int is_create, FT_HANDLE *, int nodesize, int basementnodesize, enum toku_compression_method compression_method, CACHETABLE, TOKUTXN, int(*)(DB *,const DBT*,const DBT*)) __attribute__ ((warn_unused_result));
 
@@ -128,7 +130,8 @@ void toku_ft_handle_get_fanout(FT_HANDLE, unsigned int *fanout);
 void toku_ft_set_bt_compare(FT_HANDLE ft_handle, ft_compare_func cmp_func);
 const toku::comparator &toku_ft_get_comparator(FT_HANDLE ft_handle);
 
-void toku_ft_set_redirect_callback(FT_HANDLE ft_h, on_redirect_callback redir_cb, void* extra);
+typedef void (*on_redirect_callback)(FT_HANDLE ft_handle, void *extra);
+void toku_ft_set_redirect_callback(FT_HANDLE ft_handle, on_redirect_callback cb, void *extra);
 
 // How updates (update/insert/deletes) work:
 // There are two flavers of upsertdels:  Singleton and broadcast.
@@ -166,6 +169,9 @@ void toku_ft_set_redirect_callback(FT_HANDLE ft_h, on_redirect_callback redir_cb
 // Implementation note: Acquires a write lock on the entire database.
 //  This function works by sending an BROADCAST-UPDATE message containing
 //   the key and the extra.
+typedef int (*ft_update_func)(DB *db, const DBT *key, const DBT *old_val, const DBT *extra,
+                              void (*set_val)(const DBT *new_val, void *set_extra),
+                              void *set_extra);
 void toku_ft_set_update(FT_HANDLE ft_h, ft_update_func update_fun);
 
 int toku_ft_handle_open(FT_HANDLE, const char *fname_in_env,
@@ -182,6 +188,14 @@ void toku_ft_handle_close(FT_HANDLE ft_handle);
 // close an ft handle during recovery. the underlying ft must close, and will use the given lsn.
 void toku_ft_handle_close_recovery(FT_HANDLE ft_handle, LSN oplsn);
 
+// At the ydb layer, a DICTIONARY_ID uniquely identifies an open dictionary.
+// With the introduction of the loader (ticket 2216), it is possible for the file that holds
+// an open dictionary to change, so these are now separate and independent unique identifiers (see FILENUM)
+struct DICTIONARY_ID {
+    uint64_t dictid;
+};
+static const DICTIONARY_ID DICTIONARY_ID_NONE = { .dictid = 0 };
+
 int
 toku_ft_handle_open_with_dict_id(
     FT_HANDLE ft_h, 
@@ -230,7 +244,7 @@ void toku_ft_delete (FT_HANDLE ft_h, DBT *k, TOKUTXN txn);
 void toku_ft_maybe_delete (FT_HANDLE ft_h, DBT *k, TOKUTXN txn, bool oplsn_valid, LSN oplsn, bool do_logging);
 
 TXNID toku_ft_get_oldest_referenced_xid_estimate(FT_HANDLE ft_h);
-TXN_MANAGER toku_ft_get_txn_manager(FT_HANDLE ft_h);
+struct txn_manager *toku_ft_get_txn_manager(FT_HANDLE ft_h);
 
 struct txn_gc_info;
 void toku_ft_send_insert(FT_HANDLE ft_h, DBT *key, DBT *val, XIDS xids, enum ft_msg_type type, txn_gc_info *gc_info);

ft/ft-serialize.cc

@@ -89,10 +89,12 @@ PATENT RIGHTS GRANT:
 #ident "Copyright (c) 2007-2013 Tokutek Inc.  All rights reserved."
 #ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
 
+#include "ft/block_allocator.h"
 #include "ft/block_table.h"
 #include "ft/compress.h"
 #include "ft/ft.h"
 #include "ft/ft-internal.h"
+#include "ft/msg.h"
 
 // not version-sensitive because we only serialize a descriptor using the current layout_version
 uint32_t
@@ -152,7 +154,7 @@ deserialize_descriptor_from_rbuf(struct rbuf *rb, DESCRIPTOR desc, int layout_ve
     }
 
     uint32_t size;
-    bytevec data;
+    const void *data;
     rbuf_bytes(rb, &data, &size);
     toku_memdup_dbt(&desc->dbt, data, size);
 }
@@ -212,7 +214,7 @@ int deserialize_ft_versioned(int fd, struct rbuf *rb, FT *ftp, uint32_t version)
 
     //Verification of initial elements.
     //Check magic number
-    bytevec magic;
+    const void *magic;
     rbuf_literal_bytes(rb, &magic, 8);
     lazy_assert(memcmp(magic,"tokudata",8)==0);
 
@@ -234,7 +236,7 @@ int deserialize_ft_versioned(int fd, struct rbuf *rb, FT *ftp, uint32_t version)
     size = rbuf_network_int(rb);
     lazy_assert(size == rb->size);
 
-    bytevec tmp_byte_order_check;
+    const void *tmp_byte_order_check;
     lazy_assert((sizeof tmp_byte_order_check) >= 8);
     rbuf_literal_bytes(rb, &tmp_byte_order_check, 8); //Must not translate byte order
     int64_t byte_order_stored;
@@ -244,13 +246,13 @@ int deserialize_ft_versioned(int fd, struct rbuf *rb, FT *ftp, uint32_t version)
     uint64_t checkpoint_count;
     checkpoint_count = rbuf_ulonglong(rb);
     LSN checkpoint_lsn;
-    checkpoint_lsn = rbuf_lsn(rb);
+    checkpoint_lsn = rbuf_LSN(rb);
     unsigned nodesize;
     nodesize = rbuf_int(rb);
     DISKOFF translation_address_on_disk;
-    translation_address_on_disk = rbuf_diskoff(rb);
+    translation_address_on_disk = rbuf_DISKOFF(rb);
     DISKOFF translation_size_on_disk;
-    translation_size_on_disk = rbuf_diskoff(rb);
+    translation_size_on_disk = rbuf_DISKOFF(rb);
     lazy_assert(translation_address_on_disk > 0);
     lazy_assert(translation_size_on_disk > 0);
 
@@ -343,7 +345,7 @@ int deserialize_ft_versioned(int fd, struct rbuf *rb, FT *ftp, uint32_t version)
         time_of_last_optimize_begin = rbuf_ulonglong(rb);
         time_of_last_optimize_end = rbuf_ulonglong(rb);
         count_of_optimize_in_progress = rbuf_int(rb);
-        msn_at_start_of_last_completed_optimize = rbuf_msn(rb);
+        msn_at_start_of_last_completed_optimize = rbuf_MSN(rb);
     }
 
     enum toku_compression_method compression_method;
@@ -352,7 +354,7 @@ int deserialize_ft_versioned(int fd, struct rbuf *rb, FT *ftp, uint32_t version)
     if (ft->layout_version_read_from_disk >= FT_LAYOUT_VERSION_19) {
         unsigned char method = rbuf_char(rb);
         compression_method = (enum toku_compression_method) method;
-        highest_unused_msn_for_upgrade = rbuf_msn(rb);
+        highest_unused_msn_for_upgrade = rbuf_MSN(rb);
     } else {
         // we hard coded zlib until 5.2, then quicklz in 5.2
         if (ft->layout_version_read_from_disk < FT_LAYOUT_VERSION_18) {
@@ -365,7 +367,7 @@ int deserialize_ft_versioned(int fd, struct rbuf *rb, FT *ftp, uint32_t version)
     MSN max_msn_in_ft;
     max_msn_in_ft = ZERO_MSN;  // We'll upgrade it from the root node later if necessary
     if (ft->layout_version_read_from_disk >= FT_LAYOUT_VERSION_21) {
-        max_msn_in_ft = rbuf_msn(rb);
+        max_msn_in_ft = rbuf_MSN(rb);
     }
 
     (void) rbuf_int(rb); //Read in checksum and ignore (already verified).
@@ -552,7 +554,7 @@ int deserialize_ft_from_fd_into_rbuf(int fd,
     rbuf_init(rb, prefix, prefix_size);
 
     //Check magic number
-    bytevec magic;
+    const void *magic;
     rbuf_literal_bytes(rb, &magic, 8);
     if (memcmp(magic,"tokudata",8)!=0) {
         if ((*(uint64_t*)magic) == 0) {
@@ -626,7 +628,7 @@ int deserialize_ft_from_fd_into_rbuf(int fd,
     }
 
     //Verify byte order
-    bytevec tmp_byte_order_check;
+    const void *tmp_byte_order_check;
     lazy_assert((sizeof toku_byte_order_host) == 8);
     rbuf_literal_bytes(rb, &tmp_byte_order_check, 8); //Must not translate byte order
     int64_t byte_order_stored;
@@ -638,7 +640,7 @@ int deserialize_ft_from_fd_into_rbuf(int fd,
 
     //Load checkpoint count
     *checkpoint_count = rbuf_ulonglong(rb);
-    *checkpoint_lsn = rbuf_lsn(rb);
+    *checkpoint_lsn = rbuf_LSN(rb);
     //Restart at beginning during regular deserialization
     rb->ndone = 0;
 

ft/ft-test-helpers.cc

@@ -89,13 +89,12 @@ PATENT RIGHTS GRANT:
 #ident "Copyright (c) 2007-2013 Tokutek Inc.  All rights reserved."
 #ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
 
-#include "ft-cachetable-wrappers.h"
-#include "ft-flusher.h"
-#include "ft-internal.h"
-#include "ft.h"
-#include "node.h"
-#include "fttypes.h"
-#include "ule.h"
+#include "ft/ft.h"
+#include "ft/ft-cachetable-wrappers.h"
+#include "ft/ft-internal.h"
+#include "ft/ft-flusher.h"
+#include "ft/node.h"
+#include "ft/ule.h"
 
 // dummymsn needed to simulate msn because messages are injected at a lower level than toku_ft_root_put_msg()
 #define MIN_DUMMYMSN ((MSN) {(uint64_t)1 << 62})

ft/ft-verify.cc

@@ -109,18 +109,18 @@ compare_pairs (FT_HANDLE ft_handle, const DBT *a, const DBT *b) {
 }
 
 static int 
-compare_pair_to_key (FT_HANDLE ft_handle, const DBT *a, bytevec key, ITEMLEN keylen) {
+compare_pair_to_key (FT_HANDLE ft_handle, const DBT *a, const void *key, uint32_t keylen) {
     DBT y;
     return ft_handle->ft->cmp(a, toku_fill_dbt(&y, key, keylen));
 }
 
 static int
-verify_msg_in_child_buffer(FT_HANDLE ft_handle, enum ft_msg_type type, MSN msn, bytevec key, ITEMLEN keylen, bytevec UU(data), ITEMLEN UU(datalen), XIDS UU(xids), const DBT *lesser_pivot, const DBT *greatereq_pivot)
+verify_msg_in_child_buffer(FT_HANDLE ft_handle, enum ft_msg_type type, MSN msn, const void *key, uint32_t keylen, const void *UU(data), uint32_t UU(datalen), XIDS UU(xids), const DBT *lesser_pivot, const DBT *greatereq_pivot)
     __attribute__((warn_unused_result));
 
 UU()
 static int
-verify_msg_in_child_buffer(FT_HANDLE ft_handle, enum ft_msg_type type, MSN msn, bytevec key, ITEMLEN keylen, bytevec UU(data), ITEMLEN UU(datalen), XIDS UU(xids), const DBT *lesser_pivot, const DBT *greatereq_pivot) {
+verify_msg_in_child_buffer(FT_HANDLE ft_handle, enum ft_msg_type type, MSN msn, const void *key, uint32_t keylen, const void *UU(data), uint32_t UU(datalen), XIDS UU(xids), const DBT *lesser_pivot, const DBT *greatereq_pivot) {
     int result = 0;
     if (msn.msn == ZERO_MSN.msn)
         result = EINVAL;
@@ -328,8 +328,8 @@ struct verify_msg_fn {
         XIDS xid = msg.xids();
         const void *key = msg.kdbt()->data;
         const void *data = msg.vdbt()->data;
-        ITEMLEN keylen = msg.kdbt()->size;
-        ITEMLEN datalen = msg.vdbt()->size;
+        uint32_t keylen = msg.kdbt()->size;
+        uint32_t datalen = msg.vdbt()->size;
 
         int r = verify_msg_in_child_buffer(ft_handle, type, msn, key, keylen, data, datalen, xid,
                                            curr_less_pivot,

ft/ft.h

@@ -91,13 +91,15 @@ PATENT RIGHTS GRANT:
 #ident "Copyright (c) 2007-2013 Tokutek Inc.  All rights reserved."
 #ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
 
-#include "fttypes.h"
-#include "ybt.h"
 #include <db.h>
-#include "cachetable.h"
-#include "log.h"
-#include "ft-ops.h"
-#include "compress.h"
+
+#include "ft/cachetable.h"
+#include "ft/ft-ops.h"
+#include "ft/log.h"
+#include "ft/ybt.h"
+
+typedef struct ft *FT;
+typedef struct ft_options *FT_OPTIONS;
 
 // unlink a ft from the filesystem with or without a txn.
 // if with a txn, then the unlink happens on commit.
@@ -173,9 +175,17 @@ void toku_ft_update_cmp_descriptor(FT ft);
 DESCRIPTOR toku_ft_get_descriptor(FT_HANDLE ft_handle);
 DESCRIPTOR toku_ft_get_cmp_descriptor(FT_HANDLE ft_handle);
 
+typedef struct {
+    // delta versions in basements could be negative
+    int64_t numrows;
+    int64_t numbytes;
+} STAT64INFO_S, *STAT64INFO;
+static const STAT64INFO_S ZEROSTATS = { .numrows = 0, .numbytes = 0};
+
 void toku_ft_update_stats(STAT64INFO headerstats, STAT64INFO_S delta);
 void toku_ft_decrease_stats(STAT64INFO headerstats, STAT64INFO_S delta);
 
+typedef void (*remove_ft_ref_callback)(FT ft, void *extra);
 void toku_ft_remove_reference(FT ft,
                               bool oplsn_valid, LSN oplsn,
                               remove_ft_ref_callback remove_ref, void *extra);
@@ -220,5 +230,5 @@ struct toku_product_name_strings_struct {
 extern struct toku_product_name_strings_struct toku_product_name_strings;
 extern int tokudb_num_envs;
 
-int toku_keycompare (bytevec key1, ITEMLEN key1len, bytevec key2, ITEMLEN key2len);
+int toku_keycompare (const void *key1, uint32_t key1len, const void *key2, uint32_t key2len);
 int toku_builtin_compare_fun (DB *, const DBT *, const DBT*) __attribute__((__visibility__("default")));

ft/ft_node-serialize.cc

@@ -1131,7 +1131,7 @@ read_compressed_sub_block(struct rbuf *rb, struct sub_block *sb)
     int r = 0;
     sb->compressed_size = rbuf_int(rb);
     sb->uncompressed_size = rbuf_int(rb);
-    bytevec* cp = (bytevec*)&sb->compressed_ptr;
+    const void **cp = (const void **) &sb->compressed_ptr;
     rbuf_literal_bytes(rb, cp, sb->compressed_size);
     sb->xsum = rbuf_int(rb);
     // let's check the checksum
@@ -1212,7 +1212,7 @@ deserialize_ftnode_info(
     struct rbuf rb;
     rbuf_init(&rb, (unsigned char *) sb->uncompressed_ptr, data_size);
 
-    node->max_msn_applied_to_node_on_disk = rbuf_msn(&rb);
+    node->max_msn_applied_to_node_on_disk = rbuf_MSN(&rb);
     (void)rbuf_int(&rb);
     node->flags = rbuf_int(&rb);
     node->height = rbuf_int(&rb);
@@ -1488,7 +1488,7 @@ deserialize_ftnode_header_from_rbuf_if_small_enough (FTNODE *ftnode,
         goto cleanup;
     }
 
-    bytevec magic;
+    const void *magic;
     rbuf_literal_bytes(rb, &magic, 8);
     if (memcmp(magic, "tokuleaf", 8)!=0 &&
         memcmp(magic, "tokunode", 8)!=0) {
@@ -1556,8 +1556,8 @@ deserialize_ftnode_header_from_rbuf_if_small_enough (FTNODE *ftnode,
     }
 
     // Finish reading compressed the sub_block
-    bytevec* cp;
-    cp = (bytevec*)&sb_node_info.compressed_ptr;
+    const void **cp;
+    cp = (const void **) &sb_node_info.compressed_ptr;
     rbuf_literal_bytes(rb, cp, sb_node_info.compressed_size);
     sb_node_info.xsum = rbuf_int(rb);
     // let's check the checksum
@@ -1954,7 +1954,7 @@ deserialize_and_upgrade_ftnode(FTNODE node,
     // Re-read the magic field from the previous call, since we are
     // restarting with a fresh rbuf.
     {
-        bytevec magic;
+        const void *magic;
         rbuf_literal_bytes(&rb, &magic, 8);              // 1. magic
     }
 
@@ -2036,7 +2036,7 @@ deserialize_ftnode_from_rbuf(
 
     // now start reading from rbuf
     // first thing we do is read the header information
-    bytevec magic;
+    const void *magic;
     rbuf_literal_bytes(rb, &magic, 8);
     if (memcmp(magic, "tokuleaf", 8)!=0 &&
         memcmp(magic, "tokunode", 8)!=0) {
@@ -2561,7 +2561,7 @@ deserialize_rollback_log_from_rbuf (BLOCKNUM blocknum, ROLLBACK_LOG_NODE *log_p,
     }
 
     //printf("Deserializing %lld datasize=%d\n", off, datasize);
-    bytevec magic;
+    const void *magic;
     rbuf_literal_bytes(rb, &magic, 8);
     lazy_assert(!memcmp(magic, "tokuroll", 8));
 
@@ -2594,7 +2594,7 @@ deserialize_rollback_log_from_rbuf (BLOCKNUM blocknum, ROLLBACK_LOG_NODE *log_p,
     while (rb->ndone < rb->size) {
         struct roll_entry *item;
         uint32_t rollback_fsize = rbuf_int(rb); //Already read 4.  Rest is 4 smaller
-        bytevec item_vec;
+        const void *item_vec;
         rbuf_literal_bytes(rb, &item_vec, rollback_fsize-4);
         unsigned char* item_buf = (unsigned char*)item_vec;
         r = toku_parse_rollback(item_buf, rollback_fsize-4, &item, &result->rollentry_arena);

ft/fttypes.h

@@ -1,196 +0,0 @@
-/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */
-// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4:
-
-#ident "$Id$"
-/*
-COPYING CONDITIONS NOTICE:
-
-  This program is free software; you can redistribute it and/or modify
-  it under the terms of version 2 of the GNU General Public License as
-  published by the Free Software Foundation, and provided that the
-  following conditions are met:
-
-      * Redistributions of source code must retain this COPYING
-        CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the
-        DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the
-        PATENT MARKING NOTICE (below), and the PATENT RIGHTS
-        GRANT (below).
-
-      * Redistributions in binary form must reproduce this COPYING
-        CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the
-        DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the
-        PATENT MARKING NOTICE (below), and the PATENT RIGHTS
-        GRANT (below) in the documentation and/or other materials
-        provided with the distribution.
-
-  You should have received a copy of the GNU General Public License
-  along with this program; if not, write to the Free Software
-  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
-  02110-1301, USA.
-
-COPYRIGHT NOTICE:
-
-  TokuDB, Tokutek Fractal Tree Indexing Library.
-  Copyright (C) 2007-2013 Tokutek, Inc.
-
-DISCLAIMER:
-
-  This program is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-UNIVERSITY PATENT NOTICE:
-
-  The technology is licensed by the Massachusetts Institute of
-  Technology, Rutgers State University of New Jersey, and the Research
-  Foundation of State University of New York at Stony Brook under
-  United States of America Serial No. 11/760379 and to the patents
-  and/or patent applications resulting from it.
-
-PATENT MARKING NOTICE:
-
-  This software is covered by US Patent No. 8,185,551.
-  This software is covered by US Patent No. 8,489,638.
-
-PATENT RIGHTS GRANT:
-
-  "THIS IMPLEMENTATION" means the copyrightable works distributed by
-  Tokutek as part of the Fractal Tree project.
-
-  "PATENT CLAIMS" means the claims of patents that are owned or
-  licensable by Tokutek, both currently or in the future; and that in
-  the absence of this license would be infringed by THIS
-  IMPLEMENTATION or by using or running THIS IMPLEMENTATION.
-
-  "PATENT CHALLENGE" shall mean a challenge to the validity,
-  patentability, enforceability and/or non-infringement of any of the
-  PATENT CLAIMS or otherwise opposing any of the PATENT CLAIMS.
-
-  Tokutek hereby grants to you, for the term and geographical scope of
-  the PATENT CLAIMS, a non-exclusive, no-charge, royalty-free,
-  irrevocable (except as stated in this section) patent license to
-  make, have made, use, offer to sell, sell, import, transfer, and
-  otherwise run, modify, and propagate the contents of THIS
-  IMPLEMENTATION, where such license applies only to the PATENT
-  CLAIMS.  This grant does not include claims that would be infringed
-  only as a consequence of further modifications of THIS
-  IMPLEMENTATION.  If you or your agent or licensee institute or order
-  or agree to the institution of patent litigation against any entity
-  (including a cross-claim or counterclaim in a lawsuit) alleging that
-  THIS IMPLEMENTATION constitutes direct or contributory patent
-  infringement, or inducement of patent infringement, then any rights
-  granted to you under this License shall terminate as of the date
-  such litigation is filed.  If you or your agent or exclusive
-  licensee institute or order or agree to the institution of a PATENT
-  CHALLENGE, then Tokutek may terminate any rights granted to you
-  under this License.
-*/
-
-#pragma once
-
-#ident "Copyright (c) 2007-2013 Tokutek Inc.  All rights reserved."
-#ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
-
-#include <sys/types.h>
-#ifndef _XOPEN_SOURCE
-#define _XOPEN_SOURCE 500
-#endif
-#define _FILE_OFFSET_BITS 64
-
-#include "toku_assert.h"
-#include <db.h>
-#include <inttypes.h>
-
-
-// Use the C++ bool and constants (true false), rather than BOOL, TRUE, and FALSE.
-
-typedef struct ft_handle *FT_HANDLE;
-typedef struct ftnode *FTNODE;
-typedef struct ftnode_disk_data *FTNODE_DISK_DATA;
-typedef struct ftnode_leaf_basement_node *BASEMENTNODE;
-typedef struct ftnode_nonleaf_childinfo *NONLEAF_CHILDINFO;
-typedef struct sub_block *SUB_BLOCK;
-typedef struct ft *FT;
-typedef struct ft_header *FT_HEADER;
-typedef struct ft_options *FT_OPTIONS;
-
-typedef unsigned int ITEMLEN;
-typedef const void *bytevec;
-
-typedef int64_t DISKOFF;  /* Offset in a disk. -1 is the NULL pointer. */
-typedef uint64_t TXNID;
-
-typedef struct txnid_pair_s {
-    TXNID parent_id64;
-    TXNID child_id64;
-} TXNID_PAIR;
-
-
-#define TXNID_NONE_LIVING ((TXNID)0)
-#define TXNID_NONE        ((TXNID)0)
-#define TXNID_MAX         ((TXNID)-1)
-
-static const TXNID_PAIR TXNID_PAIR_NONE = { .parent_id64 = TXNID_NONE, .child_id64 = TXNID_NONE };
-
-typedef struct blocknum_s { int64_t b; } BLOCKNUM; // make a struct so that we will notice type problems.
-typedef struct gid_s { uint8_t *gid; } GID; // the gid is of size [DB_GID_SIZE]
-typedef TOKU_XA_XID *XIDP; // this is the type that's passed to the logger code (so that we don't have to copy all 152 bytes when only a subset are even valid.)
-#define ROLLBACK_NONE     ((BLOCKNUM){0})
-
-static inline BLOCKNUM make_blocknum(int64_t b) { BLOCKNUM result={b}; return result; }
-
-typedef struct {
-    uint32_t len;
-    char *data;
-} BYTESTRING;
-
-/* Log Sequence Number (LSN)
- * Make the LSN be a struct instead of an integer so that we get better type checking. */
-typedef struct __toku_lsn { uint64_t lsn; } LSN;
-#define ZERO_LSN ((LSN){0})
-#define MAX_LSN  ((LSN){UINT64_MAX})
-
-/* Message Sequence Number (MSN)
- * Make the MSN be a struct instead of an integer so that we get better type checking. */
-typedef struct __toku_msn { uint64_t msn; } MSN;
-#define ZERO_MSN ((MSN){0})                 // dummy used for message construction, to be filled in when msg is applied to tree
-#define MIN_MSN  ((MSN){(uint64_t)1 << 62})  // first 2^62 values reserved for messages created before Dr. No (for upgrade)
-#define MAX_MSN  ((MSN){UINT64_MAX})
-
-typedef struct {
-    int64_t numrows;           // delta versions in basements could be negative
-    int64_t numbytes;
-} STAT64INFO_S, *STAT64INFO;
-
-static const STAT64INFO_S ZEROSTATS = {0,0};
-
-/* At the ft layer, a FILENUM uniquely identifies an open file.
- * At the ydb layer, a DICTIONARY_ID uniquely identifies an open dictionary.
- * With the introduction of the loader (ticket 2216), it is possible for the file that holds
- * an open dictionary to change, so these are now separate and independent unique identifiers.
- */
-typedef struct {uint32_t fileid;} FILENUM;
-#define FILENUM_NONE ((FILENUM){UINT32_MAX})
-
-typedef struct {uint64_t dictid;} DICTIONARY_ID;
-#define DICTIONARY_ID_NONE ((DICTIONARY_ID){0})
-
-typedef struct {
-    uint32_t num;
-    FILENUM  *filenums;
-} FILENUMS;
-
-typedef struct tokulogger *TOKULOGGER;
-typedef struct txn_manager *TXN_MANAGER;
-typedef struct tokutxn    *TOKUTXN;
-
-typedef struct xids_t *XIDS;
-
-typedef int (*ft_compare_func)(DB *, const DBT *, const DBT *);
-typedef void (*setval_func)(const DBT *, void *);
-typedef int (*ft_update_func)(DB *, const DBT *, const DBT *, const DBT *, setval_func, void *);
-typedef void (*remove_ft_ref_callback)(FT, void*);
-typedef void (*on_redirect_callback)(FT_HANDLE, void*);
-
-#define UU(x) x __attribute__((__unused__))

ft/ftverify.cc

@@ -94,18 +94,17 @@ PATENT RIGHTS GRANT:
 // fractal tree file, one block at a time.
 ////////////////////////////////////////////////////////////////////
 
-#include "fttypes.h"
-#include "ft-internal.h"
-#include "node.h"
-#include "ft_layout_version.h"
-#include "block_table.h"
-#include "rbuf.h"
-#include "sub_block.h"
+#include "portability/toku_assert.h"
+#include "portability/toku_list.h"
+#include "portability/toku_portability.h"
 
-#include <toku_assert.h>
-#include <toku_list.h>
-#include <toku_portability.h>
-#include <util/threadpool.h>
+#include "ft/block_allocator.h"
+#include "ft/ft-internal.h"
+#include "ft/ft_layout_version.h"
+#include "ft/node.h"
+#include "ft/rbuf.h"
+#include "ft/sub_block.h"
+#include "util/threadpool.h"
 
 #include <fcntl.h>
 #include <math.h>

ft/le-cursor.h

@@ -91,7 +91,7 @@ PATENT RIGHTS GRANT:
 #ident "Copyright (c) 2010-2013 Tokutek Inc.  All rights reserved."
 #ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
 
-#include "ft-ops.h"
+#include "ft/ft-internal.h"
 
 // A leaf entry cursor (LE_CURSOR) is a special type of FT_CURSOR that visits all of the leaf entries in a tree
 // and returns the leaf entry to the caller.  It maintains a copy of the key that it was last positioned over to

ft/loader/dbufio.cc

@@ -89,16 +89,17 @@ PATENT RIGHTS GRANT:
 #ident "Copyright (c) 2010-2013 Tokutek Inc.  All rights reserved."
 #ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
 
-#include "loader/dbufio.h"
-#include "fttypes.h"
-#include <toku_assert.h>
 #include <errno.h>
-#include <unistd.h>
-#include "memory.h"
 #include <string.h>
+#include <unistd.h>
+
+#include "portability/toku_assert.h"
+#include "portability/memory.h"
+
+#include "ft/ft.h"
+#include "ft/ft-internal.h"
+#include "loader/dbufio.h"
 #include "loader/loader-internal.h"
-#include "ft-internal.h"
-#include "ft.h"
 
 struct dbufio_file {
     // i/o thread owns these

ft/loader/loader-internal.h

@@ -91,13 +91,15 @@ PATENT RIGHTS GRANT:
 #ident "Copyright (c) 2010-2013 Tokutek Inc.  All rights reserved."
 
 #include <db.h>
-#include "fttypes.h"
+
+#include "portability/toku_pthread.h"
+
+#include "loader/dbufio.h"
 #include "loader/loader.h"
 #include "util/queue.h"
-#include <toku_pthread.h>
-#include "loader/dbufio.h"
 
-enum { EXTRACTOR_QUEUE_DEPTH = 2,
+enum {
+    EXTRACTOR_QUEUE_DEPTH = 2,
     FILE_BUFFER_SIZE  = 1<<24,
     MIN_ROWSET_MEMORY = 1<<23,
     MIN_MERGE_FANIN   = 2,
@@ -109,7 +111,6 @@ enum { EXTRACTOR_QUEUE_DEPTH = 2,
     MAX_UNCOMPRESSED_BUF = MIN_MERGE_BUF_SIZE
 };
 
-
 /* These functions are exported to allow the tests to compile. */
 
 /* These structures maintain a collection of all the open temporary files used by the loader. */

ft/loader/loader.h

@@ -92,7 +92,10 @@ PATENT RIGHTS GRANT:
 #ident "Copyright (c) 2007-2013 Tokutek Inc.  All rights reserved."
 #ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
 
+#include "ft/txn.h"
 #include "ft/cachetable.h"
+#include "ft/comparator.h"
+#include "ft/ft-ops.h"
 
 // The loader callbacks are C functions and need to be defined as such
 

ft/log-internal.h

@@ -103,7 +103,6 @@ PATENT RIGHTS GRANT:
 #include "txn.h"
 #include "txn_manager.h"
 #include "rollback_log_node_cache.h"
-#include "txn_child_manager.h"
 
 #include <portability/toku_pthread.h>
 
@@ -182,8 +181,6 @@ struct tokulogger {
     tokutime_t time_spent_writing_to_disk; // how much tokutime did we spend writing to disk?
     uint64_t num_wait_buf_long;            // how many times we waited >= 100ms for the in buf
 
-    void (*remove_finalize_callback) (DICTIONARY_ID, void*);  // ydb-level callback to be called when a transaction that ...
-    void * remove_finalize_callback_extra;                    // ... deletes a file is committed or when one that creates a file is aborted.
     CACHEFILE rollback_cachefile;
     rollback_log_node_cache rollback_cache;
     TXN_MANAGER txn_manager;
@@ -192,100 +189,6 @@ struct tokulogger {
 int toku_logger_find_next_unused_log_file(const char *directory, long long *result);
 int toku_logger_find_logfiles (const char *directory, char ***resultp, int *n_logfiles);
 
-struct txn_roll_info {
-    // these are number of rollback nodes and rollback entries for this txn.
-    //
-    // the current rollback node below has sequence number num_rollback_nodes - 1
-    // (because they are numbered 0...num-1). often, the current rollback is
-    // already set to this block num, which means it exists and is available to
-    // log some entries. if the current rollback is NONE and the number of
-    // rollback nodes for this transaction is non-zero, then we will use
-    // the number of rollback nodes to know which sequence number to assign
-    // to a new one we create
-    uint64_t num_rollback_nodes;
-    uint64_t num_rollentries;
-    uint64_t num_rollentries_processed;
-    uint64_t rollentry_raw_count;  // the total count of every byte in the transaction and all its children.
-
-    // spilled rollback nodes are rollback nodes that were gorged by this
-    // transaction, retired, and saved in a list.
-
-    // the spilled rollback head is the block number of the first rollback node
-    // that makes up the rollback log chain
-    BLOCKNUM spilled_rollback_head;
-    // the spilled rollback is the block number of the last rollback node that
-    // makes up the rollback log chain. 
-    BLOCKNUM spilled_rollback_tail;
-    // the current rollback node block number we may use. if this is ROLLBACK_NONE,
-    // then we need to create one and set it here before using it.
-    BLOCKNUM current_rollback; 
-};
-
-struct tokutxn {
-    // These don't change after create:
-
-    TXNID_PAIR txnid;
-
-    uint64_t snapshot_txnid64; // this is the lsn of the snapshot
-    const TXN_SNAPSHOT_TYPE snapshot_type;
-    const bool for_recovery;
-    const TOKULOGGER logger;
-    const TOKUTXN parent;
-    // The child txn is protected by the child_txn_manager lock
-    // and by the user contract. The user contract states (and is
-    // enforced at the ydb layer) that a child txn should not be created
-    // while another child exists. The txn_child_manager will protect
-    // other threads from trying to read this value while another
-    // thread commits/aborts the child
-    TOKUTXN child;
-    // statically allocated child manager, if this 
-    // txn is a root txn, this manager will be used and set to 
-    // child_manager for this transaction and all of its children
-    txn_child_manager child_manager_s;
-    // child manager for this transaction, all of its children,
-    // and all of its ancestors
-    txn_child_manager* child_manager;
-    // These don't change but they're created in a way that's hard to make
-    // strictly const.
-    DB_TXN *container_db_txn; // reference to DB_TXN that contains this tokutxn
-    xid_omt_t *live_root_txn_list; // the root txns live when the root ancestor (self if a root) started.
-    XIDS xids; // Represents the xid list
-
-    TOKUTXN snapshot_next;
-    TOKUTXN snapshot_prev;
-
-    bool begin_was_logged;
-    bool declared_read_only; // true if the txn was declared read only when began
-    // These are not read until a commit, prepare, or abort starts, and
-    // they're "monotonic" (only go false->true) during operation:
-    bool do_fsync;
-    bool force_fsync_on_commit;  //This transaction NEEDS an fsync once (if) it commits.  (commit means root txn)
-
-    // Not used until commit, prepare, or abort starts:
-    LSN do_fsync_lsn;
-    TOKU_XA_XID xa_xid; // for prepared transactions
-    TXN_PROGRESS_POLL_FUNCTION progress_poll_fun;
-    void *progress_poll_fun_extra;
-
-    toku_mutex_t txn_lock;
-    // Protected by the txn lock:
-    omt<FT> open_fts; // a collection of the fts that we touched.  Indexed by filenum.
-    struct txn_roll_info roll_info; // Info used to manage rollback entries
-
-    // mutex that protects the transition of the state variable
-    // the rest of the variables are used by the txn code and 
-    // hot indexing to ensure that when hot indexing is processing a 
-    // leafentry, a TOKUTXN cannot dissappear or change state out from
-    // underneath it
-    toku_mutex_t state_lock;
-    toku_cond_t state_cond;
-    TOKUTXN_STATE state;
-    uint32_t num_pin; // number of threads (all hot indexes) that want this
-                      // txn to not transition to commit or abort
-    uint64_t client_id;
-};
-typedef struct tokutxn    *TOKUTXN;
-
 static inline int
 txn_has_current_rollback_log(TOKUTXN txn) {
     return txn->roll_info.current_rollback.b != ROLLBACK_NONE.b;

ft/log.h

@@ -92,16 +92,17 @@ PATENT RIGHTS GRANT:
 #ident "Copyright (c) 2007-2013 Tokutek Inc.  All rights reserved."
 #ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
 
-#include <toku_portability.h>
-#include <errno.h>
 #include <db.h>
+#include <errno.h>
 
-#include "fttypes.h"
-#include "memory.h"
-#include "logger.h"
-#include "rollback.h"
-#include "recover.h"
-#include "txn.h"
+#include "portability/memory.h"
+#include "portability/toku_portability.h"
+
+#include "ft/logger.h"
+#include "ft/rollback.h"
+#include "ft/recover.h"
+#include "ft/txn.h"
+#include "util/bytestring.h"
 
 struct roll_entry;
 

ft/logformat.cc

@@ -854,9 +854,9 @@ int main (int argc, const char *const argv[]) {
     fprintf2(cf, hf, "#ident \"Copyright (c) 2007-2013 Tokutek Inc.  All rights reserved.\"\n");
     fprintf2(cf, pf, "#include <stdint.h>\n");
     fprintf2(cf, pf, "#include <sys/time.h>\n");
-    fprintf2(cf, pf, "#include <ft/fttypes.h>\n");
     fprintf2(cf, pf, "#include <ft/log-internal.h>\n");
     fprintf(hf, "#include <ft/ft-internal.h>\n");
+    fprintf(hf, "#include <util/bytestring.h>\n");
     fprintf(hf, "#include <util/memarena.h>\n");
     generate_enum();
     generate_log_struct();

ft/logger.cc

@@ -171,7 +171,6 @@ int toku_logger_create (TOKULOGGER *resultp) {
     result->write_log_files = true;
     result->trim_log_files = true;
     result->directory=0;
-    result->remove_finalize_callback = NULL;
     // fd is uninitialized on purpose
     // ct is uninitialized on purpose
     result->lg_max = 100<<20; // 100MB default

ft/logger.h

@@ -92,9 +92,11 @@ PATENT RIGHTS GRANT:
 #ident "Copyright (c) 2007-2013 Tokutek Inc.  All rights reserved."
 #ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
 
-#include "ft/fttypes.h"
-#include "ft/ft-internal.h"
+#include "ft/block_table.h"
 #include "ft/ft_layout_version.h"
+#include "ft/txn.h"
+
+typedef struct tokulogger *TOKULOGGER;
 
 enum {
     TOKU_LOG_VERSION_1 = 1,
@@ -110,15 +112,15 @@ int toku_logger_open (const char *directory, TOKULOGGER logger);
 int toku_logger_open_with_last_xid(const char *directory, TOKULOGGER logger, TXNID last_xid);
 void toku_logger_shutdown(TOKULOGGER logger);
 int toku_logger_close(TOKULOGGER *loggerp);
-void toku_logger_initialize_rollback_cache(TOKULOGGER logger, FT ft);
-int toku_logger_open_rollback(TOKULOGGER logger, CACHETABLE cachetable, bool create);
+void toku_logger_initialize_rollback_cache(TOKULOGGER logger, struct ft *ft);
+int toku_logger_open_rollback(TOKULOGGER logger, struct cachetable *ct, bool create);
 void toku_logger_close_rollback(TOKULOGGER logger);
 bool toku_logger_rollback_is_open (TOKULOGGER); // return true iff the rollback is open.
 
 void toku_logger_fsync (TOKULOGGER logger);
 void toku_logger_fsync_if_lsn_not_fsynced(TOKULOGGER logger, LSN lsn);
 int toku_logger_is_open(TOKULOGGER logger);
-void toku_logger_set_cachetable (TOKULOGGER logger, CACHETABLE ct);
+void toku_logger_set_cachetable (TOKULOGGER logger, struct cachetable *ct);
 int toku_logger_set_lg_max(TOKULOGGER logger, uint32_t lg_max);
 int toku_logger_get_lg_max(TOKULOGGER logger, uint32_t *lg_maxp);
 int toku_logger_set_lg_bsize(TOKULOGGER logger, uint32_t bsize);
@@ -139,10 +141,24 @@ int toku_logger_restart(TOKULOGGER logger, LSN lastlsn);
 // given LSN and delete them.
 void toku_logger_maybe_trim_log(TOKULOGGER logger, LSN oldest_open_lsn);
 
+// At the ft layer, a FILENUM uniquely identifies an open file.
+struct FILENUM {
+    uint32_t fileid;
+};
+static const FILENUM FILENUM_NONE = { .fileid = UINT32_MAX };
+
+struct FILENUMS {
+    uint32_t num;
+    FILENUM *filenums;
+};
+
 void toku_logger_log_fcreate(TOKUTXN txn, const char *fname, FILENUM filenum, uint32_t mode, uint32_t flags, uint32_t nodesize, uint32_t basementnodesize, enum toku_compression_method compression_method);
 void toku_logger_log_fdelete(TOKUTXN txn, FILENUM filenum);
 void toku_logger_log_fopen(TOKUTXN txn, const char * fname, FILENUM filenum, uint32_t treeflags);
 
+// the log generation code requires a typedef if we want to pass by pointer
+typedef TOKU_XA_XID *XIDP;
+
 int toku_fread_uint8_t (FILE *f, uint8_t *v, struct x1764 *mm, uint32_t *len);
 int toku_fread_uint32_t_nocrclen (FILE *f, uint32_t *v);
 int toku_fread_uint32_t (FILE *f, uint32_t *v, struct x1764 *checksum, uint32_t *len);
@@ -258,4 +274,63 @@ void toku_logger_get_status(TOKULOGGER logger, LOGGER_STATUS s);
 
 int toku_get_version_of_logs_on_disk(const char *log_dir, bool *found_any_logs, uint32_t *version_found);
 
-TXN_MANAGER toku_logger_get_txn_manager(TOKULOGGER logger);
+struct txn_manager *toku_logger_get_txn_manager(TOKULOGGER logger);
+
+// For serialize / deserialize
+
+#include "ft/wbuf.h"
+
+static inline void wbuf_nocrc_FILENUM(struct wbuf *wb, FILENUM fileid) {
+    wbuf_nocrc_uint(wb, fileid.fileid);
+}
+
+static inline void wbuf_FILENUM(struct wbuf *wb, FILENUM fileid) {
+    wbuf_uint(wb, fileid.fileid);
+}
+
+static inline void wbuf_nocrc_FILENUMS(struct wbuf *wb, FILENUMS v) {
+    wbuf_nocrc_uint(wb, v.num);
+    for (uint32_t i = 0; i < v.num; i++) {
+        wbuf_nocrc_FILENUM(wb, v.filenums[i]);
+    }
+}
+
+static inline void wbuf_FILENUMS(struct wbuf *wb, FILENUMS v) {
+    wbuf_uint(wb, v.num);
+    for (uint32_t i = 0; i < v.num; i++) {
+        wbuf_FILENUM(wb, v.filenums[i]);
+    }
+}
+
+static inline void wbuf_nocrc_XIDP (struct wbuf *w, TOKU_XA_XID *xid) {
+    wbuf_nocrc_uint32_t(w, xid->formatID);
+    wbuf_nocrc_uint8_t(w, xid->gtrid_length);
+    wbuf_nocrc_uint8_t(w, xid->bqual_length);
+    wbuf_nocrc_literal_bytes(w, xid->data, xid->gtrid_length+xid->bqual_length);
+}
+
+#include "ft/rbuf.h"
+
+static inline void rbuf_FILENUM(struct rbuf *rb, FILENUM *filenum) {
+    filenum->fileid = rbuf_int(rb);
+}
+static inline void rbuf_ma_FILENUM(struct rbuf *rb, memarena *UU(ma), FILENUM *filenum) {
+    rbuf_FILENUM(rb, filenum);
+}
+
+static inline void rbuf_FILENUMS(struct rbuf *rb, FILENUMS *filenums) {
+    filenums->num = rbuf_int(rb);
+    XMALLOC_N(filenums->num, filenums->filenums);
+    for (uint32_t i = 0; i < filenums->num; i++) {
+        rbuf_FILENUM(rb, &(filenums->filenums[i]));
+    }
+}
+
+static inline void rbuf_ma_FILENUMS(struct rbuf *rb, memarena *ma, FILENUMS *filenums) {
+    rbuf_ma_uint32_t(rb, ma, &(filenums->num));
+    filenums->filenums = (FILENUM *) ma->malloc_from_arena(filenums->num * sizeof(FILENUM));
+    assert(filenums->filenums != NULL);
+    for (uint32_t i = 0; i < filenums->num; i++) {
+        rbuf_ma_FILENUM(rb, ma, &(filenums->filenums[i]));
+    }
+}

ft/msg.cc

@@ -90,7 +90,6 @@ PATENT RIGHTS GRANT:
 
 #include "portability/toku_portability.h"
 
-#include "ft/fttypes.h"
 #include "ft/msg.h"
 #include "ft/xids.h"
 #include "ft/ybt.h"
@@ -102,11 +101,11 @@ ft_msg::ft_msg(const DBT *key, const DBT *val, enum ft_msg_type t, MSN m, XIDS x
 }
 
 ft_msg ft_msg::deserialize_from_rbuf(struct rbuf *rb, XIDS *x, bool *is_fresh) {
-    bytevec keyp, valp;
-    ITEMLEN keylen, vallen;
+    const void *keyp, *valp;
+    uint32_t keylen, vallen;
     enum ft_msg_type t = (enum ft_msg_type) rbuf_char(rb);
     *is_fresh = rbuf_char(rb);
-    MSN m = rbuf_msn(rb);
+    MSN m = rbuf_MSN(rb);
     xids_create_from_buffer(rb, x);
     rbuf_bytes(rb, &keyp, &keylen);
     rbuf_bytes(rb, &valp, &vallen);
@@ -116,8 +115,8 @@ ft_msg ft_msg::deserialize_from_rbuf(struct rbuf *rb, XIDS *x, bool *is_fresh) {
 }
 
 ft_msg ft_msg::deserialize_from_rbuf_v13(struct rbuf *rb, MSN m, XIDS *x) {
-    bytevec keyp, valp;
-    ITEMLEN keylen, vallen;
+    const void *keyp, *valp;
+    uint32_t keylen, vallen;
     enum ft_msg_type t = (enum ft_msg_type) rbuf_char(rb);
     xids_create_from_buffer(rb, x);
     rbuf_bytes(rb, &keyp, &keylen);
@@ -169,3 +168,4 @@ void ft_msg::serialize_to_wbuf(struct wbuf *wb, bool is_fresh) const {
     wbuf_nocrc_bytes(wb, _key.data, _key.size);
     wbuf_nocrc_bytes(wb, _val.data, _val.size);
 }
+

ft/msg.h

@@ -94,9 +94,24 @@ PATENT RIGHTS GRANT:
 
 #pragma once
 
+#include <db.h>
+
+#include "portability/toku_assert.h"
+#include "portability/toku_stdint.h"
+
 #ident "Copyright (c) 2007-2013 Tokutek Inc.  All rights reserved."
 #ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
 
+// Message Sequence Number (MSN)
+typedef struct __toku_msn { uint64_t msn; } MSN;
+
+// dummy used for message construction, to be filled in when msg is applied to tree
+static const MSN ZERO_MSN = { .msn = 0 };
+
+// first 2^62 values reserved for messages created before Dr. No (for upgrade)
+static const MSN MIN_MSN = { .msn = 1ULL << 62 };
+static const MSN MAX_MSN = { .msn = UINT64_MAX };
+
 /* tree command types */
 enum ft_msg_type {
     FT_NONE = 0,
@@ -214,3 +229,18 @@ private:
     MSN _msn;
     XIDS _xids;
 };
+
+// For serialize / deserialize
+
+#include "ft/wbuf.h"
+
+static inline void wbuf_MSN(struct wbuf *wb, MSN msn) {
+    wbuf_ulonglong(wb, msn.msn);
+}
+
+#include "ft/rbuf.h"
+
+static inline MSN rbuf_MSN(struct rbuf *rb) {
+    MSN msn = { .msn = rbuf_ulonglong(rb) };
+    return msn;
+}

ft/msg_buffer.cc

@@ -225,8 +225,8 @@ void message_buffer::enqueue(const ft_msg &msg, bool is_fresh, int32_t *offset)
         int next_2 = next_power_of_two(need_space_total);
         _resize(next_2);
     }
-    ITEMLEN keylen = msg.kdbt()->size;
-    ITEMLEN datalen = msg.vdbt()->size;
+    uint32_t keylen = msg.kdbt()->size;
+    uint32_t datalen = msg.vdbt()->size;
     struct buffer_entry *entry = get_buffer_entry(_memory_used);
     entry->type = (unsigned char) msg.type();
     entry->msn = msg.msn();
@@ -256,13 +256,13 @@ bool message_buffer::get_freshness(int32_t offset) const {
 
 ft_msg message_buffer::get_message(int32_t offset, DBT *keydbt, DBT *valdbt) const {
     struct buffer_entry *entry = get_buffer_entry(offset);
-    ITEMLEN keylen = entry->keylen;
-    ITEMLEN vallen = entry->vallen;
+    uint32_t keylen = entry->keylen;
+    uint32_t vallen = entry->vallen;
     enum ft_msg_type type = (enum ft_msg_type) entry->type;
     MSN msn = entry->msn;
     const XIDS xids = (XIDS) &entry->xids_s;
-    bytevec key = xids_get_end_of_array(xids);
-    bytevec val = (uint8_t *) key + entry->keylen;
+    const void *key = xids_get_end_of_array(xids);
+    const void *val = (uint8_t *) key + entry->keylen;
     return ft_msg(toku_fill_dbt(keydbt, key, keylen), toku_fill_dbt(valdbt, val, vallen), type, msn, xids);
 }
 

ft/msg_buffer.h

@@ -88,10 +88,9 @@ PATENT RIGHTS GRANT:
 
 #pragma once
 
-#include "ft/fttypes.h"
-#include "ft/xids-internal.h"
-#include "ft/xids.h"
 #include "ft/msg.h"
+#include "ft/xids.h"
+#include "ft/xids-internal.h"
 #include "ft/ybt.h"
 
 class message_buffer {

ft/node.cc

@@ -1576,7 +1576,7 @@ static void bnc_insert_msg(NONLEAF_CHILDINFO bnc, const ft_msg &msg, bool is_fre
 }
 
 // This is only exported for tests.
-void toku_bnc_insert_msg(NONLEAF_CHILDINFO bnc, const void *key, ITEMLEN keylen, const void *data, ITEMLEN datalen, enum ft_msg_type type, MSN msn, XIDS xids, bool is_fresh, const toku::comparator &cmp)
+void toku_bnc_insert_msg(NONLEAF_CHILDINFO bnc, const void *key, uint32_t keylen, const void *data, uint32_t datalen, enum ft_msg_type type, MSN msn, XIDS xids, bool is_fresh, const toku::comparator &cmp)
 {
     DBT k, v;
     ft_msg msg(toku_fill_dbt(&k, key, keylen), toku_fill_dbt(&v, data, datalen), type, msn, xids);

ft/node.h

@@ -88,10 +88,9 @@ PATENT RIGHTS GRANT:
 
 #pragma once
 
-#include "ft/comparator.h"
-#include "ft/cachetable.h"
 #include "ft/bndata.h"
-#include "ft/fttypes.h"
+#include "ft/comparator.h"
+#include "ft/ft.h"
 #include "ft/msg_buffer.h"
 
 /* Pivot keys.
@@ -242,6 +241,7 @@ struct ftnode {
     struct ftnode_partition *bp;
     struct ctpair *ct_pair;
 };
+typedef struct ftnode *FTNODE;
 
 // data of an available partition of a leaf ftnode
 struct ftnode_leaf_basement_node {
@@ -251,6 +251,7 @@ struct ftnode_leaf_basement_node {
     bool stale_ancestor_messages_applied;
     STAT64INFO_S stat64_delta;      // change in stat64 counters since basement was last written to disk
 };
+typedef struct ftnode_leaf_basement_node *BASEMENTNODE;
 
 enum pt_state {  // declare this to be packed so that when used below it will only take 1 byte.
     PT_INVALID = 0,
@@ -277,6 +278,7 @@ struct ftnode_nonleaf_childinfo {
     off_omt_t stale_message_tree;
     uint64_t flow[2];  // current and last checkpoint
 };
+typedef struct ftnode_nonleaf_childinfo *NONLEAF_CHILDINFO;
     
 typedef struct ftnode_child_pointer {
     union {
@@ -298,6 +300,9 @@ struct ftnode_disk_data {
     uint32_t start;
     uint32_t size;
 };
+typedef struct ftnode_disk_data *FTNODE_DISK_DATA;
+
+// TODO: Turn these into functions instead of macros
 #define BP_START(node_dd,i) ((node_dd)[i].start)
 #define BP_SIZE(node_dd,i) ((node_dd)[i].size)
 
@@ -463,7 +468,7 @@ unsigned int toku_bnc_nbytesinbuf(NONLEAF_CHILDINFO bnc);
 int toku_bnc_n_entries(NONLEAF_CHILDINFO bnc);
 long toku_bnc_memory_size(NONLEAF_CHILDINFO bnc);
 long toku_bnc_memory_used(NONLEAF_CHILDINFO bnc);
-void toku_bnc_insert_msg(NONLEAF_CHILDINFO bnc, const void *key, ITEMLEN keylen, const void *data, ITEMLEN datalen, enum ft_msg_type type, MSN msn, XIDS xids, bool is_fresh, const toku::comparator &cmp);
+void toku_bnc_insert_msg(NONLEAF_CHILDINFO bnc, const void *key, uint32_t keylen, const void *data, uint32_t datalen, enum ft_msg_type type, MSN msn, XIDS xids, bool is_fresh, const toku::comparator &cmp);
 void toku_bnc_empty(NONLEAF_CHILDINFO bnc);
 void toku_bnc_flush_to_child(FT ft, NONLEAF_CHILDINFO bnc, FTNODE child, TXNID parent_oldest_referenced_xid_known);
 bool toku_bnc_should_promote(FT ft, NONLEAF_CHILDINFO bnc) __attribute__((const, nonnull));
@@ -514,8 +519,6 @@ void toku_ft_leaf_apply_msg(const toku::comparator &cmp, ft_update_func update_f
                             const ft_msg &msg, txn_gc_info *gc_info,
                             uint64_t *workdone, STAT64INFO stats_to_update);
 
-CACHETABLE_WRITE_CALLBACK get_write_callbacks_for_node(FT ft);
-
 //
 // Message management for orthopush
 //
@@ -602,7 +605,7 @@ static inline void set_BLB(FTNODE node, int i, BASEMENTNODE bn) {
     p->u.leaf = bn;
 }
 
-static inline SUB_BLOCK BSB(FTNODE node, int i) {
+static inline struct sub_block *BSB(FTNODE node, int i) {
     paranoid_invariant(i >= 0);
     paranoid_invariant(i < node->n_children);
     FTNODE_CHILD_POINTER p = node->bp[i].ptr;
@@ -610,7 +613,7 @@ static inline SUB_BLOCK BSB(FTNODE node, int i) {
     return p.u.subblock;
 }
 
-static inline void set_BSB(FTNODE node, int i, SUB_BLOCK sb) {
+static inline void set_BSB(FTNODE node, int i, struct sub_block *sb) {
     paranoid_invariant(i >= 0);
     paranoid_invariant(i < node->n_children);
     FTNODE_CHILD_POINTER *p = &node->bp[i].ptr;

ft/pivotkeys.cc

@@ -235,7 +235,7 @@ void ftnode_pivot_keys::deserialize_from_rbuf(struct rbuf *rb, int n) {
     XMALLOC_N_ALIGNED(64, _num_pivots, _dbt_keys);
     bool keys_same_size = true;
     for (int i = 0; i < _num_pivots; i++) {
-        bytevec pivotkeyptr;
+        const void *pivotkeyptr;
         uint32_t size;
         rbuf_bytes(rb, &pivotkeyptr, &size);
         toku_memdup_dbt(&_dbt_keys[i], pivotkeyptr, size);

ft/rbuf.h

@@ -94,7 +94,6 @@ PATENT RIGHTS GRANT:
 
 #include <string.h>
 
-#include "ft/fttypes.h"
 #include "portability/memory.h"
 #include "portability/toku_assert.h"
 #include "portability/toku_htonl.h"
@@ -159,14 +158,14 @@ static unsigned int rbuf_int (struct rbuf *r) {
 #endif
 }
 
-static inline void rbuf_literal_bytes (struct rbuf *r, bytevec *bytes, unsigned int n_bytes) {
+static inline void rbuf_literal_bytes (struct rbuf *r, const void **bytes, unsigned int n_bytes) {
     *bytes =   &r->buf[r->ndone];
     r->ndone+=n_bytes;
     assert(r->ndone<=r->size);
 }
 
 /* Return a pointer into the middle of the buffer. */
-static inline void rbuf_bytes (struct rbuf *r, bytevec *bytes, unsigned int *n_bytes)
+static inline void rbuf_bytes (struct rbuf *r, const void **bytes, unsigned int *n_bytes)
 {
     *n_bytes = rbuf_int(r);
     rbuf_literal_bytes(r, bytes, *n_bytes);
@@ -182,28 +181,6 @@ static inline signed long long rbuf_longlong (struct rbuf *r) {
     return (signed long long)rbuf_ulonglong(r);
 }
 
-static inline DISKOFF rbuf_diskoff (struct rbuf *r) {
-    return rbuf_ulonglong(r);
-}
-
-static inline LSN rbuf_lsn (struct rbuf *r) {
-    LSN lsn = {rbuf_ulonglong(r)};
-    return lsn;
-}
-
-static inline MSN rbuf_msn (struct rbuf *r) {
-    MSN msn = {rbuf_ulonglong(r)};
-    return msn;
-}
-
-static inline BLOCKNUM rbuf_blocknum (struct rbuf *r) {
-    BLOCKNUM result = make_blocknum(rbuf_longlong(r));
-    return result;
-}
-static inline void rbuf_ma_BLOCKNUM (struct rbuf *r, memarena *ma __attribute__((__unused__)), BLOCKNUM *blocknum) {
-    *blocknum = rbuf_blocknum(r);
-}
-
 static inline void rbuf_ma_uint32_t (struct rbuf *r, memarena *ma __attribute__((__unused__)), uint32_t *num) {
     *num = rbuf_int(r);
 }
@@ -212,52 +189,6 @@ static inline void rbuf_ma_uint64_t (struct rbuf *r, memarena *ma __attribute__(
     *num = rbuf_ulonglong(r);
 }
 
-
-static inline void rbuf_TXNID (struct rbuf *r, TXNID *txnid) {
-    *txnid = rbuf_ulonglong(r);
-}
-
-static inline void rbuf_TXNID_PAIR (struct rbuf *r, TXNID_PAIR *txnid) {
-    txnid->parent_id64 = rbuf_ulonglong(r);
-    txnid->child_id64 = rbuf_ulonglong(r);
-}
-
-static inline void rbuf_ma_TXNID (struct rbuf *r, memarena *ma __attribute__((__unused__)), TXNID *txnid) {
-    rbuf_TXNID(r, txnid);
-}
-
-static inline void rbuf_ma_TXNID_PAIR (struct rbuf *r, memarena *ma __attribute__((__unused__)), TXNID_PAIR *txnid) {
-    rbuf_TXNID_PAIR(r, txnid);
-}
-
-static inline void rbuf_FILENUM (struct rbuf *r, FILENUM *filenum) {
-    filenum->fileid = rbuf_int(r);
-}
-static inline void rbuf_ma_FILENUM (struct rbuf *r, memarena *ma __attribute__((__unused__)), FILENUM *filenum) {
-    rbuf_FILENUM(r, filenum);
-}
-
-// 2954
-// Don't try to use the same space, malloc it
-static inline void rbuf_FILENUMS(struct rbuf *r, FILENUMS *filenums) {
-    filenums->num = rbuf_int(r);
-    filenums->filenums = (FILENUM *) toku_malloc( filenums->num * sizeof(FILENUM) );
-    assert(filenums->filenums != NULL);
-    for (uint32_t i=0; i < filenums->num; i++) {
-        rbuf_FILENUM(r, &(filenums->filenums[i]));
-    }
-}
-
-// 2954
-static inline void rbuf_ma_FILENUMS (struct rbuf *r, memarena *ma __attribute__((__unused__)), FILENUMS *filenums) {
-    rbuf_ma_uint32_t(r, ma, &(filenums->num));
-    filenums->filenums = (FILENUM *) ma->malloc_from_arena(filenums->num * sizeof(FILENUM));
-    assert(filenums->filenums != NULL);
-    for (uint32_t i=0; i < filenums->num; i++) {
-        rbuf_ma_FILENUM(r, ma, &(filenums->filenums[i]));
-    }
-}
-
 // Don't try to use the same space, malloc it
 static inline void rbuf_BYTESTRING (struct rbuf *r, BYTESTRING *bs) {
     bs->len  = rbuf_int(r);

ft/recover.h

@@ -92,14 +92,15 @@ PATENT RIGHTS GRANT:
 #ident "Copyright (c) 2007-2013 Tokutek Inc.  All rights reserved."
 #ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
 
-#include <toku_portability.h>
+#include <db.h>
 #include <errno.h>
 
-#include <db.h>
-#include <util/x1764.h>
+#include "portability/memory.h"
+#include "portability/toku_portability.h"
 
-#include "fttypes.h"
-#include "memory.h"
+#include "ft/comparator.h"
+#include "ft/ft-ops.h"
+#include "util/x1764.h"
 
 typedef void (*prepared_txn_callback_t)(DB_ENV*, TOKUTXN);
 typedef void (*keep_cachetable_callback_t)(DB_ENV*, CACHETABLE);
@@ -109,7 +110,7 @@ typedef void (*keep_cachetable_callback_t)(DB_ENV*, CACHETABLE);
 int tokudb_recover (DB_ENV *env,
 		    prepared_txn_callback_t    prepared_txn_callback,
 		    keep_cachetable_callback_t keep_cachetable_callback,
-		    TOKULOGGER logger,
+		    struct tokulogger *logger,
 		    const char *env_dir, const char *log_dir,
                     ft_compare_func bt_compare,
                     ft_update_func update_function,

ft/roll.cc

@@ -96,7 +96,6 @@ PATENT RIGHTS GRANT:
 #include "ft.h"
 #include "ft-ops.h"
 #include "log-internal.h"
-//#include "txn_manager.h"
 #include "xids.h"
 #include "rollback-apply.h"
 

ft/rollback-apply.cc

@@ -89,9 +89,8 @@ PATENT RIGHTS GRANT:
 #ident "Copyright (c) 2007-2013 Tokutek Inc.  All rights reserved."
 #ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
 
-#include "fttypes.h"
-#include "log-internal.h"
-#include "rollback-apply.h"
+#include "ft/log-internal.h"
+#include "ft/rollback-apply.h"
 
 static void
 poll_txn_progress_function(TOKUTXN txn, uint8_t is_commit, uint8_t stall_for_checkpoint) {

ft/rollback-ct-callbacks.cc

@@ -94,7 +94,6 @@ PATENT RIGHTS GRANT:
 
 #include "ft/block_table.h"
 #include "ft/ft-internal.h"
-#include "ft/fttypes.h"
 #include "ft/rollback.h"
 #include "ft/rollback-ct-callbacks.h"
 

ft/rollback-ct-callbacks.h

@@ -92,9 +92,7 @@ PATENT RIGHTS GRANT:
 #ident "Copyright (c) 2007-2013 Tokutek Inc.  All rights reserved."
 #ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
 
-
-#include "cachetable.h"
-#include "fttypes.h"
+#include "ft/cachetable.h"
 
 void toku_rollback_flush_callback(CACHEFILE cachefile, int fd, BLOCKNUM logname, void *rollback_v, void** UU(disk_data), void *extraargs, PAIR_ATTR size, PAIR_ATTR* new_size, bool write_me, bool keep_me, bool for_checkpoint, bool UU(is_clone));
 int toku_rollback_fetch_callback(CACHEFILE cachefile, PAIR p, int fd, BLOCKNUM logname, uint32_t fullhash, void **rollback_pv,  void** UU(disk_data), PAIR_ATTR *sizep, int * UU(dirtyp), void *extraargs);

ft/rollback.h

@@ -92,8 +92,9 @@ PATENT RIGHTS GRANT:
 #ident "Copyright (c) 2007-2013 Tokutek Inc.  All rights reserved."
 #ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
 
-#include "ft/sub_block.h"
 #include "ft/cachetable.h"
+#include "ft/sub_block.h"
+#include "ft/txn.h"
 
 #include "util/memarena.h"
 
@@ -137,7 +138,7 @@ void *toku_memdup_in_rollback(ROLLBACK_LOG_NODE log, const void *v, size_t len);
 // if necessary.
 void toku_maybe_spill_rollbacks(TOKUTXN txn, ROLLBACK_LOG_NODE log);
 
-void toku_txn_maybe_note_ft (TOKUTXN txn, FT ft);
+void toku_txn_maybe_note_ft (TOKUTXN txn, struct ft *ft);
 int toku_logger_txn_rollback_stats(TOKUTXN txn, struct txn_stat *txn_stat);
 
 int toku_find_xid_by_xid (const TXNID &xid, const TXNID &xidfind);

ft/sub_block.cc

@@ -89,21 +89,21 @@ PATENT RIGHTS GRANT:
 #ident "Copyright (c) 2007-2013 Tokutek Inc.  All rights reserved."
 #ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
 
-#include "compress.h"
-#include "sub_block.h"
-#include "quicklz.h"
-
-#include <memory.h>
-#include <toku_assert.h>
-#include <toku_portability.h>
-#include <util/threadpool.h>
-#include <util/x1764.h>
-
+#include <errno.h>
 #include <stdio.h>
 #include <string.h>
-#include <errno.h>
 #include <zlib.h>
 
+#include "portability/memory.h"
+#include "portability/toku_assert.h"
+#include "portability/toku_portability.h"
+
+#include "ft/compress.h"
+#include "ft/sub_block.h"
+#include "ft/quicklz.h"
+#include "util/threadpool.h"
+#include "util/x1764.h"
+
 SUB_BLOCK sub_block_creat(void) {
     SUB_BLOCK XMALLOC(sb);
     sub_block_init(sb);

ft/sub_block.h

@@ -92,9 +92,9 @@ PATENT RIGHTS GRANT:
 #ident "Copyright (c) 2007-2013 Tokutek Inc.  All rights reserved."
 #ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
 
-#include "compress.h"
-#include "fttypes.h"
+#include "ft/compress.h"
 
+// TODO: Clean this abstraciton up
 static const int max_sub_blocks = 8;
 static const int target_sub_block_size = 512 * 1024;
 static const int max_basement_nodes = 32;

ft/tests/block_allocator_test.cc

@@ -88,7 +88,6 @@ PATENT RIGHTS GRANT:
 
 #ident "Copyright (c) 2007-2013 Tokutek Inc.  All rights reserved."
 
-
 #include "test.h"
 
 static void ba_alloc_at(block_allocator *ba, uint64_t size, uint64_t offset) {

ft/tests/ft-test-cursor-2.cc

@@ -96,7 +96,7 @@ static const char *fname = TOKU_TEST_FILENAME;
 static TOKUTXN const null_txn = 0;
 
 static int
-save_data (ITEMLEN UU(keylen), bytevec UU(key), ITEMLEN vallen, bytevec val, void *v, bool lock_only) {
+save_data (uint32_t UU(keylen), const void *UU(key), uint32_t vallen, const void *val, void *v, bool lock_only) {
     if (lock_only) return 0;
     assert(key!=NULL);
     void **CAST_FROM_VOIDP(vp, v);

ft/tests/ft-test-cursor.cc

@@ -442,7 +442,7 @@ static void test_ft_cursor_rwalk(int n) {
 }
 
 static int
-ascending_key_string_checkf (ITEMLEN keylen, bytevec key, ITEMLEN UU(vallen), bytevec UU(val), void *v, bool lock_only)
+ascending_key_string_checkf (uint32_t keylen, const void *key, uint32_t UU(vallen), const void *UU(val), void *v, bool lock_only)
 // the keys are strings.  Verify that they keylen matches the key, that the keys are ascending.  Use (char**)v  to hold a
 // malloc'd previous string.
 {

ft/tests/ft-test5.cc

@@ -127,7 +127,7 @@ static void test5 (void) {
 	    if (i%1000==0 && verbose) { printf("r"); fflush(stdout); }
 	    snprintf(key, 100, "key%d", rk);
 	    snprintf(valexpected, 100, "val%d", values[rk]);
-	    struct check_pair pair = {(ITEMLEN) (1+strlen(key)), key, (ITEMLEN) (1+strlen(valexpected)), valexpected, 0};
+	    struct check_pair pair = {(uint32_t) (1+strlen(key)), key, (uint32_t) (1+strlen(valexpected)), valexpected, 0};
 	    r = toku_ft_lookup(t, toku_fill_dbt(&k, key, 1+strlen(key)), lookup_checkf, &pair);
 	    assert(r==0);
 	    assert(pair.call_count==1);

ft/tests/ftloader-test.cc

@@ -350,7 +350,7 @@ static void verify_dbfile(int n, int sorted_keys[], const char *sorted_vals[], c
     size_t userdata = 0;
     int i;
     for (i=0; i<n; i++) {
-	struct check_pair pair = {sizeof sorted_keys[i], &sorted_keys[i], (ITEMLEN) strlen(sorted_vals[i]), sorted_vals[i], 0};
+	struct check_pair pair = {sizeof sorted_keys[i], &sorted_keys[i], (uint32_t) strlen(sorted_vals[i]), sorted_vals[i], 0};
         r = toku_ft_cursor_get(cursor, NULL, lookup_checkf, &pair, DB_NEXT);
         if (r != 0) {
 	    assert(pair.call_count ==0);

ft/tests/le-cursor-provdel.cc

@@ -97,7 +97,7 @@ PATENT RIGHTS GRANT:
 
 
 static int
-get_next_callback(ITEMLEN keylen, bytevec key, ITEMLEN vallen UU(), bytevec val UU(), void *extra, bool lock_only) {
+get_next_callback(uint32_t keylen, const void *key, uint32_t vallen UU(), const void *val UU(), void *extra, bool lock_only) {
     DBT *CAST_FROM_VOIDP(key_dbt, extra);
     if (!lock_only) {
         toku_dbt_set(keylen, key, key_dbt, NULL);

ft/tests/le-cursor-right.cc

@@ -101,7 +101,7 @@ PATENT RIGHTS GRANT:
 static TOKUTXN const null_txn = 0;
 
 static int
-get_next_callback(ITEMLEN keylen, bytevec key, ITEMLEN vallen UU(), bytevec val UU(), void *extra, bool lock_only) {
+get_next_callback(uint32_t keylen, const void *key, uint32_t vallen UU(), const void *val UU(), void *extra, bool lock_only) {
     DBT *CAST_FROM_VOIDP(key_dbt, extra);
     if (!lock_only) {
         toku_dbt_set(keylen, key, key_dbt, NULL);

ft/tests/le-cursor-walk.cc

@@ -99,7 +99,7 @@ PATENT RIGHTS GRANT:
 static TOKUTXN const null_txn = 0;
 
 static int
-get_next_callback(ITEMLEN keylen, bytevec key, ITEMLEN vallen UU(), bytevec val UU(), void *extra, bool lock_only) {
+get_next_callback(uint32_t keylen, const void *key, uint32_t vallen UU(), const void *val UU(), void *extra, bool lock_only) {
     DBT *CAST_FROM_VOIDP(key_dbt, extra);
     if (!lock_only) {
         toku_dbt_set(keylen, key, key_dbt, NULL);

ft/tests/test-leafentry-child-txn.cc

@@ -91,10 +91,9 @@ PATENT RIGHTS GRANT:
 #include <string.h>
 
 #include "test.h"
-#include "fttypes.h"
 
-#include "ule.h"
-#include "ule-internal.h"
+#include "ft/ule.h"
+#include "ft/ule-internal.h"
 
 static void init_empty_ule(ULE ule) {
     ule->num_cuxrs = 0;

ft/tests/test-leafentry-nested.cc

@@ -91,10 +91,9 @@ PATENT RIGHTS GRANT:
 #include <string.h>
 
 #include "test.h"
-#include "fttypes.h"
 
-#include "ule.h"
-#include "ule-internal.h"
+#include "ft/ule.h"
+#include "ft/ule-internal.h"
 
 enum {MAX_SIZE = 256};
 static XIDS nested_xids[MAX_TRANSACTION_RECORDS];

ft/tests/test.h

@@ -101,16 +101,17 @@ PATENT RIGHTS GRANT:
 #include <string.h>
 #include <portability/toku_path.h>
 
-#include "ft.h"
-#include "node.h"
-#include "block_table.h"
-#include "log-internal.h"
-#include "logger.h"
-#include "fttypes.h"
-#include "ft-ops.h"
-#include "cursor.h"
-#include "cachetable.h"
-#include "cachetable-internal.h"
+#include "ft/ft.h"
+#include "ft/node.h"
+#include "ft/block_allocator.h"
+#include "ft/block_table.h"
+#include "ft/log-internal.h"
+#include "ft/logger.h"
+#include "ft/ft-ops.h"
+#include "ft/cursor.h"
+#include "ft/cachetable.h"
+#include "ft/cachetable-internal.h"
+#include "util/bytestring.h"
 
 #define CKERR(r) ({ int __r = r; if (__r!=0) fprintf(stderr, "%s:%d error %d %s\n", __FILE__, __LINE__, __r, strerror(r)); assert(__r==0); })
 #define CKERR2(r,r2) do { if (r!=r2) fprintf(stderr, "%s:%d error %d %s, expected %d\n", __FILE__, __LINE__, r, strerror(r), r2); assert(r==r2); } while (0)
@@ -121,7 +122,7 @@ PATENT RIGHTS GRANT:
     fflush(stderr); \
 } while (0)
 
-const ITEMLEN len_ignore = 0xFFFFFFFF;
+const uint32_t len_ignore = 0xFFFFFFFF;
 
 static const prepared_txn_callback_t NULL_prepared_txn_callback         __attribute__((__unused__)) = NULL;
 static const keep_cachetable_callback_t  NULL_keep_cachetable_callback  __attribute__((__unused__)) = NULL;
@@ -155,14 +156,14 @@ last_dummymsn(void) {
 
 
 struct check_pair {
-    ITEMLEN keylen;  // A keylen equal to 0xFFFFFFFF means don't check the keylen or the key.
-    bytevec key;     // A NULL key means don't check the key.
-    ITEMLEN vallen;  // Similarly for vallen and null val.
-    bytevec val;
+    uint32_t keylen;  // A keylen equal to 0xFFFFFFFF means don't check the keylen or the key.
+    const void *key;     // A NULL key means don't check the key.
+    uint32_t vallen;  // Similarly for vallen and null val.
+    const void *val;
     int call_count;
 };
 static int
-lookup_checkf (ITEMLEN keylen, bytevec key, ITEMLEN vallen, bytevec val, void *pair_v, bool lock_only) {
+lookup_checkf (uint32_t keylen, const void *key, uint32_t vallen, const void *val, void *pair_v, bool lock_only) {
     if (!lock_only) {
         struct check_pair *pair = (struct check_pair *) pair_v;
         if (key!=NULL) {
@@ -187,8 +188,8 @@ ft_lookup_and_check_nodup (FT_HANDLE t, const char *keystring, const char *valst
 {
     DBT k;
     toku_fill_dbt(&k, keystring, strlen(keystring) + 1);
-    struct check_pair pair = {(ITEMLEN) (1+strlen(keystring)), keystring,
-                              (ITEMLEN) (1+strlen(valstring)), valstring,
+    struct check_pair pair = {(uint32_t) (1+strlen(keystring)), keystring,
+                              (uint32_t) (1+strlen(valstring)), valstring,
 			      0};
     int r = toku_ft_lookup(t, &k, lookup_checkf, &pair);
     assert(r==0);
@@ -200,7 +201,7 @@ ft_lookup_and_fail_nodup (FT_HANDLE t, char *keystring)
 {
     DBT k;
     toku_fill_dbt(&k, keystring, strlen(keystring) + 1);
-    struct check_pair pair = {(ITEMLEN) (1+strlen(keystring)), keystring,
+    struct check_pair pair = {(uint32_t) (1+strlen(keystring)), keystring,
 			      0, 0,
 			      0};
     int r = toku_ft_lookup(t, &k, lookup_checkf, &pair);

ft/tests/test3856.cc

@@ -110,7 +110,7 @@ string_cmp(DB* UU(db), const DBT *a, const DBT *b)
 }
 
 static int
-found(ITEMLEN UU(keylen), bytevec key, ITEMLEN UU(vallen), bytevec UU(val), void *UU(extra), bool lock_only)
+found(uint32_t UU(keylen), const void *key, uint32_t UU(vallen), const void *UU(val), void *UU(extra), bool lock_only)
 {
     assert(key != NULL && !lock_only);
     return 0;

ft/tests/test_logcursor.cc

@@ -92,7 +92,6 @@ PATENT RIGHTS GRANT:
 
 #include "logcursor.h"
 #include "test.h"
-#include "fttypes.h"
 
 #if defined(HAVE_LIMITS_H)
 # include <limits.h>

ft/tests/upgrade_test_simple.cc

@@ -102,7 +102,7 @@ PATENT RIGHTS GRANT:
 static TOKUTXN const null_txn = NULL;
 
 static int
-noop_getf(ITEMLEN UU(keylen), bytevec UU(key), ITEMLEN UU(vallen), bytevec UU(val), void *extra, bool UU(lock_only))
+noop_getf(uint32_t UU(keylen), const void *UU(key), uint32_t UU(vallen), const void *UU(val), void *extra, bool UU(lock_only))
 {
     int *CAST_FROM_VOIDP(calledp, extra);
     (*calledp)++;

ft/tests/ybt-test.cc

@@ -111,11 +111,11 @@ static void ybt_test0 (void) {
     toku_init_dbt(&t0);
     toku_init_dbt(&t1);
     {
-	bytevec temp1 = "hello";
+	const void *temp1 = "hello";
         toku_dbt_set(6, temp1, &t0, &v0);
     }
     {
-        bytevec temp2 = "foo";
+        const void *temp2 = "foo";
 	toku_dbt_set(  4, temp2, &t1, &v1);
     }
     assert(t0.size==6);
@@ -124,7 +124,7 @@ static void ybt_test0 (void) {
     assert(strcmp((char*)t1.data, "foo")==0);
 
     {
-        bytevec temp3 = "byebye";
+        const void *temp3 = "byebye";
 	toku_dbt_set(7, temp3, &t1, &v0);      /* Use v0, not v1 */
     }
     // This assertion would be wrong, since v0 may have been realloc'd, and t0.data may now point
@@ -141,7 +141,7 @@ static void ybt_test0 (void) {
     t0.flags = DB_DBT_USERMEM;
     t0.ulen  = 0;
     {
-        bytevec temp4 = "hello";
+        const void *temp4 = "hello";
 	toku_dbt_set(6, temp4, &t0, 0);
     }
     assert(t0.data==0);
@@ -152,7 +152,7 @@ static void ybt_test0 (void) {
     t0.flags = DB_DBT_REALLOC;
     cleanup(&v0);
     {
-        bytevec temp5 = "internationalization";
+        const void *temp5 = "internationalization";
 	toku_dbt_set(21, temp5, &t0, &v0);
     }
     assert(v0.data==0); /* Didn't change v0 */
@@ -160,7 +160,7 @@ static void ybt_test0 (void) {
     assert(strcmp((char*)t0.data, "internationalization")==0);
 
     {
-        bytevec temp6 = "provincial";
+        const void *temp6 = "provincial";
 	toku_dbt_set(11, temp6, &t0, &v0);
     }
     assert(t0.size==11);

ft/tokuconst.h

@@ -99,5 +99,7 @@ PATENT RIGHTS GRANT:
  * root transaction (id 0).
  */
 
-enum {MAX_NESTED_TRANSACTIONS = 253};
-enum {MAX_TRANSACTION_RECORDS = MAX_NESTED_TRANSACTIONS + 1};
+enum {
+    MAX_NESTED_TRANSACTIONS = 253,
+    MAX_TRANSACTION_RECORDS = MAX_NESTED_TRANSACTIONS + 1
+};

ft/tokuftdump.cc

@@ -101,7 +101,6 @@ PATENT RIGHTS GRANT:
 #include "ft/block_table.h"
 #include "ft/cachetable.h"
 #include "ft/ft.h"
-#include "ft/fttypes.h"
 #include "ft/ft-internal.h"
 #include "ft/node.h"
 
@@ -124,9 +123,9 @@ static void format_time(const uint64_t time_int, char *buf) {
     buf[24] = 0;
 }
 
-static void print_item(bytevec val, ITEMLEN len) {
+static void print_item(const void *val, uint32_t len) {
     printf("\"");
-    ITEMLEN i;
+    uint32_t i;
     for (i=0; i<len; i++) {
         unsigned char ch = ((unsigned char*)val)[i];
         if (isprint(ch) && ch!='\\' && ch!='"') {
@@ -285,8 +284,8 @@ static void dump_node(int fd, BLOCKNUM blocknum, FT ft) {
                         XIDS xids = msg.xids();
                         const void *key = msg.kdbt()->data;
                         const void *data = msg.vdbt()->data;
-                        ITEMLEN keylen = msg.kdbt()->size;
-                        ITEMLEN datalen = msg.vdbt()->size;
+                        uint32_t keylen = msg.kdbt()->size;
+                        uint32_t datalen = msg.vdbt()->size;
                         printf("    msn=%" PRIu64 " (0x%" PRIx64 ") ", msn.msn, msn.msn);
                         printf("    TYPE=");
                         switch (type) {

ft/txn.cc

@@ -97,7 +97,6 @@ PATENT RIGHTS GRANT:
 #include "ule.h"
 #include "rollback-apply.h"
 #include "txn_manager.h"
-#include "txn_child_manager.h"
 #include <util/status.h>
 
 ///////////////////////////////////////////////////////////////////////////////////

ft/txn.h

@@ -92,17 +92,61 @@ PATENT RIGHTS GRANT:
 #ident "Copyright (c) 2007-2013 Tokutek Inc.  All rights reserved."
 #ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
 
-#include "txn_manager.h"
+#include "portability/toku_stdint.h"
 
-void txn_status_init(void);
-void txn_status_destroy(void);
+#include "ft/block_table.h"
+#include "ft/txn_state.h"
+#include "util/omt.h"
 
+typedef uint64_t TXNID;
+
+typedef struct tokutxn *TOKUTXN;
+
+#define TXNID_NONE_LIVING ((TXNID)0)
+#define TXNID_NONE        ((TXNID)0)
+#define TXNID_MAX         ((TXNID)-1)
+
+typedef struct txnid_pair_s {
+    TXNID parent_id64;
+    TXNID child_id64;
+} TXNID_PAIR;
+
+static const TXNID_PAIR TXNID_PAIR_NONE = { .parent_id64 = TXNID_NONE, .child_id64 = TXNID_NONE };
+
+// We include the child manager here beacuse it uses the TXNID / TOKUTXN types
+#include "ft/txn_child_manager.h"
+
+/* Log Sequence Number (LSN)
+ * Make the LSN be a struct instead of an integer so that we get better type checking. */
+typedef struct __toku_lsn { uint64_t lsn; } LSN;
+static const LSN ZERO_LSN = { .lsn = 0 };
+static const LSN MAX_LSN = { .lsn = UINT64_MAX };
+
+//
+// Types of snapshots that can be taken by a tokutxn
+//  - TXN_SNAPSHOT_NONE: means that there is no snapshot. Reads do not use snapshot reads.
+//                       used for SERIALIZABLE and READ UNCOMMITTED
+//  - TXN_SNAPSHOT_ROOT: means that all tokutxns use their root transaction's snapshot
+//                       used for REPEATABLE READ
+//  - TXN_SNAPSHOT_CHILD: means that each child tokutxn creates its own snapshot
+//                        used for READ COMMITTED
+//
+
+typedef enum __TXN_SNAPSHOT_TYPE { 
+    TXN_SNAPSHOT_NONE=0,
+    TXN_SNAPSHOT_ROOT=1,
+    TXN_SNAPSHOT_CHILD=2
+} TXN_SNAPSHOT_TYPE;
+
+typedef toku::omt<struct tokutxn *> txn_omt_t;
+typedef toku::omt<TXNID> xid_omt_t;
+typedef toku::omt<struct referenced_xid_tuple, struct referenced_xid_tuple *> rx_omt_t;
 
 inline bool txn_pair_is_none(TXNID_PAIR txnid) {
     return txnid.parent_id64 == TXNID_NONE && txnid.child_id64 == TXNID_NONE;
 }
 
-inline bool txn_needs_snapshot(TXN_SNAPSHOT_TYPE snapshot_type, TOKUTXN parent) {
+inline bool txn_needs_snapshot(TXN_SNAPSHOT_TYPE snapshot_type, struct tokutxn *parent) {
     // we need a snapshot if the snapshot type is a child or
     // if the snapshot type is root and we have no parent.
     // Cases that we don't need a snapshot: when snapshot type is NONE
@@ -110,29 +154,131 @@ inline bool txn_needs_snapshot(TXN_SNAPSHOT_TYPE snapshot_type, TOKUTXN parent)
     return (snapshot_type != TXN_SNAPSHOT_NONE && (parent==NULL || snapshot_type == TXN_SNAPSHOT_CHILD));
 }
 
-void toku_txn_lock(TOKUTXN txn);
-void toku_txn_unlock(TOKUTXN txn);
+struct tokulogger;
 
-uint64_t toku_txn_get_root_id(TOKUTXN txn);
-bool txn_declared_read_only(TOKUTXN txn);
+struct txn_roll_info {
+    // these are number of rollback nodes and rollback entries for this txn.
+    //
+    // the current rollback node below has sequence number num_rollback_nodes - 1
+    // (because they are numbered 0...num-1). often, the current rollback is
+    // already set to this block num, which means it exists and is available to
+    // log some entries. if the current rollback is NONE and the number of
+    // rollback nodes for this transaction is non-zero, then we will use
+    // the number of rollback nodes to know which sequence number to assign
+    // to a new one we create
+    uint64_t num_rollback_nodes;
+    uint64_t num_rollentries;
+    uint64_t num_rollentries_processed;
+    uint64_t rollentry_raw_count;  // the total count of every byte in the transaction and all its children.
+
+    // spilled rollback nodes are rollback nodes that were gorged by this
+    // transaction, retired, and saved in a list.
+
+    // the spilled rollback head is the block number of the first rollback node
+    // that makes up the rollback log chain
+    BLOCKNUM spilled_rollback_head;
+
+    // the spilled rollback is the block number of the last rollback node that
+    // makes up the rollback log chain. 
+    BLOCKNUM spilled_rollback_tail;
+
+    // the current rollback node block number we may use. if this is ROLLBACK_NONE,
+    // then we need to create one and set it here before using it.
+    BLOCKNUM current_rollback; 
+};
+
+struct tokutxn {
+    // These don't change after create:
+
+    TXNID_PAIR txnid;
+
+    uint64_t snapshot_txnid64; // this is the lsn of the snapshot
+    const TXN_SNAPSHOT_TYPE snapshot_type;
+    const bool for_recovery;
+    struct tokulogger *const logger;
+    struct tokutxn *const parent;
+    // The child txn is protected by the child_txn_manager lock
+    // and by the user contract. The user contract states (and is
+    // enforced at the ydb layer) that a child txn should not be created
+    // while another child exists. The txn_child_manager will protect
+    // other threads from trying to read this value while another
+    // thread commits/aborts the child
+    struct tokutxn *child;
+
+    // statically allocated child manager, if this 
+    // txn is a root txn, this manager will be used and set to 
+    // child_manager for this transaction and all of its children
+    txn_child_manager child_manager_s;
+
+    // child manager for this transaction, all of its children,
+    // and all of its ancestors
+    txn_child_manager* child_manager;
+
+    // These don't change but they're created in a way that's hard to make
+    // strictly const.
+    DB_TXN *container_db_txn; // reference to DB_TXN that contains this tokutxn
+    xid_omt_t *live_root_txn_list; // the root txns live when the root ancestor (self if a root) started.
+    struct xids_t *xids; // Represents the xid list
+
+    struct tokutxn *snapshot_next;
+    struct tokutxn *snapshot_prev;
+
+    bool begin_was_logged;
+    bool declared_read_only; // true if the txn was declared read only when began
+
+    // These are not read until a commit, prepare, or abort starts, and
+    // they're "monotonic" (only go false->true) during operation:
+    bool do_fsync;
+    bool force_fsync_on_commit;  //This transaction NEEDS an fsync once (if) it commits.  (commit means root txn)
+
+    // Not used until commit, prepare, or abort starts:
+    LSN do_fsync_lsn;
+    TOKU_XA_XID xa_xid; // for prepared transactions
+    TXN_PROGRESS_POLL_FUNCTION progress_poll_fun;
+    void *progress_poll_fun_extra;
+
+    toku_mutex_t txn_lock;
+    // Protected by the txn lock:
+    toku::omt<struct ft*> open_fts; // a collection of the fts that we touched.  Indexed by filenum.
+    struct txn_roll_info roll_info; // Info used to manage rollback entries
+
+    // mutex that protects the transition of the state variable
+    // the rest of the variables are used by the txn code and 
+    // hot indexing to ensure that when hot indexing is processing a 
+    // leafentry, a TOKUTXN cannot dissappear or change state out from
+    // underneath it
+    toku_mutex_t state_lock;
+    toku_cond_t state_cond;
+    TOKUTXN_STATE state;
+    uint32_t num_pin; // number of threads (all hot indexes) that want this
+                      // txn to not transition to commit or abort
+    uint64_t client_id;
+};
+typedef struct tokutxn *TOKUTXN;
+
+void toku_txn_lock(struct tokutxn *txn);
+void toku_txn_unlock(struct tokutxn *txn);
+
+uint64_t toku_txn_get_root_id(struct tokutxn *txn);
+bool txn_declared_read_only(struct tokutxn *txn);
 
 int toku_txn_begin_txn (
     DB_TXN  *container_db_txn,
-    TOKUTXN parent_tokutxn, 
-    TOKUTXN *tokutxn, 
-    TOKULOGGER logger,
+    struct tokutxn *parent_tokutxn, 
+    struct tokutxn **tokutxn, 
+    struct tokulogger *logger,
     TXN_SNAPSHOT_TYPE snapshot_type,
     bool read_only
     );
 
-DB_TXN * toku_txn_get_container_db_txn (TOKUTXN tokutxn);
-void toku_txn_set_container_db_txn (TOKUTXN, DB_TXN*);
+DB_TXN * toku_txn_get_container_db_txn (struct tokutxn *tokutxn);
+void toku_txn_set_container_db_txn(struct tokutxn *txn, DB_TXN *db_txn);
 
 // toku_txn_begin_with_xid is called from recovery and has no containing DB_TXN 
 int toku_txn_begin_with_xid (
-    TOKUTXN parent_tokutxn, 
-    TOKUTXN *tokutxn, 
-    TOKULOGGER logger, 
+    struct tokutxn *parent_tokutxn, 
+    struct tokutxn **tokutxn, 
+    struct tokulogger *logger, 
     TXNID_PAIR xid, 
     TXN_SNAPSHOT_TYPE snapshot_type,
     DB_TXN *container_db_txn,
@@ -140,43 +286,43 @@ int toku_txn_begin_with_xid (
     bool read_only
     );
 
-void toku_txn_update_xids_in_txn(TOKUTXN txn, TXNID xid);
+void toku_txn_update_xids_in_txn(struct tokutxn *txn, TXNID xid);
 
-int toku_txn_load_txninfo (TOKUTXN txn, struct txninfo *info);
+int toku_txn_load_txninfo (struct tokutxn *txn, struct txninfo *info);
 
-int toku_txn_commit_txn (TOKUTXN txn, int nosync,
+int toku_txn_commit_txn (struct tokutxn *txn, int nosync,
                          TXN_PROGRESS_POLL_FUNCTION poll, void *poll_extra);
-int toku_txn_commit_with_lsn(TOKUTXN txn, int nosync, LSN oplsn,
+int toku_txn_commit_with_lsn(struct tokutxn *txn, int nosync, LSN oplsn,
                              TXN_PROGRESS_POLL_FUNCTION poll, void *poll_extra);
 
-int toku_txn_abort_txn(TOKUTXN txn,
+int toku_txn_abort_txn(struct tokutxn *txn,
                        TXN_PROGRESS_POLL_FUNCTION poll, void *poll_extra);
-int toku_txn_abort_with_lsn(TOKUTXN txn, LSN oplsn,
+int toku_txn_abort_with_lsn(struct tokutxn *txn, LSN oplsn,
                             TXN_PROGRESS_POLL_FUNCTION poll, void *poll_extra);
 
-void toku_txn_prepare_txn (TOKUTXN txn, TOKU_XA_XID *xid);
+void toku_txn_prepare_txn (struct tokutxn *txn, TOKU_XA_XID *xid);
 // Effect: Do the internal work of preparing a transaction (does not log the prepare record).
 
-void toku_txn_get_prepared_xa_xid (TOKUTXN, TOKU_XA_XID *);
+void toku_txn_get_prepared_xa_xid(struct tokutxn *txn, TOKU_XA_XID *xa_xid);
 // Effect: Fill in the XID information for a transaction.  The caller allocates the XID and the function fills in values.
 
-void toku_txn_maybe_fsync_log(TOKULOGGER logger, LSN do_fsync_lsn, bool do_fsync);
+void toku_txn_maybe_fsync_log(struct tokulogger *logger, LSN do_fsync_lsn, bool do_fsync);
 
-void toku_txn_get_fsync_info(TOKUTXN ttxn, bool* do_fsync, LSN* do_fsync_lsn);
+void toku_txn_get_fsync_info(struct tokutxn *ttxn, bool* do_fsync, LSN* do_fsync_lsn);
 
 // Complete and destroy a txn
-void toku_txn_close_txn(TOKUTXN txn);
+void toku_txn_close_txn(struct tokutxn *txn);
 
 // Remove a txn from any live txn lists
-void toku_txn_complete_txn(TOKUTXN txn);
+void toku_txn_complete_txn(struct tokutxn *txn);
 
 // Free the memory of a txn
-void toku_txn_destroy_txn(TOKUTXN txn);
+void toku_txn_destroy_txn(struct tokutxn *txn);
 
-XIDS toku_txn_get_xids (TOKUTXN);
+struct xids_t *toku_txn_get_xids(struct tokutxn *txn);
 
 // Force fsync on commit
-void toku_txn_force_fsync_on_commit(TOKUTXN txn);
+void toku_txn_force_fsync_on_commit(struct tokutxn *txn);
 
 typedef enum {
     TXN_BEGIN,             // total number of transactions begun (does not include recovered txns)
@@ -195,33 +341,30 @@ void toku_txn_get_status(TXN_STATUS s);
 
 bool toku_is_txn_in_live_root_txn_list(const xid_omt_t &live_root_txn_list, TXNID xid);
 
-TXNID toku_get_oldest_in_live_root_txn_list(TOKUTXN txn);
+TXNID toku_get_oldest_in_live_root_txn_list(struct tokutxn *txn);
 
-#include "txn_state.h"
-
-TOKUTXN_STATE toku_txn_get_state(TOKUTXN txn);
+TOKUTXN_STATE toku_txn_get_state(struct tokutxn *txn);
 
 struct tokulogger_preplist {
     TOKU_XA_XID xid;
     DB_TXN *txn;
 };
-int toku_logger_recover_txn (TOKULOGGER logger, struct tokulogger_preplist preplist[/*count*/], long count, /*out*/ long *retp, uint32_t flags);
+int toku_logger_recover_txn (struct tokulogger *logger, struct tokulogger_preplist preplist[/*count*/], long count, /*out*/ long *retp, uint32_t flags);
 
-void toku_maybe_log_begin_txn_for_write_operation(TOKUTXN txn);
+void toku_maybe_log_begin_txn_for_write_operation(struct tokutxn *txn);
 
 // Return whether txn (or it's descendents) have done no work.
-bool toku_txn_is_read_only(TOKUTXN txn);
+bool toku_txn_is_read_only(struct tokutxn *txn);
 
-void toku_txn_lock_state(TOKUTXN txn);
-void toku_txn_unlock_state(TOKUTXN txn);
-void toku_txn_pin_live_txn_unlocked(TOKUTXN txn);
-void toku_txn_unpin_live_txn(TOKUTXN txn);
+void toku_txn_lock_state(struct tokutxn *txn);
+void toku_txn_unlock_state(struct tokutxn *txn);
+void toku_txn_pin_live_txn_unlocked(struct tokutxn *txn);
+void toku_txn_unpin_live_txn(struct tokutxn *txn);
 
-bool toku_txn_has_spilled_rollback(TOKUTXN txn);
-
-uint64_t toku_txn_get_client_id(TOKUTXN txn);
-void toku_txn_set_client_id(TOKUTXN txn, uint64_t client_id);
+bool toku_txn_has_spilled_rollback(struct tokutxn *txn);
 
+uint64_t toku_txn_get_client_id(struct tokutxn *txn);
+void toku_txn_set_client_id(struct tokutxn *txn, uint64_t client_id);
 
 //
 // This function is used by the leafentry iterators.
@@ -234,4 +377,57 @@ void toku_txn_set_client_id(TOKUTXN txn, uint64_t client_id);
 // For the above to NOT be true:
 //  - id > context->snapshot_txnid64 OR id is in context's live root transaction list
 //
-int toku_txn_reads_txnid(TXNID txnid, TOKUTXN txn);
+int toku_txn_reads_txnid(TXNID txnid, struct tokutxn *txn);
+
+void txn_status_init(void);
+
+void txn_status_destroy(void);
+
+// For serialize / deserialize
+
+#include "ft/wbuf.h"
+
+static inline void wbuf_TXNID(struct wbuf *wb, TXNID txnid) {
+    wbuf_ulonglong(wb, txnid);
+}
+
+static inline void wbuf_nocrc_TXNID(struct wbuf *wb, TXNID txnid) {
+    wbuf_nocrc_ulonglong(wb, txnid);
+}
+
+static inline void wbuf_nocrc_TXNID_PAIR(struct wbuf *wb, TXNID_PAIR txnid) {
+    wbuf_nocrc_ulonglong(wb, txnid.parent_id64);
+    wbuf_nocrc_ulonglong(wb, txnid.child_id64);
+}
+
+static inline void wbuf_nocrc_LSN(struct wbuf *wb, LSN lsn) {
+    wbuf_nocrc_ulonglong(wb, lsn.lsn);
+}
+
+static inline void wbuf_LSN(struct wbuf *wb, LSN lsn) {
+    wbuf_ulonglong(wb, lsn.lsn);
+}
+
+#include "ft/rbuf.h"
+
+static inline void rbuf_TXNID(struct rbuf *rb, TXNID *txnid) {
+    *txnid = rbuf_ulonglong(rb);
+}
+
+static inline void rbuf_TXNID_PAIR(struct rbuf *rb, TXNID_PAIR *txnid) {
+    txnid->parent_id64 = rbuf_ulonglong(rb);
+    txnid->child_id64 = rbuf_ulonglong(rb);
+}
+
+static inline void rbuf_ma_TXNID(struct rbuf *rb, memarena *UU(ma), TXNID *txnid) {
+    rbuf_TXNID(rb, txnid);
+}
+
+static inline void rbuf_ma_TXNID_PAIR (struct rbuf *r, memarena *ma __attribute__((__unused__)), TXNID_PAIR *txnid) {
+    rbuf_TXNID_PAIR(r, txnid);
+}
+
+static inline LSN rbuf_LSN(struct rbuf *rb) {
+    LSN lsn = { .lsn = rbuf_ulonglong(rb) };
+    return lsn;
+}

ft/txn_child_manager.h

@@ -89,11 +89,13 @@ PATENT RIGHTS GRANT:
 
 #pragma once
 
+// We should be including ft/txn.h here but that header includes this one,
+// so we don't.
+#include "portability/toku_pthread.h"
+
 #ident "Copyright (c) 2007-2013 Tokutek Inc.  All rights reserved."
 #ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
 
-#include "txn_manager.h"
-
 class txn_child_manager {
 public:
     void init (TOKUTXN root);
@@ -104,7 +106,7 @@ public:
     void suspend();
     void resume();
     void find_tokutxn_by_xid_unlocked(TXNID_PAIR xid, TOKUTXN* result);
-    int iterate(txn_mgr_iter_callback cb, void* extra);
+    int iterate(int (*cb)(TOKUTXN txn, void *extra), void* extra);
 
 private:
     TXNID m_last_xid;

ft/txn_manager.h

@@ -92,11 +92,12 @@ PATENT RIGHTS GRANT:
 #ident "Copyright (c) 2007-2013 Tokutek Inc.  All rights reserved."
 #ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
 
-#include <toku_portability.h>
-#include <util/omt.h>
-#include "fttypes.h"
-#include <portability/toku_pthread.h>
-#include <util/omt.h>
+#include "portability/toku_portability.h"
+#include "portability/toku_pthread.h"
+
+#include "ft/txn.h"
+
+typedef struct txn_manager *TXN_MANAGER;
 
 struct referenced_xid_tuple {
     TXNID begin_id;
@@ -104,10 +105,6 @@ struct referenced_xid_tuple {
     uint32_t references;
 };
 
-typedef toku::omt<TOKUTXN> txn_omt_t;
-typedef toku::omt<TXNID> xid_omt_t;
-typedef toku::omt<struct referenced_xid_tuple, struct referenced_xid_tuple *> rx_omt_t;
-
 struct txn_manager {
     toku_mutex_t txn_manager_lock;  // a lock protecting this object
     txn_omt_t live_root_txns; // a sorted tree.
@@ -190,22 +187,6 @@ TXNID toku_txn_manager_get_oldest_living_xid(TXN_MANAGER txn_manager);
 
 TXNID toku_txn_manager_get_oldest_referenced_xid_estimate(TXN_MANAGER txn_manager);
 
-//
-// Types of snapshots that can be taken by a tokutxn
-//  - TXN_SNAPSHOT_NONE: means that there is no snapshot. Reads do not use snapshot reads.
-//                       used for SERIALIZABLE and READ UNCOMMITTED
-//  - TXN_SNAPSHOT_ROOT: means that all tokutxns use their root transaction's snapshot
-//                       used for REPEATABLE READ
-//  - TXN_SNAPSHOT_CHILD: means that each child tokutxn creates its own snapshot
-//                        used for READ COMMITTED
-//
-
-typedef enum __TXN_SNAPSHOT_TYPE { 
-    TXN_SNAPSHOT_NONE=0,
-    TXN_SNAPSHOT_ROOT=1,
-    TXN_SNAPSHOT_CHILD=2
-} TXN_SNAPSHOT_TYPE;
-
 void toku_txn_manager_handle_snapshot_create_for_child_txn(
     TOKUTXN txn,
     TXN_MANAGER txn_manager,

ft/ule.cc

@@ -102,27 +102,27 @@ PATENT RIGHTS GRANT:
 // See design documentation for nested transactions at
 // TokuWiki/Imp/TransactionsOverview.
 
-#include <toku_portability.h>
-#include "ft/fttypes.h"
+#include "portability/toku_portability.h"
+
 #include "ft/ft-internal.h"
-#include "ft/msg.h"
 #include "ft/leafentry.h"
 #include "ft/logger.h"
+#include "ft/msg.h"
 #include "ft/txn.h"
 #include "ft/txn_manager.h"
 #include "ft/ule.h"
 #include "ft/ule-internal.h"
 #include "ft/xids.h"
-#include <util/omt.h>
-#include <util/status.h>
-#include <util/scoped_malloc.h>
-#include <util/partitioned_counter.h>
+#include "util/bytestring.h"
+#include "util/omt.h"
+#include "util/partitioned_counter.h"
+#include "util/scoped_malloc.h"
+#include "util/status.h"
 
 #define ULE_DEBUG 0
 
 static uint32_t ule_get_innermost_numbytes(ULE ule, uint32_t keylen);
 
-
 ///////////////////////////////////////////////////////////////////////////////////
 // Engine status
 //

ft/wbuf.h

@@ -94,12 +94,10 @@ PATENT RIGHTS GRANT:
 #include <memory.h>
 #include <string.h>
 
-#include <portability/toku_htonl.h>
-#include <util/x1764.h>
+#include "portability/toku_htonl.h"
 
-#include "fttypes.h"
-
-#define CRC_INCR
+#include "util/bytestring.h"
+#include "util/x1764.h"
 
 /* When serializing a value, write it into a buffer. */
 /* This code requires that the buffer be big enough to hold whatever you put into it. */
@@ -113,13 +111,13 @@ struct wbuf {
     struct x1764  checksum;    // The checksum state
 };
 
-static inline void wbuf_nocrc_init (struct wbuf *w, void *buf, DISKOFF size) {
+static inline void wbuf_nocrc_init (struct wbuf *w, void *buf, unsigned int size) {
     w->buf = (unsigned char *) buf;
     w->size = size;
     w->ndone = 0;
 }
 
-static inline void wbuf_init (struct wbuf *w, void *buf, DISKOFF size) {
+static inline void wbuf_init (struct wbuf *w, void *buf, unsigned int size) {
     wbuf_nocrc_init(w, buf, size);
     toku_x1764_init(&w->checksum);
 }
@@ -194,7 +192,7 @@ static inline uint8_t* wbuf_nocrc_reserve_literal_bytes(struct wbuf *w, uint32_t
     return dest;
 }
 
-static inline void wbuf_nocrc_literal_bytes(struct wbuf *w, bytevec bytes_bv, uint32_t nbytes) {
+static inline void wbuf_nocrc_literal_bytes(struct wbuf *w, const void *bytes_bv, uint32_t nbytes) {
     const unsigned char *bytes = (const unsigned char *) bytes_bv;
 #if 0
     { int i; for (i=0; i<nbytes; i++) wbuf_nocrc_char(w, bytes[i]); }
@@ -205,17 +203,17 @@ static inline void wbuf_nocrc_literal_bytes(struct wbuf *w, bytevec bytes_bv, ui
 #endif
 }
 
-static inline void wbuf_literal_bytes(struct wbuf *w, bytevec bytes_bv, uint32_t nbytes) {
+static inline void wbuf_literal_bytes(struct wbuf *w, const void *bytes_bv, uint32_t nbytes) {
     wbuf_nocrc_literal_bytes(w, bytes_bv, nbytes);
     toku_x1764_add(&w->checksum, &w->buf[w->ndone-nbytes], nbytes);
 }
 
-static void wbuf_nocrc_bytes (struct wbuf *w, bytevec bytes_bv, uint32_t nbytes) {
+static void wbuf_nocrc_bytes (struct wbuf *w, const void *bytes_bv, uint32_t nbytes) {
     wbuf_nocrc_uint(w, nbytes);
     wbuf_nocrc_literal_bytes(w, bytes_bv, nbytes);
 }
 
-static void wbuf_bytes (struct wbuf *w, bytevec bytes_bv, uint32_t nbytes) {
+static void wbuf_bytes (struct wbuf *w, const void *bytes_bv, uint32_t nbytes) {
     wbuf_uint(w, nbytes);
     wbuf_literal_bytes(w, bytes_bv, nbytes);
 }
@@ -262,73 +260,3 @@ static inline void wbuf_nocrc_uint32_t (struct wbuf *w, uint32_t v) {
 static inline void wbuf_uint32_t (struct wbuf *w, uint32_t v) {
     wbuf_uint(w, v);
 }
-
-static inline void wbuf_DISKOFF (struct wbuf *w, DISKOFF off) {
-    wbuf_ulonglong(w, (uint64_t)off);
-}
-
-static inline void wbuf_BLOCKNUM (struct wbuf *w, BLOCKNUM b) {
-    wbuf_ulonglong(w, b.b);
-}
-static inline void wbuf_nocrc_BLOCKNUM (struct wbuf *w, BLOCKNUM b) {
-    wbuf_nocrc_ulonglong(w, b.b);
-}
-
-static inline void wbuf_nocrc_TXNID (struct wbuf *w, TXNID tid) {
-    wbuf_nocrc_ulonglong(w, tid);
-}
-
-static inline void wbuf_nocrc_TXNID_PAIR (struct wbuf *w, TXNID_PAIR tid) {
-    wbuf_nocrc_ulonglong(w, tid.parent_id64);
-    wbuf_nocrc_ulonglong(w, tid.child_id64);
-}
-
-
-static inline void wbuf_TXNID (struct wbuf *w, TXNID tid) {
-    wbuf_ulonglong(w, tid);
-}
-
-static inline void wbuf_nocrc_XIDP (struct wbuf *w, XIDP xid) {
-    wbuf_nocrc_uint32_t(w, xid->formatID);
-    wbuf_nocrc_uint8_t(w, xid->gtrid_length);
-    wbuf_nocrc_uint8_t(w, xid->bqual_length);
-    wbuf_nocrc_literal_bytes(w, xid->data, xid->gtrid_length+xid->bqual_length);
-}
-
-static inline void wbuf_nocrc_LSN (struct wbuf *w, LSN lsn) {
-    wbuf_nocrc_ulonglong(w, lsn.lsn);
-}
-
-static inline void wbuf_LSN (struct wbuf *w, LSN lsn) {
-    wbuf_ulonglong(w, lsn.lsn);
-}
-
-static inline void wbuf_MSN (struct wbuf *w, MSN msn) {
-    wbuf_ulonglong(w, msn.msn);
-}
-
-static inline void wbuf_nocrc_FILENUM (struct wbuf *w, FILENUM fileid) {
-    wbuf_nocrc_uint(w, fileid.fileid);
-}
-
-static inline void wbuf_FILENUM (struct wbuf *w, FILENUM fileid) {
-    wbuf_uint(w, fileid.fileid);
-}
-
-// 2954
-static inline void wbuf_nocrc_FILENUMS (struct wbuf *w, FILENUMS v) {
-    wbuf_nocrc_uint(w, v.num);
-    uint32_t i;
-    for (i = 0; i < v.num; i++) {
-        wbuf_nocrc_FILENUM(w, v.filenums[i]);
-    }
-}
-
-// 2954
-static inline void wbuf_FILENUMS (struct wbuf *w, FILENUMS v) {
-    wbuf_uint(w, v.num);
-    uint32_t i;
-    for (i = 0; i < v.num; i++) {
-        wbuf_FILENUM(w, v.filenums[i]);
-    }
-}

ft/xids-internal.h

@@ -97,7 +97,9 @@ PATENT RIGHTS GRANT:
 // ids[num_xids - 1] is the innermost transaction.
 // Should only be accessed by accessor functions xids_xxx, not directly.
 
-#include <portability/toku_stdint.h>
+#include "portability/toku_stdint.h"
+
+#include "ft/txn.h"
 
 // If the xids struct is unpacked, the compiler aligns the ids[] and we waste a lot of space
 typedef struct __attribute__((__packed__)) xids_t {

ft/xids.cc

@@ -101,18 +101,16 @@ PATENT RIGHTS GRANT:
  *       host order.
  */
 
-
 #include <errno.h>
 #include <string.h>
 
-#include <toku_portability.h>
-#include "fttypes.h"
-#include "xids.h"
-#include "xids-internal.h"
-#include "toku_assert.h"
-#include "memory.h"
-#include <toku_htod.h>
+#include "portability/memory.h"
+#include "portability/toku_assert.h"
+#include "portability/toku_htod.h"
+#include "portability/toku_portability.h"
 
+#include "ft/xids.h"
+#include "ft/xids-internal.h"
 
 /////////////////////////////////////////////////////////////////////////////////
 //  This layer of abstraction (xids_xxx) understands xids<> and nothing else.
@@ -131,12 +129,10 @@ PATENT RIGHTS GRANT:
 // 
 //
 
-
 // This is the xids list for a transactionless environment.
 // It is also the initial state of any xids list created for
 // nested transactions.
 
-
 XIDS
 xids_get_root_xids(void) {
     static const struct xids_t root_xids = {
@@ -153,7 +149,6 @@ xids_can_create_child(XIDS xids) {
     return (xids->num_xids + 1) != MAX_TRANSACTION_RECORDS;
 }
 
-
 int
 xids_create_unknown_child(XIDS parent_xids, XIDS *xids_p) {
     // Postcondition:
@@ -211,14 +206,12 @@ xids_create_from_buffer(struct rbuf *rb,		// xids list for parent transaction
     *xids_p = xids;
 }
 
-
 void
 xids_destroy(XIDS *xids_p) {
     if (*xids_p != xids_get_root_xids()) toku_free(*xids_p);
     *xids_p = NULL;
 }
 
-
 // Return xid at requested position.  
 // If requesting an xid out of range (which will be the case if xids array is empty)
 // then return 0, the xid of the root transaction.
@@ -236,7 +229,6 @@ xids_get_num_xids(XIDS xids) {
     return rval;
 }
 
-
 // Return innermost xid 
 TXNID 
 xids_get_innermost_xid(XIDS xids) {
@@ -281,7 +273,6 @@ xids_get_serialize_size(XIDS xids){
     return rval;
 }
 
-
 unsigned char *
 xids_get_end_of_array(XIDS xids) {
     TXNID *r = xids->ids + xids->num_xids;

ft/xids.h

@@ -103,9 +103,12 @@ PATENT RIGHTS GRANT:
 #ident "Copyright (c) 2007-2013 Tokutek Inc.  All rights reserved."
 #ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
 
-#include "rbuf.h"
-#include "wbuf.h"
-#include "tokuconst.h"
+#include "ft/txn.h"
+#include "ft/rbuf.h"
+#include "ft/wbuf.h"
+#include "ft/tokuconst.h"
+
+typedef struct xids_t *XIDS;
 
 // Retrieve an XIDS representing the root transaction.
 XIDS xids_get_root_xids(void);

ft/ybt.cc

@@ -94,7 +94,6 @@ PATENT RIGHTS GRANT:
 
 #include "portability/memory.h"
 
-#include "ft/fttypes.h"
 #include "ft/ybt.h"
 
 DBT *
@@ -187,7 +186,7 @@ toku_destroy_dbt(DBT *dbt) {
 }
 
 DBT *
-toku_fill_dbt(DBT *dbt, bytevec k, ITEMLEN len) {
+toku_fill_dbt(DBT *dbt, const void *k, uint32_t len) {
     toku_init_dbt(dbt);
     dbt->size=len;
     dbt->data=(char*)k;
@@ -246,7 +245,7 @@ dbt_realloc(DBT *dbt) {
 }
 
 int
-toku_dbt_set (ITEMLEN len, bytevec val, DBT *d, struct simple_dbt *sdbt) {
+toku_dbt_set (uint32_t len, const void *val, DBT *d, struct simple_dbt *sdbt) {
 // sdbt is the static value used when flags==0
 // Otherwise malloc or use the user-supplied memory, as according to the flags in d->flags.
     int r;

ft/ybt.h

@@ -92,8 +92,6 @@ PATENT RIGHTS GRANT:
 #ident "Copyright (c) 2007-2013 Tokutek Inc.  All rights reserved."
 #ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
 
-// fttypes.h must be first to make 64-bit file mode work right in linux.
-#include "fttypes.h"
 #include <db.h>
 
 // TODO: John
@@ -109,7 +107,7 @@ DBT *toku_init_dbt_flags(DBT *, uint32_t flags);
 
 void toku_destroy_dbt(DBT *);
 
-DBT *toku_fill_dbt(DBT *dbt, bytevec k, ITEMLEN len);
+DBT *toku_fill_dbt(DBT *dbt, const void *k, uint32_t len);
 
 DBT *toku_memdup_dbt(DBT *dbt, const void *k, size_t len);
 
@@ -117,9 +115,9 @@ DBT *toku_copyref_dbt(DBT *dst, const DBT src);
 
 DBT *toku_clone_dbt(DBT *dst, const DBT &src);
 
-int toku_dbt_set(ITEMLEN len, bytevec val, DBT *d, struct simple_dbt *sdbt);
+int toku_dbt_set(uint32_t len, const void *val, DBT *d, struct simple_dbt *sdbt);
 
-int toku_dbt_set_value(DBT *, bytevec *val, ITEMLEN vallen, void **staticptrp, bool ybt1_disposable);
+int toku_dbt_set_value(DBT *, const void **val, uint32_t vallen, void **staticptrp, bool ybt1_disposable);
 
 void toku_sdbt_cleanup(struct simple_dbt *sdbt);
 

locktree/lock_request.cc

@@ -91,6 +91,7 @@ PATENT RIGHTS GRANT:
 
 #include <toku_race_tools.h>
 
+#include <ft/txn.h>
 #include <ft/ybt.h>
 
 #include "locktree.h"

locktree/lock_request.h

@@ -92,14 +92,13 @@ PATENT RIGHTS GRANT:
 #ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
 
 #include <db.h>
-#include <toku_pthread.h>
 
-#include <ft/fttypes.h>
-#include <ft/comparator.h>
+#include "portability/toku_pthread.h"
 
-#include "locktree.h"
-#include "txnid_set.h"
-#include "wfg.h"
+#include "locktree/locktree.h"
+#include "locktree/txnid_set.h"
+#include "locktree/wfg.h"
+#include "ft/comparator.h"
 
 namespace toku {
 

locktree/locktree.h

@@ -95,7 +95,7 @@ PATENT RIGHTS GRANT:
 #include <toku_time.h>
 #include <toku_pthread.h>
 
-#include <ft/fttypes.h>
+#include <ft/ft-ops.h> // just for DICTIONARY_ID..
 #include <ft/comparator.h>
 
 #include <util/omt.h>

locktree/treenode.h

@@ -94,6 +94,7 @@ PATENT RIGHTS GRANT:
 #include <memory.h>
 #include <string.h>
 
+#include <ft/txn.h>
 #include <ft/comparator.h>
 
 #include <portability/toku_pthread.h>

locktree/txnid_set.h

@@ -91,9 +91,9 @@ PATENT RIGHTS GRANT:
 #ident "Copyright (c) 2007-2013 Tokutek Inc.  All rights reserved."
 #ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
 
-#include <ft/fttypes.h>
+#include "ft/txn.h"
 
-#include <util/omt.h>
+#include "util/omt.h"
 
 namespace toku {
 

locktree/wfg.h

@@ -91,11 +91,8 @@ PATENT RIGHTS GRANT:
 #ident "Copyright (c) 2007-2013 Tokutek Inc.  All rights reserved."
 #ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
 
-#include <ft/fttypes.h>
-
-#include <util/omt.h>
-
-#include "txnid_set.h"
+#include "locktree/txnid_set.h"
+#include "util/omt.h"
 
 namespace toku {
 

src/indexer.cc

@@ -539,7 +539,7 @@ struct le_cursor_extra {
 // cachetable pair locks. because no txn can commit on this db, read
 // the provisional info for the newly read ule.
 static int
-le_cursor_callback(ITEMLEN keylen, bytevec key, ITEMLEN UU(vallen), bytevec val, void *extra, bool lock_only) {
+le_cursor_callback(uint32_t keylen, const void *key, uint32_t UU(vallen), const void *val, void *extra, bool lock_only) {
     if (lock_only || val == NULL) {
         ; // do nothing if only locking. do nothing if val==NULL, means DB_NOTFOUND
     } else {

src/tests/hotindexer-undo-do-test.cc

@@ -96,8 +96,6 @@ PATENT RIGHTS GRANT:
 #include "test.h"
 
 #include <ft/tokuconst.h>
-#include <ft/fttypes.h>
-#include <ft/leafentry.h>
 #include <ft/ule.h>
 #include <ft/ule-internal.h>
 #include <ft/le-cursor.h>

src/ydb-internal.h

@@ -97,7 +97,6 @@ PATENT RIGHTS GRANT:
 #include <ft/cachetable.h>
 #include <ft/cursor.h>
 #include <ft/comparator.h>
-#include <ft/fttypes.h>
 #include <ft/logger.h>
 #include <ft/txn.h>
 

src/ydb_cursor.cc

@@ -248,7 +248,7 @@ query_context_with_input_init(QUERY_CONTEXT_WITH_INPUT context, DBC *c, uint32_t
     context->input_val = val;
 }
 
-static int c_getf_first_callback(ITEMLEN keylen, bytevec key, ITEMLEN vallen, bytevec val, void *extra, bool);
+static int c_getf_first_callback(uint32_t keylen, const void *key, uint32_t vallen, const void *val, void *extra, bool);
 
 static void 
 c_query_context_init(QUERY_CONTEXT context, DBC *c, uint32_t flag, YDB_CALLBACK_FUNCTION f, void *extra) {
@@ -291,7 +291,7 @@ c_getf_first(DBC *c, uint32_t flag, YDB_CALLBACK_FUNCTION f, void *extra) {
 
 //result is the result of the query (i.e. 0 means found, DB_NOTFOUND, etc..)
 static int
-c_getf_first_callback(ITEMLEN keylen, bytevec key, ITEMLEN vallen, bytevec val, void *extra, bool lock_only) {
+c_getf_first_callback(uint32_t keylen, const void *key, uint32_t vallen, const void *val, void *extra, bool lock_only) {
     QUERY_CONTEXT      super_context = (QUERY_CONTEXT) extra;
     QUERY_CONTEXT_BASE context       = &super_context->base;
 
@@ -318,7 +318,7 @@ c_getf_first_callback(ITEMLEN keylen, bytevec key, ITEMLEN vallen, bytevec val,
     return r;
 }
 
-static int c_getf_last_callback(ITEMLEN keylen, bytevec key, ITEMLEN vallen, bytevec val, void *extra, bool);
+static int c_getf_last_callback(uint32_t keylen, const void *key, uint32_t vallen, const void *val, void *extra, bool);
 
 static int
 c_getf_last(DBC *c, uint32_t flag, YDB_CALLBACK_FUNCTION f, void *extra) {
@@ -342,7 +342,7 @@ c_getf_last(DBC *c, uint32_t flag, YDB_CALLBACK_FUNCTION f, void *extra) {
 
 //result is the result of the query (i.e. 0 means found, DB_NOTFOUND, etc..)
 static int
-c_getf_last_callback(ITEMLEN keylen, bytevec key, ITEMLEN vallen, bytevec val, void *extra, bool lock_only) {
+c_getf_last_callback(uint32_t keylen, const void *key, uint32_t vallen, const void *val, void *extra, bool lock_only) {
     QUERY_CONTEXT      super_context = (QUERY_CONTEXT) extra;
     QUERY_CONTEXT_BASE context       = &super_context->base;
 
@@ -369,7 +369,7 @@ c_getf_last_callback(ITEMLEN keylen, bytevec key, ITEMLEN vallen, bytevec val, v
     return r;
 }
 
-static int c_getf_next_callback(ITEMLEN keylen, bytevec key, ITEMLEN vallen, bytevec val, void *extra, bool);
+static int c_getf_next_callback(uint32_t keylen, const void *key, uint32_t vallen, const void *val, void *extra, bool);
 
 static int
 c_getf_next(DBC *c, uint32_t flag, YDB_CALLBACK_FUNCTION f, void *extra) {
@@ -398,7 +398,7 @@ c_getf_next(DBC *c, uint32_t flag, YDB_CALLBACK_FUNCTION f, void *extra) {
 
 //result is the result of the query (i.e. 0 means found, DB_NOTFOUND, etc..)
 static int
-c_getf_next_callback(ITEMLEN keylen, bytevec key, ITEMLEN vallen, bytevec val, void *extra, bool lock_only) {
+c_getf_next_callback(uint32_t keylen, const void *key, uint32_t vallen, const void *val, void *extra, bool lock_only) {
     QUERY_CONTEXT      super_context = (QUERY_CONTEXT) extra;
     QUERY_CONTEXT_BASE context       = &super_context->base;
 
@@ -428,7 +428,7 @@ c_getf_next_callback(ITEMLEN keylen, bytevec key, ITEMLEN vallen, bytevec val, v
     return r;
 }
 
-static int c_getf_prev_callback(ITEMLEN keylen, bytevec key, ITEMLEN vallen, bytevec val, void *extra, bool);
+static int c_getf_prev_callback(uint32_t keylen, const void *key, uint32_t vallen, const void *val, void *extra, bool);
 
 static int
 c_getf_prev(DBC *c, uint32_t flag, YDB_CALLBACK_FUNCTION f, void *extra) {
@@ -457,7 +457,7 @@ c_getf_prev(DBC *c, uint32_t flag, YDB_CALLBACK_FUNCTION f, void *extra) {
 
 //result is the result of the query (i.e. 0 means found, DB_NOTFOUND, etc..)
 static int
-c_getf_prev_callback(ITEMLEN keylen, bytevec key, ITEMLEN vallen, bytevec val, void *extra, bool lock_only) {
+c_getf_prev_callback(uint32_t keylen, const void *key, uint32_t vallen, const void *val, void *extra, bool lock_only) {
     QUERY_CONTEXT      super_context = (QUERY_CONTEXT) extra;
     QUERY_CONTEXT_BASE context       = &super_context->base;
 
@@ -486,7 +486,7 @@ c_getf_prev_callback(ITEMLEN keylen, bytevec key, ITEMLEN vallen, bytevec val, v
     return r;
 }
 
-static int c_getf_current_callback(ITEMLEN keylen, bytevec key, ITEMLEN vallen, bytevec val, void *extra, bool);
+static int c_getf_current_callback(uint32_t keylen, const void *key, uint32_t vallen, const void *val, void *extra, bool);
 
 static int
 c_getf_current(DBC *c, uint32_t flag, YDB_CALLBACK_FUNCTION f, void *extra) {
@@ -503,7 +503,7 @@ c_getf_current(DBC *c, uint32_t flag, YDB_CALLBACK_FUNCTION f, void *extra) {
 
 //result is the result of the query (i.e. 0 means found, DB_NOTFOUND, etc..)
 static int
-c_getf_current_callback(ITEMLEN keylen, bytevec key, ITEMLEN vallen, bytevec val, void *extra, bool lock_only) {
+c_getf_current_callback(uint32_t keylen, const void *key, uint32_t vallen, const void *val, void *extra, bool lock_only) {
     QUERY_CONTEXT      super_context = (QUERY_CONTEXT) extra;
     QUERY_CONTEXT_BASE context       = &super_context->base;
 
@@ -523,7 +523,7 @@ c_getf_current_callback(ITEMLEN keylen, bytevec key, ITEMLEN vallen, bytevec val
     return r;
 }
 
-static int c_getf_set_callback(ITEMLEN keylen, bytevec key, ITEMLEN vallen, bytevec val, void *extra, bool);
+static int c_getf_set_callback(uint32_t keylen, const void *key, uint32_t vallen, const void *val, void *extra, bool);
 
 int
 toku_c_getf_set(DBC *c, uint32_t flag, DBT *key, YDB_CALLBACK_FUNCTION f, void *extra) {
@@ -548,7 +548,7 @@ toku_c_getf_set(DBC *c, uint32_t flag, DBT *key, YDB_CALLBACK_FUNCTION f, void *
 
 //result is the result of the query (i.e. 0 means found, DB_NOTFOUND, etc..)
 static int
-c_getf_set_callback(ITEMLEN keylen, bytevec key, ITEMLEN vallen, bytevec val, void *extra, bool lock_only) {
+c_getf_set_callback(uint32_t keylen, const void *key, uint32_t vallen, const void *val, void *extra, bool lock_only) {
     QUERY_CONTEXT_WITH_INPUT super_context = (QUERY_CONTEXT_WITH_INPUT) extra;
     QUERY_CONTEXT_BASE       context       = &super_context->base;
 
@@ -576,7 +576,7 @@ c_getf_set_callback(ITEMLEN keylen, bytevec key, ITEMLEN vallen, bytevec val, vo
     return r;
 }
 
-static int c_getf_set_range_callback(ITEMLEN keylen, bytevec key, ITEMLEN vallen, bytevec val, void *extra, bool);
+static int c_getf_set_range_callback(uint32_t keylen, const void *key, uint32_t vallen, const void *val, void *extra, bool);
 
 static int
 c_getf_set_range(DBC *c, uint32_t flag, DBT *key, YDB_CALLBACK_FUNCTION f, void *extra) {
@@ -601,7 +601,7 @@ c_getf_set_range(DBC *c, uint32_t flag, DBT *key, YDB_CALLBACK_FUNCTION f, void
 
 //result is the result of the query (i.e. 0 means found, DB_NOTFOUND, etc..)
 static int
-c_getf_set_range_callback(ITEMLEN keylen, bytevec key, ITEMLEN vallen, bytevec val, void *extra, bool lock_only) {
+c_getf_set_range_callback(uint32_t keylen, const void *key, uint32_t vallen, const void *val, void *extra, bool lock_only) {
     QUERY_CONTEXT_WITH_INPUT super_context = (QUERY_CONTEXT_WITH_INPUT) extra;
     QUERY_CONTEXT_BASE       context       = &super_context->base;
 
@@ -653,7 +653,7 @@ c_getf_set_range_with_bound(DBC *c, uint32_t flag, DBT *key, DBT *key_bound, YDB
     return r;
 }
 
-static int c_getf_set_range_reverse_callback(ITEMLEN keylen, bytevec key, ITEMLEN vallen, bytevec val, void *extra, bool);
+static int c_getf_set_range_reverse_callback(uint32_t keylen, const void *key, uint32_t vallen, const void *val, void *extra, bool);
 
 static int
 c_getf_set_range_reverse(DBC *c, uint32_t flag, DBT *key, YDB_CALLBACK_FUNCTION f, void *extra) {
@@ -678,7 +678,7 @@ c_getf_set_range_reverse(DBC *c, uint32_t flag, DBT *key, YDB_CALLBACK_FUNCTION
 
 //result is the result of the query (i.e. 0 means found, DB_NOTFOUND, etc..)
 static int
-c_getf_set_range_reverse_callback(ITEMLEN keylen, bytevec key, ITEMLEN vallen, bytevec val, void *extra, bool lock_only) {
+c_getf_set_range_reverse_callback(uint32_t keylen, const void *key, uint32_t vallen, const void *val, void *extra, bool lock_only) {
     QUERY_CONTEXT_WITH_INPUT super_context = (QUERY_CONTEXT_WITH_INPUT) extra;
     QUERY_CONTEXT_BASE       context       = &super_context->base;
 

src/ydb_db.cc

@@ -96,7 +96,6 @@ PATENT RIGHTS GRANT:
 #include <ft/ft.h>
 #include <ft/ft-flusher.h>
 #include <ft/checkpoint.h>
-#include <ft/log_header.h>
 
 #include "ydb_cursor.h"
 #include "ydb_row_lock.h"
@@ -943,7 +942,7 @@ struct last_key_extra {
 };
 
 static int
-db_get_last_key_callback(ITEMLEN keylen, bytevec key, ITEMLEN vallen UU(), bytevec val UU(), void *extra, bool lock_only) {
+db_get_last_key_callback(uint32_t keylen, const void *key, uint32_t vallen UU(), const void *val UU(), void *extra, bool lock_only) {
     if (!lock_only) {
         DBT keydbt;
         toku_fill_dbt(&keydbt, key, keylen);

util/bytestring.h

@@ -0,0 +1,96 @@
+/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */
+// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4:
+
+/*
+COPYING CONDITIONS NOTICE:
+
+  This program is free software; you can redistribute it and/or modify
+  it under the terms of version 2 of the GNU General Public License as
+  published by the Free Software Foundation, and provided that the
+  following conditions are met:
+
+      * Redistributions of source code must retain this COPYING
+        CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the
+        DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the
+        PATENT MARKING NOTICE (below), and the PATENT RIGHTS
+        GRANT (below).
+
+      * Redistributions in binary form must reproduce this COPYING
+        CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the
+        DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the
+        PATENT MARKING NOTICE (below), and the PATENT RIGHTS
+        GRANT (below) in the documentation and/or other materials
+        provided with the distribution.
+
+  You should have received a copy of the GNU General Public License
+  along with this program; if not, write to the Free Software
+  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+  02110-1301, USA.
+
+COPYRIGHT NOTICE:
+
+  TokuDB, Tokutek Fractal Tree Indexing Library.
+  Copyright (C) 2014 Tokutek, Inc.
+
+DISCLAIMER:
+
+  This program is distributed in the hope that it will be useful, but
+  WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+  General Public License for more details.
+
+UNIVERSITY PATENT NOTICE:
+
+  The technology is licensed by the Massachusetts Institute of
+  Technology, Rutgers State University of New Jersey, and the Research
+  Foundation of State University of New York at Stony Brook under
+  United States of America Serial No. 11/760379 and to the patents
+  and/or patent applications resulting from it.
+
+PATENT MARKING NOTICE:
+
+  This software is covered by US Patent No. 8,185,551.
+  This software is covered by US Patent No. 8,489,638.
+
+PATENT RIGHTS GRANT:
+
+  "THIS IMPLEMENTATION" means the copyrightable works distributed by
+  Tokutek as part of the Fractal Tree project.
+
+  "PATENT CLAIMS" means the claims of patents that are owned or
+  licensable by Tokutek, both currently or in the future; and that in
+  the absence of this license would be infringed by THIS
+  IMPLEMENTATION or by using or running THIS IMPLEMENTATION.
+
+  "PATENT CHALLENGE" shall mean a challenge to the validity,
+  patentability, enforceability and/or non-infringement of any of the
+  PATENT CLAIMS or otherwise opposing any of the PATENT CLAIMS.
+
+  Tokutek hereby grants to you, for the term and geographical scope of
+  the PATENT CLAIMS, a non-exclusive, no-charge, royalty-free,
+  irrevocable (except as stated in this section) patent license to
+  make, have made, use, offer to sell, sell, import, transfer, and
+  otherwise run, modify, and propagate the contents of THIS
+  IMPLEMENTATION, where such license applies only to the PATENT
+  CLAIMS.  This grant does not include claims that would be infringed
+  only as a consequence of further modifications of THIS
+  IMPLEMENTATION.  If you or your agent or licensee institute or order
+  or agree to the institution of patent litigation against any entity
+  (including a cross-claim or counterclaim in a lawsuit) alleging that
+  THIS IMPLEMENTATION constitutes direct or contributory patent
+  infringement, or inducement of patent infringement, then any rights
+  granted to you under this License shall terminate as of the date
+  such litigation is filed.  If you or your agent or exclusive
+  licensee institute or order or agree to the institution of a PATENT
+  CHALLENGE, then Tokutek may terminate any rights granted to you
+  under this License.
+*/
+
+#pragma once
+
+#include "portability/toku_stdint.h"
+
+struct BYTESTRING {
+    uint32_t len;
+    char *data;
+};