Merge changes from Antidote Refractor to integrate Gingko

.gitignore

@@ -35,3 +35,5 @@ edoc/
 #docker local builds uses a tmp dir to fetch build scripts
 docker-tmpdir*
 *.coverdata
+*.iml
+_checkouts

Makefile

@@ -14,7 +14,7 @@ distclean: clean relclean
 	$(REBAR) clean --all
 
 shell: rel
-	export NODE_NAME=antidote@127.0.0.1 ; \
+	export NODE_NAME=antidote@192.168.1.4 ; \
 	export COOKIE=antidote ; \
 	export ROOT_DIR_PREFIX=$$NODE_NAME/ ; \
 	_build/default/rel/antidote/bin/antidote console ${ARGS}
@@ -56,7 +56,7 @@ else
 	${REBAR} ct --dir test/singledc --cover_export_name=singledc
 endif
 
-multidc: 
+multidc:
 ifdef SUITE
 	${REBAR} ct --dir test/multidc --suite ${SUITE}
 else

_checkouts/gingko

@@ -0,0 +1 @@
+/Users/pandey/work/gingko/

config/sys.config.src

@@ -6,12 +6,12 @@
     {logger, [
 
       {handler, default, logger_std_h,
-        #{level => info,
+        #{level => error,
           formatter => {logger_formatter, #{single_line => false, max_size => 2048}},
           config => #{type => standard_io}}},
 
       {handler, debug, logger_std_h,
-        #{level => info,
+        #{level => error,
           formatter => {logger_formatter, #{single_line => true, max_size => 2048}},
           config => #{type => {file, "${ROOT_DIR_PREFIX}${LOGGER_DIR_PREFIX}logger_logs/info.log"}}}},
 
@@ -34,7 +34,7 @@
     %% has to be an exponent of 2
     %% low number will decrease file accesses (good for testing) and boot time
     %% high number enables scaling and generates smaller log files
-    {ring_creation_size, 64}
+    {ring_creation_size, 16}
   ]},
 
 

include/antidote.hrl

@@ -101,26 +101,31 @@
 }).
 -type commit_log_payload() :: #commit_log_payload{}.
 
+-record(start_log_payload, {}).
+
 -record(update_log_payload, {
     key :: key(),
     bucket :: bucket(),
     type :: type(),
     op :: op()
 }).
+
 -type update_log_payload() :: #update_log_payload{}.
 
 -record(abort_log_payload, {}).
 
 -record(prepare_log_payload, {prepare_time :: non_neg_integer()}).
 
--type any_log_payload() :: #update_log_payload{}
+-type any_log_payload() :: #start_log_payload{}
+                         | #update_log_payload{}
                          | #commit_log_payload{}
                          | #abort_log_payload{}
                          | #prepare_log_payload{}.
 
 -record(log_operation, {
     tx_id :: txid(),
-    op_type :: update
+    op_type :: start
+             | update
              | prepare
              | commit
              | abort

include/gingko.hrl

@@ -0,0 +1,131 @@
+%TODO link to antidote_crdt types
+
+-type key() :: term().
+-type op() :: {update, {key(), type(), term()}} | {read, {key(), type()}}. %TODO what is the term in update?
+-type type() :: atom(). 
+-type downstream_record() :: term(). 
+-type snapshot() :: term().
+-type txid() :: term().
+
+
+%% Module names to be used across implementation.
+-define(CACHE_DAEMON, cache_daemon).
+-define(LOG_INDEX_DAEMON, log_index_daemon).
+-define(LOGGING_MASTER, gingko_op_log).
+
+
+%% Version of log records being used
+-define(LOG_RECORD_VERSION, 0).
+
+-record(op_number, {
+    %% TODO 19 undefined is required here, because of the use in inter_dc_log_sender_vnode.
+    %% The use there should be refactored.
+    node :: undefined | {node(), dcid()},
+    global :: undefined | non_neg_integer(),
+    local :: undefined | non_neg_integer()
+}).
+
+-type op_name() :: atom().
+-type op_param() :: term().
+-type effect() :: term().
+-type dcid() :: 'undefined' | {atom(),tuple()}. %% TODO, is this the only structure that is returned by riak_core_ring:cluster_name(Ring)?
+-type snapshot_time() :: 'undefined' | vectorclock:vectorclock().
+-type clock_time() :: non_neg_integer().
+-type dc_and_commit_time() :: {dcid(), clock_time()}.
+-type op_num() :: non_neg_integer().
+-type op_id() :: {op_num(), node()}.
+-type payload() :: term().
+-type partition_id() :: ets:tid() | integer(). % TODO 19 adding integer basically makes the tid type non-opaque, because some places of the code depend on it being an integer. This dependency should be removed, if possible.
+-type log_id() :: [partition_id()].
+%%chash:index_as_int() is the same as riak_core_apl:index().
+%%If it is changed in the future this should be fixed also.
+-type index_node() :: {chash:index_as_int(), node()}.
+
+
+-record(materialized_snapshot, {
+    %% This is the opid of the latest op in the list
+    %% of ops for this key included in this snapshot
+    %% before an op that was not included, so to a new
+    %% snapshot will be generated by starting from this op
+    last_op_id :: op_num(),
+    value :: snapshot()
+}).
+
+
+-record(clocksi_payload, {
+    key :: key(),
+    type :: type(),
+    op_param :: effect(),
+    snapshot_time :: snapshot_time(),
+    commit_time :: dc_and_commit_time(),
+    txid :: txid()
+}).
+
+-type clocksi_payload() :: #clocksi_payload{}.
+-record(commit_log_payload, {
+    commit_time :: dc_and_commit_time(),
+    snapshot_time :: snapshot_time()
+}).
+
+-record(update_log_payload, {
+    key :: key(),
+    type :: type(),
+    op :: op()
+}).
+
+-type reason() :: term().
+-type preflist() :: riak_core_apl:preflist().
+-type cache_id() :: ets:tab().
+
+
+-record(abort_log_payload, {}).
+
+-record(prepare_log_payload, {prepare_time :: non_neg_integer()}).
+
+-type any_log_payload() :: #update_log_payload{}
+                         | #commit_log_payload{}
+                         | #abort_log_payload{}
+                         | #prepare_log_payload{}.
+
+-record(log_operation, {
+    tx_id :: txid(),
+    op_type :: update
+             | prepare
+             | commit
+             | abort
+             | noop,
+    log_payload :: any_log_payload()
+}).
+
+
+%% The way records are stored in the log.
+-record(log_record, {
+    %% The version of the log record, for backwards compatibility
+    version :: non_neg_integer(),
+    log_operation :: #log_operation{}
+}).
+
+-type continuation() :: disk_log:continuation() | start.
+
+
+-record(log_read, {
+  log_entry :: {integer(), #log_record{}},
+  continuation:: continuation()
+}).
+
+
+-record(log_index,{
+  key :: atom(),
+  snapshot_time:: snapshot_time(),
+  continuation:: continuation()
+}).
+
+
+-record(cache_object, {
+  key:: atom(),
+  type:: antidote_crdt:typ(),
+  snapshot_time:: snapshot_time(),
+  snapshot:: snapshot(),
+  count:: integer()
+
+}).

rebar.config

@@ -9,7 +9,7 @@
     antidote_pb_codec,
     antidotec_pb,
     vectorclock,
-
+    gingko,
     %% efficient inter-dc messaging
     {erlzmq, {git, "https://github.com/zeromq/erlzmq2", {ref, "573d583"}}},
 

rebar.lock

@@ -1,6 +1,6 @@
 {"1.2.0",
 [{<<"accept">>,{pkg,<<"accept">>,<<"0.3.5">>},2},
- {<<"antidote_crdt">>,{pkg,<<"antidote_crdt">>,<<"0.1.2">>},0},
+ {<<"antidote_crdt">>,{pkg,<<"antidote_crdt">>,<<"0.1.3">>},0},
  {<<"antidote_pb_codec">>,{pkg,<<"antidote_pb_codec">>,<<"0.1.2">>},0},
  {<<"antidote_stats">>,
   {git,"https://github.com/AntidoteDB/antidote_stats",
@@ -24,7 +24,7 @@
 [
 {pkg_hash,[
  {<<"accept">>, <<"B33B127ABCA7CC948BBE6CAA4C263369ABF1347CFA9D8E699C6D214660F10CD1">>},
- {<<"antidote_crdt">>, <<"A92A5ED8918D87AD22557825743C6EAC69DD6089D536E1BF5F9AC80992FA97F8">>},
+ {<<"antidote_crdt">>, <<"FBAD3C65BA21E781C9E3D064C483D42F0CC52FC7145F2C317A918C31224270BB">>},
  {<<"antidote_pb_codec">>, <<"ECF51F08EE1FEE0D6E82D1B4AE68811A89660A3D65DB90694A84275683AEF106">>},
  {<<"antidotec_pb">>, <<"40CD2A0A5F63284E6BB46C84B9D806A7682C79117CFC96154082A4C80759DD01">>},
  {<<"elli">>, <<"089218762A7FF3D20AE81C8E911BD0F73EE4EE0ED85454226D1FC6B4FFF3B4F6">>},
@@ -37,7 +37,7 @@
  {<<"vectorclock">>, <<"6C4A9D44895F51BB99910DBE31FC691BF05FA6B2BF84986F6E3BDE4BD18F6CBA">>}]},
 {pkg_hash_ext,[
  {<<"accept">>, <<"11B18C220BCC2EAB63B5470C038EF10EB6783BCB1FCDB11AA4137DEFA5AC1BB8">>},
- {<<"antidote_crdt">>, <<"F01570A02CB75EE2F580512E25D70CA206E86F83399080A378BA1E8FB76AE21E">>},
+ {<<"antidote_crdt">>, <<"5625673BB3227F5A77795C995F3139FCB5DC58117B1AB7AAC14A6FDDE58D7C6B">>},
  {<<"antidote_pb_codec">>, <<"B8F655B5C2D074DEB1E170C9EE607A157B577F2DB064D212643487AEA684BB5B">>},
  {<<"antidotec_pb">>, <<"FEA4286BEEF84E65EA27983E65AD8347016F4657306D468224309C44AD28521C">>},
  {<<"elli">>, <<"698B13B33D05661DB9FE7EFCBA41B84825A379CCE86E486CF6AFF9285BE0CCF8">>},

src/antidote.app.src

@@ -1,32 +1,33 @@
 %% -*- erlang -*-
 {application, antidote, [
-    {description, "A transactional CRDT database"},
-    {vsn, "0.2.2"},
-    {applications, [
-        kernel,
-        ranch,
-        stdlib,
-        riak_core,
-        erlzmq,
-        runtime_tools,
-        tools,
-        antidote_stats
-    ]},
-    {included_applications, [
-        vectorclock,
-        antidote_pb_codec,
-        antidote_crdt
-    ]},
+  {description, "A transactional CRDT database"},
+  {vsn, "0.2.2"},
+  {applications, [
+    kernel,
+    ranch,
+    stdlib,
+    riak_core,
+    erlzmq,
+    runtime_tools,
+    tools,
+    antidote_stats
+  ]},
+  {included_applications, [
+    vectorclock,
+    antidote_pb_codec,
+    antidote_crdt,
+    gingko
+  ]},
 
-    {mod, {antidote_app, []}},
+  {mod, {antidote_app, []}},
 
-    % Default values for sys.config options used for system tests which do not load config files
-    {env, [{txn_cert, true}, {txn_prot, clocksi}, {recover_from_log, true},
-        {recover_meta_data_on_start, true}, {sync_log, false},
-        {enable_logging, true},
-        {auto_start_read_servers, true},
-        {data_dir, "data_antidote"},
-        {stats, true},
-        {extended_stats, true}
-    ]}
+  % Default values for sys.config options used for system tests which do not load config files
+  {env, [{txn_cert, true}, {txn_prot, clocksi}, {recover_from_log, true},
+    {recover_meta_data_on_start, true}, {sync_log, false},
+    {enable_logging, true},
+    {auto_start_read_servers, true},
+    {data_dir, "data_antidote"},
+    {stats, true},
+    {extended_stats, true}
+  ]}
 ]}.