Skip to content

Commit

Permalink
Nhse d32 nhskv.i17 (#19)
Browse files Browse the repository at this point in the history
* Make 2i timeout 503

Avoid issue when an {error, timeout} may be passed as {error, {error, timeout} - resulting in general 500 error

* Encoding use JSON library for OTP PR (#20)

* Encoding use JSON library for OTP PR

Uses the PR'd Json library to OTP for encoding, and passing in a bespoke override on the encode function to avoid the double-pass to convert into a map.

This appears to be notably faster than mochijson2 and thoas.

* Tidy-up

Focus on proposed JSON library for OTP.

The riak_kv_clusteraae_fsm has also been reverted back to mochijson2.  The format of leveled_tictac:export_tree isn't compatible with the OTP json library - rather than over-complicate the PR, as performance is not critical for these queries, keep the old implementation.

* Test integer index

* Extend unit test

* Switch to raw OTP PR eaed0f7

Without compatibility changes

* Correct filename

* Compatibility changes to OTP JSON

Compatibility changes required for OTP 22 - 26

* Update and include quickcheck properties (#21)

* Update and include quickcheck properties

* Output too verbose with a collect.

* Update comment as per review

---------

Co-authored-by: Thomas Arts <thomas.arts@quviq.com>
  • Loading branch information
martinsumner and ThomasArts authored Mar 28, 2024
1 parent b7b690f commit aec41e6
Show file tree
Hide file tree
Showing 9 changed files with 1,685 additions and 52 deletions.
56 changes: 56 additions & 0 deletions eqc/json_eqc.erl
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
-module(json_eqc).

-include_lib("eqc/include/eqc.hrl").
-compile([export_all, nowarn_export_all]).

prop_roundtrip() ->
in_parallel(?FORALL(Obj, object(), equals(Obj, decode(encode(Obj))))).

prop_escape_all() ->
in_parallel(
?FORALL(Str, object(),
begin
Encoded = iolist_to_binary(encode_escape_all(Str)),
all_ascii(Encoded) andalso equals(Str, decode(Encoded))
end)).

printable_string() ->
Chars = oneof([
oneof("\n\r\t\v\b\f\e\d"),
choose(16#20, 16#7E),
choose(16#A0, 16#D7FF),
choose(16#E000, 16#FFFD),
choose(16#10000, 16#10FFFF)
]),
?LET(L, list(Chars), unicode:characters_to_binary(L)).

object() -> ?SIZED(Size, object(Size)).

object(0) ->
oneof([integer(), real(), printable_string(), bool(), null]);
object(Size) ->
?LAZY(oneof([object(0),
?LETSHRINK(Objects, list(object(Size div 4)), Objects),
map(printable_string(), object(Size div 4))
])).

integer() -> choose(-16#8fffffff, 16#ffffffff).

%% Helpers

all_ascii(Bin) ->
lists:all(fun(C) -> C < 128 end, binary_to_list(Bin)).

decode(IOList) ->
riak_kv_wm_json:decode(iolist_to_binary(IOList)).

%% May produce an iolist
encode(Str) ->
riak_kv_wm_json:encode(Str).

encode_escape_all(Term) ->
Encode = fun
(Binary, _) when is_binary(Binary) -> riak_kv_wm_json:encode_binary_escape_all(Binary);
(Other, Encode) -> riak_kv_wm_json:encode_value(Other, Encode)
end,
riak_kv_wm_json:encode(Term, Encode).
189 changes: 189 additions & 0 deletions include/riak_kv_wm_json.hrl
Original file line number Diff line number Diff line change
@@ -0,0 +1,189 @@
%%
%% %CopyrightBegin%
%%
%% Copyright Ericsson AB 2024. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%
%% %CopyrightEnd%
%%

%% A lot of the macros below use multi-value comparisons where
%% range checks would have worked just fine. This is because
%% the compiler & JIT can emit better code in some cases when
%% multiple clauses are to be dispatched based on such sets
%% of values. They'll generate an efficient "jump table",
%% which gets to the correct clause in one go, rather
%% than going through a set of comparisons.
%% However, this might not always be the bext way (see is_0_to_9),
%% so as always with any performance work - measure, don't guess!

-define(is_1_to_9(X),
X =:= $1 orelse
X =:= $2 orelse
X =:= $3 orelse
X =:= $4 orelse
X =:= $5 orelse
X =:= $6 orelse
X =:= $7 orelse
X =:= $8 orelse
X =:= $9
).

-define(is_0_to_9(X), X >= $0 andalso X =< $9).

-define(is_ws(X), X =:= $\s; X =:= $\t; X =:= $\r; X =:= $\n).

-define(is_ascii_escape(Byte),
Byte =:= 0 orelse
Byte =:= 1 orelse
Byte =:= 2 orelse
Byte =:= 3 orelse
Byte =:= 4 orelse
Byte =:= 5 orelse
Byte =:= 6 orelse
Byte =:= 7 orelse
Byte =:= 8 orelse
Byte =:= 9 orelse
Byte =:= 10 orelse
Byte =:= 11 orelse
Byte =:= 12 orelse
Byte =:= 13 orelse
Byte =:= 14 orelse
Byte =:= 15 orelse
Byte =:= 16 orelse
Byte =:= 17 orelse
Byte =:= 18 orelse
Byte =:= 19 orelse
Byte =:= 20 orelse
Byte =:= 21 orelse
Byte =:= 22 orelse
Byte =:= 23 orelse
Byte =:= 24 orelse
Byte =:= 25 orelse
Byte =:= 26 orelse
Byte =:= 27 orelse
Byte =:= 28 orelse
Byte =:= 29 orelse
Byte =:= 30 orelse
Byte =:= 31 orelse
Byte =:= 34 orelse
Byte =:= 92
).
-define(is_ascii_plain(Byte),
Byte =:= 32 orelse
Byte =:= 33 orelse
Byte =:= 35 orelse
Byte =:= 36 orelse
Byte =:= 37 orelse
Byte =:= 38 orelse
Byte =:= 39 orelse
Byte =:= 40 orelse
Byte =:= 41 orelse
Byte =:= 42 orelse
Byte =:= 43 orelse
Byte =:= 44 orelse
Byte =:= 45 orelse
Byte =:= 46 orelse
Byte =:= 47 orelse
Byte =:= 48 orelse
Byte =:= 49 orelse
Byte =:= 50 orelse
Byte =:= 51 orelse
Byte =:= 52 orelse
Byte =:= 53 orelse
Byte =:= 54 orelse
Byte =:= 55 orelse
Byte =:= 56 orelse
Byte =:= 57 orelse
Byte =:= 58 orelse
Byte =:= 59 orelse
Byte =:= 60 orelse
Byte =:= 61 orelse
Byte =:= 62 orelse
Byte =:= 63 orelse
Byte =:= 64 orelse
Byte =:= 65 orelse
Byte =:= 66 orelse
Byte =:= 67 orelse
Byte =:= 68 orelse
Byte =:= 69 orelse
Byte =:= 70 orelse
Byte =:= 71 orelse
Byte =:= 72 orelse
Byte =:= 73 orelse
Byte =:= 74 orelse
Byte =:= 75 orelse
Byte =:= 76 orelse
Byte =:= 77 orelse
Byte =:= 78 orelse
Byte =:= 79 orelse
Byte =:= 80 orelse
Byte =:= 81 orelse
Byte =:= 82 orelse
Byte =:= 83 orelse
Byte =:= 84 orelse
Byte =:= 85 orelse
Byte =:= 86 orelse
Byte =:= 87 orelse
Byte =:= 88 orelse
Byte =:= 89 orelse
Byte =:= 90 orelse
Byte =:= 91 orelse
Byte =:= 93 orelse
Byte =:= 94 orelse
Byte =:= 95 orelse
Byte =:= 96 orelse
Byte =:= 97 orelse
Byte =:= 98 orelse
Byte =:= 99 orelse
Byte =:= 100 orelse
Byte =:= 101 orelse
Byte =:= 102 orelse
Byte =:= 103 orelse
Byte =:= 104 orelse
Byte =:= 105 orelse
Byte =:= 106 orelse
Byte =:= 107 orelse
Byte =:= 108 orelse
Byte =:= 109 orelse
Byte =:= 110 orelse
Byte =:= 111 orelse
Byte =:= 112 orelse
Byte =:= 113 orelse
Byte =:= 114 orelse
Byte =:= 115 orelse
Byte =:= 116 orelse
Byte =:= 117 orelse
Byte =:= 118 orelse
Byte =:= 119 orelse
Byte =:= 120 orelse
Byte =:= 121 orelse
Byte =:= 122 orelse
Byte =:= 123 orelse
Byte =:= 124 orelse
Byte =:= 125 orelse
Byte =:= 126 orelse
Byte =:= 127
).

-define(are_all_ascii_plain(B1, B2, B3, B4, B5, B6, B7, B8),
(?is_ascii_plain(B1)) andalso
(?is_ascii_plain(B2)) andalso
(?is_ascii_plain(B3)) andalso
(?is_ascii_plain(B4)) andalso
(?is_ascii_plain(B5)) andalso
(?is_ascii_plain(B6)) andalso
(?is_ascii_plain(B7)) andalso
(?is_ascii_plain(B8))
).
25 changes: 24 additions & 1 deletion priv/riak_kv.schema
Original file line number Diff line number Diff line change
Expand Up @@ -1517,11 +1517,34 @@
{default, disabled}
]}.

%% @doc Default Secondary Index Timeout (milliseconds)
%% Set the default timeout for running secondary index queries in millisceconds
%% This is the time to run the query, and collate the results - the encoding of
%% those results for dispatch back to the client is outside of the scope of
%% this timeout. Setting to 0 (default) implies no timeout.
%% On hitting the timeout a 503 error will be returned on the HTTP API
{mapping, "secondary_index_timeout", "riak_kv.secondary_index_timeout", [
{datatype, integer},
{default, 0}
]}.

%% @doc Json library for 2i results
%% Prior to Riak KV 3.2.1 the mochijson2 library was used to json encode 2i
%% results. For larger results, the performance of this library is slower
%% when compared to alternatives (e.g. otp). A back-ported version of the
%% OTP 27 proposed Json library is now the default - but may return keys in a
%% different order (i.e. the continuation key may now be before the results
%% key). To reverse back to mochijson, then reset to mochijson here.
{mapping, "secondary_index_json", "riak_kv.secondary_index_json", [
{datatype, {enum, [otp, mochijson]}},
{default, otp}
]}.

%% @doc For $key index queries, should keys which are tombstones be returned.
%% This config will only make a difference with the leveled backend, it is
%% ignored on other backends. Disable to change default behaviour and stop
%% returning keys of tombstones in $key queries
{mapping, "dollarkey_readtombs", "riak_kv.dollarkey_readtombs", [
{datatype, {flag, enabled, disabled}},
{default, enabled}
]}.
]}.
9 changes: 6 additions & 3 deletions src/riak_client.erl
Original file line number Diff line number Diff line change
Expand Up @@ -1187,8 +1187,10 @@ wait_for_query_results(ReqId, Timeout) ->
wait_for_query_results(ReqId, Timeout, Acc) ->
receive
{ReqId, done} -> {ok, lists:flatten(lists:reverse(Acc))};
{ReqId,{results, Res}} -> wait_for_query_results(ReqId, Timeout, [Res | Acc]);
{ReqId, Error} -> {error, Error}
{ReqId, {results, Res}} ->
wait_for_query_results(ReqId, Timeout, [Res | Acc]);
{ReqId, {error, Error}} -> {error, Error};
{ReqId, UnexpectedMsg} -> {error, UnexpectedMsg}
after Timeout ->
{error, timeout}
end.
Expand All @@ -1200,7 +1202,8 @@ wait_for_query_results(ReqId, Timeout, Acc) ->
wait_for_fold_results(ReqId, Timeout) ->
receive
{ReqId, {results, Results}} -> {ok, Results};
{ReqId, Error} -> {error, Error}
{ReqId, {error, Error}} -> {error, Error};
{ReqId, UnexpectedMsg} -> {error, UnexpectedMsg}
after Timeout ->
{error, timeout}
end.
Expand Down
21 changes: 10 additions & 11 deletions src/riak_index.erl
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@

%% See GH610, this default is for backwards compat, so 2i behaves as
%% it did before the FSM timeout bug was "fixed"
-define(DEFAULT_TIMEOUT, infinity).
-define(DEFAULT_TIMEOUT, 0).

%% @type data_type_defs() = [data_type_def()].
%% @type data_type_def() = {MatchFunction :: function(), ParseFunction :: function()}.
Expand Down Expand Up @@ -503,17 +503,16 @@ decode_continuation(undefined) ->
decode_continuation(Bin) ->
binary_to_term(base64:decode(Bin)).

%% @doc add the `timeout' option tuple to the
%% `Opts' proplist. if `Timeout' is undefined
%% then the `app.config' property
%% `{riak_kv, seconady_index timeout}' is used
%% If that config property is defined, then
%% a default of `infinity' is used.
%% We use `infinity' as the default to
%% match the behavior pre 1.4
%% @doc add the `timeout' option tuple to the `Opts' proplist. if `Timeout' is
%% undefined then the `app.config' property `{riak_kv, seconady_index timeout}'
%% is used. If that config property is undefined, then a default of `infinity'
%% (a timeout of 0 will be translated to infinity) is used.
%% We use `infinity' as the default to match the behavior pre 1.4
add_timeout_opt(undefined, Opts) ->
Timeout = app_helper:get_env(riak_kv, secondary_index_timeout, ?DEFAULT_TIMEOUT),
[{timeout, Timeout} | Opts];
Timeout =
app_helper:get_env(
riak_kv, secondary_index_timeout, ?DEFAULT_TIMEOUT),
add_timeout_opt(Timeout, Opts);
add_timeout_opt(0, Opts) ->
[{timeout, infinity} | Opts];
add_timeout_opt(Timeout, Opts) ->
Expand Down
1 change: 0 additions & 1 deletion src/riak_kv_clusteraae_fsm.erl
Original file line number Diff line number Diff line change
Expand Up @@ -1059,4 +1059,3 @@ convert_validate_test() ->
[IQ1, IQ2, IQ3, IQ4, IQ5, IQ6, IQ7]).

-endif.

Loading

0 comments on commit aec41e6

Please sign in to comment.