Merge branches 'wrap-no-underspecs', 'nowarn_dialyzer-when-always-tra…

…nslated' and 'kianmeng/fix-typos'

ChangeLog

@@ -173,7 +173,7 @@ Version 4.14.2
 Version 4.14.1
 2020-Sep-05
 
-        * Add a utility gpb:decode_packet/3 mimicing erlang:decode_packet/3,
+        * Add a utility gpb:decode_packet/3 mimicking erlang:decode_packet/3,
           but for varint length encoded framing of data.
         * Document the public API in gpb.erl.
         * Document gpb_defs:fetch_imports, it is part of the API.
@@ -515,7 +515,7 @@ Version 4.2.1
 2018-Jul-13
 
         * Fix proto3 bug in encoding of default values for enums when
-        given as 0 instead of as atom. Thansk to Jean-Alexandre Barszcz
+        given as 0 instead of as atom. Thanks to Jean-Alexandre Barszcz
         for reporting.
 
 Version 4.2.0
@@ -528,7 +528,7 @@ Version 4.2.0
         for asking the question that prompted this.
 
         * Add an option {maps_oneof,flat}. For maps, it changes the
-        Erlang representaiton of oneof fields from #{..., c => {x,Value}, ...}
+        Erlang representation of oneof fields from #{..., c => {x,Value}, ...}
         to #{..., x => Value}, ...}
 
         * Generate type specifications for records, and refer to them. This
@@ -564,7 +564,7 @@ Version 4.1.6
 2018-May-08
 
         * Make the Erlang 21 stacktrace syntax choice overridable.
-        In addition to the targetting version described in 4.1.5 below,
+        In addition to the targeting version described in 4.1.5 below,
         make it possible to override the stacktrace selection:
 
         When targeting Erlang 21 or later, generate code that uses the new
@@ -594,7 +594,7 @@ Version 4.1.3
         * Build changes: Provide better errors when building
         outside of a git repo, and provide a helper script,
         helers/export-from-git, to extract gpb from git in a way such
-        that it can be compiled outside fo git.
+        that it can be compiled outside of git.
 
 Version 4.1.2
 2018-Jan-26
@@ -670,7 +670,7 @@ Version 3.28.1
         allowed to be packed, such as string, bytes and sub
         messages. Such fields were not allowed to be be packed i the
         first place, but gpb never said anything so this could go
-        unnoticed. And gpb packed them in an invalid fasion, that was
+        unnoticed. And gpb packed them in an invalid fashion, that was
         probably incompaitble with other protobuf parsers/encoders.
 
         For such fields, this introduces a POTENTIAL INCOMPATIBILITY,
@@ -777,7 +777,7 @@ Version 3.27.0
 2017-May-28
 
         * Add support for groups. Thanks to David Weinstein for prompting
-        this to get implementated.
+        this to get implemented.
 
 Version 3.26.8
 2017-Mar-24
@@ -1457,7 +1457,7 @@ Version 3.3
         * Fix several errors in the gpb.app.src so that it works with
         Elixir and Relex, thanks to Paulo Almeida.
         * When decoding, reject messages with invalid varints.
-        Invaid varints are encoded with too many bits. Previously,
+        Invalid varints are encoded with too many bits. Previously,
         decoding a varint with very many bits (ie: with very many leading
         bytes > 128) would cause the Erlang VM to construct a very large
         number when decoding the varint. This would cause the Erlang VM to
@@ -1488,7 +1488,7 @@ Version 3.1
         * Add possibility to make the generated code compile-time
         independent of gpb by generating the introspection information to
         proplists instead of to #field{} records. The latter required
-        including gpb.hrl at compiletime. (There's no run-time dependency)
+        including gpb.hrl at compile time. (There's no run-time dependency)
         * Add possibility to prefix message/record and names and module
         names to avoid name collisions.
         * Add -type to the command line options, thanks to Louis-Philippe

README.md

@@ -450,7 +450,7 @@ Interaction with rebar
 For info on how to use gpb with rebar3, see
 https://rebar3.org/docs/configuration/plugins/#protocol-buffers
 
-Compatiblity with rebar2
+Compatibility with rebar2
 ------------------------
 
 In rebar there is support for gpb since version 2.6.0. See the
@@ -603,7 +603,7 @@ have been created using the mk-versioned-archive script, so it is
 possible to just unpack and build directly.
 
 If you use Github's automatic _Source code_ zip or tar.gz archives,
-you will need to either create a `gpb.vsn` file as described aboce,
+you will need to either create a `gpb.vsn` file as described above,
 or re-create a versioned archive using the `mk-versioned-archive`
 script and the `--override-version=<x>` option (or possibly the
 or the `--override-version-from-cwd-path` option if the directory name

build/mk_version_hrl

@@ -90,7 +90,7 @@ first_line_no_nl([C | Tl]) -> [C | first_line_no_nl(Tl)];
 first_line_no_nl("") -> "".
 
 file_err(FunName, FileName, Reason) ->
-    io:format(standard_error, "file:~p(~s) excute failed!, Reason :~s ~n",
+    io:format(standard_error, "file:~p(~s) execute failed!, Reason :~s ~n",
               [FunName, FileName, file:format_error(Reason)]),
     halt(1).
 

src/gpb.erl

@@ -1925,7 +1925,7 @@ version_format_test() ->
     ?assertError(_, assert_version_format("y")),
     ?assertError(_, assert_version_format("2.1-4z-gb996fbe")),
     ?assertError(_, assert_version_format("2.1-z-gb996fbe")),
-    %% malplaced dots
+    %% misplaced dots
     ?assertError(_, assert_version_format(".")),
     ?assertError(_, assert_version_format(".2")),
     ?assertError(_, assert_version_format("..2")),

src/gpb_analyzer.erl

@@ -43,8 +43,9 @@ analyze_defs(Defs, Sources, Renamings, Opts) ->
     Translations = compute_translations(Defs, Opts),
     KnownMsgSize = find_msgsizes_known_at_compile_time(MapsAsMsgs ++ Defs),
     UnknownsInfo = gpb_lib:defs_contains_fields_for_unknows(Defs),
+    UsedTypes = find_used_types(Defs),
     #anres{source_filenames    = Sources,
-           used_types          = find_used_types(Defs),
+           used_types          = UsedTypes,
            known_msg_size      = KnownMsgSize,
            fixlen_types        = find_fixlen_types(MapsAsMsgs ++ Defs),
            num_packed_fields   = find_num_packed_fields(MapsAsMsgs ++ Defs),
@@ -54,6 +55,8 @@ analyze_defs(Defs, Sources, Renamings, Opts) ->
            maps_as_msgs        = ProtoDefsVsnElem ++ MapsAsMsgs ++ MapMsgEnums,
            dec_maps_as_msgs    = ProtoDefsVsnElem ++ DMapsAsMsgs ++ MapMsgEnums,
            translations        = Translations,
+           types_only_via_translations = find_types_used_only_via_translations(
+                                           UsedTypes, Defs, Translations),
            map_types           = MapTypes,
            map_value_types     = compute_map_value_types(MapTypes),
            group_occurrences   = find_group_occurrences(Defs),
@@ -750,6 +753,60 @@ fetch_op_translation(Op, Translations, Default, Ctxt) ->
 is_repeated_elem_path([_MsgName,_FName,[]]) -> true;
 is_repeated_elem_path(_) -> false.
 
+find_types_used_only_via_translations(UsedTypes, Defs, Translations) ->
+    %% For types used only via a translation, include a
+    %% -dialyzer({nowarn_function, <encoder_function>/<arity>}).
+    %%
+    %% This is since when used only from translations,
+    %% dialyzer may find that some general-purpose execution paths
+    %% in such encoder functions will never succeed, and if so will warn.
+    %%
+    %% But there is nothing the user can do. So emit that dialyzer attr.
+
+    %% Defs may be big, but UsedTypes and Translations are usually smaller
+    %% or much smaller
+    gpb_lib:fold_msg_or_group_fields_o(
+      fun(_MsgOrGroup,
+          MsgName,
+          #?gpb_field{type=Type, name=FName, occurrence=Occurrence},
+          Oneof,
+          Acc) ->
+              case is_scalar_type(Type) andalso sets:is_element(Type, Acc) of
+                  true ->
+                      FieldPath =
+                          case {Oneof, Occurrence} of
+                              {false, repeated}  -> [MsgName, FName, []];
+                              {false, _}         -> [MsgName, FName];
+                              {{true,CFName}, _} -> [MsgName, CFName, FName]
+                          end,
+                      case has_enc_dec_translation(FieldPath, Translations) of
+                          true -> Acc;
+                          false -> sets:del_element(Type, Acc)
+                      end;
+                  false ->
+                      Acc
+              end
+      end,
+      UsedTypes,
+      Defs).
+
+has_enc_dec_translation(FieldPath, Translations) ->
+    case dict:find(FieldPath, Translations) of
+        {ok, FieldTranslations}  ->
+            lists:any(fun({encode, _}) -> true;
+                         ({decode, _}) -> true;
+                         ({decode_init_default, _}) -> true;
+                         ({decode_repeated_add_elem, _}) -> true;
+                         ({decode_repeated_finalize, _}) -> true;
+                         ({merge, _}) -> false;
+                         ({verify, _}) -> false;
+                         ({type_spec, _}) -> false
+                      end,
+                      FieldTranslations);
+        error ->
+            false
+    end.
+
 find_group_occurrences(Defs) ->
     gpb_lib:fold_msg_or_group_fields_o(
       fun(_msg_or_group, _MsgName,

src/gpb_compile.erl

@@ -978,7 +978,7 @@ file(File) ->
 %%     `{erlc_compile_options, string()}'</h4>
 %%
 %% If the the `{erlc_compile_options,string()}' option is set,
-%% then the genereted code will contain a directive `-compile([String]).'
+%% then the generated code will contain a directive `-compile([String]).'
 %%
 %% Corresponding command line option:
 %% <a href="#cmdline-option-erlc_compile_options">-erlc_compile_options</a>.
@@ -1322,7 +1322,7 @@ file(File) ->
 %%   <dd>Call `Mod:Fn(Term) -> _' to verify an unpacked `Term'.
 %%       If `Term' (`$1') is valid, the function is expected to just return
 %%       any value, which is ignored and discarded.
-%%       If `Term' is invalid, the function is exptected to not
+%%       If `Term' is invalid, the function is expected to not
 %%       return anything, but instead either crash, call
 %%       `erlang:error/1', or `throw/1' or `exit/1'.  with the
 %%       reason for error.
@@ -2669,7 +2669,7 @@ c() ->
 %%       <a href="#option-defs_as_proplists">defs_as_proplists</a></dd>
 %%   <dt><a id="cmdline-option-defs-as-maps"/>
 %%       `-defs-as-maps'</dt>
-%%     <dd>Specifies that proto defintions from the generated code
+%%     <dd>Specifies that proto definitions from the generated code
 %%       are to be returned as maps. Otherwise, they will be lists
 %%       of tuples and records (or proplists if the `-pldefs' option
 %%       is specified)<br/>
@@ -2785,13 +2785,13 @@ c() ->
 %%   <dt><a id="cmdline-option-translate_type"/>
 %%       `-translate_type TMsFs'</dt>
 %%     <dd>Call functions in `TMsFs' to pack, unpack, merge and verify
-%%       for the specifed type. The `TMsFs' is a string on the
+%%       for the specified type. The `TMsFs' is a string on the
 %%       following format: `type=Type,e=Mod:Fn,d=Mod:Fn[,m=Mod:Fn][,V=Mod:Fn]'.
 %%       The Type and specified modules and functions are called and used
 %%       as follows:
 %%       <dl>
 %%         <dt>`type=Type'</dt>
-%%         <dd>Specfies that the translations apply to fields of type.
+%%         <dd>Specifies that the translations apply to fields of type.
 %%             The `Type' may be either of:
 %%             `msg:MsgName' (after any renaming operations),
 %%             `enum:EnumName', `int32', `int64', `uint32', `uint64',
@@ -2813,7 +2813,7 @@ c() ->
 %%         <dd>Call `Mod:Fn(Term) -> _' to verify an unpacked `Term'.
 %%             If `Term' is valid, the function is expected to just return
 %%             any value, which is ignored and discarded.
-%%             If `Term' is invalid, the function is exptected to not
+%%             If `Term' is invalid, the function is expected to not
 %%             return anything, but instead either crash, call
 %%             `erlang:error/1', or `throw/1' or `exit/1' with the
 %%             reason for error.
@@ -3260,7 +3260,7 @@ opt_specs() ->
       "       instead of #field{} records, to make the generated code\n"
       "       completely free of even compile-time dependencies to gpb.\n"},
      {"defs-as-maps", undefined, defs_as_maps, "\n"
-      "        Specifies that proto defintions from the generated code\n"
+      "        Specifies that proto definitions from the generated code\n"
       "        are to be returned as maps. Otherwise, they will be lists\n"
       "        of tuples and records (or proplists if the -pldefs option\n"
       "        is specified)\n"},
@@ -3364,7 +3364,7 @@ opt_specs() ->
       "       Erlang protobuffs library:\n"
       "       * encode/1 and encode_MsgName/1\n"
       "       * decode/2 and decode_MsgName/1\n"},
-     {{section, "Quering dependencies"}},
+     {{section, "Querying dependencies"}},
      {"M", {'opt_value()', makefile_rules}, list_deps, "\n"
       "       Generate Makefile rule(s) for dependencies.\n"
       "       No code is generated unless -MMD.\n"},
@@ -4505,6 +4505,8 @@ nepp2_nl(<<"-ifdef", Rest/binary>>, NState, N, Acc) ->
     nepp2_ifdef(Rest, ifdef, NState, N, Acc);
 nepp2_nl(<<"-ifndef", Rest/binary>>, NState, N, Acc) ->
     nepp2_ifdef(Rest, ifndef, NState, N, Acc);
+nepp2_nl(<<"-if", Rest/binary>>, NState, N, Acc) ->
+    nepp2_if(Rest, NState, N, Acc);
 nepp2_nl(<<"-else.\n", Rest/binary>>, #nepp{depth=1}=NState, N, Acc) ->
     nepp2_skip(Rest, NState, N+1, Acc);
 nepp2_nl(<<"-endif.\n", Rest/binary>>, #nepp{depth=1}=NState, N, Acc) ->
@@ -4547,12 +4549,22 @@ nepp2_ifdef(Rest, SkipCond, #nepp{depth=Depth, defs=Ds}=NState, N, Acc) ->
     {Sym, Rest2} = read_until(Rest1, ")", ""),
     {_,   Rest3} = read_until(Rest2, "\n", ""),
     {Txt, Rest4, NState2, N2} =
-    case {dict:is_key(parse_term(Sym), Ds), SkipCond} of
-        {true,  ifdef}  -> nepp2_nl(Rest3, NState#nepp{depth=1}, N+1, []);
-        {false, ifndef} -> nepp2_nl(Rest3, NState#nepp{depth=1}, N+1, []);
-        _ -> nepp2_skip(Rest3, NState#nepp{depth=1}, N+1, [])
+        case {dict:is_key(parse_term(Sym), Ds), SkipCond} of
+            {true,  ifdef}  -> nepp2_nl(Rest3, NState#nepp{depth=1}, N+1, []);
+            {false, ifndef} -> nepp2_nl(Rest3, NState#nepp{depth=1}, N+1, []);
+            _ -> nepp2_skip(Rest3, NState#nepp{depth=1}, N+1, [])
+        end,
+    nepp2_nl(Rest4, NState2#nepp{depth=Depth}, N2, lists:reverse(Txt, Acc)).
 
-    end,
+nepp2_if(Rest, #nepp{depth=Depth, defs=Ds}=NState, N, Acc) ->
+    {_,    Rest1} = read_until(Rest,  "(", ""),
+    {Cond, Rest2} = read_until(Rest1, ")", ""),
+    {_,    Rest3} = read_until(Rest2, "\n", ""),
+    {Txt,  Rest4, NState2, N2} =
+        case nepp2_eval_cond(Cond, Ds) of
+            true  -> nepp2_nl(Rest3, NState#nepp{depth=1}, N+1, []);
+            false -> nepp2_skip(Rest3, NState#nepp{depth=1}, N+1, [])
+        end,
     nepp2_nl(Rest4, NState2#nepp{depth=Depth}, N2, lists:reverse(Txt, Acc)).
 
 nepp2_skip(<<"-else.\n", Rest/binary>>, #nepp{depth=Depth}=NState, N, Acc) ->
@@ -4574,6 +4586,28 @@ nepp2_skip(<<$\n, Rest/binary>>, NState, N, Acc) ->
 nepp2_skip(<<_, Rest/binary>>, NState, N, Acc) ->
     nepp2_skip(Rest, NState, N, Acc).
 
+nepp2_eval_cond(Str, Ds) ->
+    {ok, Tokens, End} = erl_scan:string(Str),
+    Tokens2 = nepp2_simple_expand(Tokens, Ds),
+    {ok, Exprs} = erl_parse:parse_exprs(Tokens2 ++ [{dot, End}]),
+    InitBinds = erl_eval:new_bindings(),
+    {value, Result, _Binds} = erl_eval:exprs(Exprs, InitBinds),
+    Result.
+
+nepp2_simple_expand([{'?', _}, {var, _, Sym} | Rest], Ds) ->
+    nepp2_assert_not_parameterized(Sym, Rest),
+    Val = dict:fetch(Sym, Ds),
+    [erl_parse:abstract(Val) | nepp2_simple_expand(Rest, Ds)];
+nepp2_simple_expand([Tok | Rest], Ds) ->
+    [Tok | nepp2_simple_expand(Rest, Ds)];
+nepp2_simple_expand([], _Ds) ->
+    [].
+
+nepp2_assert_not_parameterized(Sym, [{'(', _} | _]) ->
+    error({not_implemeted, nepp2, parameterized_macros, Sym});
+nepp2_assert_not_parameterized(_Sym, _) ->
+    ok.
+
 read_until(<<C, Rest/binary>>, Delims, Acc) ->
     case lists:member(C, Delims) of
         true  -> {lists:reverse(Acc), Rest};

src/gpb_compile.hrl

@@ -15,6 +15,7 @@
           maps_as_msgs,       % :: list() % same format as `Defs'
           dec_maps_as_msgs,   % :: list() % ditto, for decoding
           translations,       % :: dict:dict(), %% FieldPath -> TranslationOps
+          types_only_via_translations,% :: sets:set(gpb_field_type()),
           map_types,          % :: sets:set({map,_,_})
           map_value_types,    % :: {boolean(), boolean()} % submsgs?/nonsubmsgs?
           group_occurrences,  % :: dict:dict() %% GroupName -> repeated | ...

src/gpb_decoders_lib.erl

@@ -236,7 +236,7 @@ field_infos(MsgDef, FlattenOnoef) ->
              {FName, optional};
          #?gpb_field{name=FName, occurrence=defaulty} ->
              %% On finalization, treat proto3 (scalar) fields as
-             %% requried, to avoid redundant if F == '$undef' -> ...
+             %% required, to avoid redundant if F == '$undef' -> ...
              %% checks;  it can never be '$undef' since it has a
              %% type-default. (except for sub messages and onoef)
              {FName, required};
@@ -266,7 +266,7 @@ run_morph_ops([], Fns) ->
 %% underscore.
 %%
 %% The intention is that code generation can issue match patterns for
-%% all fields of a message, or all auxilliary working parameters and not
+%% all fields of a message, or all auxiliary working parameters and not
 %% have to worry about underscoring unused ones to avoid warning messages
 %% about unused variables.
 underscore_unused_vars() ->

src/gpb_defs.erl

@@ -1492,7 +1492,7 @@ to_fields_msg2(MsgName, MsgElems, OccHandler, Errors0) ->
 %%% -- Version and conversions ------------
 
 %% NB:
-%% Be quite explict here about the structure.  Don't rely so much on
+%% Be quite explicit here about the structure.  Don't rely so much on
 %% external helpers, such as gpb:is_msg_proto3/2, because helpers are
 %% for the latest defs version, while this may (in some future) be
 %% operating on a format that is not the latest any more.