From 0a23e0f74b40423612631c5dc19cf1a9870279fc Mon Sep 17 00:00:00 2001 From: Prawn Date: Sat, 4 Sep 2021 13:38:46 +1200 Subject: [PATCH] Custom "ignore" option for fields you might not want (#71) * FieldOption to ignore * New sample in the Makefile * Few changes in the generated samples * Feature flag to enable the ignore logic * Unit tests Co-authored-by: Chrusty <> --- Makefile | 1 + README.md | 5 +++-- internal/converter/converter.go | 18 ++++++++++++------ internal/converter/converter_test.go | 7 +++++++ internal/converter/testdata/hidden_fields.go | 15 +++++++++++++++ .../testdata/proto/HiddenFields.proto | 10 ++++++++++ internal/converter/types.go | 7 +++++++ jsonschemas/Enumception.jsonschema | 3 ++- jsonschemas/GoogleValue.jsonschema | 3 ++- jsonschemas/HiddenFields.jsonschema | 13 +++++++++++++ jsonschemas/NestedMessage.jsonschema | 2 +- jsonschemas/WellKnown.jsonschema | 9 ++++++--- options.proto | 7 +++++++ 13 files changed, 86 insertions(+), 14 deletions(-) create mode 100644 internal/converter/testdata/hidden_fields.go create mode 100644 internal/converter/testdata/proto/HiddenFields.proto create mode 100644 jsonschemas/HiddenFields.jsonschema create mode 100644 options.proto diff --git a/Makefile b/Makefile index 67667e14..0042ec5b 100644 --- a/Makefile +++ b/Makefile @@ -44,6 +44,7 @@ samples: @PATH=./bin:$$PATH; protoc --jsonschema_out=jsonschemas --proto_path=${PROTO_PATH} ${PROTO_PATH}/Proto2Required.proto || echo "No messages found (Proto2Required.proto)" @PATH=./bin:$$PATH; protoc --jsonschema_out=jsonschemas --proto_path=${PROTO_PATH} ${PROTO_PATH}/Proto2NestedMessage.proto || echo "No messages found (Proto2NestedMessage.proto)" @PATH=./bin:$$PATH; protoc --jsonschema_out=jsonschemas --proto_path=${PROTO_PATH} ${PROTO_PATH}/GoogleValue.proto || echo "No messages found (GoogleValue.proto)" + @PATH=./bin:$$PATH; protoc --jsonschema_out=exclude_ignored_fields:jsonschemas -I. --proto_path=${PROTO_PATH} ${PROTO_PATH}/HiddenFields.proto || echo "No messages found (HiddenFields.proto)" @PATH=./bin:$$PATH; protoc --jsonschema_out=enforce_oneof:jsonschemas --proto_path=${PROTO_PATH} ${PROTO_PATH}/OneOf.proto || echo "No messages found (OneOf.proto)" @PATH=./bin:$$PATH; protoc --jsonschema_out=all_fields_required:jsonschemas --proto_path=${PROTO_PATH} ${PROTO_PATH}/Proto2NestedObject.proto || echo "No messages found (Proto2NestedObject.proto)" @PATH=./bin:$$PATH; protoc -I /usr/include --jsonschema_out=jsonschemas --proto_path=${PROTO_PATH} ${PROTO_PATH}/WellKnown.proto || echo "No messages found (WellKnown.proto)" diff --git a/README.md b/README.md index 9d34cd0b..91f3e62f 100644 --- a/README.md +++ b/README.md @@ -39,8 +39,9 @@ protoc \ # The protobuf compiler |`debug`| Enable debug logging | |`disallow_additional_properties`| Disallow additional properties in schema | |`disallow_bigints_as_strings`| Disallow big integers as strings | -|`enforce_oneof` | Interpret Proto "oneOf" clauses | -|`json_fieldnames` | Use JSON field names only | +|`enforce_oneof`| Interpret Proto "oneOf" clauses | +|`exclude_ignored_fields`| Omit fields marked with the custom "ignore" option | +|`json_fieldnames`| Use JSON field names only | |`prefix_schema_files_with_package`| Prefix the output filename with package | |`proto_and_json_fieldnames`| Use proto and JSON field names | diff --git a/internal/converter/converter.go b/internal/converter/converter.go index 87aa899a..e9decf61 100644 --- a/internal/converter/converter.go +++ b/internal/converter/converter.go @@ -17,15 +17,17 @@ import ( ) const ( - messageDelimiter = "+" + ignoredFieldOption = "50505:1" + messageDelimiter = "+" ) // Converter is everything you need to convert protos to JSONSchemas: type Converter struct { - Flags ConverterFlags - logger *logrus.Logger - sourceInfo *sourceCodeInfo - messageTargets []string + Flags ConverterFlags + ignoredFieldOption string + logger *logrus.Logger + sourceInfo *sourceCodeInfo + messageTargets []string } // ConverterFlags control the behaviour of the converter: @@ -35,6 +37,7 @@ type ConverterFlags struct { DisallowAdditionalProperties bool DisallowBigIntsAsStrings bool EnforceOneOf bool + ExcludeIgnoredFields bool PrefixSchemaFilesWithPackage bool UseJSONFieldnamesOnly bool UseProtoAndJSONFieldNames bool @@ -43,7 +46,8 @@ type ConverterFlags struct { // New returns a configured *Converter: func New(logger *logrus.Logger) *Converter { return &Converter{ - logger: logger, + ignoredFieldOption: ignoredFieldOption, + logger: logger, } } @@ -82,6 +86,8 @@ func (c *Converter) parseGeneratorParameters(parameters string) { c.Flags.DisallowBigIntsAsStrings = true case "enforce_oneof": c.Flags.EnforceOneOf = true + case "exclude_ignored_fields": + c.Flags.ExcludeIgnoredFields = true case "json_fieldnames": c.Flags.UseJSONFieldnamesOnly = true case "prefix_schema_files_with_package": diff --git a/internal/converter/converter_test.go b/internal/converter/converter_test.go index 60c579d8..9642521c 100644 --- a/internal/converter/converter_test.go +++ b/internal/converter/converter_test.go @@ -268,6 +268,12 @@ func configureSampleProtos() map[string]sampleProto { FilesToGenerate: []string{"OneOf.proto"}, ProtoFileName: "OneOf.proto", }, + "HiddenFields": { + Flags: ConverterFlags{ExcludeIgnoredFields: true}, + ExpectedJSONSchema: []string{testdata.HiddenFields}, + FilesToGenerate: []string{"options.proto", "HiddenFields.proto"}, + ProtoFileName: "HiddenFields.proto", + }, } } @@ -284,6 +290,7 @@ func mustReadProtoFiles(t *testing.T, includePath string, filenames ...string) * args = append(args, "--descriptor_set_out=/dev/stdout") args = append(args, "--include_source_info") args = append(args, "--include_imports") + args = append(args, "-I../../") args = append(args, "--proto_path="+includePath) args = append(args, filenames...) cmd := exec.Command(protocBinary, args...) diff --git a/internal/converter/testdata/hidden_fields.go b/internal/converter/testdata/hidden_fields.go new file mode 100644 index 00000000..d283fdc9 --- /dev/null +++ b/internal/converter/testdata/hidden_fields.go @@ -0,0 +1,15 @@ +package testdata + +const HiddenFields = `{ + "$schema": "http://json-schema.org/draft-04/schema#", + "properties": { + "visible1": { + "type": "string" + }, + "visible2": { + "type": "string" + } + }, + "additionalProperties": true, + "type": "object" +}` diff --git a/internal/converter/testdata/proto/HiddenFields.proto b/internal/converter/testdata/proto/HiddenFields.proto new file mode 100644 index 00000000..8b7abd3b --- /dev/null +++ b/internal/converter/testdata/proto/HiddenFields.proto @@ -0,0 +1,10 @@ +syntax = "proto3"; +package samples; +import "options.proto"; + +message HiddenFields { + string visible1 = 1 [(protoc.gen.jsonschema.ignore) = false]; + string visible2 = 2; + string hidden1 = 3 [(protoc.gen.jsonschema.ignore) = true]; + string hidden2 = 4 [deprecated = true, (protoc.gen.jsonschema.ignore) = true]; +} diff --git a/internal/converter/types.go b/internal/converter/types.go index 366c0025..b6f9bd52 100644 --- a/internal/converter/types.go +++ b/internal/converter/types.go @@ -495,6 +495,13 @@ func (c *Converter) recursiveConvertMessageType(curPkg *ProtoPackage, msg *descr c.logger.WithField("message_str", proto.MarshalTextString(msg)).Trace("Converting message") for _, fieldDesc := range msg.GetField() { + + // Look for our custom "ignore" field-option (and hope that nobody else happens to be using our number): + if c.Flags.ExcludeIgnoredFields && strings.Contains(fieldDesc.GetOptions().String(), c.ignoredFieldOption) { + c.logger.WithField("field_name", fieldDesc.GetName()).WithField("message_name", msg.GetName()).Debug("Omitting ignored field") + continue + } + recursedJSONSchemaType, err := c.convertField(curPkg, fieldDesc, msg, duplicatedMessages) if err != nil { c.logger.WithError(err).WithField("field_name", fieldDesc.GetName()).WithField("message_name", msg.GetName()).Error("Failed to convert field") diff --git a/jsonschemas/Enumception.jsonschema b/jsonschemas/Enumception.jsonschema index 4e7048db..315afb68 100644 --- a/jsonschemas/Enumception.jsonschema +++ b/jsonschemas/Enumception.jsonschema @@ -34,13 +34,14 @@ }, "payload": { "$ref": "samples.PayloadMessage", - "additionalProperties": true + "additionalProperties": false }, "payloads": { "items": { "$schema": "http://json-schema.org/draft-04/schema#", "$ref": "samples.PayloadMessage" }, + "additionalProperties": false, "type": "array" }, "importedEnum": { diff --git a/jsonschemas/GoogleValue.jsonschema b/jsonschemas/GoogleValue.jsonschema index 7e164d66..709e6bd5 100644 --- a/jsonschemas/GoogleValue.jsonschema +++ b/jsonschemas/GoogleValue.jsonschema @@ -18,7 +18,8 @@ { "type": "string" } - ] + ], + "description": "`Value` represents a dynamically typed value which can be either\n null, a number, a string, a boolean, a recursive struct value, or a\n list of values. A producer of value is expected to set one of that\n variants, absence of any variant indicates an error.\n\n The JSON representation for `Value` is JSON value." } }, "additionalProperties": true, diff --git a/jsonschemas/HiddenFields.jsonschema b/jsonschemas/HiddenFields.jsonschema new file mode 100644 index 00000000..043da7ab --- /dev/null +++ b/jsonschemas/HiddenFields.jsonschema @@ -0,0 +1,13 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "properties": { + "visible1": { + "type": "string" + }, + "visible2": { + "type": "string" + } + }, + "additionalProperties": true, + "type": "object" +} \ No newline at end of file diff --git a/jsonschemas/NestedMessage.jsonschema b/jsonschemas/NestedMessage.jsonschema index 88cf76ee..03724ba1 100644 --- a/jsonschemas/NestedMessage.jsonschema +++ b/jsonschemas/NestedMessage.jsonschema @@ -43,7 +43,7 @@ ] } }, - "additionalProperties": true, + "additionalProperties": false, "type": "object" }, "description": { diff --git a/jsonschemas/WellKnown.jsonschema b/jsonschemas/WellKnown.jsonschema index ac4d47da..940b4bbf 100644 --- a/jsonschemas/WellKnown.jsonschema +++ b/jsonschemas/WellKnown.jsonschema @@ -20,13 +20,16 @@ }, "list_of_integers": { "items": { - "type": "integer" + "type": "integer", + "description": "Wrapper message for `int32`.\n\n The JSON representation for `Int32Value` is JSON number." }, "type": "array" }, "duration": { - "additionalProperties": true, - "type": "string" + "pattern": "^([0-9]+\\.?[0-9]*|\\.[0-9]+)s$", + "type": "string", + "description": "This is a duration:", + "format": "regex" } }, "additionalProperties": true, diff --git a/options.proto b/options.proto new file mode 100644 index 00000000..95116e53 --- /dev/null +++ b/options.proto @@ -0,0 +1,7 @@ +syntax = "proto3"; +package protoc.gen.jsonschema; +import "google/protobuf/descriptor.proto"; + +extend google.protobuf.FieldOptions { + optional bool ignore = 50505; +}