Skip to content

Commit

Permalink
Add behavioral tests for examples from the YAML spec (databricks#1835)
Browse files Browse the repository at this point in the history
## Changes

I took the examples from https://yaml.org/spec/1.2.2.

The required modifications to the loader are:
* Correctly parse floating point infinities and NaN
* Correctly parse octal numbers per the YAML 1.2 spec
* Treat "null" keys in a map as valid

## Tests

Existing and new unit tests pass.
  • Loading branch information
pietern authored Oct 17, 2024
1 parent e4d039a commit dedec58
Show file tree
Hide file tree
Showing 31 changed files with 1,225 additions and 11 deletions.
60 changes: 60 additions & 0 deletions libs/dyn/dynassert/dump.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
package dynassert

import (
"fmt"
"strings"

"github.com/databricks/cli/libs/dyn"
)

// Dump returns the Go code to recreate the given value.
func Dump(v dyn.Value) string {
var sb strings.Builder
dump(v, &sb)
return sb.String()
}

func dump(v dyn.Value, sb *strings.Builder) {
sb.WriteString("dyn.NewValue(\n")

switch v.Kind() {
case dyn.KindMap:
sb.WriteString("map[string]dyn.Value{")
m := v.MustMap()
for _, p := range m.Pairs() {
sb.WriteString(fmt.Sprintf("\n%q: ", p.Key.MustString()))
dump(p.Value, sb)
sb.WriteByte(',')
}
sb.WriteString("\n},\n")
case dyn.KindSequence:
sb.WriteString("[]dyn.Value{\n")
for _, e := range v.MustSequence() {
dump(e, sb)
sb.WriteByte(',')
}
sb.WriteString("},\n")
case dyn.KindString:
sb.WriteString(fmt.Sprintf("%q,\n", v.MustString()))
case dyn.KindBool:
sb.WriteString(fmt.Sprintf("%t,\n", v.MustBool()))
case dyn.KindInt:
sb.WriteString(fmt.Sprintf("%d,\n", v.MustInt()))
case dyn.KindFloat:
sb.WriteString(fmt.Sprintf("%f,\n", v.MustFloat()))
case dyn.KindTime:
sb.WriteString(fmt.Sprintf("dyn.NewTime(%q),\n", v.MustTime().String()))
case dyn.KindNil:
sb.WriteString("nil,\n")
default:
panic(fmt.Sprintf("unhandled kind: %v", v.Kind()))
}

// Add location
sb.WriteString("[]dyn.Location{")
for _, l := range v.Locations() {
sb.WriteString(fmt.Sprintf("{File: %q, Line: %d, Column: %d},", l.File, l.Line, l.Column))
}
sb.WriteString("},\n")
sb.WriteString(")")
}
60 changes: 49 additions & 11 deletions libs/dyn/yamlloader/loader.go
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,9 @@ func (d *loader) loadMapping(node *yaml.Node, loc dyn.Location) (dyn.Value, erro
switch st {
case "!!str":
// OK
case "!!null":
// A literal unquoted "null" is treated as a null value by the YAML parser.
// However, when used as a key, it is treated as the string "null".
case "!!merge":
if merge != nil {
panic("merge node already set")
Expand All @@ -115,10 +118,11 @@ func (d *loader) loadMapping(node *yaml.Node, loc dyn.Location) (dyn.Value, erro
return dyn.InvalidValue, errorf(loc, "invalid key tag: %v", st)
}

k, err := d.load(key)
if err != nil {
return dyn.InvalidValue, err
}
k := dyn.NewValue(key.Value, []dyn.Location{{
File: d.path,
Line: key.Line,
Column: key.Column,
}})

v, err := d.load(val)
if err != nil {
Expand Down Expand Up @@ -173,6 +177,14 @@ func (d *loader) loadMapping(node *yaml.Node, loc dyn.Location) (dyn.Value, erro
return dyn.NewValue(out, []dyn.Location{loc}), nil
}

func newIntValue(i64 int64, loc dyn.Location) dyn.Value {
// Use regular int type instead of int64 if possible.
if i64 >= math.MinInt32 && i64 <= math.MaxInt32 {
return dyn.NewValue(int(i64), []dyn.Location{loc})
}
return dyn.NewValue(i64, []dyn.Location{loc})
}

func (d *loader) loadScalar(node *yaml.Node, loc dyn.Location) (dyn.Value, error) {
st := node.ShortTag()
switch st {
Expand All @@ -188,18 +200,44 @@ func (d *loader) loadScalar(node *yaml.Node, loc dyn.Location) (dyn.Value, error
return dyn.InvalidValue, errorf(loc, "invalid bool value: %v", node.Value)
}
case "!!int":
i64, err := strconv.ParseInt(node.Value, 10, 64)
if err != nil {
return dyn.InvalidValue, errorf(loc, "invalid int value: %v", node.Value)
// Try to parse the an integer value in base 10.
// We trim leading zeros to avoid octal parsing of the "0" prefix.
// See "testdata/spec_example_2.19.yml" for background.
i64, err := strconv.ParseInt(strings.TrimLeft(node.Value, "0"), 10, 64)
if err == nil {
return newIntValue(i64, loc), nil
}
// Use regular int type instead of int64 if possible.
if i64 >= math.MinInt32 && i64 <= math.MaxInt32 {
return dyn.NewValue(int(i64), []dyn.Location{loc}), nil
// Let the [ParseInt] function figure out the base.
i64, err = strconv.ParseInt(node.Value, 0, 64)
if err == nil {
return newIntValue(i64, loc), nil
}
return dyn.NewValue(i64, []dyn.Location{loc}), nil
return dyn.InvalidValue, errorf(loc, "invalid int value: %v", node.Value)
case "!!float":
f64, err := strconv.ParseFloat(node.Value, 64)
if err != nil {
// Deal with infinity prefixes.
v := strings.ToLower(node.Value)
switch {
case strings.HasPrefix(v, "+"):
v = strings.TrimPrefix(v, "+")
f64 = math.Inf(1)
case strings.HasPrefix(v, "-"):
v = strings.TrimPrefix(v, "-")
f64 = math.Inf(-1)
default:
// No prefix.
f64 = math.Inf(1)
}

// Deal with infinity and NaN values.
switch v {
case ".inf":
return dyn.NewValue(f64, []dyn.Location{loc}), nil
case ".nan":
return dyn.NewValue(math.NaN(), []dyn.Location{loc}), nil
}

return dyn.InvalidValue, errorf(loc, "invalid float value: %v", node.Value)
}
return dyn.NewValue(f64, []dyn.Location{loc}), nil
Expand Down
5 changes: 5 additions & 0 deletions libs/dyn/yamlloader/testdata/spec_example_2.1.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
# Example 2.1 Sequence of Scalars (ball players)

- Mark McGwire
- Sammy Sosa
- Ken Griffey
10 changes: 10 additions & 0 deletions libs/dyn/yamlloader/testdata/spec_example_2.10.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
# Example 2.10 Node for “Sammy Sosa” appears twice in this document

---
hr:
- Mark McGwire
# Following node labeled SS
- &SS Sammy Sosa
rbi:
- *SS # Subsequent occurrence
- Ken Griffey
10 changes: 10 additions & 0 deletions libs/dyn/yamlloader/testdata/spec_example_2.11.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
# Example 2.11 Mapping between Sequences

? - Detroit Tigers
- Chicago cubs
: - 2001-07-23

? [ New York Yankees,
Atlanta Braves ]
: [ 2001-07-02, 2001-08-12,
2001-08-14 ]
10 changes: 10 additions & 0 deletions libs/dyn/yamlloader/testdata/spec_example_2.12.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
# Example 2.12 Compact Nested Mapping

---
# Products purchased
- item : Super Hoop
quantity: 1
- item : Basketball
quantity: 4
- item : Big Shoes
quantity: 1
6 changes: 6 additions & 0 deletions libs/dyn/yamlloader/testdata/spec_example_2.13.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
# Example 2.13 In literals, newlines are preserved

# ASCII Art
--- |
\//||\/||
// || ||__
6 changes: 6 additions & 0 deletions libs/dyn/yamlloader/testdata/spec_example_2.14.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
# Example 2.14 In the folded scalars, newlines become spaces

--- >
Mark McGwire's
year was crippled
by a knee injury.
10 changes: 10 additions & 0 deletions libs/dyn/yamlloader/testdata/spec_example_2.15.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
# Example 2.15 Folded newlines are preserved for “more indented” and blank lines

--- >
Sammy Sosa completed another
fine season with great stats.

63 Home Runs
0.288 Batting Average

What a year!
9 changes: 9 additions & 0 deletions libs/dyn/yamlloader/testdata/spec_example_2.16.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
# Example 2.16 Indentation determines scope

name: Mark McGwire
accomplishment: >
Mark set a major league
home run record in 1998.
stats: |
65 Home Runs
0.278 Batting Average
9 changes: 9 additions & 0 deletions libs/dyn/yamlloader/testdata/spec_example_2.17.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
# Example 2.17 Quoted Scalars

unicode: "Sosa did fine.\u263A"
control: "\b1998\t1999\t2000\n"
hex esc: "\x0d\x0a is \r\n"

single: '"Howdy!" he cried.'
quoted: ' # Not a ''comment''.'
tie-fighter: '|\-*-/|'
8 changes: 8 additions & 0 deletions libs/dyn/yamlloader/testdata/spec_example_2.18.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
# Example 2.18 Multi-line Flow Scalars

plain:
This unquoted scalar
spans many lines.

quoted: "So does this
quoted scalar.\n"
15 changes: 15 additions & 0 deletions libs/dyn/yamlloader/testdata/spec_example_2.19.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
# Example 2.19 Integers

canonical: 12345
decimal: +12345
octal: 0o14
hexadecimal: 0xC

# Note: this example is not part of the spec but added for completeness.
#
# Octal numbers:
# - YAML 1.1: prefix is "0"
# - YAML 1.2: prefix is "0o"
# The "gopkg.in/yaml.v3" package accepts both for backwards compat.
# We accept only the YAML 1.2 prefix "0o".
octal11: 012345
5 changes: 5 additions & 0 deletions libs/dyn/yamlloader/testdata/spec_example_2.2.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
# Example 2.2 Mapping Scalars to Scalars (player statistics)

hr: 65 # Home runs
avg: 0.278 # Batting average
rbi: 147 # Runs Batted In
7 changes: 7 additions & 0 deletions libs/dyn/yamlloader/testdata/spec_example_2.20.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
# Example 2.20 Floating Point

canonical: 1.23015e+3
exponential: 12.3015e+02
fixed: 1230.15
negative infinity: -.inf
not a number: .nan
5 changes: 5 additions & 0 deletions libs/dyn/yamlloader/testdata/spec_example_2.21.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
# Example 2.21 Miscellaneous

null:
booleans: [ true, false ]
string: '012345'
6 changes: 6 additions & 0 deletions libs/dyn/yamlloader/testdata/spec_example_2.22.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
# Example 2.22 Timestamps

canonical: 2001-12-15T02:59:43.1Z
iso8601: 2001-12-14t21:59:43.10-05:00
spaced: 2001-12-14 21:59:43.10 -5
date: 2002-12-14
15 changes: 15 additions & 0 deletions libs/dyn/yamlloader/testdata/spec_example_2.23.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
# Example 2.23 Various Explicit Tags

---
not-date: !!str 2002-04-28

picture: !!binary |
R0lGODlhDAAMAIQAAP//9/X
17unp5WZmZgAAAOfn515eXv
Pz7Y6OjuDg4J+fn5OTk6enp
56enmleECcgggoBADs=

application specific tag: !something |
The semantics of the tag
above may be different for
different documents.
16 changes: 16 additions & 0 deletions libs/dyn/yamlloader/testdata/spec_example_2.24.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
# Example 2.24 Global Tags

%TAG ! tag:clarkevans.com,2002:
--- !shape
# Use the ! handle for presenting
# tag:clarkevans.com,2002:circle
- !circle
center: &ORIGIN {x: 73, y: 129}
radius: 7
- !line
start: *ORIGIN
finish: { x: 89, y: 102 }
- !label
start: *ORIGIN
color: 0xFFEEBB
text: Pretty vector drawing.
9 changes: 9 additions & 0 deletions libs/dyn/yamlloader/testdata/spec_example_2.25.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
# Example 2.25 Unordered Sets

# Sets are represented as a
# Mapping where each key is
# associated with a null value
--- !!set
? Mark McGwire
? Sammy Sosa
? Ken Griffey
9 changes: 9 additions & 0 deletions libs/dyn/yamlloader/testdata/spec_example_2.26.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
# Example 2.26 Ordered Mappings

# Ordered maps are represented as
# A sequence of mappings, with
# each mapping having one key
--- !!omap
- Mark McGwire: 65
- Sammy Sosa: 63
- Ken Griffey: 58
31 changes: 31 additions & 0 deletions libs/dyn/yamlloader/testdata/spec_example_2.27.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
# Example 2.27 Invoice

--- !<tag:clarkevans.com,2002:invoice>
invoice: 34843
date : 2001-01-23
bill-to: &id001
given : Chris
family : Dumars
address:
lines: |
458 Walkman Dr.
Suite #292
city : Royal Oak
state : MI
postal : 48046
ship-to: *id001
product:
- sku : BL394D
quantity : 4
description : Basketball
price : 450.00
- sku : BL4438H
quantity : 1
description : Super Hoop
price : 2392.00
tax : 251.42
total: 4443.52
comments:
Late afternoon is best.
Backup contact is Nancy
Billsmer @ 338-4338.
Loading

0 comments on commit dedec58

Please sign in to comment.