Multiline string support (#156)

* refactor dotenv parser in order to support multi-line variable values declaration

Signed-off-by: x1unix <denis0051@gmail.com>

* Add multi-line var values test case and update comment test

Signed-off-by: x1unix <denis0051@gmail.com>

* Expand fixture tests to include multiline strings

* Update go versions to test against

* Switch to GOINSECURE for power8 CI task

* When tests fail, show source version of string (inc special chars)

* Update parser.go

Co-authored-by: Austin Sasko <austintyler0239@yahoo.com>

* Fix up bad merge

* Add a full fixture for comments for extra piece of mind

* Fix up some lint/staticcheck recommendations

* Test against go 1.19 too

Signed-off-by: x1unix <denis0051@gmail.com>
Co-authored-by: x1unix <denis0051@gmail.com>
Co-authored-by: Austin Sasko <austintyler0239@yahoo.com>

.github/workflows/ci.yml

@@ -8,7 +8,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        go: [ '1.18', '1.17', '1.16', '1.15' ]
+        go: [ '1.19', '1.18', '1.17', '1.16', '1.15' ]
         os: [ ubuntu-latest, macOS-latest, windows-latest ]
     name: ${{ matrix.os }} Go ${{ matrix.go }} Tests
     steps:

fixtures/comments.env

@@ -0,0 +1,4 @@
+# Full line comment
+foo=bar # baz
+bar=foo#baz
+baz="foo"#bar

fixtures/quoted.env

@@ -7,3 +7,13 @@ OPTION_F="2"
 OPTION_G=""
 OPTION_H="\n"
 OPTION_I = "echo 'asd'"
+OPTION_J='line 1
+line 2'
+OPTION_K='line one
+this is \'quoted\'
+one more line'
+OPTION_L="line 1
+line 2"
+OPTION_M="line one
+this is \"quoted\"
+one more line"

godotenv.go

@@ -14,10 +14,10 @@
 package godotenv
 
 import (
-	"bufio"
 	"errors"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"os"
 	"os/exec"
 	"regexp"
@@ -28,6 +28,16 @@ import (
 
 const doubleQuoteSpecialChars = "\\\n\r\"!$`"
 
+// Parse reads an env file from io.Reader, returning a map of keys and values.
+func Parse(r io.Reader) (map[string]string, error) {
+	data, err := ioutil.ReadAll(r)
+	if err != nil {
+		return nil, err
+	}
+
+	return UnmarshalBytes(data)
+}
+
 // Load will read your env file(s) and load them into ENV for this process.
 //
 // Call this function as close as possible to the start of your program (ideally in main).
@@ -96,37 +106,17 @@ func Read(filenames ...string) (envMap map[string]string, err error) {
 	return
 }
 
-// Parse reads an env file from io.Reader, returning a map of keys and values.
-func Parse(r io.Reader) (envMap map[string]string, err error) {
-	envMap = make(map[string]string)
-
-	var lines []string
-	scanner := bufio.NewScanner(r)
-	for scanner.Scan() {
-		lines = append(lines, scanner.Text())
-	}
-
-	if err = scanner.Err(); err != nil {
-		return
-	}
-
-	for _, fullLine := range lines {
-		if !isIgnoredLine(fullLine) {
-			var key, value string
-			key, value, err = parseLine(fullLine, envMap)
-
-			if err != nil {
-				return
-			}
-			envMap[key] = value
-		}
-	}
-	return
-}
-
 // Unmarshal reads an env file from a string, returning a map of keys and values.
 func Unmarshal(str string) (envMap map[string]string, err error) {
-	return Parse(strings.NewReader(str))
+	return UnmarshalBytes([]byte(str))
+}
+
+// UnmarshalBytes parses env file from byte slice of chars, returning a map of keys and values.
+func UnmarshalBytes(src []byte) (map[string]string, error) {
+	out := make(map[string]string)
+	err := parseBytes(src, out)
+
+	return out, err
 }
 
 // Exec loads env vars from the specified filenames (empty map falls back to default)
@@ -137,7 +127,9 @@ func Unmarshal(str string) (envMap map[string]string, err error) {
 // If you want more fine grained control over your command it's recommended
 // that you use `Load()` or `Read()` and the `os/exec` package yourself.
 func Exec(filenames []string, cmd string, cmdArgs []string) error {
-	Load(filenames...)
+	if err := Load(filenames...); err != nil {
+		return err
+	}
 
 	command := exec.Command(cmd, cmdArgs...)
 	command.Stdin = os.Stdin
@@ -161,8 +153,7 @@ func Write(envMap map[string]string, filename string) error {
 	if err != nil {
 		return err
 	}
-	file.Sync()
-	return err
+	return file.Sync()
 }
 
 // Marshal outputs the given environment as a dotenv-formatted environment file.
@@ -202,7 +193,7 @@ func loadFile(filename string, overload bool) error {
 
 	for key, value := range envMap {
 		if !currentEnv[key] || overload {
-			os.Setenv(key, value)
+			_ = os.Setenv(key, value)
 		}
 	}
 
@@ -259,15 +250,15 @@ func parseLine(line string, envMap map[string]string) (key string, value string,
 	}
 
 	if len(splitString) != 2 {
-		err = errors.New("Can't separate key from value")
+		err = errors.New("can't separate key from value")
 		return
 	}
 
 	// Parse the key
 	key = splitString[0]
-	if strings.HasPrefix(key, "export") {
-		key = strings.TrimPrefix(key, "export")
-	}
+
+	key = strings.TrimPrefix(key, "export")
+
 	key = strings.TrimSpace(key)
 
 	key = exportRegex.ReplaceAllString(splitString[0], "$1")
@@ -343,11 +334,6 @@ func expandVariables(v string, m map[string]string) string {
 	})
 }
 
-func isIgnoredLine(line string) bool {
-	trimmedLine := strings.TrimSpace(line)
-	return len(trimmedLine) == 0 || strings.HasPrefix(trimmedLine, "#")
-}
-
 func doubleQuoteEscape(line string) string {
 	for _, c := range doubleQuoteSpecialChars {
 		toReplace := "\\" + string(c)

godotenv_test.go

@@ -35,7 +35,7 @@ func loadEnvAndCompareValues(t *testing.T, loader func(files ...string) error, e
 		envValue := os.Getenv(k)
 		v := expectedValues[k]
 		if envValue != v {
-			t.Errorf("Mismatch for key '%v': expected '%v' got '%v'", k, v, envValue)
+			t.Errorf("Mismatch for key '%v': expected '%#v' got '%#v'", k, v, envValue)
 		}
 	}
 }
@@ -189,6 +189,10 @@ func TestLoadQuotedEnv(t *testing.T) {
 		"OPTION_G": "",
 		"OPTION_H": "\n",
 		"OPTION_I": "echo 'asd'",
+		"OPTION_J": "line 1\nline 2",
+		"OPTION_K": "line one\nthis is \\'quoted\\'\none more line",
+		"OPTION_L": "line 1\nline 2",
+		"OPTION_M": "line one\nthis is \"quoted\"\none more line",
 	}
 
 	loadEnvAndCompareValues(t, Load, envFileName, expectedValues, noopPresets)
@@ -271,6 +275,34 @@ func TestExpanding(t *testing.T) {
 
 }
 
+func TestVariableStringValueSeparator(t *testing.T) {
+	input := "TEST_URLS=\"stratum+tcp://stratum.antpool.com:3333\nstratum+tcp://stratum.antpool.com:443\""
+	want := map[string]string{
+		"TEST_URLS": "stratum+tcp://stratum.antpool.com:3333\nstratum+tcp://stratum.antpool.com:443",
+	}
+	got, err := Parse(strings.NewReader(input))
+	if err != nil {
+		t.Error(err)
+	}
+
+	if len(got) != len(want) {
+		t.Fatalf(
+			"unexpected value:\nwant:\n\t%#v\n\ngot:\n\t%#v", want, got)
+	}
+
+	for k, wantVal := range want {
+		gotVal, ok := got[k]
+		if !ok {
+			t.Fatalf("key %q doesn't present in result", k)
+		}
+		if wantVal != gotVal {
+			t.Fatalf(
+				"mismatch in %q value:\nwant:\n\t%s\n\ngot:\n\t%s", k,
+				wantVal, gotVal)
+		}
+	}
+}
+
 func TestActualEnvVarsAreLeftAlone(t *testing.T) {
 	os.Clearenv()
 	os.Setenv("OPTION_A", "actualenv")
@@ -377,33 +409,38 @@ func TestParsing(t *testing.T) {
 }
 
 func TestLinesToIgnore(t *testing.T) {
-	// it 'ignores empty lines' do
-	// expect(env("\n \t  \nfoo=bar\n \nfizz=buzz")).to eql('foo' => 'bar', 'fizz' => 'buzz')
-	if !isIgnoredLine("\n") {
-		t.Error("Line with nothing but line break wasn't ignored")
-	}
-
-	if !isIgnoredLine("\r\n") {
-		t.Error("Line with nothing but windows-style line break wasn't ignored")
-	}
-
-	if !isIgnoredLine("\t\t ") {
-		t.Error("Line full of whitespace wasn't ignored")
-	}
-
-	// it 'ignores comment lines' do
-	// expect(env("\n\n\n # HERE GOES FOO \nfoo=bar")).to eql('foo' => 'bar')
-	if !isIgnoredLine("# comment") {
-		t.Error("Comment wasn't ignored")
-	}
-
-	if !isIgnoredLine("\t#comment") {
-		t.Error("Indented comment wasn't ignored")
+	cases := map[string]struct {
+		input string
+		want  string
+	}{
+		"Line with nothing but line break": {
+			input: "\n",
+		},
+		"Line with nothing but windows-style line break": {
+			input: "\r\n",
+		},
+		"Line full of whitespace": {
+			input: "\t\t ",
+		},
+		"Comment": {
+			input: "# Comment",
+		},
+		"Indented comment": {
+			input: "\t # comment",
+		},
+		"non-ignored value": {
+			input: `export OPTION_B='\n'`,
+			want:  `export OPTION_B='\n'`,
+		},
 	}
 
-	// make sure we're not getting false positives
-	if isIgnoredLine(`export OPTION_B='\n'`) {
-		t.Error("ignoring a perfectly valid line to parse")
+	for n, c := range cases {
+		t.Run(n, func(t *testing.T) {
+			got := string(getStatementStart([]byte(c.input)))
+			if got != c.want {
+				t.Errorf("Expected:\t %q\nGot:\t %q", c.want, got)
+			}
+		})
 	}
 }
 
@@ -424,6 +461,17 @@ func TestErrorParsing(t *testing.T) {
 	}
 }
 
+func TestComments(t *testing.T) {
+	envFileName := "fixtures/comments.env"
+	expectedValues := map[string]string{
+		"foo": "bar",
+		"bar": "foo#baz",
+		"baz": "foo",
+	}
+
+	loadEnvAndCompareValues(t, Load, envFileName, expectedValues, noopPresets)
+}
+
 func TestWrite(t *testing.T) {
 	writeAndCompare := func(env string, expected string) {
 		envMap, _ := Unmarshal(env)