fix(query): countLines, IgnoreLines and fileCommands

e2e/fixtures/E2E_CLI_083_RESULT.json

@@ -0,0 +1,28 @@
+{
+	"kics_version": "development",
+	"files_scanned": 4,
+	"lines_scanned": 89,
+	"files_parsed": 3,
+	"lines_parsed": 86,
+	"lines_ignored": 3,
+	"files_failed_to_scan": 0,
+	"queries_total": 1,
+	"queries_failed_to_execute": 0,
+	"queries_failed_to_compute_similarity_id": 0,
+	"scan_id": "console",
+	"severity_counters": {
+		"HIGH": 0,
+		"INFO": 0,
+		"LOW": 0,
+		"MEDIUM": 0,
+		"TRACE": 0
+	},
+	"total_counter": 0,
+	"total_bom_resources": 0,
+	"start": "2024-02-06T12:29:45.3845776Z",
+	"end": "2024-02-06T12:29:49.5261723Z",
+	"paths": [
+		"/path/test/fixtures/helm_ignore"
+	],
+	"queries": []
+}

e2e/fixtures/E2E_CLI_084_RESULT.json

@@ -0,0 +1,28 @@
+{
+	"kics_version": "development",
+	"files_scanned": 4,
+	"lines_scanned": 89,
+	"files_parsed": 3,
+	"lines_parsed": 34,
+	"lines_ignored": 55,
+	"files_failed_to_scan": 0,
+	"queries_total": 1,
+	"queries_failed_to_execute": 0,
+	"queries_failed_to_compute_similarity_id": 0,
+	"scan_id": "console",
+	"severity_counters": {
+		"HIGH": 0,
+		"INFO": 0,
+		"LOW": 0,
+		"MEDIUM": 0,
+		"TRACE": 0
+	},
+	"total_counter": 0,
+	"total_bom_resources": 0,
+	"start": "2024-02-06T12:29:45.3845776Z",
+	"end": "2024-02-06T12:29:49.5261723Z",
+	"paths": [
+    "/path/test/fixtures/helm_ignore_block"
+	],
+	"queries": []
+}

e2e/fixtures/E2E_CLI_085_RESULT.json

@@ -0,0 +1,28 @@
+{
+  "kics_version": "development",
+  "files_scanned": 4,
+  "lines_scanned": 89,
+  "files_parsed": 3,
+  "lines_parsed": 86,
+  "lines_ignored": 3,
+  "files_failed_to_scan": 0,
+  "queries_total": 1,
+  "queries_failed_to_execute": 0,
+  "queries_failed_to_compute_similarity_id": 0,
+  "scan_id": "console",
+  "severity_counters": {
+    "HIGH": 0,
+    "INFO": 0,
+    "LOW": 0,
+    "MEDIUM": 0,
+    "TRACE": 0
+  },
+  "total_counter": 0,
+  "total_bom_resources": 0,
+  "start": "2024-02-06T15:01:20.657455Z",
+  "end": "2024-02-06T15:01:25.1183483Z",
+  "paths": [
+    "/path/test/fixtures/helm_disable_query"
+  ],
+  "queries": []
+}

e2e/testcases/e2e-cli-083_helm_ignore.go

@@ -0,0 +1,27 @@
+package testcases
+
+// E2E-CLI-083 - KICS  scan
+// should perform a scan and return zero results ignoring the file
+func init() { //nolint
+	testSample := TestCase{
+		Name: "should perform a scan and return zero results ignoring the file [E2E-CLI-083]",
+		Args: args{
+			Args: []cmdArgs{
+				[]string{"scan", "-o", "/path/e2e/output",
+					"--output-name", "E2E_CLI_083_RESULT",
+					"-p", "\"/path/test/fixtures/helm_ignore\"",
+					"-i", "b7652612-de4e-4466-a0bf-1cd81f0c6063",
+				},
+			},
+			ExpectedResult: []ResultsValidation{
+				{
+					ResultsFile:    "E2E_CLI_083_RESULT",
+					ResultsFormats: []string{"json"},
+				},
+			},
+		},
+		WantStatus: []int{0},
+	}
+
+	Tests = append(Tests, testSample)
+}

e2e/testcases/e2e-cli-084_helm_ignore_block.go

@@ -0,0 +1,27 @@
+package testcases
+
+// E2E-CLI-084 - KICS  scan
+// should perform a scan and return zero results ignoring the block
+func init() { //nolint
+	testSample := TestCase{
+		Name: "should perform a scan and return zero results ignoring the block [E2E-CLI-084]",
+		Args: args{
+			Args: []cmdArgs{
+				[]string{"scan", "-o", "/path/e2e/output",
+					"--output-name", "E2E_CLI_084_RESULT",
+					"-p", "\"/path/test/fixtures/helm_ignore_block\"",
+					"-i", "b7652612-de4e-4466-a0bf-1cd81f0c6063",
+				},
+			},
+			ExpectedResult: []ResultsValidation{
+				{
+					ResultsFile:    "E2E_CLI_084_RESULT",
+					ResultsFormats: []string{"json"},
+				},
+			},
+		},
+		WantStatus: []int{0},
+	}
+
+	Tests = append(Tests, testSample)
+}

e2e/testcases/e2e-cli-085_helm_disable_query.go

@@ -0,0 +1,27 @@
+package testcases
+
+// E2E-CLI-085 - KICS  scan
+// should perform a scan and return zero results ignoring the query
+func init() { //nolint
+	testSample := TestCase{
+		Name: "should perform a scan and return zero results ignoring the query [E2E-CLI-085]",
+		Args: args{
+			Args: []cmdArgs{
+				[]string{"scan", "-o", "/path/e2e/output",
+					"--output-name", "E2E_CLI_085_RESULT",
+					"-p", "\"/path/test/fixtures/helm_disable_query\"",
+					"-i", "b7652612-de4e-4466-a0bf-1cd81f0c6063",
+				},
+			},
+			ExpectedResult: []ResultsValidation{
+				{
+					ResultsFile:    "E2E_CLI_085_RESULT",
+					ResultsFormats: []string{"json"},
+				},
+			},
+		},
+		WantStatus: []int{0},
+	}
+
+	Tests = append(Tests, testSample)
+}

pkg/detector/helm/helm_detect.go

@@ -39,7 +39,10 @@ const (
 func (d DetectKindLine) DetectLine(file *model.FileMetadata, searchKey string,
 	outputLines int, logWithFields *zerolog.Logger) model.VulnerabilityLines {
 	searchKey = fmt.Sprintf("%s.%s", strings.TrimRight(strings.TrimLeft(file.HelmID, "# "), ":"), searchKey)
-	lines := *file.LinesOriginalData
+
+	lines := make([]string, len(*file.LinesOriginalData))
+	copy(lines, *file.LinesOriginalData)
+
 	curLineRes := detectCurlLine{
 		foundRes: false,
 		lineRes:  0,

pkg/kics/resolver_sink.go

@@ -5,6 +5,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"regexp"
 	"sort"
 
 	sentryReport "github.com/Checkmarx/kics/internal/sentry"
@@ -28,8 +29,6 @@ func (s *Service) resolverSink(ctx context.Context, filename, scanID string, ope
 
 	for _, rfile := range resFiles.File {
 		s.Tracker.TrackFileFound()
-		countLines := bytes.Count(rfile.Content, []byte{'\n'}) + 1
-		s.Tracker.TrackFileFoundCountLines(countLines)
 
 		isMinified := minified.IsMinified(rfile.FileName, rfile.Content)
 		documents, err := s.Parser.Parse(rfile.FileName, rfile.Content, openAPIResolveReferences, isMinified)
@@ -40,6 +39,21 @@ func (s *Service) resolverSink(ctx context.Context, filename, scanID string, ope
 			log.Err(err).Msgf("failed to parse file content")
 			return []string{}, nil
 		}
+
+		if kind == model.KindHELM {
+			ignoreList, errorIL := s.getOriginalIgnoreLines(rfile.FileName, rfile.OriginalData, openAPIResolveReferences, isMinified)
+			if errorIL == nil {
+				documents.IgnoreLines = ignoreList
+
+				// Need to ignore #KICS_HELM_ID Line
+				documents.CountLines = bytes.Count(rfile.OriginalData, []byte{'\n'})
+			}
+		} else {
+			documents.CountLines = bytes.Count(rfile.OriginalData, []byte{'\n'}) + 1
+		}
+
+		fileCommands := s.Parser.CommentsCommands(rfile.FileName, rfile.OriginalData)
+
 		for _, document := range documents.Docs {
 			_, err = json.Marshal(document)
 			if err != nil {
@@ -67,6 +81,7 @@ func (s *Service) resolverSink(ctx context.Context, filename, scanID string, ope
 				FilePath:          rfile.FileName,
 				Content:           string(rfile.Content),
 				HelmID:            rfile.SplitID,
+				Commands:          fileCommands,
 				IDInfo:            rfile.IDInfo,
 				LinesIgnore:       documents.IgnoreLines,
 				ResolvedFiles:     documents.ResolvedFiles,
@@ -76,8 +91,22 @@ func (s *Service) resolverSink(ctx context.Context, filename, scanID string, ope
 			s.saveToFile(ctx, &file)
 		}
 		s.Tracker.TrackFileParse()
+		s.Tracker.TrackFileFoundCountLines(documents.CountLines)
 		s.Tracker.TrackFileParseCountLines(documents.CountLines - len(documents.IgnoreLines))
 		s.Tracker.TrackFileIgnoreCountLines(len(documents.IgnoreLines))
 	}
 	return resFiles.Excluded, nil
 }
+
+func (s *Service) getOriginalIgnoreLines(filename string,
+	originalFile []uint8,
+	openAPIResolveReferences, isMinified bool) (ignoreLines []int, err error) {
+	refactor := regexp.MustCompile(`.*\n?.*KICS\_HELM\_ID.+\n`).ReplaceAll(originalFile, []uint8{})
+	refactor = regexp.MustCompile(`{{-\s*(.*?)\s*}}`).ReplaceAll(refactor, []uint8{})
+
+	documentsOriginal, err := s.Parser.Parse(filename, refactor, openAPIResolveReferences, isMinified)
+	if err == nil {
+		ignoreLines = documentsOriginal.IgnoreLines
+	}
+	return
+}

pkg/model/comment_yaml.go

@@ -1,6 +1,7 @@
 package model
 
 import (
+	"reflect"
 	"strings"
 	"sync"
 
@@ -150,6 +151,7 @@ func processLine(kind yaml.Kind, content *yaml.Node, position int) (linesIgnore
 	} else {
 		nodeToIgnore = content.Content[position]
 	}
+
 	linesIgnore = append(linesIgnore, nodeToIgnore.Line-1, nodeToIgnore.Line)
 	return
 }
@@ -175,13 +177,17 @@ func processBlock(kind yaml.Kind, content []*yaml.Node, position int) (linesIgno
 // getNodeLastLine returns the last line of a node
 func getNodeLastLine(node *yaml.Node) (lastLine int) {
 	lastLine = node.Line
-	for _, content := range node.Content {
-		if content.Line > lastLine {
-			lastLine = content.Line
-		}
-		if lineContent := getNodeLastLine(content); lineContent > lastLine {
-			lastLine = lineContent
+	if len(node.Content) > 0 {
+		for _, content := range node.Content {
+			if content.Line > lastLine {
+				lastLine = content.Line
+			}
+			if lineContent := getNodeLastLine(content); lineContent > lastLine {
+				lastLine = lineContent
+			}
 		}
+	} else if reflect.TypeOf(node.Value).Kind() == reflect.String {
+		lastLine += strings.Count(node.Value, "\n")
 	}
 
 	return
@@ -190,6 +196,12 @@ func getNodeLastLine(node *yaml.Node) (lastLine int) {
 // value returns the value of the comment
 func (c *comment) value() (value CommentCommand) {
 	comment := strings.ToLower(string(*c))
+	if isHelm(comment) {
+		res := KICSGetContentCommentRgxp.FindString(comment)
+		if len(res) > 0 {
+			comment = res
+		}
+	}
 	// check if we are working with kics command
 	if KICSCommentRgxp.MatchString(comment) {
 		comment = KICSCommentRgxp.ReplaceAllString(comment, "")
@@ -200,3 +212,7 @@ func (c *comment) value() (value CommentCommand) {
 	}
 	return CommentCommand(comment)
 }
+
+func isHelm(comment string) bool {
+	return strings.Contains(comment, "helm")
+}

pkg/model/comment_yaml_test.go

@@ -1,6 +1,7 @@
 package model
 
 import (
+	"github.com/stretchr/testify/assert"
 	"sort"
 	"testing"
 
@@ -634,6 +635,30 @@ func Test_ignoreCommentsYAML(t *testing.T) {
 				},
 			},
 		},
+		{
+			name: "test_7: ignore_multiline_string",
+			want: []int{4, 5, 6, 7, 8, 9},
+			args: args{
+				&yaml.Node{
+					Kind: yaml.MappingNode,
+					Content: []*yaml.Node{
+						{
+							Kind:        yaml.ScalarNode,
+							Value:       "deploy.yml",
+							HeadComment: "# kics-scan ignore-block",
+							Line:        5,
+							Column:      3,
+						},
+						{
+							Kind:   yaml.ScalarNode,
+							Value:  "---\nfoo\n  bar: abc\nuploader-token: my-awesome-token\n",
+							Line:   5,
+							Column: 15,
+						},
+					},
+				},
+			},
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
@@ -647,3 +672,29 @@ func Test_ignoreCommentsYAML(t *testing.T) {
 		})
 	}
 }
+
+func Test_value(t *testing.T) {
+	tests := []struct {
+		name  string
+		input comment
+		want  string
+	}{
+		{
+			name:  "Should return ignore-block",
+			input: comment("# source: test/templates/deployment.yaml\n# kics-scan ignore-block\n# kics_helm_id_2:"),
+			want:  "ignore-block",
+		},
+		{
+			name:  "Should Not return ignore-block",
+			input: comment("# source: test/templates/deployment.yaml\n# kics ignore-block\n# kics_helm_id_2:"),
+			want:  "# source: test/templates/deployment.yaml\n# kics ignore-block\n# kics_helm_id_2:",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			res := tt.input.value()
+			assert.Equal(t, string(res), tt.want)
+		})
+	}
+}

pkg/model/model.go

@@ -67,7 +67,9 @@ var (
 
 var (
 	// KICSCommentRgxp is the regexp to identify if a comment is a KICS comment
-	KICSCommentRgxp = regexp.MustCompile(`^((/{2})|#|;)*\s*kics-scan\s*`)
+	KICSCommentRgxp = regexp.MustCompile(`(^|\n)((/{2})|#|;)*\s*kics-scan\s*`)
+	// KICSGetContentCommentRgxp to gets the kics comment on the hel case
+	KICSGetContentCommentRgxp = regexp.MustCompile(`(^|\n)((/{2})|#|;)*\s*kics-scan([^\n]*)\n`)
 	// KICSCommentRgxpYaml is the regexp to identify if the comment has KICS comment at the end of the comment in YAML
 	KICSCommentRgxpYaml = regexp.MustCompile(`((/{2})|#)*\s*kics-scan\s*(ignore-line|ignore-block)\s*\n*$`)
 )