From 888446ba556b10a4f6012227c98664ce165c2d3f Mon Sep 17 00:00:00 2001 From: Aleksandr Razumov Date: Mon, 3 Feb 2020 11:56:32 +0300 Subject: [PATCH 1/3] autogenerated_exclude: increase scanner buffer Some lines can be very long, so increase scanner buffer to mitigate this. Fix #954 --- pkg/result/processors/autogenerated_exclude.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pkg/result/processors/autogenerated_exclude.go b/pkg/result/processors/autogenerated_exclude.go index cc5883d96f2a..927abd714ee5 100644 --- a/pkg/result/processors/autogenerated_exclude.go +++ b/pkg/result/processors/autogenerated_exclude.go @@ -120,6 +120,12 @@ func getDoc(filePath string) (string, error) { defer file.Close() scanner := bufio.NewScanner(file) + + // Issue 954: Some lines can be very long, e.g. auto-generated + // embedded resources. Reported on file of 86.2KB. + const maxTokenSize = 512 * 1024 // 512KB should be enough + scanner.Buffer(make([]byte, maxTokenSize), maxTokenSize) + var docLines []string for scanner.Scan() { line := strings.TrimSpace(scanner.Text()) From e084395cfa65be35a4bfa20006e677feed05a729 Mon Sep 17 00:00:00 2001 From: Aleksandr Razumov Date: Mon, 3 Feb 2020 13:48:47 +0300 Subject: [PATCH 2/3] autogenerated_exclude: optimize scanner buffer --- pkg/result/processors/autogenerated_exclude.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pkg/result/processors/autogenerated_exclude.go b/pkg/result/processors/autogenerated_exclude.go index 927abd714ee5..ac7191996f94 100644 --- a/pkg/result/processors/autogenerated_exclude.go +++ b/pkg/result/processors/autogenerated_exclude.go @@ -123,8 +123,11 @@ func getDoc(filePath string) (string, error) { // Issue 954: Some lines can be very long, e.g. auto-generated // embedded resources. Reported on file of 86.2KB. - const maxTokenSize = 512 * 1024 // 512KB should be enough - scanner.Buffer(make([]byte, maxTokenSize), maxTokenSize) + const ( + naxSize = 512 * 1024 // 512KB should be enough + initialSize = 4096 // same as startBufSize in bufio + ) + scanner.Buffer(make([]byte, initialSize), naxSize) var docLines []string for scanner.Scan() { From fdd533b6616456c3650dac9952dddd1d4536fb1c Mon Sep 17 00:00:00 2001 From: Aleksandr Razumov Date: Mon, 3 Feb 2020 13:49:54 +0300 Subject: [PATCH 3/3] autogenerated_exclude: fix typo --- pkg/result/processors/autogenerated_exclude.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/result/processors/autogenerated_exclude.go b/pkg/result/processors/autogenerated_exclude.go index ac7191996f94..d804351b9f2d 100644 --- a/pkg/result/processors/autogenerated_exclude.go +++ b/pkg/result/processors/autogenerated_exclude.go @@ -124,10 +124,10 @@ func getDoc(filePath string) (string, error) { // Issue 954: Some lines can be very long, e.g. auto-generated // embedded resources. Reported on file of 86.2KB. const ( - naxSize = 512 * 1024 // 512KB should be enough + maxSize = 512 * 1024 // 512KB should be enough initialSize = 4096 // same as startBufSize in bufio ) - scanner.Buffer(make([]byte, initialSize), naxSize) + scanner.Buffer(make([]byte, initialSize), maxSize) var docLines []string for scanner.Scan() {