Utf16 bom support (#4326)

* added UTF-16 BOM support

* removed BOM removal; doesn't make a difference

pkg/decoders/utf16_test.go

@@ -27,6 +27,18 @@ func TestUTF16Decoder(t *testing.T) {
 			expected:  []byte("Hello World"),
 			expectNil: false,
 		},
+		{
+			name:      "Valid UTF-16LE input with BOM (FF FE)",
+			input:     []byte{255, 254, 72, 0, 101, 0, 108, 0, 108, 0, 111, 0, 32, 0, 87, 0, 111, 0, 114, 0, 108, 0, 100, 0},
+			expected:  []byte("Hello World"),
+			expectNil: false,
+		},
+		{
+			name:      "Valid UTF-16BE input with BOM (FE FF)",
+			input:     []byte{254, 255, 0, 72, 0, 101, 0, 108, 0, 108, 0, 111, 0, 32, 0, 87, 0, 111, 0, 114, 0, 108, 0, 100},
+			expected:  []byte("Hello World"),
+			expectNil: false,
+		},
 		{
 			name:      "Invalid UTF-16 input (it's UTF-8)",
 			input:     []byte("Hello World!"),

pkg/engine/engine.go

@@ -795,7 +795,9 @@ func (e *Engine) scannerWorker(ctx context.Context) {
 		sourceVerify := chunk.Verify
 		for _, decoder := range e.decoders {
 			decodeStart := time.Now()
-			decoded := decoder.FromChunk(chunk)
+			// This copy is needed to preserve the original chunk.Data across multiple decoders.
+			chunkCopy := *chunk
+			decoded := decoder.FromChunk(&chunkCopy)
 			decodeTime := time.Since(decodeStart).Microseconds()
 			decodeLatency.WithLabelValues(decoder.Type().String(), chunk.SourceName).Observe(float64(decodeTime))