From fb76eaf17b2b923bcc3e59314cf3605bce9a8bcd Mon Sep 17 00:00:00 2001
From: Miccah <m.castorina93@gmail.com>
Date: Tue, 13 Jun 2023 19:00:40 -0500
Subject: [PATCH] Use heuristic to choose the most likely UTF-16 decoded string
 (#1381)

* Use heuristic to choose the most likely UTF-16 decoded string

* Assume ASCII and include valid BE and LE bytes

* Remove unused code

* Assume ASCII and return nil when not utf16

---------

Co-authored-by: bill-rich <bill.rich@gmail.com>
---
 pkg/decoders/utf16.go       |  52 +++++++++++-------------------------
 pkg/decoders/utf16_test.dll | Bin 0 -> 5120 bytes
 pkg/decoders/utf16_test.go  |  27 ++++++++++++++++---
 pkg/decoders/utf8.go        |  11 ++++----
 4 files changed, 45 insertions(+), 45 deletions(-)
 create mode 100644 pkg/decoders/utf16_test.dll

diff --git a/pkg/decoders/utf16.go b/pkg/decoders/utf16.go
index c3bc8fbbc..d5599a520 100644
--- a/pkg/decoders/utf16.go
+++ b/pkg/decoders/utf16.go
@@ -3,7 +3,6 @@ package decoders
 import (
 	"bytes"
 	"encoding/binary"
-	"fmt"
 	"unicode/utf8"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
@@ -17,6 +16,9 @@ func (d *UTF16) FromChunk(chunk *sources.Chunk) *sources.Chunk {
 	}
 
 	if utf16Data, err := utf16ToUTF8(chunk.Data); err == nil {
+		if len(utf16Data) == 0 {
+			return nil
+		}
 		chunk.Data = utf16Data
 		return chunk
 	}
@@ -26,43 +28,19 @@ func (d *UTF16) FromChunk(chunk *sources.Chunk) *sources.Chunk {
 
 // utf16ToUTF8 converts a byte slice containing UTF-16 encoded data to a UTF-8 encoded byte slice.
 func utf16ToUTF8(b []byte) ([]byte, error) {
-	endianness, err := guessUTF16Endianness(b)
-	if err != nil {
-		return nil, err
-	}
-
-	buf := &bytes.Buffer{}
-	for i := 0; i < len(b); i += 2 {
-		r := rune(endianness.Uint16(b[i:]))
-		if utf8.ValidRune(r) {
-			buf.WriteRune(r)
+	var bufBE, bufLE bytes.Buffer
+	for i := 0; i < len(b)-1; i += 2 {
+		if r := rune(binary.BigEndian.Uint16(b[i:])); b[i] == 0 && utf8.ValidRune(r) {
+			if isValidByte(byte(r)) {
+				bufBE.WriteRune(r)
+			}
+		}
+		if r := rune(binary.LittleEndian.Uint16(b[i:])); b[i+1] == 0 && utf8.ValidRune(r) {
+			if isValidByte(byte(r)) {
+				bufLE.WriteRune(r)
+			}
 		}
 	}
 
-	return buf.Bytes(), nil
-}
-
-func guessUTF16Endianness(b []byte) (binary.ByteOrder, error) {
-	if len(b) < 2 || len(b)%2 != 0 {
-		return nil, fmt.Errorf("input length must be even and at least 2 bytes long")
-	}
-
-	var evenNullBytes, oddNullBytes int
-
-	for i := 0; i < len(b); i += 2 {
-		if b[i] == 0 {
-			oddNullBytes++
-		}
-		if b[i+1] == 0 {
-			evenNullBytes++
-		}
-	}
-
-	if evenNullBytes > oddNullBytes {
-		return binary.LittleEndian, nil
-	}
-	if oddNullBytes > evenNullBytes {
-		return binary.BigEndian, nil
-	}
-	return nil, fmt.Errorf("could not determine endianness")
+	return append(bufLE.Bytes(), bufBE.Bytes()...), nil
 }
diff --git a/pkg/decoders/utf16_test.dll b/pkg/decoders/utf16_test.dll
new file mode 100644
index 0000000000000000000000000000000000000000..d5ffcb1110dd5c37da1f3fa4233ab80ffc06a8d6
GIT binary patch
literal 5120
zcmeHLTZ~&r8UBygyV*F&HWyk2s=|pA+O)~qdu!OF>y4fDZt7)kS$h+bf|BFovG*kB
z;+{F)tdR(55g>S>MXHK~goJnkDpV0bEeMbhAS4u_K7fQ;E)qNxc!<PfB^the&N;R>
zZAy79j5GiIxA`wKbN-pl;)Q3ZkBE}!w{H`@gppfb!FLANAs+hB4~OV`+ixCtNn5yi
zpt5dBz3F=m-*9x(a9uCZt3vl%u5P({ak;EJUQJ|n?MfYrOs~umEogmo>u39(i@p7s
z(zHVxAxeQW7;@h{j8R9=qchMkB*M6XsrY%M>*(Nf>!VAbCEBh2b;pPdRp^_ryUci=
zXrw=4_70bbc7p!vLqwHc{(r*tPUuQ=5cK0cb25S03_vHh13;#7jp7ESnoe{n<4fNJ
zrDDqi89?WNdRiXQXMACMP*kEz5VEbk=shh@bgG9`=pT4p(KxmQ_0x5osJ4wr1C4XR
z-u|nWP8{CF$viS0A&I_E!B+B-F&{(o@Q@Nh>mG&Bsq*|OO?eAr-<ZgZWX4CvM~}1H
zezF0NBA0{Lh`xgHI^*TQx7>zgjs4iO30OY3R;C3+i`}M!r`JkFj2{O*$g)$mS7l{1
zOxMocOT^1gZ)oFmUzqD2ko*lmy)z<2Vm|XQ5A8P^<Gd`h+5?FJdS1cLQz~%}<)AV^
z-=*(s19VOMC*Y8Rzb3|CS5PR<R~23%w(xE3FOaS$7<$C84Ln5$fU_s|&k=yoc4Pa_
zBp&Th!|=`&$F`eZg-QW<A3aEa*B+)7oo9Y-e-d6aI^Ut#mZhj?fgMHP=*R|?EsYiu
z+bNmQ;GJi`A23A^1Ma3#1&=FOP_U%n83oTPXaFJ)z<bG3_!nrB?x&{|{t%AkzbHdH
zXbf;4JqCC`T>v~tCg2hJI^Z~c2k<d^8E}Sv3HVXPF9Xid6Z8sT3eZ%z8Q~`Vjy|I}
z$?M_1QK#{a<6w81j*Pf%<$WNQPc0gjJAH_x;WTYAN>htot!0a8Dz{}IoXm{pO3xNC
zx!Q6A%Mo<Ww*s+Xxq@b$s;JdOtq|a}R$GCf8P922w(w62SNKNIE9xk%Nuwb;6ct6a
z)o2)1yN6ehQaDw+U9o~*cE#`;BA7$v6<0m~O4mlLKL_7ug)c47eXFqK);-@b0tgNJ
zu3AXCZZ%p6ev6S+Q7;>H5wusaLb7W-Ojs$e3ftIJSdTo8Y{kdgnn8DBEMNnM+wKaY
zWmf#as#>-c^hl$viIg%?8YmU<E@o=BP4LF|Q?zpwX>~SI)NNsMev~l-&yV><%V@Zs
z3@lT|tk_SyuVvwHSf-HCuE8(P3`lH@q(#g0J?Yg0oSd2{V3pg_lGI5p+&)Rm)yr^C
z^i(#5L;VERNKl;&YM~Sc^a=0`x=NBRVl?p+z?r%Ntc}sa44rc51$;6kz>m>+d|pL5
zgHL9bmS_TH<QUCU8T|}2x)vpDnB)Tncd;K@u26$Us7as1XFU&(7RGf%ut1xzeuP%Y
zr8RJoH6*9WK_n8Nw1H2)2>b{grzbHxi<~TYOQH0&;IpY<=;JbGc(9XF@id^vR48jf
zHjH@&#sJtDv{3!F&5zD4&3rNcjlDO%{D%121xo6gHrS_=)(>RQ9wvtrdOS%$2VXpS
z@zZ;Ln>vvg80-h*fg#)$gQ@_0kgH@Rao;)LXfAo~?4~K2d?qUEzIRn>(Ax%wL^+N2
zXO?CwGoCLB&F0bQFy%Hrk{N-4q4#yZ$D$>hw$Wa~=Q+qTbZG2^nYg}CN~3=Dx#J=P
ze-!s1ez+yUR|GC5_W58~@Dm7cSHZTutBWl=uu!AG*j{qi1y`UNUwk`MDj|68CpUgT
z+Tse|74ztmxc4XdeHAsfybm4E9C{kEr+Z|q!uM)#TP+vM$A12+<-NbZeELg;SDzjE
z+iPXEk!||c25MV2aGKdDx2#vaoK>l1T@fJfSdeMfs#JcwF!ty%ii|zK92w()&tJQF
zqyDYe{-|Gj`4_JssJQ)uaXqHCj`4#g<X*(a%x(&<<>W{cyChT%i00h>FzNZN^Z%!*
zgxY5v&#e7ehZT%lmm_?sO}-7E_3lsxC)8hmi-=DO(G(t-&*X_-;6oX;7ZMY|&Z0P7
z1e~Q+U?sfJmq3>Q=fc-I{Wy8+&G3c~ZN<0!$;eLpmW%E3SVY%g)j&y^LqV}oLP{tq
zb(9uvhlpoYAT_|1kQ*p3Vc}5<^kQ-+V$~2+04)nzjkmG%IzrM=AmiTzRe|FTkK&@C
z2zw}qP35DFj0{jhna53`BplZ$IFm|0j<g7_ROQ*i?B2NUoEQ3u=bgnoA2YjojN&yl
z(&_AHC*tCj2$ZgiIP9Ltw%X6&g0b-<I*ivM-inyhP%}8QCN3dfnFcNd{`*Ep&nx@O
z5uaC$?Z@#k6|TVD{Dk?eV7>?6Ev!M%6CEM9t?M>|cesYr+pB-)Ds<N%UgrYr@CE2#
zw`^pg<D_g|x$YcZKm6S~L-eQk_<_Flqn}Ql+;nVxBYb7$(x}^MUAU%K!($+qUaQOv
zPo{Ml7;eq5Jy+z?Z6VVqr&FmZBYM^8FyP8uy5+kkq`58}Lk>If+hN%BoD+t0G8?05
z-7#FNE@TkjIm+8~O4mEKN;QF3Y0%yh5Z@|k-NikWOE0!@Y1x*k9%dP%*-U4ZfxvIc
zfS-i#GMce8PlIPE%ogsSc1VKZizi!%QPfs&FW>>%5b|Bx9FJ@&yB#wq=4Oj;;svoG
zY~5y<OB=G}Zg^LOpVnJe!NeO3D_A#dDZ+%5jcmtAH<l>uY;W9C*-rY?si|zdG1J)L
X_h8eb|1(g3Z@&lg`+rLBxB~wM=zb&c

literal 0
HcmV?d00001

diff --git a/pkg/decoders/utf16_test.go b/pkg/decoders/utf16_test.go
index 92c13dd8d..9a05e0113 100644
--- a/pkg/decoders/utf16_test.go
+++ b/pkg/decoders/utf16_test.go
@@ -2,6 +2,7 @@ package decoders
 
 import (
 	"bytes"
+	"os"
 	"testing"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
@@ -35,8 +36,8 @@ func TestUTF16Decoder(t *testing.T) {
 		{
 			name:      "Invalid UTF-16 input (odd length)",
 			input:     []byte{72, 0, 101, 0, 108, 0, 108, 0, 111, 0, 32, 0, 87, 0, 111, 0, 114, 0, 108, 0, 0},
-			expected:  nil,
-			expectNil: true,
+			expected:  []byte("Hello Worl"),
+			expectNil: false,
 		},
 	}
 
@@ -57,12 +58,32 @@ func TestUTF16Decoder(t *testing.T) {
 				return
 			}
 			if !bytes.Equal(decodedChunk.Data, tc.expected) {
-				t.Errorf("Expected decoded data: %v, got: %v", tc.expected, decodedChunk.Data)
+				t.Errorf("Expected decoded data: %s, got: %s", tc.expected, decodedChunk.Data)
 			}
 		})
 	}
 }
 
+func TestDLL(t *testing.T) {
+	data, err := os.ReadFile("utf16_test.dll")
+	if err != nil {
+		t.Errorf("Failed to read test data: %v", err)
+		return
+	}
+
+	chunk := &sources.Chunk{Data: data}
+	decoder := &UTF16{}
+	decodedChunk := decoder.FromChunk(chunk)
+	if decodedChunk == nil {
+		t.Errorf("Expected chunk with data, got nil")
+		return
+	}
+	if !bytes.Contains(decodedChunk.Data, []byte("aws_secret_access_key")) {
+		t.Errorf("Expected chunk to have aws_secret_access_key")
+		return
+	}
+}
+
 func BenchmarkUtf16ToUtf8(b *testing.B) {
 	// Example UTF-16LE encoded data
 	data := []byte{72, 0, 101, 0, 108, 0, 108, 0, 111, 0, 32, 0, 87, 0, 111, 0, 114, 0, 108, 0, 100, 0}
diff --git a/pkg/decoders/utf8.go b/pkg/decoders/utf8.go
index 63a8ea437..93090e71f 100644
--- a/pkg/decoders/utf8.go
+++ b/pkg/decoders/utf8.go
@@ -26,11 +26,6 @@ func (d *UTF8) FromChunk(chunk *sources.Chunk) *sources.Chunk {
 // extacting contigous portions of printable characters that we care
 // about from some bytes
 func extractSubstrings(b []byte) []byte {
-	isValidByte := func(c byte) bool {
-		// https://www.rapidtables.com/code/text/ascii-table.html
-		// split on anything that is not ascii space through tilde
-		return c > 31 && c < 127
-	}
 
 	field := make([]byte, len(b))
 	fieldLen := 0
@@ -53,3 +48,9 @@ func extractSubstrings(b []byte) []byte {
 
 	return buf.Bytes()
 }
+
+func isValidByte(c byte) bool {
+	// https://www.rapidtables.com/code/text/ascii-table.html
+	// split on anything that is not ascii space through tilde
+	return c > 31 && c < 127
+}