Compare commits

..
Author SHA1 Message Date
Graham Steffaniak fe4ebbf86b fix(utils): read only the first 8KB in IsTextFile
IsTextFile used os.ReadFile, which loaded the entire file into
memory and only then sliced off the first 8KB for inspection.
Checking a multi-GB file therefore read the whole file just to
look at its head — and IsTextFile runs on every file-watcher
event, so a large file change could spike memory for no reason.

Read at most sampleSize bytes with os.Open + io.ReadFull instead.
Behavior is unchanged: the checks still see exactly the first
8KB (or the whole file when it is smaller), empty files are
still text, and open/read errors are still returned.

Add test cases locking in the sampling contract: a large text
file is text, and binary content past the first 8KB does not
affect the result.
2026-11-15 10:05:52 -05:00
2 changed files with 27 additions and 9 deletions

No files matched your search

+15 -9
View File
@@ -1,6 +1,7 @@
package utils
import (
"io"
"net/http"
"os"
"unicode/utf8"
@@ -14,19 +15,24 @@ import (
func IsTextFile(realPath string) (bool, error) {
const sampleSize = 8192 // Read first 8KB to check
const maxNullByteRatio = 0.05 // Reject if more than 5% null bytes
// Read sample from file
content, err := os.ReadFile(realPath)
// Read only the first sampleSize bytes. The previous implementation
// used os.ReadFile, which loaded the entire file into memory and
// only then sliced off the first 8KB — so checking a multi-GB file
// (e.g. on a file-watcher event) read the whole file for no reason.
f, err := os.Open(realPath)
if err != nil {
return false, err
}
// Empty files are considered text
if len(content) == 0 {
return true, nil
defer f.Close()
buf := make([]byte, sampleSize)
n, err := io.ReadFull(f, buf)
if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF {
return false, err
}
// Use sample for large files to avoid reading entire file into memory
sample := content
if len(content) > sampleSize {
sample = content[:sampleSize]
sample := buf[:n]
// Empty files are considered text
if len(sample) == 0 {
return true, nil
}
// Check 1: Reject PDFs. A PDF with a long ASCII text layer looks like text
// for far more than sampleSize bytes.
+12
View File
@@ -27,6 +27,18 @@ func TestIsTextFile(t *testing.T) {
content: append([]byte("%PDF-1.3\n"+asciiPageStream), 0xFF, 0xD8, 0xFF, 0xE0),
want: false,
},
{
// Only the first 8KB is inspected, so a large text file is text.
name: "large text file larger than the sample",
content: []byte(strings.Repeat("hello world\n", 10000)),
want: true,
},
{
// Likewise, binary content past the first 8KB does not matter.
name: "binary content only after the sample",
content: append([]byte(strings.Repeat("plain text\n", 750)), make([]byte, 64)...),
want: true,
},
}
for _, tt := range tests {