Compare commits

...
79 Commits
Author SHA1 Message Date
Graham Steffaniak 885adcb3a2 Beta/v2.1.5 (#3389) 2026-11-13 19:32:18 -05:00
Graham Steffaniak aaa34e9bb3 Fix generation counters (#3391) 2026-11-13 18:17:06 -06:00
Graham Steffaniak 844f8f9fdd address coderabbit blocker comments (#3390) 2026-11-13 18:03:02 -06:00
Graham Steffaniak 9aba7b55ca updated dependencies (#3387) 2026-11-13 15:54:19 -06:00
Graham Steffaniak 7964b30e5d Fix checksum authenticated users on shares (#3386) 2026-11-13 15:19:53 -06:00
Graham Steffaniak 618de3fbcc Fix GHSA-823x-gh7f-grmf (#3385) 2026-11-13 14:49:39 -06:00
Graham Steffaniak a68fa0ea15 fix GHSA-jg32-ppcc-ph62 archive extraction symlink gap (#3384) 2026-11-13 14:14:10 -06:00
Graham Steffaniak 3bf3d2ce92 fix GHSA-2546-vf4j-vh9v symlink previews (#3383) 2026-11-13 14:03:44 -06:00
Graham Steffaniak f6cc16d1c8 fix subtitles GHSA-hfvf-2639-p2mq (#3382) 2026-11-13 13:45:38 -06:00
Graham Steffaniak 8a1b735e0d add more cache invalidaton for realpath (#3381) 2026-11-13 13:00:15 -06:00
A-d-r-iandGraham Steffaniak 10a26832dd fix(i18n): improve French translations (#3356)
Co-authored-by: Graham Steffaniak <42989099+gtsteffaniak@users.noreply.github.com>
2026-11-13 11:44:14 -06:00
Kevin B. 0dfab77d6d update frontend deps v2.1.5 (#3378) 2026-11-13 11:43:57 -06:00
Vladimir 7bd9c67eb7 fix(sidebar): keep source selector stationary during horizontal tree scrolling (#3368) 2026-11-13 11:40:09 -06:00
009be1a06d fix(http): restore Unix socket reverse-proxy behavior after #3309 (#3365)
Co-authored-by: Cursor Agent <cursoragent@cursor.com>
Co-authored-by: Graham Steffaniak <gtsteffaniak@users.noreply.github.com>
2026-11-13 11:39:42 -06:00
danielroizen 272a0fab9d fix(preview): respect EXIF orientation on embedded JPEG thumbnails (#3358) 2026-11-13 11:39:28 -06:00
Lykinandxunguangzlj-cloud 63a517de1f fix(picker): open the parent folder on a single click (#3355)
Co-authored-by: xunguangzlj-cloud <293371022+xunguangzlj-cloud@users.noreply.github.com>
2026-11-13 11:39:04 -06:00
Lykinandxunguangzlj-cloud e39b534531 fix(users): show password-change requirement in edit preferences (#3354)
Co-authored-by: xunguangzlj-cloud <293371022+xunguangzlj-cloud@users.noreply.github.com>
2026-11-13 11:38:14 -06:00
Graham Steffaniak 5d25220e94 fix advanced user toggle duplicate (#3353) 2026-11-07 10:52:59 -06:00
8355682773 fix(upload): don't pause while the server saves, and make the stall timeout a setting (#3346)
Co-authored-by: FandacqkaCorp <275647464+FandacqkaCorp@users.noreply.github.com>
Co-authored-by: Claude Sonnet 5.5 <noreply@anthropic.com>
Co-authored-by: Graham Steffaniak <42989099+gtsteffaniak@users.noreply.github.com>
2026-11-07 10:49:55 -06:00
Graham Steffaniak 48c1ce709b updated changelog v2.1.5 (#3350) 2026-11-06 17:34:04 -06:00
Graham Steffaniak 43682dc809 fix(users): keep requested profile values when creating a user (#3349) 2026-11-06 16:09:36 -06:00
Xyleme-git 80fba42076 Update fr.json (#3343) 2026-11-06 15:51:28 -06:00
Hromad-aandClaude Opus 5.5 a01ab6901c i18n(cz): fix Czech translations (#3333)
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-06 15:27:14 -06:00
Geekice1andGeekice1 e652538ea8 i18n: fix and improve Italian translation (it.json) (#3332)
Co-authored-by: Geekice1 <geekice1@users.noreply.github.com>
2026-11-06 15:26:49 -06:00
Peter Gao c974da7002 fix(i18n):Update Chinese translations for UI elements (#3330) 2026-11-06 15:26:37 -06:00
Konstantinos Chatzikokolakis eaee271b96 Square image previews: avoid the full-resolution crop copy in imaging.Fill (#3340) 2026-11-06 15:13:20 -06:00
HaneandClaude Opus 5.5 0d6b592046 fix(upload): serialize directory creation before uploading files (#3321)
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-06 15:10:52 -06:00
1791bb7a36 feat(cache): add server.cacheDirMaxSizeMB to cap the preview cache size (#3314)
Co-authored-by: FandacqkaCorp <275647464+FandacqkaCorp@users.noreply.github.com>
Co-authored-by: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-11-06 15:08:40 -06:00
86d4994152 fix(preview): open RAW, HEIC and rendered documents in the viewer when previews are disabled (#3313)
Co-authored-by: FandacqkaCorp <275647464+FandacqkaCorp@users.noreply.github.com>
Co-authored-by: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-11-06 15:03:43 -06:00
Graham Steffaniak 8415d5f1ff Fix install button (#3312) 2026-10-31 08:27:12 -05:00
Graham Steffaniak 85f934208c replace button with link 2026-10-31 08:20:44 -05:00
1022ecafef fix(frontend): address CodeRabbit follow-ups from #3276 and #3303 (#3311)
Co-authored-by: FandacqkaCorp <275647464+FandacqkaCorp@users.noreply.github.com>
Co-authored-by: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-10-31 08:17:44 -05:00
Graham Steffaniak 062ac9fb2d Beta/v2.1.4 (#3307) 2026-10-30 19:14:33 -05:00
Graham SteffaniakandGraham Steffaniak cecdd79fcc Restrict Unix socket forwarded-header trust with peer credentials (#3309)
Co-authored-by: Graham Steffaniak <gtsteffaniak@users.noreply.github.com>
2026-10-30 18:40:10 -05:00
Graham Steffaniak ac50ecfd18 fix: CodeRabbit follow-ups from #3307 (quick-win bugfixes) (#3308) 2026-10-30 17:56:50 -05:00
Graham Steffaniak 71cc2f3f1b fixed linting 2026-10-30 16:30:00 -05:00
Graham Steffaniak 1b3f47f176 fix sensitive data GHSA-892f-9h5p-mfc8 (#3306) 2026-10-30 16:25:25 -05:00
Graham Steffaniak 219b64b5ec addressed deep copy issue 2026-10-30 15:37:45 -05:00
Graham Steffaniak 3ef4928ed7 fix: prevent visible border in generated maskable PWA icons (#3300) 2026-10-30 15:33:33 -05:00
Graham Steffaniak df0a5ad10c fix sensitive data GHSA-892f-9h5p-mfc8 2026-10-30 15:22:46 -05:00
Graham Steffaniak a43d0ab2d3 updated changelog 2026-10-30 15:05:34 -05:00
Graham Steffaniak 9de09e5ec8 feat(search): sidebar File actions and close button in advanced search (#3303) 2026-10-30 15:02:51 -05:00
Graham Steffaniak 9d38c686a1 updated changelog 2026-10-30 15:01:15 -05:00
Graham Steffaniak 1cfca112e4 fix empty view after login if source named 'Files' (#3304) 2026-10-30 14:51:04 -05:00
Graham Steffaniak f540e22dc9 fix(quotas): complete delete accounting for bulk and nested resets (#3305) 2026-10-30 13:26:38 -05:00
Graham Steffaniak dade856ebe fix(move-copy): require choosing a destination when none can be inferred (advanced search) (#3276) 2026-10-30 13:19:15 -05:00
Graham SteffaniakandCursor 2f1da21253 fix(quotas): complete delete accounting for bulk and nested resets
Follow-up to #3235 / #3163: reset accounted folder quotas inside deleted
directories without index lookups, wire the same prepare/apply path into
bulk and public delete handlers, and dedupe scope quota selection.

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-10-30 13:04:16 -05:00
Kevin B. de8d2acd43 fix empty view after login 2026-10-30 13:04:15 -05:00
Graham Steffaniak 6375be202e fix: release nested folder quotas when deleting their ancestor (#3235) 2026-10-30 12:59:12 -05:00
FandacqkaCorpandClaude Sonnet 5.5 816b99cee0 feat(search): add a close button to advanced search
On tool pages the top bar button only toggles the sidebar, so the only ways
out of advanced search were the sidebar or the browser back button.

Add an icon-only grey X at the top-left of the advanced search page (first
element of the tools wrapper); the top bar button is left as it is. It clears
the selection and goes back to the page the search was opened from, or to
/files when there is none (e.g. the URL was opened directly).

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-10-30 16:14:12 +01:00
FandacqkaCorpandClaude Sonnet 5.5 bf349a7dcd feat(search): show the sidebar File actions in advanced search
The sidebar File actions button only exists in the listing, so in advanced
search the actions on the selected results (select all, download, copy, move)
were only reachable through the context menu, which is hard to find, especially
on touch devices.

Show the same sidebar button in advanced search. There it opens the context
menu on the current selection instead of the create-only menu, and it doesn't
depend on create permissions (the menu already checks permissions per
source). The menu now always offers Select multiple in advanced search, so
single results can be selected when "open with a single click" is on, and hides
the create (+) toggle, since results have no single folder to create in.

The button still honours the "hide sidebar file actions" preference and isn't
shown in shares.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-10-30 15:55:48 +01:00
Matt Van Horn db6504bf80 fix: prevent visible border in generated maskable PWA icons
Maskable icon generation now alpha-composites the resized artwork over
the opaque background using `OverlayCenter`. Added tests covering
transparent padding and opaque edges, verifying the output dimensions,
opacity, and background pixels outside the artwork.

Fixes #3187
2026-10-30 03:59:21 -07:00
FandacqkaCorpandClaude Sonnet 5.5 2019d573f8 fix(move-copy): require choosing a destination when there is no source to fall back to
In advanced search the open request is the list of results and has no source.
The Copy/Move prompt then showed "Choose a source" but, if the button was
pressed without choosing anything, silently sent the items to the root of the
first selected item's source.

Disable the Copy/Move button until a destination source is chosen in that
case (no destination yet and no source in the open request, outside shares),
and use the existing "Choose a source" text as its tooltip. Normal listings
and shares are unchanged.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-10-30 11:05:37 +01:00
Matt Van Horn d716fc15ec test: share the nested quota delete fixture between API and WebDAV tests
TestResourceDeleteNestedQuota and TestWebDAVDeleteNestedQuota seeded the
same files, quotas and usage deltas. Move that setup into
seedNestedQuotaFixture and call it from both.
2026-10-30 00:28:00 -07:00
Graham Steffaniak 56e16608e4 fix(frontend): keep video loading spinner off when the fetch stalls mid-playback (#3289) 2026-10-29 19:19:47 -05:00
Graham Steffaniak c471fc135b fix(frontend): use native video fullscreen on iPhone (#3288) 2026-10-29 19:18:33 -05:00
Graham Steffaniak 5112cf87b1 fix(analytics): report freebsd instead of "other" in system.os (#3284) 2026-10-29 19:17:23 -05:00
Graham Steffaniak 3d7bac47ee fix(stream): play single-file share media on Windows (#3271) 2026-10-29 19:10:22 -05:00
vulragrag-star 2a38d2fa1e fix(users): enforce lockPassword on self-service password changes (#3283) 2026-10-29 19:01:34 -05:00
Matt Van Horn 82e110c08d test: cover scoped users in the API nested quota delete test
The API path is scope-relative while quota paths and the index are
source-relative. Run the test for a user scoped to /scoped as well as /,
which fails if the delete looks up the unscoped path.
2026-10-29 08:14:00 -07:00
Matt Van Horn f4c96078a1 Merge remote-tracking branch 'upstream/main' into fix/3163-release-nested-folder-quotas-on-delete
# Conflicts:
#	backend/internal/web/resource.go
#	backend/internal/web/webdav.go
#	backend/internal/web/webdav_test.go
2026-10-29 08:14:00 -07:00
975adedabb Address directory indexing bugs (#3292)
Co-authored-by: Lucas Siqueira <lucas.morais.siqueira@gmail.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
2026-10-28 10:29:42 -05:00
Graham Steffaniak 14ec0eba0d Fix indexing regressions (#3260) 2026-10-27 19:41:12 -05:00
Lucas de Morais SiqueiraandGraham Steffaniak 307d5f6591 fix(indexing): skip unreadable directory entries instead of failing the listing (#3184)
Co-authored-by: Graham Steffaniak <42989099+gtsteffaniak@users.noreply.github.com>
2026-10-27 19:25:01 -05:00
Get The Fork Off ff9df43356 fix(frontend): keep video loading spinner off when the fetch stalls mid-playback 2026-10-27 20:50:30 -03:00
Get The Fork Off 802e91aa05 fix(frontend): use native video fullscreen on iPhone 2026-10-27 20:48:46 -03:00
NefelibateLGandZCode Scout 28ef44cdec fix(analytics): report freebsd instead of "other" in system.os
The analytics snapshot classified every GOOS outside {linux, windows,
darwin} as "other", so the shipped freebsd-amd64 release binary
reported "os": "other" in the analytics payload even though
backend/.goreleaser.yaml builds freebsd as a first-class target
(#3244).

Add freebsd to the mapSystemOS whitelist, matching the release build
matrix, and add a table-driven test covering all four shipped GOOS
values plus the unsupported and empty fallbacks.

Fixes #3244

Co-authored-by: ZCode Scout <noreply@zcode.local>
Signed-off-by: NefelibateLG <3453510037@qq.com>
2026-10-27 09:16:06 +08:00
sarankumar1325 0ee0f35301 fix(stream): play single-file share media on Windows
shareRelativeDisplayName ran filepath.Base on slash-separated paths. On
Windows filepath.Base("/") returns "\", which passed the "/" check, so a
single-file share streamed with file=/ got "\" as its name instead of the
shared file's name. IsMediaStreamFile then rejected it and the stream
endpoint answered 403 "stream endpoint supports audio and video only".

Use path.Base, since both the share-relative path and the share path are
slash-separated.

Fixes #3270
2026-10-25 11:30:00 +05:30
Graham Steffaniak cc99aa542e updated with symlink extraction change (#3265) 2026-10-23 15:36:09 -05:00
8514da4a9a fix(auth): API token permission handling for customized tokens (#3255)
Co-authored-by: Cursor Agent <cursoragent@cursor.com>
Co-authored-by: Graham Steffaniak <gtsteffaniak@users.noreply.github.com>
2026-10-23 14:48:18 -05:00
Graham SteffaniakandDevin 4625b89528 fix webp preview support (#3259)
Co-authored-by: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>
2026-10-22 15:34:57 -05:00
Jay Hemnani b2a07196b7 fix(archive): extract tar hard links instead of dropping them (#3234) 2026-10-22 15:26:16 -05:00
Graham Steffaniak 0a8698ee4d updated changelog 2026-10-22 15:24:08 -05:00
ec8ad36c95 Feature/html preview navigation and media (#3131)
Co-authored-by: Graham Steffaniak <graham.steffaniak@autodesk.com>
Co-authored-by: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>
2026-10-22 15:13:31 -05:00
vulragrag-star 2c0c1b4bb2 fix(frontend): show the share banner to public visitors (#3250) 2026-10-22 13:38:49 -05:00
Thorsten Liborius 6fa71bd13d fix(frontend): show correct relative expiration time for sharesFix/share expiry relative time (#3247) 2026-10-21 18:47:44 -05:00
Graham Steffaniakandcoderabbitai[bot] 06aa16e7d7 fix raw embedded image support - rtf rw2 raw rwl (#3243)
Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com>
2026-10-20 17:43:38 -05:00
Matt Van Horn bb45ebcce1 fix: avoid shadowing err in nested quota delete tests 2026-10-20 10:41:27 -07:00
Matt Van Horn 26665efc13 fix: release nested folder quotas when deleting their ancestor
Adds shared preparation and application helpers for API and WebDAV
deletion accounting. Before deletion removes index metadata, each nested
accounted quota is measured independently, while covering, scope, and
share quotas use the deleted item's size. Adjustments are applied only
after deletion succeeds, and unknown sizes are omitted. API deletion
also derives the source index path from the item's real path.

Fixes #3163
2026-10-20 04:39:51 -07:00
198 changed files with 10027 additions and 2772 deletions

No files matched your search

+64 -1
View File
@@ -2,6 +2,69 @@
All notable changes to this project will be documented in this file. For commit guidelines, please refer to [Standard Version](https://github.com/conventional-changelog/standard-version).
## v2.1.5
**Security**:
- [High] Server-side archive create (`POST /api/resources/archive`) now resolves the full destination path for writes via symlink-scoped mutation resolution, closing arbitrary host file overwrite through an in-scope symlink leaf (GHSA-jg32-ppcc-ph62) reporter @manus-use
- [Moderate] `GET /api/media/subtitles` no longer follows symlink external subtitle sidecars beside the video file, and public share requests resolve sidecar paths under the share prefix before read (GHSA-hfvf-2639-p2mq) reporter @manus-use
- [Moderate] Directory motion-preview now resolves symlinked children within the caller's scope, closing the incomplete GHSA-g2q5 remediation path through `getDirectoryPreview` (GHSA-2546-vf4j-vh9v) reporter @manus-use
- [Moderate] RealPathCache entries for affected index paths are invalidated after symlink-scoped mutations so stale resolved paths cannot bypass containment on subsequent create, write, delete, or move operations (GHSA-j764-qg3m-rhv7 follow-up, #3381)
- [Moderate] File watcher, download, view, upload, copy, and WebDAV now enforce per-path access rules on symlink-resolved targets, so DENY rules and scope boundaries cannot be bypassed via a link inside an allowed folder (GHSA-823x-gh7f-grmf) reporters @ch4rlesexe @manus-use @zohuyhieuzo03
**New Features**:
- feat(cache): add server.cacheDirMaxSizeMB to cap the preview cache size (#3314)
- Square image previews: avoid the full-resolution crop copy in imaging.Fill (#3340)
**Bug Fixes**:
- fix(http): Unix socket hardening — default `http.socketFileMode` is `0660`; the socket is created under a restrictive umask and then `chmod`/`chown` apply your configured mode; on filesystem sockets, forwarded headers require `http.unixSocketTrustedUIDs` and/or `http.unixSocketTrustedGIDs`, with `http.unixSocketTrustAllLocalPeers: true` as an explicit opt-in for pre-#3309 trust-all behavior (startup warning) (#3309)
- fix(users): save the prompt close button and new file templates when editing a user (#3342) (#3323)
- fix(upload): serialize directory creation before uploading files (#3321)
- fix(preview): open RAW, HEIC and rendered documents in the viewer when previews are disabled (#3313)
- fix(users): keep requested profile values when creating a user
- fix advanced user toggle duplicate (#3353) (#3324)
- fix(upload): don't pause while the server saves, and make the stall timeout a setting (#3346)
- fix(preview): respect EXIF orientation on embedded JPEG thumbnails (#3358)
- fix(picker): open the parent folder on a single click (#3355)
- fix(users): show password-change requirement in edit preferences (#3354)
- fix(sidebar): keep source selector stationary during horizontal tree scrolling (#3368)
- fix(http): restore Unix socket reverse-proxy behavior after #3309 hardening (#3365)
- fix(share): file checksum generation on public shares uses `GET /public/api/resources?checksum=…` with the share hash and session cookie; anonymous visitors receive 401 and no longer see the checksum section in the file info dialog (#3360)
**Upgrade notes** (Unix socket listeners):
- If you listen on a filesystem `http.socket` with `http.trustProxyHeaders: true`, set `http.unixSocketTrustedUIDs` and/or `http.unixSocketTrustedGIDs` to the UID/GID(s) from Linux peer credentials (`SO_PEERCRED`) that may set `X-Forwarded-*`. A GID allowlist trusts every peer with that GID (for example when the proxy and FileBrowser share a group), not only one process. If you relied on empty allowlists before, set `http.unixSocketTrustAllLocalPeers: true` (a warning is logged).
- Default `http.socketFileMode` is `0660`; use optional `http.socketGroup` when the proxy runs as another user in the same group.
## v2.1.4
**Security**:
- [Low] `GET /api/settings` and `GET /api/settings?property=auth` no longer return cleartext secrets (`auth.key`, `totpSecret`, `adminPassword`, OIDC `clientSecret`, LDAP bind password, recaptcha secret, OnlyOffice secret); secret fields are replaced with `**hidden**` in the response, and the YAML export now redacts the recaptcha and LDAP secrets whose comments lacked the `secret:` marker (GHSA-892f-9h5p-mfc8) reporter @paulmossman
**New Features**:
- feat(search): sidebar File actions and close button in advanced search (#3303) (#2479) (#3303)
**Bug Fixes**:
- Deleting a directory now releases folder quotas rooted inside it instead of leaving them charged for files that no longer exist; applies to API and WebDAV deletes (#3163) (#3235)
- Bulk and public share deletes now release accounted quota usage; nested folder quotas reset when an ancestor is removed (#3163) (#3305)
- Directory listings skip entries that return permission denied or not-found on per-entry stat instead of failing the whole folder (e.g. unreadable NTFS junctions on CIFS); skipped entries are omitted from the index like `ls -l` (#3184)
- Preview generation for webp broken (#3254) (#3259)
- Share banner never shows to public visitors, returns bannerUrl but not banner (#3246) (#3250)
- fix html/markdown relative file preview support (#3131) (#3129) (#2355)
- fix(archive): extract tar hard links instead of dropping them (#3234)
- fix(auth): apply stored permission caps for customized API tokens with no global flags enabled (#3255)
- Archive extraction writes regular and hard-link entries at the entry path instead of following an existing symlink at that path (tar and zip) (#3265)
- fix(move-copy): require choosing a destination source in advanced search when none can be inferred, instead of silently copying to the source root (#3276)
- fix(stream): single-file shares can now stream their own media on Windows by using slash-based path handling (#3270) (#3271)
- fix(users): `lockPassword` is now enforced on self-service password changes; non-admins get 403 while admin resets still work (#3279) (#3283)
- fix(analytics): the FreeBSD release binary now reports `freebsd` in `system.os` instead of `other` (#3244) (#3284)
- fix(frontend): share expiration now reads backend Unix-seconds timestamps correctly instead of showing dates decades off (#3247)
- fix(frontend): video on iPhone now uses the native fullscreen player instead of the in-page fallback (#3113) (#3288)
- fix(frontend): the video loading spinner no longer stays on for the rest of playback when the fetch stalls but the buffer can continue (#3287) (#3289)
- fix(frontend): the share banner is now shown to public visitors, using `bannerUrl` instead of the never-set `banner` field (#3246) (#3250)
- fix(preview): camera RAW files (Panasonic `.rw2`/`.raw`/`.rwl`, Olympus `.orf`, `.cr2`/`.dng`) extract embedded previews from TIFF strips/SubIFDs and no longer return 415 when EXIF decoding fails (#3219) (#3243)
- fix(indexing): restore lazy listing persistence with atomic folder-size deltas and ancestor propagation, and force a full scan for scanners that missed their slot while the process was down (#3260)
- fix(indexing): directory listings skip entries that cannot be accessed or have vanished instead of failing the whole listing; remaining errors still fail and fully-vanished nonempty directories return an error (#3292)
- fix(frontend): redirects from `/files` stay inside the files area so a source named "Files" no longer leaves an empty view after login (#3167) (#3304)
## v2.1.3
**Security**:
@@ -15,7 +78,7 @@ All notable changes to this project will be documented in this file. For commit
- [Moderate] Authenticated `GET /api/resources/view` no longer accepts view tokens minted for a public share when the caller lacks view permission on the requested source (GHSA-wxj5-g3qp-g732) reporter @BlueN0r
- [Low] SSE source-update and user-targeted delivery hold the client registry lock through nonblocking sends, preventing concurrent map mutation and send-on-closed-channel panics when Realtime clients connect, disconnect, or shut down (GHSA-p9wm-f5jj-75pj) reporter @tao0845
- [High] Incomplete symlink-containment remediation: an authenticated non-admin user could create a relative symlink via archive extraction, relocate it with WebDAV MOVE, and write through it to arbitrary host paths with the service account's privileges (GHSA-3j46-5w28-f8p2) reporter @Saku0512
- [Moderate] Source-root containment bypass: create, write, delete, and move operations now resolve symlinks uncached and enforce the caller's scope, closing escapes through pre-existing out-of-source symlinks on upload, PUT, PATCH, delete, and WebDAV endpoints (GHSA-j764-qg3m-rhv7) reporter @chaoming0625
- [Moderate] Source-root containment bypass: create, write, delete, and move operations now resolve symlinks uncached and enforce the caller's scope, closing API escapes through pre-existing out-of-source symlinks on upload, PUT, PATCH, delete, and WebDAV endpoints (GHSA-j764-qg3m-rhv7) reporter @chaoming0625
- [Moderate] Preview reads resolve symlinks within the caller's scope so authenticated preview cannot follow an out-of-scope symlink target (GHSA-g2q5-r6ch-v25f) reporter @tao0845
- [Moderate] Archive downloads and extraction no longer read or write through symlinks that resolve outside the caller's share or user scope; escaping archive-walk entries are skipped (GHSA-qqf3-2wvq-q4w9) reporter @SmokinCaterpillar
- [Moderate] Share directory-archive downloads now apply the share owner's access rules to every entry, closing a bypass where anonymous share visitors could download admin-denied content (GHSA-9fcr-gm58-p839) reporter @Tan-JunWei
+20 -20
View File
@@ -3,7 +3,7 @@ module github.com/gtsteffaniak/filebrowser/backend
go 1.27.0
require (
github.com/alecthomas/kong v1.16.1
github.com/alecthomas/kong v1.17.0
github.com/asdine/storm/v3 v3.2.1
github.com/coreos/go-oidc/v3 v3.21.0
github.com/coreos/go-systemd/v22 v22.7.0
@@ -12,34 +12,34 @@ require (
github.com/evanoberholster/imagemeta v1.1.0
github.com/gen2brain/go-fitz v1.28.2
github.com/go-ldap/ldap/v3 v3.4.15
github.com/go-playground/validator/v10 v10.30.5
github.com/go-playground/validator/v10 v10.30.6
github.com/go-webauthn/webauthn v0.18.2
github.com/goccy/go-yaml v1.19.2
github.com/golang-jwt/jwt/v4 v4.5.2
github.com/google/go-cmp v0.7.0
github.com/google/uuid v1.6.0
github.com/gtsteffaniak/go-cache v1.1.0
github.com/gtsteffaniak/go-ffmpeg v1.4.0
github.com/gtsteffaniak/go-ffmpeg v1.6.2
github.com/gtsteffaniak/go-logger v1.1.0
github.com/kovidgoyal/imaging v1.9.0
github.com/mattn/go-sqlite3 v1.14.52
github.com/mattn/go-sqlite3 v1.14.53
github.com/pquerna/otp v1.5.0
github.com/spf13/afero v1.15.0
github.com/stretchr/testify v1.12.1
github.com/swaggo/http-swagger v1.3.4
github.com/swaggo/swag v1.16.6
golang.org/x/crypto v0.58.0
golang.org/x/image v0.47.0
golang.org/x/mod v0.42.0
golang.org/x/net v0.61.0
golang.org/x/oauth2 v0.37.0
golang.org/x/sys v0.49.0
golang.org/x/term v0.47.0
golang.org/x/text v0.43.0
golang.org/x/crypto v0.59.0
golang.org/x/image v0.48.0
golang.org/x/mod v0.43.0
golang.org/x/net v0.62.0
golang.org/x/oauth2 v0.38.0
golang.org/x/sys v0.50.0
golang.org/x/term v0.48.0
golang.org/x/text v0.44.0
golang.org/x/time v0.16.0
golang.org/x/tools v0.52.0
gopkg.in/yaml.v3 v3.0.1
modernc.org/sqlite v1.60.1
modernc.org/sqlite v1.62.0
)
require (
@@ -49,12 +49,12 @@ require (
github.com/dsoprea/go-logging v0.0.0-20200710184922-b02d349568dd // indirect
github.com/dsoprea/go-utility/v2 v2.0.0-20221003172846-a3e1774ef349 // indirect
github.com/dustin/go-humanize v1.1.0 // indirect
github.com/ebitengine/purego v0.11.2 // indirect
github.com/fxamacker/cbor/v2 v2.9.6 // indirect
github.com/ebitengine/purego v0.11.3 // indirect
github.com/fxamacker/cbor/v2 v2.10.0 // indirect
github.com/gabriel-vasile/mimetype v1.4.16 // indirect
github.com/go-asn1-ber/asn1-ber v1.5.8 // indirect
github.com/go-errors/errors v1.5.1 // indirect
github.com/go-jose/go-jose/v4 v4.1.6 // indirect
github.com/go-jose/go-jose/v4 v4.1.9 // indirect
github.com/go-openapi/jsonpointer v1.0.2 // indirect
github.com/go-openapi/jsonreference v1.0.3 // indirect
github.com/go-openapi/spec v1.0.1 // indirect
@@ -70,7 +70,7 @@ require (
github.com/go-viper/mapstructure/v2 v2.5.0 // indirect
github.com/go-webauthn/x v0.3.1 // indirect
github.com/golang-jwt/jwt/v5 v5.3.1 // indirect
github.com/golang/geo v0.0.0-20260928092222-7d12f68cfadb // indirect
github.com/golang/geo v0.0.0-20261106073233-5e8d6967632a // indirect
github.com/golang/protobuf v1.5.4 // indirect
github.com/golang/snappy v0.0.4 // indirect
github.com/google/go-tpm v0.9.8 // indirect
@@ -89,13 +89,13 @@ require (
github.com/tinylib/msgp v1.6.5 // indirect
github.com/x448/float16 v0.8.4 // indirect
go.etcd.io/bbolt v1.5.0 // indirect
go.yaml.in/yaml/v3 v3.0.5 // indirect
golang.org/x/sync v0.24.0 // indirect
go.yaml.in/yaml/v3 v3.0.6 // indirect
golang.org/x/sync v0.25.0 // indirect
google.golang.org/appengine v1.6.8 // indirect
google.golang.org/protobuf v1.36.12 // indirect
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
modernc.org/libc v1.79.0 // indirect
modernc.org/libc v1.81.0 // indirect
modernc.org/mathutil v1.7.1 // indirect
modernc.org/memory v1.12.1 // indirect
)
+46 -46
View File
@@ -8,8 +8,8 @@ github.com/Sereal/Sereal v0.0.0-20190618215532-0b8ac451a863 h1:BRrxwOZBolJN4gIwv
github.com/Sereal/Sereal v0.0.0-20190618215532-0b8ac451a863/go.mod h1:D0JMgToj/WdxCgd30Kc1UcA9E+WdZoJqeVOuYW7iTBM=
github.com/alecthomas/assert/v2 v2.11.0 h1:2Q9r3ki8+JYXvGsDyBXwH3LcJ+WK5D0gc5E8vS6K3D0=
github.com/alecthomas/assert/v2 v2.11.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k=
github.com/alecthomas/kong v1.16.1 h1:ixhCt93XkJ98kGposQ54+bl0IK6XwqB40AsMynU7Z8E=
github.com/alecthomas/kong v1.16.1/go.mod h1:wrlbXem1CWqUV5Vbmss5ISYhsVPkBb1Yo7YKJghju2I=
github.com/alecthomas/kong v1.17.0 h1:f52BVWChYMDkMkCoqkEWRuFOk/K4B5LO7xIijeLJRdk=
github.com/alecthomas/kong v1.17.0/go.mod h1:wrlbXem1CWqUV5Vbmss5ISYhsVPkBb1Yo7YKJghju2I=
github.com/alecthomas/repr v0.5.2 h1:SU73FTI9D1P5UNtvseffFSGmdNci/O6RsqzeXJtP0Qs=
github.com/alecthomas/repr v0.5.2/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4=
github.com/alexbrainman/sspi v0.0.0-20250919150558-7d374ff0d59e h1:4dAU9FXIyQktpoUAgOJK3OTFc/xug0PCXYCqU0FgDKI=
@@ -47,12 +47,12 @@ github.com/dsoprea/go-utility/v2 v2.0.0-20221003172846-a3e1774ef349 h1:DilThiXje
github.com/dsoprea/go-utility/v2 v2.0.0-20221003172846-a3e1774ef349/go.mod h1:4GC5sXji84i/p+irqghpPFZBF8tRN/Q7+700G0/DLe8=
github.com/dustin/go-humanize v1.1.0 h1:dbKTrvD0klcbBV/h4AWJdMuZogJACoMlvWIWZ5b2xWg=
github.com/dustin/go-humanize v1.1.0/go.mod h1:hc1CvRkJMsgxqjmjMQF3QNRAZBwY8AXBAzKYoSX9sFI=
github.com/ebitengine/purego v0.11.2 h1:WWiTx8rD8zYdOwSj8O/NiNbsOvSAgJMTlyV1XlbUSeU=
github.com/ebitengine/purego v0.11.2/go.mod h1:DCHPP08djqhNSoTfImcnHYQRZmd0qhakvrozqaEYhGQ=
github.com/ebitengine/purego v0.11.3 h1:OB+JLpdm7ioymhyA+6s5kbb/uOuYycAgL8NNTFc5MLg=
github.com/ebitengine/purego v0.11.3/go.mod h1:DCHPP08djqhNSoTfImcnHYQRZmd0qhakvrozqaEYhGQ=
github.com/evanoberholster/imagemeta v1.1.0 h1:0iBtgVxLORZTZAlKxRcNodBmjMfHCSFwUh8Asmi83oI=
github.com/evanoberholster/imagemeta v1.1.0/go.mod h1:nmf6le+qSyoRAXgp2psPbtfkxcEgf/AZGgai9pnN8d0=
github.com/fxamacker/cbor/v2 v2.9.6 h1:NHZPxuUjJXQjX9itA785y/5NKSlI4HQ+S7noZsmI7hs=
github.com/fxamacker/cbor/v2 v2.9.6/go.mod h1:4EaZp6ClVunETjQ4GHmGNzysDK91Lr+r6Yz6tJqzVCw=
github.com/fxamacker/cbor/v2 v2.10.0 h1:HVF3T5vYwLqO81ebPirWEPNv2SMrUMD3HLnKgEr6uWE=
github.com/fxamacker/cbor/v2 v2.10.0/go.mod h1:uSwlToFuT9BZAKRxxVm3qQC8DR4tlWl7D6pf20iRYH4=
github.com/gabriel-vasile/mimetype v1.4.16 h1:5izNFuvA37st1dDRAcA7TtmngBmRgiIPDmTtiEvjx1E=
github.com/gabriel-vasile/mimetype v1.4.16/go.mod h1:3oFFwxX8kTklZwzIl2aQEDZT0lKY8xH92ST88bAwba4=
github.com/gen2brain/go-fitz v1.28.2 h1:845G85N5TUgnq5oDqyYrW0JvehAkeo35UkkK2dJtW1M=
@@ -65,8 +65,8 @@ github.com/go-errors/errors v1.1.1/go.mod h1:psDX2osz5VnTOnFWbDeWwS7yejl+uV3FEWE
github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og=
github.com/go-errors/errors v1.5.1 h1:ZwEMSLRCapFLflTpT7NKaAc7ukJ8ZPEjzlxt8rPN8bk=
github.com/go-errors/errors v1.5.1/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og=
github.com/go-jose/go-jose/v4 v4.1.6 h1:sNRkftxFJxwsILiQukqp5z83QBSmntVwmgRY7AqBDO4=
github.com/go-jose/go-jose/v4 v4.1.6/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08=
github.com/go-jose/go-jose/v4 v4.1.9 h1:pKfeQkig1NmgK4hJrZYXLEYURXfoJB61Larw0uIW4gY=
github.com/go-jose/go-jose/v4 v4.1.9/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08=
github.com/go-ldap/ldap/v3 v3.4.15 h1:eRB4f4Ecn1POL1mO86+uUDlecmyWsdiJwh60bAUS4UY=
github.com/go-ldap/ldap/v3 v3.4.15/go.mod h1:jeD0c98EdZQCKKKKe4kTWs5aCf6y+j8KuYsYLznZSQk=
github.com/go-openapi/jsonpointer v1.0.2 h1:NbW7xT2b+l7x+JefiPGDsZWsrLk1ogrz4AHK1TTzMt8=
@@ -102,8 +102,8 @@ github.com/go-playground/locales v0.14.2 h1:d8UmcrM6Nip0hfGZKLGpAvZH37XB4TS0xzK9
github.com/go-playground/locales v0.14.2/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY=
github.com/go-playground/universal-translator v0.18.2 h1:LCsMLC9RzmbUMNUPVYD15dmcjwYAJhmX8mPZRW4rAVU=
github.com/go-playground/universal-translator v0.18.2/go.mod h1:67VZIMp5lQpDWlnStOct22q1bkdJGJqHghbOtmkawxk=
github.com/go-playground/validator/v10 v10.30.5 h1:YyCXvVShZbs2Sm3Mb53eNOlhRXctSOzW5QJAouCTZL4=
github.com/go-playground/validator/v10 v10.30.5/go.mod h1:wEqiaov48pXX1kjhc3Da8y0M0Dtg/BK7gurFBLgwFrQ=
github.com/go-playground/validator/v10 v10.30.6 h1:E+mymlPNRzAMNz1FShrRvLeGWcprF5w305eLLgWieuo=
github.com/go-playground/validator/v10 v10.30.6/go.mod h1:jPLL8SKCXGe/g87QmEyNw3NO28nmGt5GCnZnufm7rj4=
github.com/go-viper/mapstructure/v2 v2.5.0 h1:vM5IJoUAy3d7zRSVtIwQgBj7BiWtMPfmPEgAXnvj1Ro=
github.com/go-viper/mapstructure/v2 v2.5.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
github.com/go-webauthn/webauthn v0.18.2 h1:0BeftmEHU7i3Dv0VFwBtidy/ba37Vcdjvqst9EYu8Sk=
@@ -119,8 +119,8 @@ github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArs
github.com/golang/geo v0.0.0-20190916061304-5b978397cfec/go.mod h1:QZ0nwyI2jOfgRAoBvP+ab5aRr7c9x7lhGEJrKvBwjWI=
github.com/golang/geo v0.0.0-20200319012246-673a6f80352d/go.mod h1:QZ0nwyI2jOfgRAoBvP+ab5aRr7c9x7lhGEJrKvBwjWI=
github.com/golang/geo v0.0.0-20210211234256-740aa86cb551/go.mod h1:QZ0nwyI2jOfgRAoBvP+ab5aRr7c9x7lhGEJrKvBwjWI=
github.com/golang/geo v0.0.0-20260928092222-7d12f68cfadb h1:z//sB+BhCYNvUrk19E8aWqNx+BBom8p/uD/nh95N/5U=
github.com/golang/geo v0.0.0-20260928092222-7d12f68cfadb/go.mod h1:Mymr9kRGDc64JPr03TSZmuIBODZ3KyswLzm1xL0HFA8=
github.com/golang/geo v0.0.0-20261106073233-5e8d6967632a h1:GLMvsMlp+kWJbqjyIav7/LBks3hMVyANhPVy+tSDA5M=
github.com/golang/geo v0.0.0-20261106073233-5e8d6967632a/go.mod h1:bJozU6ZRa8d29WpXkTQZ42lWCOJ+SC3XA8sXnLqA5IU=
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
@@ -137,14 +137,14 @@ github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY=
github.com/google/go-tpm-tools v0.3.13-0.20230620182252-4639ecce2aba h1:qJEJcuLzH5KDR0gKc0zcktin6KSAwL7+jWKBYceddTc=
github.com/google/go-tpm-tools v0.3.13-0.20230620182252-4639ecce2aba/go.mod h1:EFYHy8/1y2KfgTAsx7Luu7NGhoxtuVHnNo8jE7FikKc=
github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3 h1:LMLX+LgTNWpfvCBdFebv6EsYotImrt/Ppc5cXIriCSo=
github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3/go.mod h1:jl5iWTm0/hd5PjEYEOuwAJ57L/CibdZfrqZ5XA5GrCk=
github.com/google/pprof v0.0.0-20261016150608-1facf8f91b50 h1:PlNWJRxu5APSV0pWqfIsZ0qt9OOGHSWfZQ4zvx+WgWs=
github.com/google/pprof v0.0.0-20261016150608-1facf8f91b50/go.mod h1:yUo1AMMpsPIDFGuwDK8nLcf8+vvAd5l2O6JDZ6hpwrM=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/gtsteffaniak/go-cache v1.1.0 h1:ni/2cJpCxpGmaMZd92iTDIGOy+yUcMcxQuW7bSKhkCE=
github.com/gtsteffaniak/go-cache v1.1.0/go.mod h1:rHUZvf4huhMX/bnNK4/oQ3sNAE2fYC2jFpAWZiBPWac=
github.com/gtsteffaniak/go-ffmpeg v1.4.0 h1:XkNFlx6koCCh8cFXn9HxYEEpvLlbnxmIvWX+5HLuN+8=
github.com/gtsteffaniak/go-ffmpeg v1.4.0/go.mod h1:FefhoD+KZzHWqBaUpntcKp6lXtUzdFW00F74axnENUQ=
github.com/gtsteffaniak/go-ffmpeg v1.6.2 h1:gtvBg3G4PHO1Z/ZIPEK/XP4XT/g+l9S0Rpmmq6v5tnA=
github.com/gtsteffaniak/go-ffmpeg v1.6.2/go.mod h1:FefhoD+KZzHWqBaUpntcKp6lXtUzdFW00F74axnENUQ=
github.com/gtsteffaniak/go-logger v1.1.0 h1:baFds+iibuLLNpNLwygz32bDqdsunI6dGWcdxwMIVKI=
github.com/gtsteffaniak/go-logger v1.1.0/go.mod h1:rapapTju4D7bwt6h4JOdZNUXDjnp0WkTBd6TN5yWglI=
github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8=
@@ -185,8 +185,8 @@ github.com/leodido/go-urn v1.5.0 h1:pLqT2kq1zpHW/1D18QMjMpdtX7cekxqtJJjg5ANyWw0=
github.com/leodido/go-urn v1.5.0/go.mod h1:9BORnCDhdPBJNDEX+w1bJisa8yOKYi116VeO96s4ifE=
github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI=
github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A=
github.com/mattn/go-sqlite3 v1.14.52 h1:wVbm2Qnf4OXkqhBTSPuCRZDRnxfbVrrmiCEroVdog8U=
github.com/mattn/go-sqlite3 v1.14.52/go.mod h1:6JTjA44L93a0QCyJef5YvlPoKXntQPjzWv5gtm9sB6w=
github.com/mattn/go-sqlite3 v1.14.53 h1:mnTM0owzUMrKuLo+0w51NUiB0ir2Fo7kFXlPt/l3BGs=
github.com/mattn/go-sqlite3 v1.14.53/go.mod h1:6JTjA44L93a0QCyJef5YvlPoKXntQPjzWv5gtm9sB6w=
github.com/ncruces/go-strftime v1.1.0 h1:vtPmPWMkb539RGV9WpWT9EwJGvrOCKvMNI4obWL47Hk=
github.com/ncruces/go-strftime v1.1.0/go.mod h1:yVaq7iwCN09vHRrKBrrflXT0M9tTMG2KmZ9Fj7xeG/g=
github.com/philhofer/fwd v1.2.1 h1:UtpS73q8bFgy/yH3gCq8LSNvj5gv/FowNOiWZa7uKwQ=
@@ -229,17 +229,17 @@ go.etcd.io/bbolt v1.5.0 h1:S7GAl7Fxv12yohbwFfIbQCGDWbQbtDGPET4P/bD4lxU=
go.etcd.io/bbolt v1.5.0/go.mod h1:mkltfYE5aUHQxUct9N9V+Kp7aSjFqjgrhcXIS70Lrdk=
go.uber.org/mock v0.6.0 h1:hyF9dfmbgIX5EfOdasqLsWD6xqpNZlXblLB/Dbnwv3Y=
go.uber.org/mock v0.6.0/go.mod h1:KiVJ4BqZJaMj4svdfmHM0AUx4NJYO8ZNpPnZn1Z+BBU=
go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw=
go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg=
go.yaml.in/yaml/v3 v3.0.6 h1:zcDfvjXEqTQmpjniUBLc91xYJMB2c4QOFxEF/LCEckM=
go.yaml.in/yaml/v3 v3.0.6/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.58.0 h1:COkYLr4k7nDI4r6QtMML2AfzeIswWHAJthLS6K/M/54=
golang.org/x/crypto v0.58.0/go.mod h1:Xh+kl5A+M0gMiWDytH2Hhr5Z7WbNCpv/eB/EFgfSk+w=
golang.org/x/image v0.47.0 h1:za/8gVz++PPZqmZJBNSISICN5Rrj5zixP/8K++fgmqo=
golang.org/x/image v0.47.0/go.mod h1:IkN5o4A7vq55vXN8FWjjnhg9Rs2GOtffqsSi9mWAKYE=
golang.org/x/crypto v0.59.0 h1:xS2jhqYbzNrpvkWP2YTDf4TJwBy3ykR03i8vFDxEc7c=
golang.org/x/crypto v0.59.0/go.mod h1:veDyUwATGs3c+vN3VfTbzViMGXeTBk3WyhBOkn+F/4U=
golang.org/x/image v0.48.0 h1:1AvuPcuoOTJpA7s4VZ5umrb90T3So0rkB/bSGmxkxik=
golang.org/x/image v0.48.0/go.mod h1:mmtg64iqJKGqEgy+r13dfybl85tmxUd2lMnmaiSdWYY=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/mod v0.42.0 h1:ICeL3t2L30ljSPHHxDnR4fqVcBt/zR2zJAb9juavmQg=
golang.org/x/mod v0.42.0/go.mod h1:u5fzIrRInkL8MaO1cbRkUHT64v303YIT5KEqpNr7jRs=
golang.org/x/mod v0.43.0 h1:MFuq5J6s35W9D3IjkwyWc9vHh0nENOrt5K8Iof4ofmg=
golang.org/x/mod v0.43.0/go.mod h1:4ec1JnuyBz5+FIVhg86UsA39YG8BpnrZdJYSNHmGzHY=
golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20191105084925-a882066a44e0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
@@ -251,14 +251,14 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.0.0-20221002022538-bcab6841153b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.61.0 h1:RR+6j/BTBCrggOLugxa+66R8FvoN6MHlzDlZIbN9YFw=
golang.org/x/net v0.61.0/go.mod h1:WgDCOTH8iwtB66T4/GfI94AHo/hhdb7Hah9NDseQfiI=
golang.org/x/oauth2 v0.37.0 h1:JUlcxA8oAtauLfiH8FX2/FkAWHAdi0QtGCGc+hofE98=
golang.org/x/oauth2 v0.37.0/go.mod h1:IxwZNxUULJmpBFf9K/9NTMSIfZZuvuTy1gGxhigP/58=
golang.org/x/net v0.62.0 h1:18ELalfM5+XNRwzM2nYqXsZQ9ByS1uhj2O/U7+5jqRY=
golang.org/x/net v0.62.0/go.mod h1:WgDCOTH8iwtB66T4/GfI94AHo/hhdb7Hah9NDseQfiI=
golang.org/x/oauth2 v0.38.0 h1:uW7XQbcu1eoiQS7uMJvLOjPMl04bTbmY1VeRPxvbspI=
golang.org/x/oauth2 v0.38.0/go.mod h1:IxwZNxUULJmpBFf9K/9NTMSIfZZuvuTy1gGxhigP/58=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.24.0 h1:iqiDrGoDgyuvAKrhLQthUEzU/R378N78gXbBjxed7oA=
golang.org/x/sync v0.24.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0=
golang.org/x/sync v0.25.0 h1:dJnifr0nMV8/SBnBvdo4eSGGnrc3O4FFx6IqvtLcmRs=
golang.org/x/sync v0.25.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -270,21 +270,21 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.49.0 h1:XbzkgYJHdqh/8m2Uu0W/dQv8nktxx4BFHp1M0gROTXA=
golang.org/x/sys v0.49.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
golang.org/x/sys v0.50.0 h1:dkKZEFTEKaLr1sjhs0WiKqoi8Jj5nHbS3Njmx1UoMJs=
golang.org/x/sys v0.50.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
golang.org/x/term v0.47.0 h1:cpyOwQkr2Nzxe2p6KnrTG7Ei2wtzjsuSjhpXXepZ/BM=
golang.org/x/term v0.47.0/go.mod h1:efAfAuG6RKABvg1ArGebRkvX6LXul14SM67o2e4mUCY=
golang.org/x/term v0.48.0 h1:G6UU1eiSB8nl3DzeUGt7Mg+2SctI8tPS6Z5/mBt6cOk=
golang.org/x/term v0.48.0/go.mod h1:BRDLHCZr2SUjIeBHHpRyQeC9xzSAVbzY2rvKgafnHf4=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.43.0 h1:1QivrlhwAsnMOcqgOdl9my7s9OyZl8O5/UGUTJVXoWc=
golang.org/x/text v0.43.0/go.mod h1:hbSIYA/amXcRXTFSZqh8tWHEoqRvvbKPJ2uz/KU+a+s=
golang.org/x/text v0.44.0 h1:YItdp/mQNGnsmrgpcdA5yQ3bnRxCfo/Tj/iByGW9GGo=
golang.org/x/text v0.44.0/go.mod h1:S4FR1waQ9LbjQTls97SyRjDaUUVkaM+cXwwuG/G+MwY=
golang.org/x/time v0.16.0 h1:vMb6ptszcQMkcwiRTAuNNU50gom6++Q/6gY2hDM6VDE=
golang.org/x/time v0.16.0/go.mod h1:rVKOqvZeKvrDKTQiAHJ7wmwP0RzleSphoEA9RcdLA0s=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -311,10 +311,10 @@ gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
modernc.org/cc/v4 v4.30.3 h1:aDq8gIX1X7+VFgO/JYYji6pLWL2H4tv3g8WatT3KCys=
modernc.org/cc/v4 v4.30.3/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI=
modernc.org/ccgo/v4 v4.37.0 h1:BAuJCUtIHIxFIYaFhqXIw5YaoMBlNCUlFovejHfMip4=
modernc.org/ccgo/v4 v4.37.0/go.mod h1:/8627VhwWtAbqUXmPORxFQU5SgLrAx2jl9seLw9GBhY=
modernc.org/cc/v4 v4.34.0 h1:ooKvVkbVJApIi1dVGl9R0cxQz+UKixXd+zjUH6l8af0=
modernc.org/cc/v4 v4.34.0/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI=
modernc.org/ccgo/v4 v4.39.0 h1:Dk2UbgzY/DwlSKNHAJeO1hxGkkjiY3S1sgqSCrlDhzU=
modernc.org/ccgo/v4 v4.39.0/go.mod h1:/0Re1N2uIR+YRP3B57BDD2XW+zUAnxqlT5GZfSP+vBE=
modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM=
modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU=
modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI=
@@ -323,8 +323,8 @@ modernc.org/gc/v3 v3.2.0 h1:/Y3LGDMQaoYf55PebP1qsZ5FDGgAZyG1RYKEHeQzh1o=
modernc.org/gc/v3 v3.2.0/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY=
modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks=
modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI=
modernc.org/libc v1.79.0 h1:9Dvvv++bAMSkSuBzuOFAatbTV5OIoq6Y6FohmD3e7q8=
modernc.org/libc v1.79.0/go.mod h1:uT3kR61NsbeKJSLqq83GJwSnZMVLlSjyAz4DfTaBy1M=
modernc.org/libc v1.81.0 h1:Kx11pb+4ETDPgkN9P6nHNTeVfrS+1w46LoIDbYcZEtM=
modernc.org/libc v1.81.0/go.mod h1:xlOEexcpOVlRiVw/CDpTqkXxG2uXXGF4i8uR7BAM9sY=
modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU=
modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg=
modernc.org/memory v1.12.1 h1:nFMiWrpStgZczNl6XI9GnIk/rWhYIyHGUaR04pGbp9g=
@@ -333,8 +333,8 @@ modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg=
modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns=
modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w=
modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE=
modernc.org/sqlite v1.60.1 h1:/blz53O951KWFOso4QQvEs/Fq6cDBKLtMVrYNSeJVKw=
modernc.org/sqlite v1.60.1/go.mod h1:1dIoEagfDE72QytD5scH1lxARtaUgKgHC/NuApA27r0=
modernc.org/sqlite v1.62.0 h1:gMaCjSM1qpl+iOjiWhR1LYP3zqrYDI+5GYLGt6veP2g=
modernc.org/sqlite v1.62.0/go.mod h1:Jzjh1mt4s0HuQ0KvtB1LIuHiGRmU2D5pgj8dUhW7/SE=
modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0=
modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A=
modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y=
+9 -5
View File
@@ -3,7 +3,6 @@ package activity
import (
"net"
"net/http"
"strings"
"github.com/gtsteffaniak/filebrowser/backend/pkg/settings"
)
@@ -11,14 +10,19 @@ import (
func remoteIP(r *http.Request) string {
cfg := &settings.Config
// Forwarded client headers are only honored when the socket peer is a
// configured trusted proxy; otherwise a direct client could spoof its IP.
trustedPeer := cfg.Http.TrustProxyHeaders && settings.IsTrustedProxyPeer(r)
xff := r.Header.Get("X-Forwarded-For")
if cfg.Http.TrustProxyHeaders && xff != "" {
ips := strings.Split(xff, ",")
return strings.TrimSpace(ips[0])
if trustedPeer && xff != "" {
if ip := settings.TrustedProxyClientIP(xff); ip != "" {
return ip
}
}
xri := r.Header.Get("X-Real-IP")
if cfg.Http.TrustProxyHeaders && xri != "" {
if trustedPeer && xri != "" {
return xri
}
@@ -9,7 +9,10 @@ import (
"io"
"os"
"path/filepath"
"sort"
"sync"
"sync/atomic"
"time"
"github.com/gtsteffaniak/filebrowser/backend/internal/adapters/fs/fileutils"
"github.com/gtsteffaniak/go-logger/logger"
@@ -21,9 +24,28 @@ type Cache interface {
Set(key string, value []byte) error
}
const (
// evictTargetPercent is the size, as a percentage of the limit, that eviction shrinks the cache to.
// Stopping below the limit avoids running an eviction on every write once the cache is full.
evictTargetPercent = 90
// touchInterval is how old a file's modification time has to be before a read refreshes it. Reads
// refresh it so the files in use are the last ones evicted, but not on every read.
touchInterval = time.Hour
)
// FileCache struct for file-based caching
type FileCache struct {
dir string
// maxBytes is the size limit of the cache in bytes. 0 means no limit.
maxBytes int64
// size is the total size of the cached files in bytes. Only tracked when maxBytes > 0.
size atomic.Int64
// sizeMu keeps size consistent with the files on disk: stores and deletes hold it for reading while
// they change a file and its size, and eviction holds it for writing while it measures the directory,
// so a store can never fall between a measurement and the size being set from it.
sizeMu sync.RWMutex
// evicting is set while an eviction is running so only one runs at a time.
evicting atomic.Bool
// granular locks
scopedLocks struct {
sync.Mutex
@@ -33,6 +55,12 @@ type FileCache struct {
}
func NewFileCache(dir string) (*FileCache, error) {
return NewFileCacheWithLimit(dir, 0)
}
// NewFileCacheWithLimit is like NewFileCache but keeps the cache under maxBytes, deleting the least
// recently used files when it grows past it. 0 means no limit.
func NewFileCacheWithLimit(dir string, maxBytes int64) (*FileCache, error) {
cacheDir := filepath.Join(dir, "diskcache")
// Migrate existing cache files from old structure (if any)
@@ -44,7 +72,13 @@ func NewFileCache(dir string) (*FileCache, error) {
if err := os.MkdirAll(cacheDir, fileutils.PermDir); err != nil {
return nil, fmt.Errorf("failed to create cache directory: %w", err)
}
return &FileCache{dir: cacheDir}, nil
cache := &FileCache{dir: cacheDir}
if maxBytes > 0 {
cache.maxBytes = maxBytes
// measures the cache and trims it right away if it is already over the limit
cache.evictPass()
}
return cache, nil
}
// migrateOldCacheStructure moves cache files from old structure (dir/a/...) to new structure (dir/diskcache/a/...)
@@ -99,19 +133,50 @@ func (f *FileCache) Store(ctx context.Context, key string, value []byte) error {
return err
}
if f.maxBytes > 0 {
f.sizeMu.RLock()
defer f.sizeMu.RUnlock()
}
previousSize := f.fileSize(fileName)
if err := os.WriteFile(fileName, value, fileutils.PermFile); err != nil {
// a failed write can leave a truncated or partial file, so count what is actually there
f.sizeChanged(f.fileSize(fileName) - previousSize)
return err
}
f.sizeChanged(int64(len(value)) - previousSize)
return nil
}
// fileSize returns the size of a cached file, or 0 if it doesn't exist or the size isn't tracked.
func (f *FileCache) fileSize(fileName string) int64 {
if f.maxBytes <= 0 {
return 0
}
info, err := os.Stat(fileName)
if err != nil {
return 0
}
return info.Size()
}
// sizeChanged adds delta to the tracked size and starts an eviction if that put the cache over its limit.
func (f *FileCache) sizeChanged(delta int64) {
if f.maxBytes <= 0 {
return
}
if f.size.Add(delta) > f.maxBytes {
f.evictAsync()
}
}
func (f *FileCache) Load(ctx context.Context, key string) (value []byte, exist bool, err error) {
r, ok, err := f.open(key)
if err != nil || !ok {
return nil, ok, err
}
defer r.Close()
f.touch(r)
value, err = io.ReadAll(r)
if err != nil {
@@ -125,13 +190,119 @@ func (f *FileCache) Delete(ctx context.Context, key string) error {
mu.Lock()
defer mu.Unlock()
if f.maxBytes > 0 {
f.sizeMu.RLock()
defer f.sizeMu.RUnlock()
}
fileName := f.getFileName(key)
removedSize := f.fileSize(fileName)
if err := os.Remove(fileName); err != nil && !errors.Is(err, os.ErrNotExist) {
return err
}
if removedSize > 0 {
f.size.Add(-removedSize)
}
return nil
}
// touch refreshes the modification time of a file that was just read, which is what eviction orders by.
func (f *FileCache) touch(file *os.File) {
if f.maxBytes <= 0 {
return
}
info, err := file.Stat()
if err != nil || time.Since(info.ModTime()) < touchInterval {
return
}
now := time.Now()
_ = os.Chtimes(file.Name(), now, now)
}
type cachedFile struct {
path string
size int64
modTime time.Time
}
// scan lists the cached files and their total size.
func (f *FileCache) scan() ([]cachedFile, int64) {
var files []cachedFile
var total int64
_ = filepath.WalkDir(f.dir, func(path string, entry os.DirEntry, err error) error {
if err != nil || entry.IsDir() {
return nil
}
info, err := entry.Info()
if err != nil || !info.Mode().IsRegular() {
return nil
}
files = append(files, cachedFile{path: path, size: info.Size(), modTime: info.ModTime()})
total += info.Size()
return nil
})
return files, total
}
// evictAsync starts an eviction in the background unless one is already running, so the request that
// stored the file doesn't wait for it.
func (f *FileCache) evictAsync() {
if !f.evicting.CompareAndSwap(false, true) {
return
}
go func() {
for {
removed := f.evictPass()
f.evicting.Store(false)
if removed == 0 {
return // under the limit, or nothing could be removed: don't spin
}
// Files stored while the pass was running were counted but not necessarily seen by it, and a
// store that came in just before the flag was cleared saw it set and didn't start another
// pass, so check again before stopping.
if f.size.Load() <= f.maxBytes || !f.evicting.CompareAndSwap(false, true) {
return
}
}
}()
}
// evictPass measures the cache and, if it is over the limit, deletes the least recently used files until
// it is under evictTargetPercent of the limit. Deleting a file that is being read or written is harmless:
// readers keep their open file and a removed entry is simply rebuilt the next time it is requested.
func (f *FileCache) evictPass() (removed int) {
f.sizeMu.Lock()
files, total := f.scan()
f.size.Store(total)
f.sizeMu.Unlock()
if total <= f.maxBytes {
return 0
}
target := f.maxBytes / 100 * evictTargetPercent
sort.Slice(files, func(i, j int) bool { return files[i].modTime.Before(files[j].modTime) })
var removedBytes int64
for _, file := range files {
if f.size.Load() <= target {
break
}
// Hold the lock so no store or delete changes this file between measuring it and removing it, and
// subtract the size it has now: the listing is from the scan and the file may have been rewritten since.
f.sizeMu.Lock()
info, err := os.Stat(file.path)
if err == nil {
err = os.Remove(file.path)
}
if err == nil {
f.size.Add(-info.Size())
removed++
removedBytes += info.Size()
}
f.sizeMu.Unlock()
}
logger.Debugf("preview cache limit: removed %d files (%d MB), now %d MB of %d MB", removed, removedBytes>>20, f.size.Load()>>20, f.maxBytes>>20)
return removed
}
func (f *FileCache) open(key string) (*os.File, bool, error) {
fileName := f.getFileName(key)
file, err := os.Open(fileName)
@@ -2,9 +2,12 @@ package diskcache
import (
"context"
"fmt"
"os"
"path/filepath"
"sync"
"testing"
"time"
"github.com/gtsteffaniak/filebrowser/backend/internal/adapters/fs/fileutils"
"github.com/stretchr/testify/require"
@@ -70,3 +73,168 @@ func fileExists(filename string) bool {
}
return !info.IsDir()
}
// storeAged stores a value and sets its modification time, so the eviction order is deterministic.
func storeAged(t *testing.T, cache *FileCache, key string, size int, age time.Duration) {
t.Helper()
require.NoError(t, cache.Store(context.Background(), key, make([]byte, size)))
when := time.Now().Add(-age)
require.NoError(t, os.Chtimes(cache.getFileName(key), when, when))
}
func waitFor(t *testing.T, condition func() bool) {
t.Helper()
deadline := time.Now().Add(5 * time.Second)
for !condition() {
if time.Now().After(deadline) {
t.Fatal("condition not met in time")
}
time.Sleep(10 * time.Millisecond)
}
}
func newLimitedCache(t *testing.T, maxBytes int64) *FileCache {
t.Helper()
fileutils.SetFsPermissions(0644, 0755)
cache, err := NewFileCacheWithLimit(t.TempDir(), maxBytes)
require.NoError(t, err)
return cache
}
func TestFileCacheEvictsLeastRecentlyUsedOverLimit(t *testing.T) {
const kb = 1024
cache := newLimitedCache(t, 10*kb)
// 3 files of 3KB are under the 10KB limit
storeAged(t, cache, "oldest", 3*kb, 3*time.Hour)
storeAged(t, cache, "old", 3*kb, 2*time.Hour)
storeAged(t, cache, "recent", 3*kb, time.Hour)
require.True(t, fileExists(cache.getFileName("oldest")), "nothing is removed under the limit")
// the 4th file goes over the limit: the oldest ones are removed until it is under 90% of it
require.NoError(t, cache.Store(context.Background(), "newest", make([]byte, 3*kb)))
waitFor(t, func() bool { return !fileExists(cache.getFileName("oldest")) })
waitFor(t, func() bool { return !cache.evicting.Load() })
require.False(t, fileExists(cache.getFileName("oldest")))
require.False(t, fileExists(cache.getFileName("old")))
require.True(t, fileExists(cache.getFileName("recent")))
require.True(t, fileExists(cache.getFileName("newest")))
require.LessOrEqual(t, cache.size.Load(), int64(9*kb))
}
func TestFileCacheTrimsAnOversizedCacheAtStartup(t *testing.T) {
const kb = 1024
fileutils.SetFsPermissions(0644, 0755)
dir := t.TempDir()
unlimited, err := NewFileCache(dir)
require.NoError(t, err)
storeAged(t, &FileCache{dir: unlimited.dir}, "a", 4*kb, 4*time.Hour)
storeAged(t, &FileCache{dir: unlimited.dir}, "b", 4*kb, 3*time.Hour)
storeAged(t, &FileCache{dir: unlimited.dir}, "c", 4*kb, 2*time.Hour)
cache, err := NewFileCacheWithLimit(dir, 10*kb)
require.NoError(t, err)
// 12KB over a 10KB limit: only the oldest file is removed, which already brings it under 90% (9KB)
require.False(t, fileExists(cache.getFileName("a")))
require.True(t, fileExists(cache.getFileName("b")))
require.True(t, fileExists(cache.getFileName("c")))
require.Equal(t, int64(8*kb), cache.size.Load())
}
func TestFileCacheWithoutLimitKeepsEverything(t *testing.T) {
fileutils.SetFsPermissions(0644, 0755)
cache, err := NewFileCache(t.TempDir())
require.NoError(t, err)
for i := 0; i < 5; i++ {
storeAged(t, cache, fmt.Sprintf("key-%d", i), 1<<20, time.Duration(i)*time.Hour)
}
for i := 0; i < 5; i++ {
require.True(t, fileExists(cache.getFileName(fmt.Sprintf("key-%d", i))))
}
require.Zero(t, cache.size.Load(), "size is only tracked when there is a limit")
}
func TestFileCacheReadRefreshesModTime(t *testing.T) {
cache := newLimitedCache(t, 1<<20)
storeAged(t, cache, "used", 100, 3*time.Hour)
_, ok, err := cache.Load(context.Background(), "used")
require.NoError(t, err)
require.True(t, ok)
info, err := os.Stat(cache.getFileName("used"))
require.NoError(t, err)
require.WithinDuration(t, time.Now(), info.ModTime(), time.Minute, "a file that is read is the last to be evicted")
}
func TestFileCacheDeleteAndOverwriteKeepTheSizeAccurate(t *testing.T) {
const kb = 1024
cache := newLimitedCache(t, 1<<20)
ctx := context.Background()
require.NoError(t, cache.Store(ctx, "k", make([]byte, 5*kb)))
require.Equal(t, int64(5*kb), cache.size.Load())
require.NoError(t, cache.Store(ctx, "k", make([]byte, 2*kb)))
require.Equal(t, int64(2*kb), cache.size.Load(), "overwriting replaces the old size")
require.NoError(t, cache.Delete(ctx, "k"))
require.Zero(t, cache.size.Load())
}
// Stores that land while an eviction is measuring or deleting must not be lost from the tracked size,
// and the cache must end up under its limit even though more files kept arriving during the eviction.
func TestFileCacheStaysUnderTheLimitWithConcurrentStores(t *testing.T) {
const (
kb = 1024
limit = 100 * kb
workers = 16
stores = 40
)
cache := newLimitedCache(t, limit)
var wg sync.WaitGroup
for w := 0; w < workers; w++ {
wg.Add(1)
go func(w int) {
defer wg.Done()
for i := 0; i < stores; i++ {
_ = cache.Store(context.Background(), fmt.Sprintf("worker-%d-key-%d", w, i), make([]byte, 4*kb))
}
}(w)
}
wg.Wait()
waitFor(t, func() bool { return !cache.evicting.Load() })
_, onDisk := cache.scan()
require.LessOrEqual(t, onDisk, int64(limit), "the files left on disk are within the limit")
require.Equal(t, onDisk, cache.size.Load(), "the tracked size matches what is on disk")
}
// Files that are rewritten while an eviction is running must be subtracted at the size they have when
// they are removed, not at the size the scan saw, or the tracked size drifts from the disk.
func TestFileCacheKeepsTheSizeAccurateWhenFilesAreRewrittenDuringEviction(t *testing.T) {
const (
kb = 1024
limit = 60 * kb
workers = 8
rounds = 300
keys = 40
)
cache := newLimitedCache(t, limit)
var wg sync.WaitGroup
for w := 0; w < workers; w++ {
wg.Add(1)
go func(w int) {
defer wg.Done()
for i := 0; i < rounds; i++ {
// the same keys again and again, with a different size each time
key := fmt.Sprintf("key-%d", (w+i)%keys)
_ = cache.Store(context.Background(), key, make([]byte, (1+(w*7+i)%8)*kb))
}
}(w)
}
wg.Wait()
waitFor(t, func() bool { return !cache.evicting.Load() })
_, onDisk := cache.scan()
require.Equal(t, onDisk, cache.size.Load(), "the tracked size matches what is on disk")
require.LessOrEqual(t, onDisk, int64(limit))
}
+31 -5
View File
@@ -335,6 +335,15 @@ func fileInfoFasterImpl(opts utils.FileOptions, user *users.User, s *Service) (*
if info.Type == "directory" && opts.HideFileExt != "" {
response.Files = filterFilesByExt(response.Files, opts.HideFileExt)
}
if info.Type == "directory" && opts.Expand {
// Expanded listings without extended attrs skip persistDirectoryListing
// (their placeholder preview values must not overwrite index rows), so
// refresh the index lazily — otherwise folder sizes here never update.
// Capture indexPath now: the deferred finalizeResponse strips the user
// scope from response.Path before returning.
// Coalesced per path so repeated listings do not run parallel refreshes.
idx.RefreshDirectoryAsync(idx.MakeIndexPath(response.Path, true).String())
}
return response, nil
}
// Process directory metadata if requested
@@ -646,6 +655,7 @@ func DeleteFiles(source, boundIndexPath, indexPath string, isDir bool) error {
if utils.CleanIndexPathSegment(indexPath) == "" {
return fmt.Errorf("refusing to delete source root directory: %s", indexPath)
}
deleteIndexPath := indexPath
realPath, err := index.GetRealPathForRemoval(boundIndexPath, indexPath)
if err != nil {
return err
@@ -664,9 +674,13 @@ func DeleteFiles(source, boundIndexPath, indexPath string, isDir bool) error {
return err
}
// Clear cache entries
indexing.RealPathCache.Delete(realPath)
indexing.IsDirCache.Delete(realPath + ":isdir")
if isDir {
indexing.ClearRealPathCaches()
}
index.InvalidateRealPathCache(deleteIndexPath, boundIndexPath)
if parent := utils.GetParentDirectoryPath(deleteIndexPath); parent != "" && parent != "/" {
index.InvalidateRealPathCache(parent, boundIndexPath)
}
// Remove metadata from index
deleteSuccess := index.DeleteMetadata(indexPath, isDir, isDir)
@@ -718,8 +732,7 @@ func RefreshIndex(source string, path string, isDir bool, recursive bool) error
if !Exists(realPath) {
// Directory no longer exists, remove it from the index
// This clears both Directories and DirectoriesLedger maps
indexing.RealPathCache.Delete(realPath)
indexing.IsDirCache.Delete(realPath + ":isdir")
idx.InvalidateRealPathCache(path)
idx.DeleteMetadata(path, true, false)
return nil
}
@@ -797,6 +810,9 @@ func moveResourceImpl(isSrcDir bool, sourceIndex, destIndex, realsrc, realdst st
if err != nil {
return err
}
if isSrcDir {
indexing.ClearRealPathCaches()
}
// Handle SOURCE cleanup (treat as deletion)
// Run async to avoid blocking the HTTP response
@@ -957,6 +973,11 @@ func WriteDirectory(boundIndexPath string, opts utils.FileOptions) error {
}
}
idx.InvalidateRealPathCache(opts.Path, boundIndexPath)
if parent := utils.GetParentDirectoryPath(opts.Path); parent != "" && parent != "/" {
idx.InvalidateRealPathCache(parent, boundIndexPath)
}
return nil
}
@@ -1032,6 +1053,11 @@ func writeFile(source, boundIndexPath, path string, in io.Reader) error {
}
}
idx.InvalidateRealPathCache(path, boundIndexPath)
if parent := utils.GetParentDirectoryPath(path); parent != "" && parent != "/" {
idx.InvalidateRealPathCache(parent, boundIndexPath)
}
return nil
}
+1 -1
View File
@@ -301,7 +301,7 @@ func detectRuntime() string {
func mapSystemOS(goos string) string {
switch goos {
case "linux", "windows", "darwin":
case "linux", "windows", "darwin", "freebsd":
return goos
default:
return "other"
@@ -0,0 +1,25 @@
package analytics
import "testing"
func TestMapSystemOS(t *testing.T) {
tests := []struct {
name string
goos string
want string
}{
{name: "linux", goos: "linux", want: "linux"},
{name: "windows", goos: "windows", want: "windows"},
{name: "darwin", goos: "darwin", want: "darwin"},
{name: "freebsd release build", goos: "freebsd", want: "freebsd"},
{name: "unsupported goos", goos: "plan9", want: "other"},
{name: "empty", goos: "", want: "other"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := mapSystemOS(tt.goos); got != tt.want {
t.Errorf("mapSystemOS(%q) = %q, want %q", tt.goos, got, tt.want)
}
})
}
}
+1
View File
@@ -57,6 +57,7 @@ func MakeSignedTokenAPI(user *users.User, name string, duration time.Duration, p
},
},
}
claim.Minimal = &minimal
if !minimal {
claim.Permissions = users.SanitizeTokenPermissions(perms)
}
@@ -0,0 +1,44 @@
package access
import (
"os"
"path/filepath"
"strings"
"github.com/gtsteffaniak/filebrowser/backend/internal/utils"
"github.com/gtsteffaniak/filebrowser/backend/pkg/indexing"
)
// PermittedAtRealPath reports whether username may access realPath after symlink
// resolution. DENY rules apply to the target index path, not only the link path.
func (s *Storage) PermittedAtRealPath(index *indexing.Index, username, realPath string, isDir bool) bool {
if s == nil || index == nil || username == "" || realPath == "" {
return false
}
lexical := realPath
if !strings.HasPrefix(realPath, index.Path) {
root, err := filepath.EvalSymlinks(index.Path)
if err != nil {
return false
}
rel, err := filepath.Rel(root, realPath)
if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) {
return false
}
lexical = index.Path
if rel != "." {
lexical = filepath.Join(index.Path, rel)
}
}
indexPath := index.MakeIndexPath(lexical, isDir).String()
return s.Permitted(index.Path, utils.IndexPathFromNormalized(indexPath, true), username)
}
// IsDirAtRealPath reports whether an existing path is a directory; missing paths are treated as files.
func IsDirAtRealPath(realPath string) bool {
st, err := os.Stat(realPath)
if err != nil {
return false
}
return st.IsDir()
}
@@ -0,0 +1,140 @@
package access
import (
"os"
"path/filepath"
"testing"
"github.com/gtsteffaniak/filebrowser/backend/internal/utils"
"github.com/gtsteffaniak/filebrowser/backend/pkg/indexing"
"github.com/gtsteffaniak/filebrowser/backend/pkg/settings"
)
func setupSymlinkDenyFixture(t *testing.T) (sourceRoot string, idx *indexing.Index, store *Storage, username string) {
t.Helper()
sourceRoot = filepath.Join(t.TempDir(), "data")
aliceDir := filepath.Join(sourceRoot, "alice")
bobDir := filepath.Join(sourceRoot, "bob")
for _, dir := range []string{aliceDir, bobDir} {
if err := os.MkdirAll(dir, 0o755); err != nil {
t.Fatal(err)
}
}
if err := os.WriteFile(filepath.Join(bobDir, "secret.txt"), []byte("secret"), 0o644); err != nil {
t.Fatal(err)
}
if err := os.Symlink("../bob/secret.txt", filepath.Join(aliceDir, "link.txt")); err != nil {
t.Skipf("symlinks required: %v", err)
}
if err := os.Symlink("../bob", filepath.Join(aliceDir, "bobdir")); err != nil {
t.Skipf("symlinks required: %v", err)
}
settings.Config.Server.SourceMap = map[string]*settings.Source{
sourceRoot: {Path: sourceRoot, Config: settings.SourceConfig{DenyByDefault: false}},
}
store = NewStorage(nil)
username = "scoped-user"
if err := store.DenyUser(sourceRoot, utils.IndexPathFromNormalized("/bob", true), username); err != nil {
t.Fatal(err)
}
idx = &indexing.Index{Source: settings.Source{Path: sourceRoot}}
return sourceRoot, idx, store, username
}
func TestPermittedAtRealPath_AllowsLinkPathDeniesResolvedTarget(t *testing.T) {
_, idx, store, user := setupSymlinkDenyFixture(t)
if !store.Permitted(idx.Path, utils.IndexPathFromNormalized("/alice/link.txt", false), user) {
t.Fatal("index path at the symlink should be allowed")
}
if store.Permitted(idx.Path, utils.IndexPathFromNormalized("/bob/secret.txt", false), user) {
t.Fatal("direct path under /bob should be denied")
}
realPath, _, err := idx.GetRealPathScoped("/", "/alice/link.txt")
if err != nil {
t.Fatal(err)
}
if store.PermittedAtRealPath(idx, user, realPath, false) {
t.Fatal("resolved target under denied /bob must not be permitted")
}
}
func TestPermittedAtRealPath_DeniesWriteThroughDirectorySymlink(t *testing.T) {
_, idx, store, user := setupSymlinkDenyFixture(t)
realPath, err := idx.GetRealPathForMutation("/", "/alice/bobdir/planted.txt")
if err != nil {
t.Fatal(err)
}
if store.PermittedAtRealPath(idx, user, realPath, false) {
t.Fatal("resolved write target under denied /bob must not be permitted")
}
}
func TestPermittedAtRealPath_DeniesCopySourceResolved(t *testing.T) {
_, idx, store, user := setupSymlinkDenyFixture(t)
realSrc, isDir, err := idx.GetRealPathScoped("/", "/alice/link.txt")
if err != nil {
t.Fatal(err)
}
if isDir {
t.Fatal("expected file")
}
if store.PermittedAtRealPath(idx, user, realSrc, isDir) {
t.Fatal("copy source resolved to denied target must not be permitted")
}
}
func TestPermittedAtRealPath_SymlinkedSourceRoot(t *testing.T) {
sourceRoot, _, _, user := setupSymlinkDenyFixture(t)
linkRoot := filepath.Join(filepath.Dir(sourceRoot), "root-link")
if err := os.Symlink(sourceRoot, linkRoot); err != nil {
t.Skipf("symlinks required: %v", err)
}
if _, err := filepath.EvalSymlinks(linkRoot); err != nil {
t.Fatal(err)
}
settings.Config.Server.SourceMap = map[string]*settings.Source{
linkRoot: {Path: linkRoot, Config: settings.SourceConfig{DenyByDefault: false}},
}
store := NewStorage(nil)
if err := store.DenyUser(linkRoot, utils.IndexPathFromNormalized("/bob", true), user); err != nil {
t.Fatal(err)
}
idx := &indexing.Index{Source: settings.Source{Path: linkRoot}}
realPath, _, err := idx.GetRealPathScoped("/", "/alice/link.txt")
if err != nil {
t.Fatal(err)
}
realPath, err = filepath.EvalSymlinks(realPath)
if err != nil {
t.Fatal(err)
}
if store.PermittedAtRealPath(idx, user, realPath, false) {
t.Fatal("resolved target under denied /bob must not be permitted with symlinked source root")
}
}
func TestIsDirAtRealPath(t *testing.T) {
dir := t.TempDir()
if err := os.MkdirAll(filepath.Join(dir, "nested"), 0o755); err != nil {
t.Fatal(err)
}
file := filepath.Join(dir, "file.txt")
if err := os.WriteFile(file, []byte("x"), 0o644); err != nil {
t.Fatal(err)
}
if !IsDirAtRealPath(dir) {
t.Fatal("expected directory")
}
if IsDirAtRealPath(file) {
t.Fatal("expected file")
}
if IsDirAtRealPath(filepath.Join(dir, "missing")) {
t.Fatal("missing path should not be treated as directory")
}
}
@@ -24,7 +24,11 @@ func HasAnyGlobalPermission(p Permissions) bool {
}
// IsMinimalApiToken reports whether the token inherits the owner's globals at auth time
// rather than embedding its own global permission caps in the JWT.
// rather than applying stored global permission caps.
func IsMinimalApiToken(t AuthToken) bool {
if t.Minimal != nil {
return *t.Minimal
}
// Legacy tokens lack minimal; empty stored global caps implied minimal (WebDAV-style).
return !HasAnyGlobalPermission(t.Permissions)
}
@@ -45,6 +45,14 @@ func TestIsMinimalApiToken(t *testing.T) {
t.Fatal("custom token with global caps should not be minimal")
}
if !IsMinimalApiToken(AuthToken{Permissions: Permissions{Modify: true}}) {
t.Fatal("token without global caps should be minimal")
t.Fatal("legacy token without global caps should be minimal")
}
minimal := true
if !IsMinimalApiToken(AuthToken{Minimal: &minimal}) {
t.Fatal("explicit minimal token should be minimal")
}
customized := false
if IsMinimalApiToken(AuthToken{Minimal: &customized}) {
t.Fatal("explicit customized token with empty caps should not be minimal")
}
}
+7 -4
View File
@@ -105,6 +105,8 @@ type AuthToken struct {
BelongsTo uint64 `json:"belongsTo,omitempty"` // legacy stored metadata only, never trusted for auth; identity is always the hashed_tokens lookup
IssuedAt int64 `json:"issuedAt,omitempty"`
ExpiresAt int64 `json:"expiresAt,omitempty"`
// Minimal distinguishes owner-inheriting tokens from customized caps. Nil means legacy (infer from stored permissions).
Minimal *bool `json:"minimal,omitempty"`
Permissions Permissions `json:"Permissions,omitempty"`
}
@@ -341,10 +343,11 @@ type NonAdminEditable struct {
}
type FileLoading struct {
MaxConcurrent int `json:"maxConcurrentUpload"`
UploadChunkSize int `json:"uploadChunkSizeMb"`
ClearAll bool `json:"clearAll"`
DownloadChunkSize int `json:"downloadChunkSizeMb"`
MaxConcurrent int `json:"maxConcurrentUpload"`
UploadChunkSize int `json:"uploadChunkSizeMb"`
ClearAll bool `json:"clearAll"`
DownloadChunkSize int `json:"downloadChunkSizeMb"`
UploadStallTimeoutSeconds int `json:"uploadStallTimeoutSeconds"` // seconds without progress before an upload is paused, 0 uses the default (10)
}
// SidebarLink represents a customizable link in the sidebar.
+32 -1
View File
@@ -10,6 +10,7 @@ import (
goffmpeg "github.com/gtsteffaniak/go-ffmpeg"
"github.com/gtsteffaniak/go-ffmpeg/capabilities"
"github.com/gtsteffaniak/go-ffmpeg/ops"
"github.com/gtsteffaniak/go-ffmpeg/probe"
"github.com/gtsteffaniak/go-logger/logger"
)
@@ -118,7 +119,37 @@ func (s *Service) VideoPreview(ctx context.Context, w io.Writer, opts ops.Previe
if s == nil || s.inner == nil {
return fmt.Errorf("ffmpeg service not available")
}
return s.inner.VideoPreview(ctx, w, opts)
return s.inner.CaptureFrame(ctx, w, previewOptionsToFrame(opts))
}
func previewOptionsToFrame(opts ops.PreviewOptions) ops.FrameOptions {
var at ops.FramePosition
if opts.SeekSec > 0 {
at = ops.AtSec(opts.SeekSec)
} else {
pct := opts.SeekPercent
if pct <= 0 || pct > 100 {
pct = 10
}
at = ops.AtPercent(pct)
}
q := opts.Quality
if q == 0 {
q = ops.LegacyPreviewDefaultQuality
}
return ops.FrameOptions{
Input: ops.InputSource{URL: opts.Input, StreamType: probe.StreamFile},
At: at,
DurationSec: opts.DurationSec,
Width: opts.Width,
Height: opts.Height,
ScaleMode: opts.ScaleMode,
Quality: q,
Timeout: opts.Timeout,
LightProbe: opts.LightProbe,
Accuracy: opts.Accuracy,
Format: ops.FrameJPEG,
}
}
// ExtractSubtitle returns embedded subtitle content as WebVTT.
+2 -2
View File
@@ -13,8 +13,8 @@ import (
"strings"
"github.com/gtsteffaniak/filebrowser/backend/internal/adapters/fs/fileutils"
"github.com/gtsteffaniak/filebrowser/backend/pkg/settings"
"github.com/gtsteffaniak/filebrowser/backend/internal/preview"
"github.com/gtsteffaniak/filebrowser/backend/pkg/settings"
"github.com/gtsteffaniak/go-logger/logger"
"github.com/kovidgoyal/imaging"
)
@@ -209,7 +209,7 @@ func generateMaskableIcon(sourceData []byte, icon iconSize) error {
// Android adaptive-icon content ratio (72/108 ≈ 2/3) keeps square art inside every mask shape
inner := icon.size * 2 / 3
art := imaging.Fit(img, inner, inner, imaging.Lanczos)
canvas := imaging.PasteCenter(imaging.New(icon.size, icon.size, maskableBackground(img)), art)
canvas := imaging.OverlayCenter(imaging.New(icon.size, icon.size, maskableBackground(img)), art, 1)
outputPath := filepath.Join(settings.PWAIconsCacheDir(), icon.name)
outFile, err := os.Create(outputPath)
+81
View File
@@ -1,13 +1,94 @@
package icons
import (
"bytes"
"image"
"image/color"
"image/png"
"os"
"path/filepath"
"testing"
"github.com/gtsteffaniak/filebrowser/backend/pkg/settings"
"github.com/kovidgoyal/imaging"
)
func TestGenerateMaskableIconKeepsCanvasOpaque(t *testing.T) {
originalCacheDir := settings.Config.Server.CacheDir
settings.Config.Server.CacheDir = t.TempDir()
t.Cleanup(func() { settings.Config.Server.CacheDir = originalCacheDir })
if err := os.MkdirAll(settings.PWAIconsCacheDir(), 0o755); err != nil {
t.Fatal(err)
}
tests := []struct {
name string
background color.Color
source func(*image.RGBA)
}{
{
name: "transparent padding",
background: color.White,
source: func(img *image.RGBA) {
img.Set(5, 5, color.RGBA{R: 20, G: 40, B: 200, A: 255})
},
},
{
name: "opaque edge",
background: color.NRGBA{R: 200, G: 100, B: 50, A: 255},
source: func(img *image.RGBA) {
for x := 0; x < 12; x++ {
img.Set(x, 0, color.RGBA{R: 200, G: 100, B: 50, A: 255})
}
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
source := image.NewRGBA(image.Rect(0, 0, 12, 12))
tt.source(source)
var sourceData bytes.Buffer
if err := png.Encode(&sourceData, source); err != nil {
t.Fatal(err)
}
const size = 96
icon := iconSize{size: size, name: "maskable.png", maskable: true}
if err := generateMaskableIcon(sourceData.Bytes(), icon); err != nil {
t.Fatal(err)
}
output, err := imaging.Open(filepath.Join(settings.PWAIconsCacheDir(), icon.name))
if err != nil {
t.Fatal(err)
}
if output.Bounds().Dx() != size || output.Bounds().Dy() != size {
t.Fatalf("got bounds %v, want %dx%d", output.Bounds(), size, size)
}
want, ok := color.NRGBAModel.Convert(tt.background).(color.NRGBA)
if !ok {
t.Fatalf("background color %#v is not NRGBA", tt.background)
}
for y := 0; y < size; y++ {
for x := 0; x < size; x++ {
outsideArt := x < 16 || x >= 80 || y < 16 || y >= 80
got, ok := color.NRGBAModel.Convert(output.At(x, y)).(color.NRGBA)
if !ok {
t.Fatalf("pixel (%d,%d) is not NRGBA", x, y)
}
if got.A != 255 {
t.Fatalf("pixel (%d,%d) has alpha %d, want opaque", x, y, got.A)
}
if outsideArt && got != want {
t.Fatalf("pixel (%d,%d) got %#v, want background %#v", x, y, got, want)
}
}
}
})
}
}
func TestMaskableBackgroundOpaqueCorner(t *testing.T) {
img := image.NewRGBA(image.Rect(0, 0, 12, 12))
img.Set(0, 0, color.RGBA{R: 10, G: 20, B: 30, A: 255})
+6 -5
View File
@@ -78,12 +78,10 @@ func extractTIFFEmbeddedPreview(f *os.File) ([]byte, error) {
return nil, err
}
ex, err := extimagemeta.Decode(f)
if err != nil {
return nil, err
var candidates []previewCandidate
if ex, err := extimagemeta.Decode(f); err == nil {
candidates = previewCandidates(ex)
}
candidates := previewCandidates(ex)
if scanCandidates, scanErr := collectTIFFPreviewCandidates(f); scanErr == nil {
candidates = append(candidates, scanCandidates...)
}
@@ -115,6 +113,9 @@ func previewCandidates(ex exif.Exif) []previewCandidate {
out = append(out, previewCandidate{offset: ifd.ImageOffset, length: ifd.ImageLength})
}
}
for _, sub := range ex.SubIFDs {
appendIFD(sub)
}
appendIFD(ex.IFD1)
appendIFD(ex.IFD2)
+8 -7
View File
@@ -47,15 +47,16 @@ func GetOrientation(ctx context.Context, path string) string {
default:
ex, err = extimagemeta.Decode(f)
}
if err != nil {
return ""
}
if ctx.Err() != nil {
return ""
}
if ex.IFD0.Orientation == 0 {
return ""
if err == nil && ex.IFD0.Orientation != 0 {
return tag.ValueNameFor(tag.IFD0, tag.TagOrientation, uint32(ex.IFD0.Orientation))
}
return tag.ValueNameFor(tag.IFD0, tag.TagOrientation, uint32(ex.IFD0.Orientation))
if !isHEICExtension(ext) {
if orient := extractTIFFOrientation(f); orient != 0 {
return tag.ValueNameFor(tag.IFD0, tag.TagOrientation, uint32(orient))
}
}
return ""
}
+98 -11
View File
@@ -9,11 +9,18 @@ import (
)
const (
tiffTagStripOffsets = 0x0111
tiffTagOrientation = 0x0112
tiffTagStripByteCounts = 0x0117
tiffTagSubIFDs = 0x014a
tiffTagJpgFromRaw = 0x002e
tiffTagJPEGInterchange = 0x0201
tiffTagJPEGInterchangeLen = 0x0202
tiffIFDScanLimit = 4 * 1024 * 1024
tiffMagicStandard = 42
tiffMagicPanasonicRW2 = 0x0055
tiffMagicOlympusORF1 = 0x4f52
tiffMagicOlympusORF2 = 0x5352
tiffTypeByte = 1
tiffTypeASCII = 2
tiffTypeShort = 3
@@ -36,30 +43,47 @@ type tiffTag struct {
valueOffset uint32
}
// collectTIFFPreviewCandidates walks TIFF IFDs (including SubIFDs) for embedded
// JPEG preview offset/length pairs that imagemeta does not surface.
func collectTIFFPreviewCandidates(f *os.File) ([]previewCandidate, error) {
// readTIFFScanWindow reads the bounded IFD scan window (up to tiffIFDScanLimit bytes) of f.
// It validates the 8-byte TIFF header first, so non-TIFF files such as JPEGs are rejected
// without allocating or reading the full window. ok is false for non-TIFF or too-short files.
func readTIFFScanWindow(f *os.File) (header []byte, order binary.ByteOrder, ifd0 uint32, ok bool, err error) {
info, err := f.Stat()
if err != nil {
return nil, err
return nil, nil, 0, false, err
}
scanSize := info.Size()
if scanSize > tiffIFDScanLimit {
scanSize = tiffIFDScanLimit
}
if scanSize < 8 {
return nil, nil
return nil, nil, 0, false, nil
}
header := make([]byte, scanSize)
var magic [8]byte
if _, err = f.ReadAt(magic[:], 0); err != nil {
if err == io.EOF {
return nil, nil, 0, false, nil
}
return nil, nil, 0, false, err
}
if _, _, ok = tiffHeaderIFD0(magic[:]); !ok {
return nil, nil, 0, false, nil
}
header = make([]byte, scanSize)
n, err := f.ReadAt(header, 0)
if err != nil && err != io.EOF {
return nil, err
return nil, nil, 0, false, err
}
header = header[:n]
order, ifd0, ok = tiffHeaderIFD0(header)
return header, order, ifd0, ok, nil
}
order, ifd0, ok := tiffHeaderIFD0(header)
if !ok {
return nil, nil
// collectTIFFPreviewCandidates walks TIFF IFDs (including SubIFDs) for embedded
// JPEG preview offset/length pairs that imagemeta does not surface.
func collectTIFFPreviewCandidates(f *os.File) ([]previewCandidate, error) {
header, order, ifd0, ok, err := readTIFFScanWindow(f)
if err != nil || !ok {
return nil, err
}
seen := make(map[uint32]struct{})
@@ -97,6 +121,7 @@ func collectTIFFPreviewCandidates(f *os.File) ([]previewCandidate, error) {
for _, ifd := range ifds {
out = appendPreviewCandidates(out, jpgFromRawCandidates(ifd))
out = appendPreviewCandidates(out, jpegInterchangeCandidates(ifd))
out = appendPreviewCandidates(out, stripPreviewCandidates(ifd, order))
if len(out) >= maxTIFFPreviewCandidates {
break
}
@@ -104,6 +129,24 @@ func collectTIFFPreviewCandidates(f *os.File) ([]previewCandidate, error) {
return out, nil
}
// extractTIFFOrientation reads standard TIFF IFD0 Orientation (0x0112) as a fallback
// when full EXIF decoding fails.
func extractTIFFOrientation(f *os.File) uint16 {
header, order, ifd0, ok, err := readTIFFScanWindow(f)
if err != nil || !ok || ifd0 == 0 {
return 0
}
ifd, _, _, err := parseTIFFIFD(header, order, ifd0)
if err != nil {
return 0
}
tag, ok := ifd.tags[tiffTagOrientation]
if !ok || tag.count == 0 {
return 0
}
return tiffTagFirstUint16(tag, order)
}
func appendPreviewCandidates(dst, src []previewCandidate) []previewCandidate {
remaining := maxTIFFPreviewCandidates - len(dst)
if remaining <= 0 {
@@ -144,6 +187,48 @@ func jpegInterchangeCandidates(ifd tiffIFD) []previewCandidate {
return []previewCandidate{{offset: offset, length: length}}
}
func stripPreviewCandidates(ifd tiffIFD, order binary.ByteOrder) []previewCandidate {
offTag, okOff := ifd.tags[tiffTagStripOffsets]
lenTag, okLen := ifd.tags[tiffTagStripByteCounts]
if !okOff || !okLen || offTag.count != 1 || lenTag.count != 1 {
return nil
}
offset := tiffTagScalarUint32(offTag, order)
length := tiffTagScalarUint32(lenTag, order)
if offset == 0 || length <= 4 || length > maxPreviewReadSize {
return nil
}
return []previewCandidate{{offset: offset, length: length}}
}
func tiffTagScalarUint32(t tiffTag, order binary.ByteOrder) uint32 {
switch t.typ {
case tiffTypeLong, tiffTypeIFD:
return t.valueOffset
case tiffTypeShort:
return uint32(tiffTagFirstUint16(t, order))
default:
return 0
}
}
func tiffTagFirstUint16(t tiffTag, order binary.ByteOrder) uint16 {
switch t.typ {
case tiffTypeShort:
if order == binary.BigEndian {
return uint16(t.valueOffset >> 16)
}
return uint16(t.valueOffset & 0xffff)
case tiffTypeLong:
if t.valueOffset > 0xffff {
return 0
}
return uint16(t.valueOffset)
default:
return 0
}
}
func tiffHeaderIFD0(data []byte) (binary.ByteOrder, uint32, bool) {
if len(data) < 8 {
return nil, 0, false
@@ -157,7 +242,9 @@ func tiffHeaderIFD0(data []byte) (binary.ByteOrder, uint32, bool) {
default:
return nil, 0, false
}
if order.Uint16(data[2:4]) != 42 {
switch order.Uint16(data[2:4]) {
case tiffMagicStandard, tiffMagicPanasonicRW2, tiffMagicOlympusORF1, tiffMagicOlympusORF2:
default:
return nil, 0, false
}
return order, order.Uint32(data[4:8]), true
+154 -3
View File
@@ -62,7 +62,7 @@ func TestCollectTIFFPreviewCandidatesIgnoresMalformedOffsets(t *testing.T) {
func TestCollectTIFFPreviewCandidatesIgnoresMalformedSubIFDOffset(t *testing.T) {
data := buildSyntheticTIFF(tiffFixture{
thumbnailJPEG: smallJPEG,
subIFDJPEG: largeJPEG,
subIFDJPEG: largeJPEG,
badJpgFromRaw: true,
})
f := openBytesFile(t, data)
@@ -114,6 +114,138 @@ func TestTiffTagUint32sSupportsIFDType(t *testing.T) {
}
}
func TestExtractEmbeddedPreviewFallbackWhenDecodeFails(t *testing.T) {
// Synthetic ARW where SubIFD offset is placed after IFD2, and a SubIFD tag
// points past EOF so extimagemeta.Decode fails with EOF, while IFD2 holds a
// valid JPEGInterchangeFormat preview.
order := binary.LittleEndian
buf := make([]byte, 32*1024)
copy(buf[0:2], []byte("II"))
order.PutUint16(buf[2:4], tiffMagicStandard)
ifd0Off := uint32(8)
order.PutUint32(buf[4:8], ifd0Off)
thumbOff := uint32(4096)
copy(buf[thumbOff:], smallJPEG)
largeOff := uint32(8192)
copy(buf[largeOff:], largeJPEG)
ifd1Off := uint32(256)
ifd2Off := uint32(512)
subIFDOff := uint32(768)
writeIFD := func(offset uint32, entries []tiffEntry, next uint32) {
off := int(offset)
order.PutUint16(buf[off:off+2], uint16(len(entries)))
entryBase := off + 2
for i, entry := range entries {
pos := entryBase + i*12
order.PutUint16(buf[pos:pos+2], entry.tag)
order.PutUint16(buf[pos+2:pos+4], entry.typ)
order.PutUint32(buf[pos+4:pos+8], entry.count)
order.PutUint32(buf[pos+8:pos+12], entry.value)
}
order.PutUint32(buf[entryBase+len(entries)*12:entryBase+len(entries)*12+4], next)
}
writeIFD(ifd0Off, []tiffEntry{
{tag: tiffTagOrientation, typ: tiffTypeShort, count: 1, value: 6},
{tag: tiffTagSubIFDs, typ: tiffTypeLong, count: 1, value: subIFDOff},
{tag: tiffTagJPEGInterchange, typ: tiffTypeLong, count: 1, value: thumbOff},
{tag: tiffTagJPEGInterchangeLen, typ: tiffTypeLong, count: 1, value: uint32(len(smallJPEG))},
}, ifd1Off)
writeIFD(ifd1Off, []tiffEntry{
{tag: tiffTagJPEGInterchange, typ: tiffTypeLong, count: 1, value: thumbOff},
{tag: tiffTagJPEGInterchangeLen, typ: tiffTypeLong, count: 1, value: uint32(len(smallJPEG))},
}, ifd2Off)
writeIFD(ifd2Off, []tiffEntry{
{tag: tiffTagJPEGInterchange, typ: tiffTypeLong, count: 1, value: largeOff},
{tag: tiffTagJPEGInterchangeLen, typ: tiffTypeLong, count: 1, value: uint32(len(largeJPEG))},
}, 0)
// SubIFD has a StripOffsets tag with count=2 pointing beyond file end so Decode fails with EOF.
writeIFD(subIFDOff, []tiffEntry{
{tag: tiffTagStripOffsets, typ: tiffTypeLong, count: 2, value: uint32(len(buf) + 4096)},
}, 0)
path := filepath.Join(t.TempDir(), "sample.arw")
if err := os.WriteFile(path, buf[:int(largeOff)+len(largeJPEG)], 0o600); err != nil {
t.Fatal(err)
}
preview, err := ExtractEmbeddedPreview(t.Context(), path)
if err != nil {
t.Fatalf("ExtractEmbeddedPreview() err = %v", err)
}
if len(preview) != len(largeJPEG) {
t.Fatalf("ExtractEmbeddedPreview() len = %d, want %d", len(preview), len(largeJPEG))
}
if got := GetOrientation(t.Context(), path); got != "Rotate 90 CW" {
t.Fatalf("GetOrientation() = %q, want %q", got, "Rotate 90 CW")
}
}
func TestExtractEmbeddedPreviewSupportsRW2AndStripPreview(t *testing.T) {
order := binary.LittleEndian
buf := make([]byte, 16*1024)
copy(buf[0:2], []byte("II"))
order.PutUint16(buf[2:4], tiffMagicPanasonicRW2)
ifd0Off := uint32(8)
order.PutUint32(buf[4:8], ifd0Off)
largeOff := uint32(4096)
copy(buf[largeOff:], largeJPEG)
entries := []tiffEntry{
{tag: tiffTagStripOffsets, typ: tiffTypeLong, count: 1, value: largeOff},
{tag: tiffTagStripByteCounts, typ: tiffTypeLong, count: 1, value: uint32(len(largeJPEG))},
}
order.PutUint16(buf[ifd0Off:ifd0Off+2], uint16(len(entries)))
for i, entry := range entries {
pos := int(ifd0Off) + 2 + i*12
order.PutUint16(buf[pos:pos+2], entry.tag)
order.PutUint16(buf[pos+2:pos+4], entry.typ)
order.PutUint32(buf[pos+4:pos+8], entry.count)
order.PutUint32(buf[pos+8:pos+12], entry.value)
}
path := filepath.Join(t.TempDir(), "sample.rw2")
if err := os.WriteFile(path, buf[:int(largeOff)+len(largeJPEG)], 0o600); err != nil {
t.Fatal(err)
}
preview, err := ExtractEmbeddedPreview(t.Context(), path)
if err != nil {
t.Fatalf("ExtractEmbeddedPreview() err = %v", err)
}
if len(preview) != len(largeJPEG) {
t.Fatalf("ExtractEmbeddedPreview() len = %d, want %d", len(preview), len(largeJPEG))
}
}
func TestExtractEmbeddedPreviewDSC02672(t *testing.T) {
path := "/Users/steffag/Downloads/DSC02672-first-4MB.ARW"
if _, err := os.Stat(path); err != nil {
t.Skip(path)
}
got, err := ExtractEmbeddedPreview(t.Context(), path)
if err != nil {
t.Fatal(err)
}
if !IsJPEG(got) {
t.Fatalf("expected JPEG preview, got %d bytes", len(got))
}
if len(got) != 2954147 {
t.Fatalf("preview size = %d, want 2954147", len(got))
}
if orient := GetOrientation(t.Context(), path); orient != "Horizontal (normal)" {
t.Fatalf("GetOrientation() = %q, want Horizontal (normal)", orient)
}
}
type tiffFixture struct {
thumbnailJPEG []byte
subIFDJPEG []byte
@@ -195,8 +327,8 @@ func buildSyntheticTIFF(fix tiffFixture) []byte {
}
type tiffEntry struct {
tag, typ uint16
count, value uint32
tag, typ uint16
count, value uint32
}
func buildChainedTIFF(chainLen int) []byte {
@@ -218,3 +350,22 @@ func buildChainedTIFF(chainLen int) []byte {
}
return buf
}
// Non-TIFF files (e.g. a JPEG reaching the orientation fallback) must be rejected from the
// 8-byte header, not after reading the whole scan window.
func TestReadTIFFScanWindowRejectsNonTIFFFromHeader(t *testing.T) {
jpeg := append([]byte{0xff, 0xd8, 0xff, 0xe0}, make([]byte, 1<<20)...)
header, _, _, ok, err := readTIFFScanWindow(openBytesFile(t, jpeg))
if err != nil || ok || header != nil {
t.Fatalf("got header=%d bytes ok=%v err=%v, want nil/false/nil", len(header), ok, err)
}
if got := extractTIFFOrientation(openBytesFile(t, jpeg)); got != 0 {
t.Fatalf("orientation = %d, want 0", got)
}
tiff := buildSyntheticTIFF(tiffFixture{thumbnailJPEG: smallJPEG})
header, _, ifd0, ok, err := readTIFFScanWindow(openBytesFile(t, tiff))
if err != nil || !ok || ifd0 == 0 || len(header) != len(tiff) {
t.Fatalf("tiff: header=%d ok=%v ifd0=%d err=%v", len(header), ok, ifd0, err)
}
}
+18
View File
@@ -1,6 +1,8 @@
package preview
import (
"bytes"
"image"
"path/filepath"
"strings"
@@ -18,3 +20,19 @@ func hasEmbeddedPreview(fileType, fileName string) bool {
func isJPEG(data []byte) bool {
return len(data) >= 2 && data[0] == 0xff && data[1] == 0xd8
}
// maxEmbeddedPreviewPixels bounds the embedded JPEG that is decoded for orientation and resizing, so a
// crafted or corrupt file can't make the decode allocate memory proportional to a huge declared size.
// Camera previews are at most full sensor resolution (a 100MP medium format JPEG is about 102M pixels).
const maxEmbeddedPreviewPixels = 150_000_000
// embeddedPreviewWithinLimits reports whether a JPEG's declared dimensions are small enough to decode.
// Only the header is read. A JPEG whose header can't be parsed is rejected too: its size can't be
// checked, and the decode steps that follow would run on it unbounded.
func embeddedPreviewWithinLimits(data []byte) bool {
cfg, _, err := image.DecodeConfig(bytes.NewReader(data))
if err != nil {
return false
}
return int64(cfg.Width)*int64(cfg.Height) <= maxEmbeddedPreviewPixels
}
+51 -1
View File
@@ -1,6 +1,11 @@
package preview
import "testing"
import (
"bytes"
"image"
"image/jpeg"
"testing"
)
func TestHasEmbeddedPreview(t *testing.T) {
if !hasEmbeddedPreview("image/x-canon-cr2", "photo.cr2") {
@@ -19,3 +24,48 @@ func TestIsJPEGMarker(t *testing.T) {
t.Fatal("expected JPEG SOI marker")
}
}
// jpegWithDeclaredSize returns a valid small JPEG whose SOF header declares the given size.
// Only the header is rewritten, so building it costs nothing even for a huge declared size.
func jpegWithDeclaredSize(t *testing.T, width, height int) []byte {
t.Helper()
var buf bytes.Buffer
if err := jpeg.Encode(&buf, image.NewGray(image.Rect(0, 0, 8, 8)), nil); err != nil {
t.Fatal(err)
}
data := buf.Bytes()
sof := bytes.Index(data, []byte{0xff, 0xc0})
if sof < 0 {
t.Fatal("no SOF0 marker in test JPEG")
}
// marker(2) length(2) precision(1) height(2) width(2)
data[sof+5], data[sof+6] = byte(height>>8), byte(height)
data[sof+7], data[sof+8] = byte(width>>8), byte(width)
return data
}
func TestEmbeddedPreviewWithinLimits(t *testing.T) {
tests := []struct {
name string
width, height int
want bool
}{
{"normal camera preview", 6000, 4000, true},
{"100MP medium format preview", 11648, 8736, true},
{"declared size far beyond any camera", 60000, 60000, false},
{"just over the limit", 12248, 12248, false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := embeddedPreviewWithinLimits(jpegWithDeclaredSize(t, tt.width, tt.height)); got != tt.want {
t.Fatalf("embeddedPreviewWithinLimits(%dx%d) = %v, want %v", tt.width, tt.height, got, tt.want)
}
})
}
}
func TestEmbeddedPreviewWithinLimitsRejectsAnUnreadableHeader(t *testing.T) {
if embeddedPreviewWithinLimits([]byte{0xff, 0xd8, 0xff, 0xe0, 0x00}) {
t.Fatal("a header that can't be read can't be size-checked, so it is rejected")
}
}
+121
View File
@@ -0,0 +1,121 @@
package preview
import (
"fmt"
"image"
"image/color"
"runtime"
"testing"
"github.com/kovidgoyal/imaging"
"github.com/stretchr/testify/require"
)
// patternYCbCr is a 4:2:0 image (what a camera JPEG decodes to) with a pattern that differs in
// every row and column, so a crop off by one pixel shows up in the resized output.
func patternYCbCr(w, h int) *image.YCbCr {
return patternYCbCrRatio(w, h, image.YCbCrSubsampleRatio420)
}
// patternYCbCrRatio is patternYCbCr with the given chroma subsampling.
func patternYCbCrRatio(w, h int, r image.YCbCrSubsampleRatio) *image.YCbCr {
img := image.NewYCbCr(image.Rect(0, 0, w, h), r)
for y := 0; y < h; y++ {
for x := 0; x < w; x++ {
img.Y[img.YOffset(x, y)] = uint8((x*7 + y*13) % 251)
}
}
for i := range img.Cb {
img.Cb[i] = uint8((i*3 + i/img.CStride) % 256)
img.Cr[i] = uint8((i + 5*(i/img.CStride)) % 256)
}
return img
}
// patternNRGBA is an NRGBA image whose pixels differ in every row and column.
func patternNRGBA(w, h int) *image.NRGBA {
img := image.NewNRGBA(image.Rect(0, 0, w, h))
for y := 0; y < h; y++ {
for x := 0; x < w; x++ {
img.SetNRGBA(x, y, color.NRGBA{uint8(x % 256), uint8(y % 256), uint8((x + y) % 256), 255})
}
}
return img
}
// requireSamePixels fails unless want and got are the same size and every pixel is equal,
// comparing from each image's own origin.
func requireSamePixels(t *testing.T, want, got image.Image) {
t.Helper()
require.Equal(t, want.Bounds().Size(), got.Bounds().Size())
wb, gb := want.Bounds().Min, got.Bounds().Min
for y := 0; y < want.Bounds().Dy(); y++ {
for x := 0; x < want.Bounds().Dx(); x++ {
wr, wg, wbl, wa := want.At(wb.X+x, wb.Y+y).RGBA()
gr, gg, gbl, ga := got.At(gb.X+x, gb.Y+y).RGBA()
if wr != gr || wg != gg || wbl != gbl || wa != ga {
t.Fatalf("pixel (%d,%d) differs: imaging.Fill %v, fillCenter %v", x, y,
[4]uint32{wr, wg, wbl, wa}, [4]uint32{gr, gg, gbl, ga})
}
}
}
}
// fillCenter must produce exactly what imaging.Fill(…, imaging.Center, …) produces: the same
// pixels and the same image type, so the encoded preview is the same bytes.
func TestFillCenter_MatchesImagingFill(t *testing.T) {
sources := map[string]image.Image{
"ycbcr landscape": patternYCbCr(1200, 801),
"ycbcr portrait": patternYCbCr(601, 1000),
"nrgba square": patternNRGBA(700, 700),
"nrgba wide": patternNRGBA(1500, 300),
"small (falls back to imaging.Fill)": patternNRGBA(90, 60),
"exact target size": patternNRGBA(256, 256),
"offset origin": patternNRGBA(900, 700).SubImage(image.Rect(37, 11, 837, 611)),
}
// Every subsampling ratio, in shapes whose centered crop has an x (or y) offset and reaches
// the source's last row (or column) -- where a SubImage's buffer ends short of a full stride.
for _, r := range []image.YCbCrSubsampleRatio{
image.YCbCrSubsampleRatio444, image.YCbCrSubsampleRatio422, image.YCbCrSubsampleRatio420,
image.YCbCrSubsampleRatio440, image.YCbCrSubsampleRatio411, image.YCbCrSubsampleRatio410,
} {
for _, sz := range [][2]int{{600, 264}, {264, 600}, {601, 263}} {
sources[fmt.Sprintf("ycbcr %v %dx%d", r, sz[0], sz[1])] = patternYCbCrRatio(sz[0], sz[1], r)
}
}
sources["nycbcra 444 600x264"] = &image.NYCbCrA{YCbCr: *patternYCbCrRatio(600, 264, image.YCbCrSubsampleRatio444), A: make([]uint8, 600*264), AStride: 600}
// 264x264 is the crop of the 600x264 and 264x600 sources itself: nothing to scale.
targets := [][2]int{{256, 256}, {320, 180}, {180, 320}, {264, 264}}
filters := map[string]imaging.ResampleFilter{
"nearest": imaging.NearestNeighbor, "box": imaging.Box, "catmullrom": imaging.CatmullRom,
}
for name, src := range sources {
for _, tg := range targets {
for fname, f := range filters {
t.Run(fmt.Sprintf("%s/%dx%d/%s", name, tg[0], tg[1], fname), func(t *testing.T) {
want := imaging.Fill(src, tg[0], tg[1], imaging.Center, f)
got := fillCenter(src, tg[0], tg[1], f)
require.Equal(t, fmt.Sprintf("%T", want), fmt.Sprintf("%T", got), "image type")
requireSamePixels(t, want, got)
})
}
}
}
}
// A square thumbnail of a large photo must not allocate a full-resolution copy of it.
func TestFillCenter_NoFullSizeCopy(t *testing.T) {
const w, h = 4000, 3000
src := patternYCbCr(w, h)
fullCopy := uint64(h * h * 4) // the square crop imaging.Fill materializes as NRGBA
var before, after runtime.MemStats
runtime.ReadMemStats(&before)
out := fillCenter(src, 256, 256, imaging.Box)
runtime.ReadMemStats(&after)
require.Equal(t, image.Pt(256, 256), out.Bounds().Size())
allocated := after.TotalAlloc - before.TotalAlloc
require.Less(t, allocated, fullCopy/4,
"fillCenter allocated %d bytes; a copy of the crop would be %d", allocated, fullCopy)
}
+86 -3
View File
@@ -7,6 +7,7 @@ import (
"image"
"image/jpeg"
"io"
"math"
exif "github.com/dsoprea/go-exif/v3"
exifcommon "github.com/dsoprea/go-exif/v3/common"
@@ -76,6 +77,18 @@ func (x Format) toImaging() imaging.Format {
}
}
// encodable reports whether imaging can encode this format.
// imaging only encodes JPEG, PNG, GIF, TIFF and BMP; webp and the netpbm
// formats are decode-only, so previews for them must be written as JPEG.
func (x Format) encodable() bool {
switch x {
case FormatJpeg, FormatPng, FormatGif, FormatTiff, FormatBmp:
return true
default:
return false
}
}
/*
ENUM(
high
@@ -186,6 +199,50 @@ func (s *Service) ResizeWithSize(ctx context.Context, in io.Reader, out io.Write
return err
}
// fillCenter is imaging.Fill(img, width, height, imaging.Center, filter) without a full-size copy
// of the source. For sources of 100px or more, imaging.Fill first crops into a new NRGBA image
// (4 bytes per pixel of the cropped area, about 144 MB for a 48 MP photo) and only then resizes.
// Here the crop is a SubImage view into the decoded pixels, so only the resize allocates. The
// crop rectangle is computed as imaging's cropAndResize computes it, so the output is the same.
//
// Two cases still go through imaging.Fill: a crop that is already the target size (nothing to
// scale, and Fill's small copy is what keeps the output type the same), and 4:4:4 YCbCr sources,
// whose scanner in imaging v1.9.0 advances one full parent stride past the end of a SubImage's
// last row and panics when the crop has an x offset and reaches the bottom row.
func fillCenter(img image.Image, width, height int, filter imaging.ResampleFilter) image.Image {
b := img.Bounds()
sub, ok := img.(interface {
SubImage(image.Rectangle) image.Image
})
if !ok || is444(img) || width <= 0 || height <= 0 || b.Dx() < 100 || b.Dy() < 100 || (b.Dx() == width && b.Dy() == height) {
return imaging.Fill(img, width, height, imaging.Center, filter)
}
cropW, cropH := b.Dx(), b.Dy()
if float64(b.Dx())/float64(b.Dy()) < float64(width)/float64(height) {
cropH = int(math.Max(1, float64(b.Dx())*float64(height)/float64(width)) + 0.5)
} else {
cropW = int(math.Max(1, float64(b.Dy())*float64(width)/float64(height)) + 0.5)
}
if cropW == width && cropH == height {
return imaging.Fill(img, width, height, imaging.Center, filter)
}
x0 := b.Min.X + (b.Dx()-cropW)/2
y0 := b.Min.Y + (b.Dy()-cropH)/2
crop := image.Rect(x0, y0, x0+cropW, y0+cropH).Intersect(b)
return imaging.Resize(sub.SubImage(crop), width, height, filter)
}
// is444 reports whether img is a YCbCr image without chroma subsampling.
func is444(img image.Image) bool {
switch p := img.(type) {
case *image.YCbCr:
return p.SubsampleRatio == image.YCbCrSubsampleRatio444
case *image.NYCbCrA:
return p.SubsampleRatio == image.YCbCrSubsampleRatio444
}
return false
}
func (s *Service) resizeWithSize(in io.Reader, out io.Writer, fileSize int64, opts ResizeOptions) error {
// Set defaults
if opts.ResizeMode == 0 {
@@ -203,7 +260,9 @@ func (s *Service) resizeWithSize(in io.Reader, out io.Writer, fileSize int64, op
var wrappedReader = in
var err error
if opts.Format == FormatHeic {
// HEIC has no encoder and webp/netpbm are decode-only in imaging; fall back
// to JPEG so preview generation does not fail with ErrUnsupportedFormat.
if !opts.Format.encodable() {
opts.Format = FormatJpeg
}
// Ensure JPEG quality is set based on quality setting
@@ -261,7 +320,7 @@ func (s *Service) resizeWithSize(in io.Reader, out io.Writer, fileSize int64, op
resampleFilter := opts.Quality.resampleFilter()
switch opts.ResizeMode {
case ResizeModeFill:
img = imaging.Fill(img, opts.Width, opts.Height, imaging.Center, resampleFilter)
img = fillCenter(img, opts.Width, opts.Height, resampleFilter)
case ResizeModeFit:
fallthrough
default:
@@ -387,8 +446,32 @@ func getEmbeddedThumbnail(in io.Reader) ([]byte, io.Reader, error) {
return nil, wrappedReader, exif.ErrNoThumbnail
}
// Read IFD1's Orientation first; fall back to IFD0's Orientation if not specified (per MWG guidelines)
orientStr := ""
tags1, err1 := ifd.FindTagWithName("Orientation")
if err1 == nil && len(tags1) > 0 {
orientStr, _ = tags1[0].FormatFirst()
}
if orientStr == "" {
tags0, err0 := index.RootIfd.FindTagWithName("Orientation")
if err0 == nil && len(tags0) > 0 {
orientStr, _ = tags0[0].FormatFirst()
}
}
thm, err := ifd.Thumbnail()
return thm, wrappedReader, err
if err != nil {
return nil, wrappedReader, err
}
if orientStr != "" && orientStr != "1" {
if !embeddedPreviewWithinLimits(thm) {
return nil, wrappedReader, fmt.Errorf("embedded thumbnail exceeds decode limits")
}
thm = applyOrientationToPreviewBytes(thm, orientStr)
}
return thm, wrappedReader, nil
}
// CreateThumbnail decodes an image and creates a fixed-size thumbnail.
+46
View File
@@ -161,6 +161,14 @@ func TestService_Resize(t *testing.T) {
},
matcher: sizeMatcher(100, 100),
},
"resize webp falls back to jpeg": {
options: ResizeOptions{Width: 100, Height: 100, ResizeMode: ResizeModeFill, Format: FormatWebp},
source: func(t *testing.T) afero.File {
t.Helper()
return openFile(t, "testdata/gray-sample.webp")
},
matcher: formatMatcher(FormatJpeg),
},
"resize with high quality": {
options: ResizeOptions{Width: 100, Height: 100, ResizeMode: ResizeModeFill, Quality: QualityHigh},
source: func(t *testing.T) afero.File {
@@ -541,3 +549,41 @@ func TestReadImageFileDimensions(t *testing.T) {
require.Greater(t, width, 0)
require.Greater(t, height, 0)
}
func TestApplyOrientationToPreviewBytes(t *testing.T) {
t.Parallel()
// Generate a 100x60 test JPEG image in memory (landscape)
img := image.NewRGBA(image.Rect(0, 0, 100, 60))
var buf bytes.Buffer
err := jpeg.Encode(&buf, img, nil)
require.NoError(t, err)
rawBytes := buf.Bytes()
tests := []struct {
orientation string
wantWidth int
wantHeight int
}{
{orientation: "1", wantWidth: 100, wantHeight: 60},
{orientation: "Horizontal (normal)", wantWidth: 100, wantHeight: 60},
{orientation: "8", wantWidth: 60, wantHeight: 100},
{orientation: "Rotate 270 CW", wantWidth: 60, wantHeight: 100},
{orientation: "6", wantWidth: 60, wantHeight: 100},
{orientation: "Rotate 90 CW", wantWidth: 60, wantHeight: 100},
{orientation: "3", wantWidth: 100, wantHeight: 60},
{orientation: "Rotate 180", wantWidth: 100, wantHeight: 60},
}
for _, tt := range tests {
t.Run(tt.orientation, func(t *testing.T) {
rotated := applyOrientationToPreviewBytes(rawBytes, tt.orientation)
require.NotEmpty(t, rotated)
cfg, _, decodeErr := image.DecodeConfig(bytes.NewReader(rotated))
require.NoError(t, decodeErr)
require.Equal(t, tt.wantWidth, cfg.Width)
require.Equal(t, tt.wantHeight, cfg.Height)
})
}
}
+5 -12
View File
@@ -65,7 +65,7 @@ func NewPreviewGenerator(concurrencyLimit int, cacheDir string) *Service {
var fileCache diskcache.Interface
// Use file cache if cacheDir is specified
var err error
fileCache, err = diskcache.NewFileCache(actualCacheDir)
fileCache, err = diskcache.NewFileCacheWithLimit(actualCacheDir, max(settings.Config.Server.CacheDirMaxSizeMB, 0)*1024*1024)
if err != nil {
logger.Error("The cache dir could not be created. Make sure the user that you executed the program with has access to create directories in the local path. See ")
logger.Fatalf("failed to create file cache path, which is now require to run the server: %v", err)
@@ -272,15 +272,6 @@ func (s *Service) generateRawPreview(ctx context.Context, file iteminfo.Extended
case previewTypeImage:
ext := strings.ToLower(filepath.Ext(file.Name))
if iteminfo.IsRawImage(ext) {
if s.ffmpegService != nil {
previewBytes, err := s.convertImageWithFFmpeg(ctx, file.RealPath, previewSize)
if err == nil && len(previewBytes) >= minPreviewSize {
return previewBytes, nil
}
if ctx.Err() != nil {
return nil, ctx.Err()
}
}
return nil, ErrUnsupportedFormat
}
return s.generateImagePreview(ctx, file, previewSize)
@@ -458,7 +449,9 @@ func GeneratePreviewWithMD5(ctx context.Context, file iteminfo.ExtendedFileInfo,
}
// Enforce file size limit for image preview generation to prevent memory exhaustion
if strings.HasPrefix(file.Type, "image") && file.Size > iteminfo.LargeFileSizeThreshold {
// Camera RAW is exempt: its preview is the embedded JPEG and the RAW data is never decoded.
if strings.HasPrefix(file.Type, "image") && file.Size > iteminfo.LargeFileSizeThreshold &&
!iteminfo.IsRawImage(strings.ToLower(filepath.Ext(file.Name))) {
message := fmt.Sprintf("Image file too large for preview: %s (size: %d bytes, limit: %d bytes)",
file.Name, file.Size, iteminfo.LargeFileSizeThreshold)
logger.Warning(message)
@@ -476,7 +469,7 @@ func GeneratePreviewWithMD5(ctx context.Context, file iteminfo.ExtendedFileInfo,
var imageBytes []byte
fromEmbeddedPreview := false
if hasEmbeddedPreview(file.Type, file.Name) {
if previewBytes, _ := imagemeta.ExtractEmbeddedPreview(ctx, file.RealPath); len(previewBytes) >= minPreviewSize && isJPEG(previewBytes) {
if previewBytes, _ := imagemeta.ExtractEmbeddedPreview(ctx, file.RealPath); len(previewBytes) >= minPreviewSize && isJPEG(previewBytes) && embeddedPreviewWithinLimits(previewBytes) {
imageBytes = previewBytes
fromEmbeddedPreview = true
if orient := imagemeta.GetOrientation(ctx, file.RealPath); orient != "" {
+2 -2
View File
@@ -13,7 +13,7 @@ type ffmpegPreviewParams struct {
func ffmpegPreviewParamsForSize(previewSize string) (ffmpegPreviewParams, error) {
if previewSize == "original" {
return ffmpegPreviewParams{quality: 10}, nil
return ffmpegPreviewParams{quality: ops.LegacyPreviewDefaultQuality}, nil
}
opts, err := getPreviewOptions(previewSize)
@@ -24,7 +24,7 @@ func ffmpegPreviewParamsForSize(previewSize string) (ffmpegPreviewParams, error)
params := ffmpegPreviewParams{
width: opts.Width,
height: opts.Height,
quality: 10, // fastest MJPEG encode (-q:v 10); visual quality comes from target dimensions
quality: ops.LegacyPreviewDefaultQuality, // 1–100 scale; equivalent to historic MJPEG -q:v 10
}
if opts.ResizeMode == ResizeModeFill {
params.scaleMode = ops.ScaleFill
@@ -25,7 +25,7 @@ func TestFfmpegPreviewParamsForSize(t *testing.T) {
wantW: 256,
wantH: 256,
wantMode: ops.ScaleFill,
wantQual: 10,
wantQual: ops.LegacyPreviewDefaultQuality,
},
{
name: "large fit",
@@ -33,7 +33,7 @@ func TestFfmpegPreviewParamsForSize(t *testing.T) {
wantW: 640,
wantH: 640,
wantMode: ops.ScaleFit,
wantQual: 10,
wantQual: ops.LegacyPreviewDefaultQuality,
},
{
name: "xlarge fit",
@@ -41,12 +41,12 @@ func TestFfmpegPreviewParamsForSize(t *testing.T) {
wantW: 1024,
wantH: 1024,
wantMode: ops.ScaleFit,
wantQual: 10,
wantQual: ops.LegacyPreviewDefaultQuality,
},
{
name: "original no scale",
size: "original",
wantQual: 10,
wantQual: ops.LegacyPreviewDefaultQuality,
},
{
name: "unsupported",
+9
View File
@@ -19,6 +19,15 @@ func AccessPermitted(sourcePath string, indexPath utils.IndexPath, username stri
return accessDb.Permitted(sourcePath, indexPath, username)
}
// AccessPermittedAtRealPath reports whether username may access realPath after symlink
// resolution on the given index. When accessDb is not initialized, access is allowed.
func AccessPermittedAtRealPath(idx *indexing.Index, username, realPath string, isDir bool) bool {
if accessDb == nil {
return true
}
return accessDb.PermittedAtRealPath(idx, username, realPath, isDir)
}
func AllowUser(sourcePath string, indexPath utils.IndexPath, username string) error {
return accessDb.AllowUser(sourcePath, indexPath, username)
}
+96 -17
View File
@@ -536,8 +536,8 @@ func ApplicableFolderQuotas(sourcePath, destIndexPath string, principalUserID ui
}
type reserveApplyPlan struct {
chk QuotaReserveCheck
existingIdx int
chk QuotaReserveCheck
existingIdx int
counterDelta int64
}
@@ -715,6 +715,98 @@ func ForceCommitSessionQuota(sessionID string, deltaBytes int64) error {
return nil
}
// DeleteAdjustments holds accounted quota updates to apply after a successful delete.
type DeleteAdjustments struct {
Deltas map[string]int64
Resets []string
}
// PrepareAccountedDelete captures quota adjustments while deleted paths are still indexed.
// Folder quotas rooted inside a deleted directory are reset; covering quotas use itemBytes.
func PrepareAccountedDelete(principal *users.User, sourceName, sourcePath, indexPath, shareHash string, isDir bool, itemBytes int64) DeleteAdjustments {
adj := DeleteAdjustments{Deltas: make(map[string]int64)}
if principal == nil {
return adj
}
indexPath = normalizeQuotaPath(indexPath)
for _, fq := range ListFolderQuotasForSource(sourcePath) {
if fq.LimitBytes <= 0 || (fq.UserID != 0 && fq.UserID != principal.ID) {
continue
}
if isDir && indexPathCovers(indexPath, fq.Path) {
if quota.EffectiveMeter(configuredFolderMeter(fq), sourceName, fq.Path) != quota.MeterAccounted {
continue
}
adj.Resets = append(adj.Resets, fq.ID)
continue
}
if !indexPathCovers(fq.Path, indexPath) {
continue
}
if quota.EffectiveMeter(configuredFolderMeter(fq), sourceName, fq.Path) != quota.MeterAccounted {
continue
}
if itemBytes > 0 {
adj.Deltas[fq.ID] = -itemBytes
}
}
if itemBytes > 0 {
for _, id := range accountedScopeQuotaIDs(principal, sourceName, sourcePath) {
adj.Deltas[id] = -itemBytes
}
if shareHash != "" {
adj.Deltas[quota.ShareQuotaID(shareHash)] = -itemBytes
}
}
return adj
}
// ApplyAccountedDelete applies prepared adjustments only after a successful deletion.
func ApplyAccountedDelete(adj DeleteAdjustments) error {
quotasMux.Lock()
defer quotasMux.Unlock()
defer signalQuotaFlush()
for id, delta := range adj.Deltas {
if err := applyCounterDeltaLocked(id, delta); err != nil {
return err
}
}
for _, id := range adj.Resets {
if err := resetAccountedUsageLocked(id); err != nil {
return err
}
}
return nil
}
func resetAccountedUsageLocked(quotaID string) error {
if err := ensureQuotaCounterMemLocked(quotaID); err != nil {
return err
}
c := quotaCounters[quotaID]
c.UsedBytes = 0
c.Dirty = true
markQuotaCounterDirty(quotaID)
return nil
}
func accountedScopeQuotaIDs(principal *users.User, sourceName, sourcePath string) []string {
var ids []string
for _, bs := range principal.BackendScopes {
if bs.Path != sourcePath || bs.Quota == nil || bs.Quota.LimitBytes <= 0 {
continue
}
configured := bs.Quota.Meter
if configured == "" {
configured = quota.MeterIndexScope
}
if quota.EffectiveMeter(configured, sourceName, scopeIndexPath(principal, sourceName)) == quota.MeterAccounted {
ids = append(ids, bs.Quota.ID)
}
}
return ids
}
// ApplyAccountedUsageDelta adjusts accounted and share quota counters for a path (negative delta frees space).
func ApplyAccountedUsageDelta(principal *users.User, sourceName, sourcePath, indexPath, shareHash string, delta int64) error {
if delta == 0 || principal == nil {
@@ -735,21 +827,8 @@ func ApplyAccountedUsageDelta(principal *users.User, sourceName, sourcePath, ind
}
}
for _, bs := range principal.BackendScopes {
if bs.Path != sourcePath || bs.Quota == nil || bs.Quota.LimitBytes <= 0 {
continue
}
sq := bs.Quota
configured := sq.Meter
if configured == "" {
configured = quota.MeterIndexScope
}
scopePath := scopeIndexPath(principal, sourceName)
effective := quota.EffectiveMeter(configured, sourceName, scopePath)
if effective != quota.MeterAccounted {
continue
}
if err := applyCounterDeltaLocked(sq.ID, delta); err != nil {
for _, id := range accountedScopeQuotaIDs(principal, sourceName, sourcePath) {
if err := applyCounterDeltaLocked(id, delta); err != nil {
return err
}
}
@@ -0,0 +1,135 @@
package state
import (
"reflect"
"slices"
"testing"
"github.com/gtsteffaniak/filebrowser/backend/internal/database/quota"
"github.com/gtsteffaniak/filebrowser/backend/internal/database/users"
"github.com/gtsteffaniak/filebrowser/backend/pkg/indexing"
)
func TestPrepareAccountedDelete(t *testing.T) {
oldFolders, oldSources, oldCounters := folderQuotasByID, folderQuotasBySource, quotaCounters
folderQuotasByID = make(map[string]*quota.FolderQuota)
folderQuotasBySource = make(map[string][]string)
quotaCounters = make(map[string]*quotaCounterMem)
t.Cleanup(func() {
folderQuotasByID, folderQuotasBySource, quotaCounters = oldFolders, oldSources, oldCounters
indexing.ClearTestIndices()
})
indexing.SetTestIndex("delete", "/source")
idx := indexing.GetIndex("delete")
idx.SetFolderSize("/public/", 60)
idx.SetFolderSize("/public/nested/", 50)
idx.SetFolderSize("/public/nested/deeper/", 30)
principal := &users.User{ID: 1, BackendScopes: []users.BackendScope{{Path: "/source", Scope: "/", Quota: &users.ScopeQuota{ID: "scope", LimitBytes: 1000, Meter: quota.MeterAccounted}}}}
for _, q := range []quota.FolderQuota{
{ID: "cover", Source: "/source", Path: "/", Meter: quota.MeterAccounted},
{ID: "exact", Source: "/source", Path: "/public", Meter: quota.MeterAccounted},
{ID: "nested", Source: "/source", Path: "/public/nested", Meter: quota.MeterAccounted, UserID: 1},
{ID: "deeper", Source: "/source", Path: "/public/nested/deeper", Meter: quota.MeterAccounted},
{ID: "missing", Source: "/source", Path: "/public/missing", Meter: quota.MeterAccounted},
{ID: "sibling", Source: "/source", Path: "/sibling", Meter: quota.MeterAccounted},
{ID: "prefix", Source: "/source", Path: "/public-other", Meter: quota.MeterAccounted},
{ID: "other-user", Source: "/source", Path: "/public/nested", Meter: quota.MeterAccounted, UserID: 2},
{ID: "other-source", Source: "/other", Path: "/public/nested", Meter: quota.MeterAccounted},
{ID: "index", Source: "/source", Path: "/public/nested", Meter: quota.MeterIndexSize},
} {
q.LimitBytes = 1000
storeFolderQuota(q)
quotaCounters[q.ID] = &quotaCounterMem{UsedBytes: 100, ReservedBytes: 7}
}
shareID := quota.ShareQuotaID("share")
quotaCounters["scope"] = &quotaCounterMem{UsedBytes: 100, ReservedBytes: 7}
quotaCounters[shareID] = &quotaCounterMem{UsedBytes: 100, ReservedBytes: 7}
wantDeltas := map[string]int64{"cover": -60, "scope": -60, shareID: -60}
wantResets := []string{"exact", "nested", "deeper", "missing"}
got := PrepareAccountedDelete(principal, "delete", "/source", "/public", "share", true, 60)
if !reflect.DeepEqual(got.Deltas, wantDeltas) {
t.Fatalf("deltas = %v, want %v", got.Deltas, wantDeltas)
}
if !slicesEqual(got.Resets, wantResets) {
t.Fatalf("resets = %v, want %v", got.Resets, wantResets)
}
for id, delta := range wantDeltas {
quotaCounters[id].UsedBytes = -delta
}
quotaCounters["cover"].UsedBytes += 40
for _, id := range wantResets {
quotaCounters[id].UsedBytes = 100
}
if err := ApplyAccountedDelete(got); err != nil {
t.Fatal(err)
}
for id, c := range quotaCounters {
expected := int64(100)
if _, ok := wantDeltas[id]; ok {
expected = 0
}
if slices.Contains(wantResets, id) {
expected = 0
}
if id == "cover" {
expected = 40
}
if c.UsedBytes != expected || c.ReservedBytes != 7 {
t.Errorf("%s: used=%d reserved=%d, want %d/7", id, c.UsedBytes, c.ReservedBytes, expected)
}
}
for _, size := range []int64{0, -1} {
got = PrepareAccountedDelete(principal, "delete", "/source", "/public", "", true, size)
wantResetsUnknown := []string{"exact", "nested", "deeper", "missing"}
if len(got.Deltas) != 0 || !slicesEqual(got.Resets, wantResetsUnknown) {
t.Fatalf("unavailable/zero ancestor: deltas=%v resets=%v", got.Deltas, got.Resets)
}
}
got = PrepareAccountedDelete(principal, "delete", "/source", "/public/nested/file", "", false, 5)
if !reflect.DeepEqual(got, DeleteAdjustments{
Deltas: map[string]int64{"cover": -5, "exact": -5, "nested": -5, "scope": -5},
}) {
t.Fatalf("file adjustments: %v", got)
}
empty := PrepareAccountedDelete(nil, "delete", "/source", "/public", "share", true, 60)
if len(empty.Deltas) != 0 || len(empty.Resets) != 0 {
t.Fatalf("nil principal: %v", empty)
}
}
func TestPrepareAccountedDeleteResetsWithoutIndexedSizes(t *testing.T) {
oldFolders, oldSources, oldCounters := folderQuotasByID, folderQuotasBySource, quotaCounters
folderQuotasByID = make(map[string]*quota.FolderQuota)
folderQuotasBySource = make(map[string][]string)
quotaCounters = make(map[string]*quotaCounterMem)
t.Cleanup(func() {
folderQuotasByID, folderQuotasBySource, quotaCounters = oldFolders, oldSources, oldCounters
indexing.ClearTestIndices()
})
indexing.SetTestIndex("delete", "/source")
principal := &users.User{ID: 1}
nested := quota.FolderQuota{ID: "nested", Source: "/source", Path: "/public/nested", Meter: quota.MeterAccounted, LimitBytes: 1000}
storeFolderQuota(nested)
quotaCounters["nested"] = &quotaCounterMem{UsedBytes: 50, ReservedBytes: 3}
got := PrepareAccountedDelete(principal, "delete", "/source", "/public", "", true, 0)
if !slicesEqual(got.Resets, []string{"nested"}) {
t.Fatalf("resets = %v, want nested reset without index sizes", got.Resets)
}
if err := ApplyAccountedDelete(got); err != nil {
t.Fatal(err)
}
if used, reserved := GetQuotaCounterSnapshot("nested"); used != 0 || reserved != 3 {
t.Fatalf("nested used=%d reserved=%d, want 0/3", used, reserved)
}
}
func slicesEqual(a, b []string) bool {
if len(a) != len(b) {
return false
}
ac := slices.Clone(a)
bc := slices.Clone(b)
slices.Sort(ac)
slices.Sort(bc)
return slices.Equal(ac, bc)
}
+12
View File
@@ -4,6 +4,7 @@ import (
"encoding/json"
"fmt"
"reflect"
"slices"
"strings"
"sync"
@@ -297,6 +298,17 @@ func ApplyUserDefaults(u *users.User) {
applyToolAccessDefaults(u)
}
// PrefillUserProfileDefaults seeds u with the effective profile defaults so a JSON decode
// overlays only the keys the request sent (encoding/json leaves absent keys untouched).
// Permissions are left zero so omitted permissions stay distinguishable; CreateUser merges them.
func PrefillUserProfileDefaults(u *users.User) {
settings.ApplyFullProfileFromDefaults(u, EffectiveUserDefaults())
// json.Unmarshal reuses an existing slice's backing array, so clone it or a request
// would overwrite the shared in-memory defaults.
u.NewFileTemplate = slices.Clone(u.NewFileTemplate)
u.Permissions = users.Permissions{}
}
func applyUserSettingsDefaults(u *users.User) {
settings.ApplyUserDefaultsFrom(u, EffectiveUserDefaults())
}
+19
View File
@@ -202,6 +202,18 @@ func UserForShareOwner(link share.Share) (users.User, error) {
// may apply default-enabled sources. In-memory/cache users always keep FrontendScopes nil; GET handlers use
// PrepForFrontend to repopulate FrontendScopes from BackendScopes for responses.
func CreateUser(user *users.User, plaintextPassword string) error {
return createUser(user, plaintextPassword, false)
}
// CreateUserKeepingProfile is CreateUser for callers whose user was seeded with
// PrefillUserProfileDefaults before decoding the request: the profile fields already hold
// defaults overlaid with the request, so they are not replaced by userDefaults again.
// Enforced defaults still win.
func CreateUserKeepingProfile(user *users.User, plaintextPassword string) error {
return createUser(user, plaintextPassword, true)
}
func createUser(user *users.User, plaintextPassword string, keepProfile bool) error {
if err := users.ValidateUsername(user.Username); err != nil {
return err
}
@@ -256,12 +268,19 @@ func CreateUser(user *users.User, plaintextPassword string) error {
requestedScopePermissions[backendScopeKeyFor(scope)] = scope.Permissions
}
}
var requestedProfile settings.UserProfile
if keepProfile {
requestedProfile = settings.ProfileFromUser(user)
}
// Apply profile/scope defaults using the final account permissions (admin flag drives scope defaults).
if permissionsRequested {
settings.ApplyUserDefaultsFromKeepingAccountPermissions(user, EffectiveUserDefaults())
} else {
applyUserSettingsDefaults(user)
}
if keepProfile {
settings.ExpandProfileIntoUser(user, requestedProfile)
}
if preserveRequirePasswordChange {
user.RequirePasswordChange = true
}
+8 -2
View File
@@ -15,12 +15,18 @@ type SubtitleTrack struct {
Embedded bool `json:"embedded"` // true for embedded subtitles, false for external files
}
// ErrSubtitleSidecarNotRegular is returned when an external subtitle path is not a regular file (e.g. symlink).
var ErrSubtitleSidecarNotRegular = errors.New("subtitle sidecar is not a regular file")
func GetSubtitleSidecarContent(subtitlePath string) (string, error) {
// validate the size is not too large
info, err := os.Stat(subtitlePath)
// Sidecars sit beside the media file; do not follow symlink leaves (see lyricsSidecar).
info, err := os.Lstat(subtitlePath)
if err != nil {
return "", err
}
if !info.Mode().IsRegular() {
return "", ErrSubtitleSidecarNotRegular
}
if info.Size() > 1024*1024*50 { // 50MB
return "", errors.New("subtitle file is too large")
}
+1
View File
@@ -22,6 +22,7 @@ import (
func setupAccessHTTPTest(t *testing.T) {
t.Helper()
lockWebGlobals(t)
dbPath := filepath.Join(t.TempDir(), "access-http-test.sqlite")
if _, err := state.Initialize(dbPath); err != nil {
t.Fatal(err)
+84 -9
View File
@@ -279,19 +279,21 @@ func archiveCreateHandler(w http.ResponseWriter, r *http.Request, d *Context) (i
if fullDestParent != fullDestFile && !state.AccessPermitted(idxTo.Path, utils.IndexPathFromNormalized(fullDestParent, true), d.User.Username) {
return http.StatusForbidden, fmt.Errorf("access denied to destination parent of %s", req.Destination)
}
parentReal, err := idxTo.GetRealPathForMutation(userScopeTo, fullDestParent)
if err != nil {
return http.StatusBadRequest, fmt.Errorf("destination parent path invalid: %v", err)
}
if err = os.MkdirAll(parentReal, fileutils.EffectiveDirPerm()); err != nil {
return http.StatusInternalServerError, fmt.Errorf("cannot create destination parent directory: %v", err)
}
destRel := strings.TrimLeft(filepath.ToSlash(req.Destination), "/")
fileName := path.Base(destRel)
if fileName == "." || fileName == "/" {
return http.StatusBadRequest, fmt.Errorf("invalid destination file name")
}
destFileReal := filepath.Join(parentReal, fileName)
destFileReal, err := idxTo.GetRealPathForMutation(userScopeTo, fullDestFile)
if err != nil {
return http.StatusBadRequest, fmt.Errorf("destination path invalid: %v", err)
}
if !state.AccessPermittedAtRealPath(idxTo, d.User.Username, destFileReal, false) {
return http.StatusForbidden, fmt.Errorf("access denied to destination %s", req.Destination)
}
if err = os.MkdirAll(filepath.Dir(destFileReal), fileutils.EffectiveDirPerm()); err != nil {
return http.StatusInternalServerError, fmt.Errorf("cannot create destination parent directory: %v", err)
}
format := strings.ToLower(strings.TrimSpace(req.Format))
if format == "" {
@@ -1198,6 +1200,26 @@ func extractArchivedDir(destPath string, mode fs.FileMode, modTime time.Time) er
return applyArchivedTimesAndPerm(destPath, mode, modTime)
}
// removeSymlinkDestination removes destPath when it is a symlink so a regular-file
// write creates the entry at destPath instead of following the link.
func removeSymlinkDestination(destPath string) error {
info, err := os.Lstat(destPath)
if err != nil {
if os.IsNotExist(err) {
return nil
}
return err
}
if info.Mode()&fs.ModeSymlink == 0 {
return nil
}
if err := os.Remove(destPath); err != nil {
logger.Debugf("unarchive: remove symlink destination: %v", err)
return fmt.Errorf("failed to write file from archive entry")
}
return nil
}
// extractArchivedRegularFile writes one regular file from src (closed by this function) and reapplies mode and mtime.
func extractArchivedRegularFile(destPath string, mode fs.FileMode, modTime time.Time, src io.ReadCloser) error {
defer func() { _ = src.Close() }()
@@ -1206,6 +1228,9 @@ func extractArchivedRegularFile(destPath string, mode fs.FileMode, modTime time.
logger.Debugf("unarchive: mkdir parent: %v", err)
return fmt.Errorf("failed to create directory from archive entry")
}
if err := removeSymlinkDestination(destPath); err != nil {
return err
}
perm := mode.Perm()
if perm == 0 {
perm = fileutils.EffectiveFilePerm()
@@ -1247,9 +1272,51 @@ func extractArchivedSymlink(destRoot, destPath, target string, modTime time.Time
logger.Debugf("unarchive: symlink: %v", err)
return fmt.Errorf("failed to create symlink from archive entry")
}
indexing.ClearRealPathCaches()
return applyArchivedTimesAndPerm(destPath, fs.ModeSymlink, modTime)
}
// extractArchivedHardLink writes a copy of a regular file extracted earlier from the same
// archive, which is what a tar hard link entry stands for.
func extractArchivedHardLink(destRoot, destPath string, h *tar.Header, extracted map[string]os.FileInfo) error {
targetPath, err := safeExtractPath(destRoot, h.Linkname)
if err != nil {
return err
}
notExtracted := fmt.Errorf("hard link %q points to %q, which is not a file earlier in the archive", h.Name, h.Linkname)
written, ok := extracted[targetPath]
if !ok {
return notExtracted
}
src, err := os.Open(targetPath)
if err != nil {
logger.Debugf("unarchive: open hard link target: %v", err)
return fmt.Errorf("failed to write file from archive entry")
}
// A later entry may have replaced the target, e.g. with a symlink to another file.
info, err := src.Stat()
if err != nil || !info.Mode().IsRegular() || !os.SameFile(info, written) {
_ = src.Close()
return notExtracted
}
// The entry may name its own target, directly (GNU tar does this for a file archived
// twice). Writing it would truncate the file it copies from. Symlinks at destPath
// are replaced by extractArchivedRegularFile instead of being left in place.
if dest, err := os.Lstat(destPath); err == nil && dest.Mode().IsRegular() && os.SameFile(dest, info) {
return src.Close()
}
return extractArchivedRegularFile(destPath, h.FileInfo().Mode(), h.ModTime, src)
}
// recordExtractedFile remembers the regular file written at destPath for later hard links.
func recordExtractedFile(extracted map[string]os.FileInfo, destPath string) {
if info, err := os.Lstat(destPath); err == nil && info.Mode().IsRegular() {
extracted[destPath] = info
} else {
delete(extracted, destPath)
}
}
func extractZip(archivePath, destDir string) error {
return extractZipWithEncoding(archivePath, destDir, "", nil)
}
@@ -1396,6 +1463,8 @@ func extractTarGz(archivePath, destDir string, policy *extractAccessPolicy) erro
defer gz.Close()
tr := tar.NewReader(gz)
// Regular files written so far, for hard links to point at.
extracted := make(map[string]os.FileInfo)
for {
h, err := tr.Next()
if err == io.EOF {
@@ -1420,6 +1489,12 @@ func extractTarGz(archivePath, destDir string, policy *extractAccessPolicy) erro
if err = extractArchivedRegularFile(destPath, mode, h.ModTime, io.NopCloser(tr)); err != nil {
return err
}
recordExtractedFile(extracted, destPath)
case tar.TypeLink:
if err = extractArchivedHardLink(destDir, destPath, h, extracted); err != nil {
return err
}
recordExtractedFile(extracted, destPath)
case tar.TypeSymlink:
if err = extractArchivedSymlink(destDir, destPath, h.Linkname, h.ModTime); err != nil {
return err
@@ -1469,7 +1544,7 @@ func preflightTarGz(archivePath, destDir string, policy *extractAccessPolicy) er
if err = preflightExtractEntry(policy, h.Name, destPath, true, false); err != nil {
return err
}
case tar.TypeReg:
case tar.TypeReg, tar.TypeLink:
if err = preflightExtractEntry(policy, h.Name, destPath, false, false); err != nil {
return err
}
@@ -0,0 +1,86 @@
package web
import (
"encoding/json"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"
"github.com/gtsteffaniak/filebrowser/backend/internal/state"
"github.com/gtsteffaniak/filebrowser/backend/internal/utils"
"github.com/gtsteffaniak/filebrowser/backend/pkg/settings"
)
func TestArchiveCreate_DeniesResolvedSymlinkDestination(t *testing.T) {
_, sourceRoot, sourceName, username := setupExtractAccessPolicyTest(t)
if err := state.DenyUser(sourceRoot, utils.IndexPathFromNormalized("/denied", true), username); err != nil {
t.Fatal(err)
}
allowedDir := filepath.Join(sourceRoot, "allowed")
deniedDir := filepath.Join(sourceRoot, "denied")
for _, dir := range []string{allowedDir, deniedDir} {
if err := os.MkdirAll(dir, 0o755); err != nil {
t.Fatal(err)
}
}
victim := filepath.Join(deniedDir, "victim")
if err := os.WriteFile(victim, []byte("keep"), 0o644); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(sourceRoot, "a.txt"), []byte("payload"), 0o644); err != nil {
t.Fatal(err)
}
link := filepath.Join(allowedDir, "report.zip")
if err := os.Symlink(filepath.Join("..", "denied", "victim"), link); err != nil {
t.Fatal(err)
}
user, err := state.GetUserByUsername(username)
if err != nil {
t.Fatal(err)
}
settings.Config.Server.SourceMap[sourceRoot].Config.DenyByDefault = false
body := `{"fromSource":"` + sourceName + `","paths":["/a.txt"],"destination":"/allowed/report.zip","format":"zip"}`
req := httptest.NewRequest(http.MethodPost, "/api/resources/archive", strings.NewReader(body))
rec := httptest.NewRecorder()
status, handlerErr := archiveCreateHandler(rec, req, &Context{User: &user})
if status != http.StatusForbidden || handlerErr == nil {
t.Fatalf("status=%d err=%v", status, handlerErr)
}
got, err := os.ReadFile(victim)
if err != nil || string(got) != "keep" {
t.Fatalf("victim content=%q err=%v", got, err)
}
}
func TestArchiveCreate_AllowsNonSymlinkDestination(t *testing.T) {
_, sourceRoot, sourceName, username := setupExtractAccessPolicyTest(t)
dest := filepath.Join(sourceRoot, "out.zip")
if err := os.WriteFile(filepath.Join(sourceRoot, "a.txt"), []byte("payload"), 0o644); err != nil {
t.Fatal(err)
}
user, err := state.GetUserByUsername(username)
if err != nil {
t.Fatal(err)
}
body := `{"fromSource":"` + sourceName + `","paths":["/a.txt"],"destination":"/out.zip","format":"zip"}`
req := httptest.NewRequest(http.MethodPost, "/api/resources/archive", strings.NewReader(body))
rec := httptest.NewRecorder()
status, handlerErr := archiveCreateHandler(rec, req, &Context{User: &user})
if status != http.StatusOK || handlerErr != nil {
t.Fatalf("status=%d err=%v body=%s", status, handlerErr, rec.Body.String())
}
var resp map[string]string
if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil {
t.Fatal(err)
}
if _, err := os.Stat(dest); err != nil {
t.Fatalf("expected archive at %s: %v", dest, err)
}
}
@@ -23,6 +23,7 @@ import (
func setupExtractAccessPolicyTest(t *testing.T) (destDir, sourceRoot, sourceName, username string) {
t.Helper()
lockWebGlobals(t)
tempDir := t.TempDir()
dbPath := filepath.Join(tempDir, "access.sqlite")
if _, err := state.Initialize(dbPath); err != nil {
+235
View File
@@ -1,11 +1,18 @@
package web
import (
"archive/tar"
"archive/zip"
"errors"
"io"
"os"
"path/filepath"
"strings"
"testing"
commonerrors "github.com/gtsteffaniak/filebrowser/backend/internal/errors"
"github.com/gtsteffaniak/filebrowser/backend/pkg/indexing"
"github.com/gtsteffaniak/filebrowser/backend/pkg/settings"
)
func TestNormalizeArchiveEntryName(t *testing.T) {
@@ -343,3 +350,231 @@ func TestExtractZipFilenameEncoding(t *testing.T) {
})
}
}
func TestExtractTarGz_hardLinks(t *testing.T) {
archivePath := filepath.Join(t.TempDir(), "links.tar.gz")
writeTestTarGz(t, archivePath, func(tw *tar.Writer) error {
if err := tw.WriteHeader(&tar.Header{Name: "proj/a.txt", Mode: 0o644, Size: 4, Typeflag: tar.TypeReg}); err != nil {
return err
}
if _, err := io.WriteString(tw, "same"); err != nil {
return err
}
return tw.WriteHeader(&tar.Header{Name: "proj/sub/b.txt", Mode: 0o644, Typeflag: tar.TypeLink, Linkname: "proj/a.txt"})
})
destDir := t.TempDir()
if err := extractTarGz(archivePath, destDir, nil); err != nil {
t.Fatalf("extract: %v", err)
}
got, err := os.ReadFile(filepath.Join(destDir, "proj", "sub", "b.txt"))
if err != nil || string(got) != "same" {
t.Fatalf("hard link entry: got %q, %v", got, err)
}
missing := filepath.Join(t.TempDir(), "missing.tar.gz")
writeTestTarGz(t, missing, func(tw *tar.Writer) error {
return tw.WriteHeader(&tar.Header{Name: "b.txt", Mode: 0o644, Typeflag: tar.TypeLink, Linkname: "a.txt"})
})
if err := extractTarGz(missing, t.TempDir(), nil); err == nil {
t.Fatal("expected an error for a hard link to a file that is not in the archive")
}
// A symlink entry replaces the extracted file before the hard link names it.
replaced := filepath.Join(t.TempDir(), "replaced.tar.gz")
writeTestTarGz(t, replaced, func(tw *tar.Writer) error {
if err := tw.WriteHeader(&tar.Header{Name: "a.txt", Mode: 0o644, Size: 4, Typeflag: tar.TypeReg}); err != nil {
return err
}
if _, err := io.WriteString(tw, "mine"); err != nil {
return err
}
if err := tw.WriteHeader(&tar.Header{Name: "a.txt", Mode: 0o777, Typeflag: tar.TypeSymlink, Linkname: "private/secret.txt"}); err != nil {
return err
}
return tw.WriteHeader(&tar.Header{Name: "b.txt", Mode: 0o644, Typeflag: tar.TypeLink, Linkname: "a.txt"})
})
replacedDest := t.TempDir()
if err := os.MkdirAll(filepath.Join(replacedDest, "private"), 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(replacedDest, "private", "secret.txt"), []byte("secret"), 0o600); err != nil {
t.Fatal(err)
}
if err := extractTarGz(replaced, replacedDest, nil); err == nil {
t.Fatal("expected an error for a hard link whose target was replaced by a symlink")
}
if got, err := os.ReadFile(filepath.Join(replacedDest, "b.txt")); err == nil {
t.Fatalf("hard link entry copied the symlink target: %q", got)
}
// A hard link names its own target, as GNU tar writes for a file archived twice,
// or names it through a symlink.
self := filepath.Join(t.TempDir(), "self.tar.gz")
writeTestTarGz(t, self, func(tw *tar.Writer) error {
if err := tw.WriteHeader(&tar.Header{Name: "a.txt", Mode: 0o644, Size: 4, Typeflag: tar.TypeReg}); err != nil {
return err
}
if _, err := io.WriteString(tw, "same"); err != nil {
return err
}
if err := tw.WriteHeader(&tar.Header{Name: "c.txt", Mode: 0o777, Typeflag: tar.TypeSymlink, Linkname: "a.txt"}); err != nil {
return err
}
if err := tw.WriteHeader(&tar.Header{Name: "a.txt", Mode: 0o644, Typeflag: tar.TypeLink, Linkname: "a.txt"}); err != nil {
return err
}
return tw.WriteHeader(&tar.Header{Name: "c.txt", Mode: 0o644, Typeflag: tar.TypeLink, Linkname: "a.txt"})
})
selfDest := t.TempDir()
if err := extractTarGz(self, selfDest, nil); err != nil {
t.Fatalf("extract: %v", err)
}
if got, err := os.ReadFile(filepath.Join(selfDest, "a.txt")); err != nil || string(got) != "same" {
t.Fatalf("hard link to itself: got %q, %v", got, err)
}
}
func TestExtractTarGz_symlinkDestination(t *testing.T) {
t.Run("pre-existing symlink at destination", func(t *testing.T) {
destDir := t.TempDir()
victim := filepath.Join(destDir, "victim.txt")
if err := os.WriteFile(victim, []byte("untouched"), 0o644); err != nil {
t.Fatal(err)
}
outPath := filepath.Join(destDir, "out.txt")
if err := os.Symlink("victim.txt", outPath); err != nil {
t.Fatal(err)
}
archivePath := filepath.Join(t.TempDir(), "out.tar.gz")
writeTestTarGz(t, archivePath, func(tw *tar.Writer) error {
if err := tw.WriteHeader(&tar.Header{Name: "out.txt", Mode: 0o644, Size: 7, Typeflag: tar.TypeReg}); err != nil {
return err
}
_, err := io.WriteString(tw, "payload")
return err
})
if err := extractTarGz(archivePath, destDir, nil); err != nil {
t.Fatalf("extract: %v", err)
}
got, err := os.ReadFile(outPath)
if err != nil || string(got) != "payload" {
t.Fatalf("out.txt: got %q, %v", got, err)
}
victimData, err := os.ReadFile(victim)
if err != nil || string(victimData) != "untouched" {
t.Fatalf("victim.txt should be unchanged: %q, %v", victimData, err)
}
info, err := os.Lstat(outPath)
if err != nil || info.Mode()&os.ModeSymlink != 0 {
t.Fatalf("out.txt should be a regular file, mode=%v err=%v", info.Mode(), err)
}
})
t.Run("symlink entry then regular file at same path", func(t *testing.T) {
destDir := t.TempDir()
other := filepath.Join(destDir, "other.txt")
if err := os.WriteFile(other, []byte("keep"), 0o644); err != nil {
t.Fatal(err)
}
archivePath := filepath.Join(t.TempDir(), "link_then_reg.tar.gz")
writeTestTarGz(t, archivePath, func(tw *tar.Writer) error {
if err := tw.WriteHeader(&tar.Header{Name: "link", Mode: 0o777, Typeflag: tar.TypeSymlink, Linkname: "other.txt"}); err != nil {
return err
}
if err := tw.WriteHeader(&tar.Header{Name: "link", Mode: 0o644, Size: 7, Typeflag: tar.TypeReg}); err != nil {
return err
}
_, err := io.WriteString(tw, "written")
return err
})
if err := extractTarGz(archivePath, destDir, nil); err != nil {
t.Fatalf("extract: %v", err)
}
linkPath := filepath.Join(destDir, "link")
got, err := os.ReadFile(linkPath)
if err != nil || string(got) != "written" {
t.Fatalf("link: got %q, %v", got, err)
}
otherData, err := os.ReadFile(other)
if err != nil || string(otherData) != "keep" {
t.Fatalf("other.txt should be unchanged: %q, %v", otherData, err)
}
})
t.Run("hard link onto symlink to source", func(t *testing.T) {
destDir := t.TempDir()
archivePath := filepath.Join(t.TempDir(), "hardlink_symlink_dest.tar.gz")
writeTestTarGz(t, archivePath, func(tw *tar.Writer) error {
if err := tw.WriteHeader(&tar.Header{Name: "a.txt", Mode: 0o644, Size: 4, Typeflag: tar.TypeReg}); err != nil {
return err
}
if _, err := io.WriteString(tw, "same"); err != nil {
return err
}
if err := tw.WriteHeader(&tar.Header{Name: "b.txt", Mode: 0o777, Typeflag: tar.TypeSymlink, Linkname: "a.txt"}); err != nil {
return err
}
return tw.WriteHeader(&tar.Header{Name: "c.txt", Mode: 0o644, Typeflag: tar.TypeLink, Linkname: "a.txt"})
})
if err := extractTarGz(archivePath, destDir, nil); err != nil {
t.Fatalf("extract: %v", err)
}
cPath := filepath.Join(destDir, "c.txt")
got, err := os.ReadFile(cPath)
if err != nil || string(got) != "same" {
t.Fatalf("c.txt: got %q, %v", got, err)
}
info, err := os.Lstat(cPath)
if err != nil || info.Mode()&os.ModeSymlink != 0 {
t.Fatalf("c.txt should be a regular file")
}
})
}
func TestExtractZip_symlinkDestination(t *testing.T) {
destDir := t.TempDir()
victim := filepath.Join(destDir, "victim.txt")
if err := os.WriteFile(victim, []byte("untouched"), 0o644); err != nil {
t.Fatal(err)
}
outPath := filepath.Join(destDir, "out.txt")
if err := os.Symlink("victim.txt", outPath); err != nil {
t.Fatal(err)
}
zipPath := filepath.Join(t.TempDir(), "out.zip")
writeTestZip(t, zipPath, func(zw *zip.Writer) error {
w, err := zw.Create("out.txt")
if err != nil {
return err
}
_, err = w.Write([]byte("payload"))
return err
})
if err := extractZip(zipPath, destDir); err != nil {
t.Fatalf("extract: %v", err)
}
got, err := os.ReadFile(outPath)
if err != nil || string(got) != "payload" {
t.Fatalf("out.txt: got %q, %v", got, err)
}
victimData, err := os.ReadFile(victim)
if err != nil || string(victimData) != "untouched" {
t.Fatalf("victim.txt should be unchanged: %q, %v", victimData, err)
}
}
func TestArchiveCreateDestination_RejectsEscapingSymlinkLeaf(t *testing.T) {
t.Parallel()
root := t.TempDir()
outside := filepath.Join(filepath.Dir(root), "outside-archive")
if err := os.MkdirAll(outside, 0o755); err != nil {
t.Fatal(err)
}
if err := os.Symlink(outside, filepath.Join(root, "report.zip")); err != nil {
t.Fatal(err)
}
idx := &indexing.Index{Source: settings.Source{Path: root}}
if _, err := idx.GetRealPathForMutation("", "/report.zip"); !errors.Is(err, commonerrors.ErrPathEscapesScope) {
t.Fatalf("expected ErrPathEscapesScope, got %v", err)
}
}
+1 -1
View File
@@ -594,7 +594,7 @@ func applyNamedApiTokenGlobalCaps(user *users.User, tokenName string) {
if !ok {
return
}
if !users.HasAnyGlobalPermission(stored.Permissions) {
if users.IsMinimalApiToken(stored) {
return
}
user.Permissions = users.IntersectGlobalPermissions(user.Permissions, stored.Permissions)
+18
View File
@@ -197,9 +197,11 @@ func TestApplyNamedApiTokenGlobalCaps(t *testing.T) {
t.Run("custom api token caps globals", func(t *testing.T) {
user := *owner
customized := false
user.Tokens = map[string]users.AuthToken{
"customized": {
Name: "customized",
Minimal: &customized,
Permissions: users.Permissions{
Admin: false,
Api: true,
@@ -216,4 +218,20 @@ func TestApplyNamedApiTokenGlobalCaps(t *testing.T) {
t.Fatalf("expected api and share, got %#v", user.Permissions)
}
})
t.Run("customized zero global caps", func(t *testing.T) {
user := *owner
customized := false
user.Tokens = map[string]users.AuthToken{
"locked-down": {
Name: "locked-down",
Minimal: &customized,
Permissions: users.Permissions{},
},
}
applyNamedApiTokenGlobalCaps(&user, "locked-down")
if user.Permissions.Admin || user.Permissions.Api || user.Permissions.Share {
t.Fatalf("customized empty caps should clear globals, got %#v", user.Permissions)
}
})
}
@@ -25,6 +25,7 @@ import (
func setupShareArchiveDownloadTest(t *testing.T) (sourceRoot string, sourceName string, owner *users.User) {
t.Helper()
lockWebGlobals(t)
dbPath := filepath.Join(t.TempDir(), "share-dl-test.sqlite")
if _, err := state.Initialize(dbPath); err != nil {
t.Fatal(err)
+12 -6
View File
@@ -187,10 +187,10 @@ func fileWatchHandler(w http.ResponseWriter, r *http.Request, d *Context) (int,
return http.StatusForbidden, fmt.Errorf("access denied to file")
}
// Get real file path
realPath, _, err := idx.GetRealPath(scopePath)
// Get real file path (scoped) and enforce ACL on the resolved target
realPath, _, err := idx.GetRealPathScoped(userScope, scopePath)
if err != nil {
return http.StatusNotFound, fmt.Errorf("file not found: %v", err)
return realPathErrStatus(err), err
}
// Get file/directory info
@@ -198,6 +198,9 @@ func fileWatchHandler(w http.ResponseWriter, r *http.Request, d *Context) (int,
if err != nil {
return http.StatusNotFound, fmt.Errorf("path not found: %v", err)
}
if !state.AccessPermittedAtRealPath(idx, d.User.Username, realPath, info.IsDir()) {
return http.StatusForbidden, fmt.Errorf("access denied to file")
}
// Get MIME type from the index if available
mimeType := "application/octet-stream"
@@ -335,10 +338,10 @@ func fileWatchSSEHandler(w http.ResponseWriter, r *http.Request, d *Context) (in
return http.StatusForbidden, fmt.Errorf("access denied to file")
}
// Get real file path
realPath, _, err := idx.GetRealPath(scopePath)
// Get real file path (scoped) and enforce ACL on the resolved target
realPath, _, err := idx.GetRealPathScoped(userScope, scopePath)
if err != nil {
return http.StatusNotFound, fmt.Errorf("path not found: %v", err)
return realPathErrStatus(err), err
}
// Get file/directory info
@@ -346,6 +349,9 @@ func fileWatchSSEHandler(w http.ResponseWriter, r *http.Request, d *Context) (in
if err != nil {
return http.StatusNotFound, fmt.Errorf("path not found: %v", err)
}
if !state.AccessPermittedAtRealPath(idx, d.User.Username, realPath, info.IsDir()) {
return http.StatusForbidden, fmt.Errorf("access denied to file")
}
// Set up SSE headers
w.Header().Set("Content-Type", "text/event-stream")
+15 -8
View File
@@ -16,11 +16,11 @@ import (
"strings"
"time"
libErrors "github.com/gtsteffaniak/filebrowser/backend/internal/errors"
"github.com/gtsteffaniak/filebrowser/backend/internal/quota"
"github.com/gtsteffaniak/filebrowser/backend/internal/database/share"
"github.com/gtsteffaniak/filebrowser/backend/internal/database/users"
libErrors "github.com/gtsteffaniak/filebrowser/backend/internal/errors"
"github.com/gtsteffaniak/filebrowser/backend/internal/preview"
"github.com/gtsteffaniak/filebrowser/backend/internal/quota"
"github.com/gtsteffaniak/filebrowser/backend/pkg/indexing/iteminfo"
"github.com/gtsteffaniak/filebrowser/backend/pkg/settings"
"github.com/gtsteffaniak/go-cache/cache"
@@ -127,7 +127,7 @@ var (
// Per-route-class token buckets (keys: IP or username). Expired entries are dropped by go-cache.
var (
authRateLimitCredentialByIP = cache.NewCache[*rate.Limiter](authLimiterEntryTTL)
authRateLimitCredentialByUsername = cache.NewCache[*rate.Limiter](authLimiterEntryTTL)
authRateLimitCredentialByUsername = cache.NewCache[*rate.Limiter](authLimiterEntryTTL)
authRateLimitModerateByIP = cache.NewCache[*rate.Limiter](authLimiterEntryTTL)
authRateLimitOIDCByIP = cache.NewCache[*rate.Limiter](authLimiterEntryTTL)
authRateLimitAuthenticatedByUser = cache.NewCache[*rate.Limiter](authLimiterEntryTTL)
@@ -256,17 +256,24 @@ func SetUserInResponseWriter(w http.ResponseWriter, user *users.User) {
func GetRemoteIP(r *http.Request) string {
cfg := &settings.Config
// Forwarded client headers are only honored when the socket peer is a
// configured trusted proxy; otherwise a direct client could spoof its IP.
trustedPeer := cfg.Http.TrustProxyHeaders && settings.IsTrustedProxyPeer(r)
xff := r.Header.Get("X-Forwarded-For")
if cfg.Http.TrustProxyHeaders && xff != "" {
// Use the rightmost hop: the trusted proxy appends the connecting client, so the
// leftmost value is attacker-controlled when clients send X-Forwarded-For.
if ip := lastForwardedValue(xff); ip != "" {
if trustedPeer && xff != "" {
// Without http.trustedProxies (Unix-socket listener) this returns the
// rightmost XFF entry, the client as recorded by the proxy. With trusted proxies
// configured, the walk skips proxy hops so multi-hop deployments
// (CDN -> reverse proxy -> filebrowser) resolve the real client IP —
// keeping per-IP rate limits and auth lockouts per client, not per edge.
if ip := settings.TrustedProxyClientIP(xff); ip != "" {
return ip
}
}
xri := r.Header.Get("X-Real-IP")
if cfg.Http.TrustProxyHeaders && xri != "" {
if trustedPeer && xri != "" {
return xri
}
+34
View File
@@ -22,3 +22,37 @@ func TestErrToStatusWrapsUnsupportedPreviewFormat(t *testing.T) {
t.Fatalf("ErrToStatus() = %d, want %d", status, http.StatusUnsupportedMediaType)
}
}
func TestIsViewerOriginalPreview(t *testing.T) {
tests := []struct {
name string
size string
file string
fileType string
docs bool // the server can render documents to an image
want bool
}{
{"Fuji RAW viewer", "original", "DSCF0308.RAF", "image/x-fuji-raf", false, true},
{"Canon RAW viewer", "original", "IMG_0001.CR3", "image/x-canon-cr3", false, true},
{"Adobe DNG viewer", "original", "a.dng", "image/x-adobe-dng", false, true},
{"HEIC viewer", "original", "a.heic", "image/heic", false, true},
{"HEIF viewer", "original", "a.heif", "image/heif", false, true},
{"rendered document viewer", "original", "a.docx", "application/vnd.openxmlformats-officedocument.wordprocessingml.document", true, true},
{"documents need the server to be able to render them", "original", "a.docx", "application/vnd.openxmlformats-officedocument.wordprocessingml.document", false, false},
{"RAW thumbnail stays disabled", "small", "DSCF0308.RAF", "image/x-fuji-raf", true, false},
{"RAW large stays disabled", "large", "DSCF0308.RAF", "image/x-fuji-raf", true, false},
{"no size stays disabled", "", "DSCF0308.RAF", "image/x-fuji-raf", true, false},
{"video poster stays disabled", "original", "a.mov", "video/quicktime", true, false},
{"audio cover stays disabled", "original", "a.mp3", "audio/mpeg", true, false},
{"JPEG is opened directly, not through previews", "original", "a.jpg", "image/jpeg", true, false},
{"PDF is opened directly, not through previews", "original", "a.pdf", "application/pdf", true, false},
{"text stays disabled", "original", "a.txt", "text/plain", true, false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := isViewerOriginalPreview(tt.size, tt.file, tt.fileType, tt.docs); got != tt.want {
t.Fatalf("isViewerOriginalPreview(%q, %q, %q, %v) = %v, want %v", tt.size, tt.file, tt.fileType, tt.docs, got, tt.want)
}
})
}
}
+57 -1
View File
@@ -2,6 +2,7 @@ package web
import (
"bytes"
"errors"
"fmt"
"net/http"
"path/filepath"
@@ -10,7 +11,9 @@ import (
"github.com/gtsteffaniak/filebrowser/backend/internal/adapters/fs/files"
"github.com/gtsteffaniak/filebrowser/backend/internal/ffmpeg"
"github.com/gtsteffaniak/filebrowser/backend/internal/state"
"github.com/gtsteffaniak/filebrowser/backend/internal/utils"
"github.com/gtsteffaniak/filebrowser/backend/pkg/indexing"
"github.com/gtsteffaniak/filebrowser/backend/pkg/settings"
)
@@ -106,9 +109,34 @@ func subtitlesHandler(w http.ResponseWriter, r *http.Request, d *Context) (int,
var content string
if !embedded {
subtitlePath := filepath.Join(filepath.Dir(fileInfo.RealPath), filepath.Base(track.Name))
var subtitlePath string
if d.Share.Hash != "" {
if d.Share.Path == "" {
return http.StatusForbidden, fmt.Errorf("share has no path scope")
}
idx := indexing.GetIndex(source)
if idx == nil {
return http.StatusInternalServerError, fmt.Errorf("source %s is not available", source)
}
bound, sidecarIndexPath, scopeErr := shareSubtitleSidecarIndexPaths(d, source, path, track.Name)
if scopeErr != nil {
return http.StatusForbidden, scopeErr
}
subtitlePath, _, err = idx.GetRealPathScoped(bound, sidecarIndexPath)
if err != nil {
return realPathErrStatus(err), err
}
if !state.AccessPermittedAtRealPath(idx, d.ShareUser.Username, subtitlePath, false) {
return http.StatusForbidden, fmt.Errorf("access denied")
}
} else {
subtitlePath = filepath.Join(filepath.Dir(fileInfo.RealPath), filepath.Base(track.Name))
}
content, err = utils.GetSubtitleSidecarContent(subtitlePath)
if err != nil {
if errors.Is(err, utils.ErrSubtitleSidecarNotRegular) {
return http.StatusForbidden, err
}
return http.StatusInternalServerError, fmt.Errorf("failed to get subtitle sidecar content: %v", err)
}
} else {
@@ -143,6 +171,34 @@ func findSubtitleTrack(subtitles []utils.SubtitleTrack, name string, embedded bo
return nil
}
// shareSubtitleSidecarIndexPaths maps middleware IndexPath (user-scope-relative) and Share.Path
// to scoped index paths for sidecar containment (see publicDeleteHandler).
func shareSubtitleSidecarIndexPaths(d *Context, sourceName, videoIndexPath, sidecarFileName string) (bound, sidecarIndexPath string, err error) {
userScope, scopeErr := d.ShareUser.GetScopeForSourceName(sourceName)
if scopeErr != nil || userScope == "" {
return "", "", fmt.Errorf("user has no access to source: %s", sourceName)
}
if d.Share.Path == "" {
return "", "", fmt.Errorf("share has no path scope")
}
shareRel := strings.TrimPrefix(d.Share.Path, userScope)
bound = utils.JoinScopedIndexPath(userScope, shareRel)
videoRel := strings.Trim(strings.TrimSpace(videoIndexPath), "/")
fullVideo := utils.JoinScopedIndexPath(userScope, videoRel)
if strings.TrimSuffix(bound, "/") == strings.TrimSuffix(fullVideo, "/") {
parent := utils.GetParentDirectoryPath(strings.TrimSuffix(bound, "/"))
if parent == "" {
parent = "/"
}
bound = parent
}
sidecarIndexPath = utils.JoinPathAsUnix(
utils.GetParentDirectoryPath(strings.TrimSuffix(fullVideo, "/")),
filepath.Base(sidecarFileName),
)
return bound, sidecarIndexPath, nil
}
// metadataHandler returns the same directory resource shape as GET /api/resources with metadata enabled,
// for client-side patching after a fast listing load.
// @Summary Directory with media metadata
@@ -0,0 +1,333 @@
package web
import (
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"
"github.com/gtsteffaniak/filebrowser/backend/internal/adapters/fs/files"
"github.com/gtsteffaniak/filebrowser/backend/internal/app"
"github.com/gtsteffaniak/filebrowser/backend/internal/database/share"
"github.com/gtsteffaniak/filebrowser/backend/internal/database/users"
"github.com/gtsteffaniak/filebrowser/backend/internal/state"
"github.com/gtsteffaniak/filebrowser/backend/internal/utils"
"github.com/gtsteffaniak/filebrowser/backend/pkg/indexing"
"github.com/gtsteffaniak/filebrowser/backend/pkg/indexing/iteminfo"
"github.com/gtsteffaniak/filebrowser/backend/pkg/settings"
)
func setupSubtitlesHTTPTest(t *testing.T) (downloadsPath string, admin *users.User) {
t.Helper()
lockWebGlobals(t)
downloadsPath = filepath.Join(t.TempDir(), "downloads")
dbPath := filepath.Join(t.TempDir(), "subtitles-http-test.sqlite")
if _, err := state.Initialize(dbPath); err != nil {
t.Fatal(err)
}
app.MustWireServices(state.Default())
t.Cleanup(func() {
state.Close()
})
settings.Config.Server.SourceMap = map[string]*settings.Source{
downloadsPath: {
Path: downloadsPath,
Name: "Downloads",
},
}
settings.Config.Server.NameToSource = map[string]*settings.Source{
"Downloads": settings.Config.Server.SourceMap[downloadsPath],
}
settings.InitializeUserResolvers()
adminUser := &users.User{
ID: 1,
FrontendUser: users.FrontendUser{
Username: "admin",
Permissions: users.Permissions{Admin: true},
},
BackendScopes: []users.BackendScope{
{Path: downloadsPath, Scope: "/"},
},
BackendSourcePermissions: map[string]users.SourceFilePermissions{
downloadsPath: {
View: true,
Download: true,
Modify: true,
Create: true,
Delete: true,
},
},
Version: users.SourcePermissionsMigrationVersion,
}
if err := state.CreateUser(adminUser, ""); err != nil {
t.Fatal(err)
}
adminUser.Permissions = users.Permissions{Admin: true}
if err := state.UpdateUser(adminUser, "", "permissions"); err != nil {
t.Fatal(err)
}
if err := state.UpdateUser(adminUser, "", "backendScopes"); err != nil {
t.Fatal(err)
}
if err := state.UpdateUser(adminUser, "", "backendSourcePermissions"); err != nil {
t.Fatal(err)
}
if err := os.MkdirAll(downloadsPath, 0755); err != nil {
t.Fatal(err)
}
indexing.SetTestIndex("Downloads", downloadsPath)
t.Cleanup(func() {
indexing.ClearTestIndices()
})
loaded, err := state.GetUserByUsername("admin")
if err != nil {
t.Fatal(err)
}
return downloadsPath, &loaded
}
func mockVideoWithExternalSubtitle(t *testing.T, trackName, videoRealPath string) {
t.Helper()
prev := files.FileInfoFasterFunc
t.Cleanup(func() { files.FileInfoFasterFunc = prev })
files.FileInfoFasterFunc = func(opts utils.FileOptions, user *users.User) (*iteminfo.ExtendedFileInfo, error) {
return &iteminfo.ExtendedFileInfo{
FileInfo: iteminfo.FileInfo{
Path: opts.Path,
ItemInfo: iteminfo.ItemInfo{
Type: "video/mp4",
},
},
Subtitles: []utils.SubtitleTrack{
{Name: trackName, Embedded: false},
},
RealPath: videoRealPath,
}, nil
}
}
func TestSubtitlesHandler_RejectsEscapingSidecarSymlink(t *testing.T) {
downloadsPath, adminUser := setupSubtitlesHTTPTest(t)
videoPath := filepath.Join(downloadsPath, "clip.mov")
mockVideoWithExternalSubtitle(t, "clip.srt", videoPath)
outside := filepath.Join(filepath.Dir(downloadsPath), "outside-secret")
if err := os.MkdirAll(outside, 0755); err != nil {
t.Fatal(err)
}
secret := filepath.Join(outside, "secret.srt")
if err := os.WriteFile(secret, []byte("SECRET_OUTSIDE_SUBTITLE"), 0644); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(videoPath, []byte("video"), 0644); err != nil {
t.Fatal(err)
}
if err := os.Symlink(secret, filepath.Join(downloadsPath, "clip.srt")); err != nil {
t.Fatal(err)
}
req := httptest.NewRequest(http.MethodGet, "/api/media/subtitles?source=Downloads&path=/clip.mov&name=clip.srt&embedded=false", nil)
rec := httptest.NewRecorder()
status, handlerErr := subtitlesHandler(rec, req, &Context{User: adminUser})
if status != http.StatusForbidden {
t.Fatalf("status=%d err=%v body=%q", status, handlerErr, rec.Body.String())
}
if strings.Contains(rec.Body.String(), "SECRET_OUTSIDE_SUBTITLE") {
t.Fatalf("response leaked outside subtitle content")
}
}
func TestSubtitlesHandler_ReturnsInScopeSidecar(t *testing.T) {
downloadsPath, adminUser := setupSubtitlesHTTPTest(t)
videoPath := filepath.Join(downloadsPath, "clip.mov")
mockVideoWithExternalSubtitle(t, "clip.srt", videoPath)
srtBody := "1\n00:00:01,000 --> 00:00:02,000\nHello in scope"
if err := os.WriteFile(videoPath, []byte("video"), 0644); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(downloadsPath, "clip.srt"), []byte(srtBody), 0644); err != nil {
t.Fatal(err)
}
req := httptest.NewRequest(http.MethodGet, "/api/media/subtitles?source=Downloads&path=/clip.mov&name=clip.srt&embedded=false", nil)
rec := httptest.NewRecorder()
status, handlerErr := subtitlesHandler(rec, req, &Context{User: adminUser})
if status != http.StatusOK || handlerErr != nil {
t.Fatalf("status=%d err=%v body=%q", status, handlerErr, rec.Body.String())
}
if !strings.Contains(rec.Body.String(), "Hello in scope") {
t.Fatalf("body=%q want in-scope subtitle text", rec.Body.String())
}
}
func TestSubtitlesHandler_ShareScopedOwnerLoadsSidecar(t *testing.T) {
sourceRoot, adminUser := setupSubtitlesHTTPTest(t)
aliceRoot := filepath.Join(sourceRoot, "users", "alice", "shared")
if err := os.MkdirAll(aliceRoot, 0755); err != nil {
t.Fatal(err)
}
srtBody := "1\n00:00:01,000 --> 00:00:02,000\nScoped share OK"
videoPath := filepath.Join(aliceRoot, "clip.mov")
if err := os.WriteFile(videoPath, []byte("video"), 0644); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(aliceRoot, "clip.srt"), []byte(srtBody), 0644); err != nil {
t.Fatal(err)
}
mockVideoWithExternalSubtitle(t, "clip.srt", videoPath)
adminUser.BackendScopes = []users.BackendScope{{Path: sourceRoot, Scope: "/users/alice"}}
req := httptest.NewRequest(http.MethodGet, "/public/api/media/subtitles?hash=h&path=clip.mov&name=clip.srt&embedded=false", nil)
rec := httptest.NewRecorder()
status, handlerErr := subtitlesHandler(rec, req, &Context{
User: adminUser,
ShareUser: adminUser,
IndexPath: "/shared/clip.mov/",
Share: share.Share{
ShareColumns: share.ShareColumns{
Hash: "h",
Path: "/users/alice/shared",
},
SourcePath: sourceRoot,
},
})
if status != http.StatusOK || handlerErr != nil {
t.Fatalf("status=%d err=%v body=%q", status, handlerErr, rec.Body.String())
}
if !strings.Contains(rec.Body.String(), "Scoped share OK") {
t.Fatalf("body=%q", rec.Body.String())
}
}
func TestSubtitlesHandler_ShareSingleFileLoadsSidecar(t *testing.T) {
sourceRoot, adminUser := setupSubtitlesHTTPTest(t)
srtBody := "single-file share subtitle"
videoPath := filepath.Join(sourceRoot, "clip.mov")
if err := os.WriteFile(videoPath, []byte("video"), 0644); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(sourceRoot, "clip.srt"), []byte(srtBody), 0644); err != nil {
t.Fatal(err)
}
mockVideoWithExternalSubtitle(t, "clip.srt", videoPath)
req := httptest.NewRequest(http.MethodGet, "/public/api/media/subtitles?hash=h&path=/&name=clip.srt&embedded=false", nil)
rec := httptest.NewRecorder()
status, handlerErr := subtitlesHandler(rec, req, &Context{
User: adminUser,
ShareUser: adminUser,
IndexPath: "/clip.mov/",
Share: share.Share{
ShareColumns: share.ShareColumns{
Hash: "h",
Path: "/clip.mov",
},
SourcePath: sourceRoot,
},
})
if status != http.StatusOK || handlerErr != nil {
t.Fatalf("status=%d err=%v body=%q", status, handlerErr, rec.Body.String())
}
if !strings.Contains(rec.Body.String(), "single-file share subtitle") {
t.Fatalf("body=%q", rec.Body.String())
}
}
func TestSubtitlesHandler_ShareDeniesSidecarSymlinkToDeniedTarget(t *testing.T) {
sourceRoot, adminUser := setupSubtitlesHTTPTest(t)
sharedDir := filepath.Join(sourceRoot, "shared")
privateDir := filepath.Join(sourceRoot, "private")
for _, dir := range []string{sharedDir, privateDir} {
if err := os.MkdirAll(dir, 0755); err != nil {
t.Fatal(err)
}
}
secret := filepath.Join(privateDir, "clip.srt")
if err := os.WriteFile(secret, []byte("SECRET_DENIED_SUBTITLE"), 0644); err != nil {
t.Fatal(err)
}
videoPath := filepath.Join(sharedDir, "clip.mov")
if err := os.WriteFile(videoPath, []byte("video"), 0644); err != nil {
t.Fatal(err)
}
if err := os.Symlink(filepath.Join("..", "private", "clip.srt"), filepath.Join(sharedDir, "clip.srt")); err != nil {
t.Fatal(err)
}
if err := state.DenyUser(sourceRoot, utils.IndexPathFromNormalized("/private", true), "admin"); err != nil {
t.Fatal(err)
}
mockVideoWithExternalSubtitle(t, "clip.srt", videoPath)
adminUser.BackendScopes = []users.BackendScope{{Path: sourceRoot, Scope: "/"}}
req := httptest.NewRequest(http.MethodGet, "/public/api/media/subtitles?hash=h&path=clip.mov&name=clip.srt&embedded=false", nil)
rec := httptest.NewRecorder()
status, handlerErr := subtitlesHandler(rec, req, &Context{
User: adminUser,
ShareUser: adminUser,
IndexPath: "/shared/clip.mov/",
Share: share.Share{
ShareColumns: share.ShareColumns{
Hash: "h",
Path: "/shared",
},
SourcePath: sourceRoot,
},
})
if status != http.StatusForbidden {
t.Fatalf("status=%d err=%v body=%q", status, handlerErr, rec.Body.String())
}
if strings.Contains(rec.Body.String(), "SECRET_DENIED_SUBTITLE") {
t.Fatalf("response leaked denied subtitle")
}
}
func TestSubtitlesHandler_ShareRejectsSidecarPastEscapingDirSymlink(t *testing.T) {
sourceRoot, adminUser := setupSubtitlesHTTPTest(t)
videoIndexPath := "/shared/link/clip.mov/"
mockVideoWithExternalSubtitle(t, "clip.srt", filepath.Join(sourceRoot, "shared", "link", "clip.mov"))
sharedDir := filepath.Join(sourceRoot, "shared")
if err := os.MkdirAll(sharedDir, 0755); err != nil {
t.Fatal(err)
}
outside := filepath.Join(filepath.Dir(sourceRoot), "outside-share")
if err := os.MkdirAll(outside, 0755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(outside, "clip.srt"), []byte("SECRET_PAST_DIR_SYMLINK"), 0644); err != nil {
t.Fatal(err)
}
if err := os.Symlink(outside, filepath.Join(sharedDir, "link")); err != nil {
t.Fatal(err)
}
req := httptest.NewRequest(http.MethodGet, "/public/api/media/subtitles?hash=h&path=link/clip.mov&name=clip.srt&embedded=false", nil)
rec := httptest.NewRecorder()
status, handlerErr := subtitlesHandler(rec, req, &Context{
User: adminUser,
ShareUser: adminUser,
IndexPath: videoIndexPath,
Share: share.Share{
ShareColumns: share.ShareColumns{
Hash: "h",
Path: "/shared",
},
SourcePath: sourceRoot,
},
})
if status != http.StatusForbidden {
t.Fatalf("status=%d err=%v body=%q", status, handlerErr, rec.Body.String())
}
if strings.Contains(rec.Body.String(), "SECRET_PAST_DIR_SYMLINK") {
t.Fatalf("response leaked subtitle past directory symlink")
}
}
+4 -1
View File
@@ -122,7 +122,10 @@ func withHashFileHelper(fn handleFunc) handleFunc {
pathWithoutUserScope = pathWithoutUserScope + "/"
}
data.IndexPath = pathWithoutUserScope
// skip file fetch for certain apis
// Skip FileInfoFaster prefetch for performance; handlers must apply equivalent
// checks (access rules + GetRealPathScoped with d.Share.Path where bytes are read).
// TODO: align all skipped public-share routes on one bound (Share.Path) or stop
// skipping FileInfoFaster here for media/metadata/items.
if (r.Method == "POST" && strings.Contains(r.URL.Path, "/resources")) ||
(r.Method == "POST" && strings.Contains(r.URL.Path, "/resources/view-token")) ||
(r.Method == "GET" && strings.Contains(r.URL.Path, "/resources/items")) ||
+85
View File
@@ -5,7 +5,9 @@ import (
"encoding/json"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"
"time"
@@ -22,6 +24,7 @@ import (
)
func setupTestEnv(t *testing.T) {
lockWebGlobals(t)
utils.BcryptCost = bcrypt.MinCost
dbPath := filepath.Join(t.TempDir(), "test.sqlite")
@@ -534,6 +537,88 @@ func newHTTPRequest(t *testing.T, hash string, requestModifiers ...func(*http.Re
return r
}
func TestPublicGetResourceChecksum(t *testing.T) {
setupTestEnv(t)
filePath := filepath.Join(t.TempDir(), "a.txt")
if err := os.WriteFile(filePath, []byte("hello\n"), 0o600); err != nil {
t.Fatal(err)
}
originalFileInfoFaster := FileInfoFasterFunc
t.Cleanup(func() { FileInfoFasterFunc = originalFileInfoFaster })
FileInfoFasterFunc = func(opts utils.FileOptions, user *users.User) (*iteminfo.ExtendedFileInfo, error) {
return &iteminfo.ExtendedFileInfo{
FileInfo: iteminfo.FileInfo{
Path: opts.Path,
ItemInfo: iteminfo.ItemInfo{
Name: "a.txt",
Type: "text/plain",
},
},
RealPath: filePath,
}, nil
}
newOwner := &users.User{
FrontendUser: users.FrontendUser{
Username: "checksum_owner",
Permissions: users.Permissions{Admin: false},
},
BackendScopes: []users.BackendScope{{Path: "/srv", Scope: "/"}},
}
if err := state.CreateUser(newOwner, ""); err != nil {
t.Fatal("failed to save user:", err)
}
ownerValue, err := state.GetUserByUsername("checksum_owner")
if err != nil {
t.Fatal(err)
}
owner := &ownerValue
if err := state.CreateShare(&share.Share{
ShareSettings: share.ShareSettings{ShareLimits: share.ShareLimits{SourceName: "srv"}},
ShareColumns: share.ShareColumns{Hash: "checksum_hash", Path: "/"},
SourcePath: "/srv",
UserID: owner.ID,
}); err != nil {
t.Fatal("failed to create share:", err)
}
originalAuthKey := settings.Config.Auth.Key
settings.Config.Auth.Key = "key"
t.Cleanup(func() { settings.Config.Auth.Key = originalAuthKey })
t.Run("anonymous", func(t *testing.T) {
recorder := httptest.NewRecorder()
req := newTestRequest(t, "checksum_hash", "", "", nil)
q := req.URL.Query()
q.Set("checksum", "md5")
req.URL.RawQuery = q.Encode()
withHashFile(publicGetResourceHandler)(recorder, req)
if recorder.Code != http.StatusUnauthorized {
t.Fatalf("expected status %d, got %d body=%s", http.StatusUnauthorized, recorder.Code, recorder.Body.String())
}
})
t.Run("authenticated", func(t *testing.T) {
token := testSessionToken(t, owner, time.Hour)
recorder := httptest.NewRecorder()
req := newTestRequest(t, "checksum_hash", "", "", map[string]string{
"Authorization": "Bearer " + token,
})
q := req.URL.Query()
q.Set("checksum", "md5")
req.URL.RawQuery = q.Encode()
withHashFile(publicGetResourceHandler)(recorder, req)
if recorder.Code != http.StatusOK {
t.Fatalf("expected status %d, got %d body=%s", http.StatusOK, recorder.Code, recorder.Body.String())
}
if !strings.Contains(recorder.Body.String(), "b1946ac92492d2347c6235b4d2611184") {
t.Fatalf("checksum missing from response: %s", recorder.Body.String())
}
})
}
func TestWithTimeoutReturnsJSONTimeoutReason(t *testing.T) {
t.Parallel()
timeout := 50 * time.Millisecond
+61 -6
View File
@@ -16,6 +16,7 @@ import (
"github.com/gtsteffaniak/filebrowser/backend/internal/adapters/fs/files"
"github.com/gtsteffaniak/filebrowser/backend/internal/preview"
"github.com/gtsteffaniak/filebrowser/backend/internal/utils"
"github.com/gtsteffaniak/filebrowser/backend/pkg/indexing"
"github.com/gtsteffaniak/filebrowser/backend/pkg/indexing/iteminfo"
"github.com/gtsteffaniak/filebrowser/backend/pkg/settings"
"github.com/gtsteffaniak/go-logger/logger"
@@ -38,6 +39,31 @@ func isClientCancellation(ctx context.Context, err error) bool {
return errors.Is(err, context.Canceled)
}
// viewerConvertedDocExtensions are the documents the viewer opens as a rendered image through the
// "original" preview (keep in sync with pdfConvertable in frontend Preview.vue).
var viewerConvertedDocExtensions = map[string]bool{
".xps": true, ".mobi": true, ".fb2": true, ".cbz": true, ".svg": true,
".docx": true, ".pptx": true, ".xlsx": true, ".hwp": true, ".hwpx": true,
}
// isViewerOriginalPreview reports whether a preview request is the viewer opening a file the browser can't
// display by itself (camera RAW, HEIC/HEIF, rendered documents). Unlike a thumbnail, without it the file can't
// be opened at all, so it is served even when thumbnails are disabled. documentRenderable tells whether the
// server can render documents of this kind to an image (see iteminfo.HasDocConvertableExtension).
func isViewerOriginalPreview(previewSize, fileName, fileType string, documentRenderable bool) bool {
if previewSize != "original" {
return false
}
ext := strings.ToLower(filepath.Ext(fileName))
if iteminfo.IsRawImage(ext) {
return true
}
if fileType == "image/heic" || fileType == "image/heif" {
return true
}
return documentRenderable && viewerConvertedDocExtensions[ext]
}
// previewHandler handles the preview request for images.
// @Summary Get image preview
// @Description Returns a preview image based on the requested path and size.
@@ -55,7 +81,9 @@ func isClientCancellation(ctx context.Context, err error) bool {
// @Failure 501 {object} map[string]string "Preview generation not implemented"
// @Router /api/resources/preview [get]
func previewHandler(w http.ResponseWriter, r *http.Request, d *Context) (int, error) {
if settings.Config.Server.DisablePreviews {
previewsDisabled := settings.Config.Server.DisablePreviews
// Only the viewer's "original" requests can pass when previews are disabled (checked below once the file is known).
if previewsDisabled && r.URL.Query().Get("size") != "original" {
return http.StatusNotImplemented, fmt.Errorf("preview is disabled")
}
path := r.URL.Query().Get("path")
@@ -85,6 +113,9 @@ func previewHandler(w http.ResponseWriter, r *http.Request, d *Context) (int, er
return ErrToStatus(err), err
}
d.FileInfo = *fileInfo
if previewsDisabled && !isViewerOriginalPreview(r.URL.Query().Get("size"), fileInfo.Name, fileInfo.Type, iteminfo.HasDocConvertableExtension(fileInfo.Name, fileInfo.Type)) {
return http.StatusNotImplemented, fmt.Errorf("preview is disabled")
}
status, err := PreviewHelperFunc(w, r, d)
if err != nil {
// Error already logged in PreviewHelperFunc or its callees
@@ -109,7 +140,9 @@ func previewHandler(w http.ResponseWriter, r *http.Request, d *Context) (int, er
// @Failure 501 {object} map[string]string "Previews disabled globally, for this share, or for upload shares"
// @Router /public/api/resources/preview [get]
func publicPreviewHandler(w http.ResponseWriter, r *http.Request, d *Context) (int, error) {
if settings.Config.Server.DisablePreviews || d.Share.DisableThumbnails {
previewSize := r.URL.Query().Get("size")
if (settings.Config.Server.DisablePreviews || d.Share.DisableThumbnails) &&
!isViewerOriginalPreview(previewSize, d.FileInfo.Name, d.FileInfo.Type, iteminfo.HasDocConvertableExtension(d.FileInfo.Name, d.FileInfo.Type)) {
return http.StatusNotImplemented, fmt.Errorf("preview is disabled")
}
if d.Share.ShareType == "upload" {
@@ -185,14 +218,36 @@ func getDirectoryPreview(ctx context.Context, r *http.Request, d *Context, frame
}
fileInfo, err := files.FileInfoFaster(
utils.FileOptions{
Path: path,
Source: source,
AlbumArt: true,
Metadata: true,
Path: path,
Source: source,
AlbumArt: true,
Metadata: true,
FollowSymlinks: true,
}, user)
if err != nil {
return nil, err
}
if d.Share.Hash != "" {
idx := indexing.GetIndex(source)
if idx == nil {
return nil, fmt.Errorf("source not available")
}
userScope, scopeErr := d.ShareUser.GetScopeForSourceName(source)
if scopeErr != nil {
return nil, scopeErr
}
shareRel := strings.TrimPrefix(d.Share.Path, userScope)
bound := utils.JoinScopedIndexPath(userScope, shareRel)
childIndexPath := utils.JoinScopedIndexPath(userScope, path)
realPath, _, rerr := idx.GetRealPathScoped(bound, childIndexPath)
if rerr != nil {
if index != 0 {
return getDirectoryPreview(ctx, r, d, 0)
}
return nil, rerr
}
fileInfo.RealPath = realPath
}
tempCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
_, previewErr := preview.GetPreviewForFile(tempCtx, *fileInfo, "small", "", 0)
cancel()
@@ -0,0 +1,70 @@
package web
import (
"context"
"errors"
"net/http/httptest"
"os"
"path/filepath"
"testing"
"github.com/gtsteffaniak/filebrowser/backend/internal/database/share"
"github.com/gtsteffaniak/filebrowser/backend/internal/database/users"
commonerrors "github.com/gtsteffaniak/filebrowser/backend/internal/errors"
"github.com/gtsteffaniak/filebrowser/backend/pkg/indexing"
"github.com/gtsteffaniak/filebrowser/backend/pkg/indexing/iteminfo"
)
func TestGetDirectoryPreview_ShareRejectsSymlinkChildOutsideShare(t *testing.T) {
sourceRoot, adminUser := setupSubtitlesHTTPTest(t)
sharedDir := filepath.Join(sourceRoot, "shared")
privateDir := filepath.Join(sourceRoot, "private")
for _, dir := range []string{sharedDir, privateDir} {
if err := os.MkdirAll(dir, 0o755); err != nil {
t.Fatal(err)
}
}
if err := os.WriteFile(filepath.Join(privateDir, "secret.jpg"), []byte{0xff, 0xd8, 0xff, 0xd9}, 0o644); err != nil {
t.Fatal(err)
}
if err := os.Symlink(filepath.Join("..", "private", "secret.jpg"), filepath.Join(sharedDir, "link.jpg")); err != nil {
t.Fatal(err)
}
ctx := context.Background()
req := httptest.NewRequest("GET", "/", nil)
adminUser.BackendScopes = []users.BackendScope{{Path: sourceRoot, Scope: "/"}}
d := &Context{
User: adminUser,
ShareUser: adminUser,
IndexPath: "/shared/",
Share: share.Share{
ShareColumns: share.ShareColumns{Hash: "h", Path: "/shared"},
SourcePath: sourceRoot,
},
FileInfo: iteminfo.ExtendedFileInfo{
Source: "Downloads",
FileInfo: iteminfo.FileInfo{
Path: "/shared/",
ItemInfo: iteminfo.ItemInfo{
Type: "directory",
},
Files: []iteminfo.ExtendedItemInfo{
{ItemInfo: iteminfo.ItemInfo{Name: "link.jpg", Type: "image/jpeg", HasPreview: true}},
},
},
},
}
idx := indexing.GetIndex("Downloads")
if idx == nil {
t.Fatal("missing index")
}
_, err := getDirectoryPreview(ctx, req, d, 0)
if err == nil {
t.Fatal("expected scope error for share child symlink outside share root")
}
if !errors.Is(err, commonerrors.ErrPathEscapesScope) {
t.Fatalf("error=%v want ErrPathEscapesScope", err)
}
}
-11
View File
@@ -19,17 +19,6 @@ func normalizeHTTPScheme(proto string) string {
}
}
func lastForwardedValue(v string) string {
parts := strings.Split(v, ",")
for i := len(parts) - 1; i >= 0; i-- {
ip := strings.TrimSpace(parts[i])
if ip != "" {
return ip
}
}
return ""
}
func firstForwardedValue(v string) string {
v = strings.TrimSpace(v)
if i := strings.Index(v, ","); i >= 0 {
+145 -1
View File
@@ -9,8 +9,12 @@ import (
)
func TestGetRemoteIPUsesRightmostForwardedFor(t *testing.T) {
lockWebGlobals(t)
orig := settings.Config.Http
t.Cleanup(func() { settings.Config.Http = orig })
settings.Config.Http.TrustProxyHeaders = true
t.Cleanup(func() { settings.Config.Http.TrustProxyHeaders = false })
// The socket peer must be a trusted proxy for forwarded headers to be honored.
settings.Config.Http.TrustedProxies = []string{"10.0.0.0/8"}
req := httptest.NewRequest("GET", "http://example.com/", nil)
req.RemoteAddr = "10.0.0.1:1234"
@@ -20,7 +24,142 @@ func TestGetRemoteIPUsesRightmostForwardedFor(t *testing.T) {
}
}
// With http.trustedProxies set, GetRemoteIP walks the XFF chain past trusted
// proxy hops so multi-hop deployments (CDN -> proxy -> filebrowser) resolve the
// real client IP instead of collapsing everyone onto the shared edge address.
func TestGetRemoteIPTrustedProxies(t *testing.T) {
lockWebGlobals(t)
orig := settings.Config.Http
t.Cleanup(func() { settings.Config.Http = orig })
settings.Config.Http.TrustProxyHeaders = true
settings.Config.Http.TrustedProxies = []string{"198.51.100.0/24", "10.0.0.0/8"}
tests := []struct {
name string
xff string
want string
}{
{"multi-hop resolves real client", "203.0.113.50, 198.51.100.7", "203.0.113.50"},
{"three hops", "192.0.2.10, 203.0.113.50, 198.51.100.7", "203.0.113.50"},
// Client-supplied XFF entries stay untrusted and cannot spoof identity.
{"spoofed prefix ignored", "1.2.3.4, 203.0.113.50, 198.51.100.7", "203.0.113.50"},
{"empty entries skipped", "203.0.113.50, , 198.51.100.7", "203.0.113.50"},
// Every hop trusted: leftmost is closest to the client.
{"all entries trusted", "198.51.100.7, 198.51.100.8", "198.51.100.7"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
req := httptest.NewRequest("GET", "http://example.com/", nil)
req.RemoteAddr = "10.0.0.1:1234"
req.Header.Set("X-Forwarded-For", tt.xff)
if got := GetRemoteIP(req); got != tt.want {
t.Fatalf("GetRemoteIP = %q, want %q", got, tt.want)
}
})
}
}
// Bare IPs (not just CIDRs) are accepted in http.trustedProxies.
func TestGetRemoteIPTrustedProxiesBareIP(t *testing.T) {
lockWebGlobals(t)
orig := settings.Config.Http
t.Cleanup(func() { settings.Config.Http = orig })
settings.Config.Http.TrustProxyHeaders = true
settings.Config.Http.TrustedProxies = []string{"198.51.100.7", "10.0.0.1"}
req := httptest.NewRequest("GET", "http://example.com/", nil)
req.RemoteAddr = "10.0.0.1:1234"
req.Header.Set("X-Forwarded-For", "203.0.113.50, 198.51.100.7")
if got := GetRemoteIP(req); got != "203.0.113.50" {
t.Fatalf("GetRemoteIP = %q, want 203.0.113.50", got)
}
}
// Without trustProxyHeaders the XFF chain is ignored entirely.
func TestGetRemoteIPIgnoresForwardedForWhenUntrusted(t *testing.T) {
lockWebGlobals(t)
orig := settings.Config.Http
t.Cleanup(func() { settings.Config.Http = orig })
settings.Config.Http.TrustProxyHeaders = false
settings.Config.Http.TrustedProxies = []string{"198.51.100.0/24"}
req := httptest.NewRequest("GET", "http://example.com/", nil)
req.RemoteAddr = "10.0.0.1:1234"
req.Header.Set("X-Forwarded-For", "203.0.113.50, 198.51.100.7")
if got := GetRemoteIP(req); got != "10.0.0.1" {
t.Fatalf("GetRemoteIP = %q, want 10.0.0.1", got)
}
}
// A direct client that is not a trusted proxy cannot spoof its address with
// forwarded headers, even when trustProxyHeaders is enabled.
func TestGetRemoteIPIgnoresForwardedHeadersFromUntrustedPeer(t *testing.T) {
lockWebGlobals(t)
orig := settings.Config.Http
t.Cleanup(func() { settings.Config.Http = orig })
settings.Config.Http.TrustProxyHeaders = true
settings.Config.Http.TrustedProxies = []string{"198.51.100.0/24"}
req := httptest.NewRequest("GET", "http://example.com/", nil)
req.RemoteAddr = "192.0.2.99:1234"
req.Header.Set("X-Forwarded-For", "1.2.3.4")
req.Header.Set("X-Real-IP", "5.6.7.8")
if got := GetRemoteIP(req); got != "192.0.2.99" {
t.Fatalf("GetRemoteIP = %q, want 192.0.2.99", got)
}
}
func TestGetRemoteIPUnixSocketHonorsForwardedWithTrustAllLocalPeers(t *testing.T) {
lockWebGlobals(t)
orig := settings.Config.Http
t.Cleanup(func() { settings.Config.Http = orig })
settings.Config.Http.TrustProxyHeaders = true
settings.Config.Http.Socket = "/var/run/filebrowser.sock"
settings.Config.Http.UnixSocketTrustAllLocalPeers = true
settings.Config.Http.TrustedProxies = nil
req := httptest.NewRequest("GET", "http://example.com/", nil)
req.RemoteAddr = "@"
req.Header.Set("X-Forwarded-For", "203.0.113.50")
if got := GetRemoteIP(req); got != "203.0.113.50" {
t.Fatalf("GetRemoteIP = %q, want 203.0.113.50", got)
}
}
// With no trusted proxies configured, forwarded headers are never honored.
func TestGetRemoteIPIgnoresForwardedHeadersWithoutTrustedProxies(t *testing.T) {
lockWebGlobals(t)
orig := settings.Config.Http
t.Cleanup(func() { settings.Config.Http = orig })
settings.Config.Http.TrustProxyHeaders = true
settings.Config.Http.TrustedProxies = nil
req := httptest.NewRequest("GET", "http://example.com/", nil)
req.RemoteAddr = "10.0.0.1:1234"
req.Header.Set("X-Forwarded-For", "203.0.113.50")
if got := GetRemoteIP(req); got != "10.0.0.1" {
t.Fatalf("GetRemoteIP = %q, want 10.0.0.1", got)
}
}
// An IPv4-mapped IPv6 trusted hop in the XFF chain must still be skipped.
func TestGetRemoteIPMappedTrustedHop(t *testing.T) {
lockWebGlobals(t)
orig := settings.Config.Http
t.Cleanup(func() { settings.Config.Http = orig })
settings.Config.Http.TrustProxyHeaders = true
settings.Config.Http.TrustedProxies = []string{"::ffff:198.51.100.0/120"}
req := httptest.NewRequest("GET", "http://example.com/", nil)
req.RemoteAddr = "198.51.100.1:1234"
req.Header.Set("X-Forwarded-For", "203.0.113.50, ::ffff:198.51.100.7")
if got := GetRemoteIP(req); got != "203.0.113.50" {
t.Fatalf("GetRemoteIP = %q, want 203.0.113.50", got)
}
}
func TestGetScheme(t *testing.T) {
lockWebGlobals(t)
t.Run("trusted proto", func(t *testing.T) {
settings.Config.Http.TrustProxyHeaders = true
t.Cleanup(func() { settings.Config.Http.TrustProxyHeaders = false })
@@ -44,6 +183,7 @@ func TestGetScheme(t *testing.T) {
}
func TestRequestSchemeTLSFallback(t *testing.T) {
lockWebGlobals(t)
settings.Config.Http.TrustProxyHeaders = false
req := httptest.NewRequest("GET", "https://example.com/", nil)
@@ -54,6 +194,7 @@ func TestRequestSchemeTLSFallback(t *testing.T) {
}
func TestRequestSchemeIgnoresInvalidProto(t *testing.T) {
lockWebGlobals(t)
settings.Config.Http.TrustProxyHeaders = true
t.Cleanup(func() { settings.Config.Http.TrustProxyHeaders = false })
@@ -65,6 +206,7 @@ func TestRequestSchemeIgnoresInvalidProto(t *testing.T) {
}
func TestRequestHostTrustProxyHeaders(t *testing.T) {
lockWebGlobals(t)
settings.Config.Http.TrustProxyHeaders = true
t.Cleanup(func() { settings.Config.Http.TrustProxyHeaders = false })
@@ -78,6 +220,7 @@ func TestRequestHostTrustProxyHeaders(t *testing.T) {
}
func TestRequestSchemeForPublicURL(t *testing.T) {
lockWebGlobals(t)
orig := settings.Config.Http.TrustProxyHeaders
t.Cleanup(func() { settings.Config.Http.TrustProxyHeaders = orig })
@@ -111,6 +254,7 @@ func TestRequestSchemeForPublicURL(t *testing.T) {
}
func TestRequestPageFullURL(t *testing.T) {
lockWebGlobals(t)
orig := settings.Config.Http
t.Cleanup(func() { settings.Config.Http = orig })
settings.Config.Http.ExternalUrl = ""
+1
View File
@@ -237,6 +237,7 @@ func TestResourcePutHandler_SuccessResponseOmitsRealPath(t *testing.T) {
func setupPutHTTPTest(t *testing.T) string {
t.Helper()
lockWebGlobals(t)
downloadsPath := filepath.Join(t.TempDir(), "downloads")
dbPath := filepath.Join(t.TempDir(), "quota-http-test.sqlite")
+28
View File
@@ -12,11 +12,13 @@ import (
"github.com/google/uuid"
"github.com/gtsteffaniak/filebrowser/backend/internal/adapters/fs/files"
"github.com/gtsteffaniak/filebrowser/backend/internal/adapters/fs/fileutils"
"github.com/gtsteffaniak/filebrowser/backend/internal/database/access"
"github.com/gtsteffaniak/filebrowser/backend/internal/database/users"
commonerrors "github.com/gtsteffaniak/filebrowser/backend/internal/errors"
"github.com/gtsteffaniak/filebrowser/backend/internal/quota"
"github.com/gtsteffaniak/filebrowser/backend/internal/state"
"github.com/gtsteffaniak/filebrowser/backend/pkg/indexing"
"github.com/gtsteffaniak/filebrowser/backend/pkg/indexing/iteminfo"
"github.com/gtsteffaniak/filebrowser/backend/pkg/settings"
"github.com/gtsteffaniak/go-logger/logger"
)
@@ -170,6 +172,10 @@ func putResourceWithQuota(w http.ResponseWriter, r *http.Request, d *Context, so
if err != nil {
return sanitizePutPathError(err)
}
permUser := accessCheckUsername(d)
if !state.AccessPermittedAtRealPath(idx, permUser, realPath, access.IsDirAtRealPath(realPath)) {
return http.StatusForbidden, fmt.Errorf("access denied to path %s", fullIndexPath)
}
if stat, statErr := os.Stat(realPath); statErr == nil && stat.IsDir() {
return http.StatusMethodNotAllowed, errPutPathIsDirectory
}
@@ -313,3 +319,25 @@ func renderQuotaError(w http.ResponseWriter, r *http.Request, err error) (int, e
}
return status, nil
}
func prepareDeleteQuotaAdjustments(d *Context, sourceName, fullIndexPath string, fileInfo *iteminfo.ExtendedFileInfo) state.DeleteAdjustments {
sourceInfo, ok := settings.Config.Server.NameToSource[sourceName]
if !ok || fileInfo == nil {
return state.DeleteAdjustments{}
}
principal := quota.PrincipalForUpload(d.User, d.Share.UserID, d.Share.Hash)
isDir := fileInfo.Type == "directory"
var deleteBytes int64
if isDir {
deleteBytes, _ = state.IndexedPathBytes(sourceName, fullIndexPath, true)
} else {
deleteBytes = fileInfo.Size
}
return state.PrepareAccountedDelete(principal, sourceName, sourceInfo.Path, fullIndexPath, d.Share.Hash, isDir, deleteBytes)
}
func applyDeleteQuotaAdjustments(adj state.DeleteAdjustments) {
if err := state.ApplyAccountedDelete(adj); err != nil {
logger.Warningf("quota adjust after delete failed: %v", err)
}
}
+1
View File
@@ -19,6 +19,7 @@ import (
func setupQuotaHTTPTest(t *testing.T) {
t.Helper()
lockWebGlobals(t)
dbPath := filepath.Join(t.TempDir(), "quota-http-test.sqlite")
if _, err := state.Initialize(dbPath); err != nil {
t.Fatal(err)
+61 -15
View File
@@ -16,6 +16,7 @@ import (
"github.com/gtsteffaniak/filebrowser/backend/internal/activity"
"github.com/gtsteffaniak/filebrowser/backend/internal/adapters/fs/files"
"github.com/gtsteffaniak/filebrowser/backend/internal/adapters/fs/fileutils"
"github.com/gtsteffaniak/filebrowser/backend/internal/database/access"
"github.com/gtsteffaniak/filebrowser/backend/internal/database/users"
"github.com/gtsteffaniak/filebrowser/backend/internal/errors"
"github.com/gtsteffaniak/filebrowser/backend/internal/preview"
@@ -209,7 +210,9 @@ func resourceGetHandler(w http.ResponseWriter, r *http.Request, d *Context) (int
// @Param path query string false "Path within the share to retrieve information for. Defaults to share root."
// @Param content query string false "Include file content if true"
// @Param metadata query string false "Extract audio/video metadata if true"
// @Param checksum query string false "Optional checksum algorithm (md5, sha1, sha256, sha512); requires a logged-in session"
// @Success 200 {object} iteminfo.FileInfo "File or directory metadata"
// @Failure 401 {object} map[string]string "Login required for checksum generation"
// @Failure 403 {object} map[string]string "Share unavailable or access denied"
// @Failure 404 {object} map[string]string "Share not found or file not found"
// @Failure 500 {object} map[string]string "Internal server error"
@@ -219,6 +222,24 @@ func publicGetResourceHandler(w http.ResponseWriter, r *http.Request, d *Context
if d.Share.ShareType == "upload" {
return http.StatusNotImplemented, fmt.Errorf("browsing is disabled for upload shares")
}
if algo := r.URL.Query().Get("checksum"); algo != "" {
if d.User == nil || d.User.Username == "" || d.User.Username == "anonymous" {
return http.StatusUnauthorized, fmt.Errorf("login required to generate checksums")
}
if d.FileInfo.Type == "directory" {
return http.StatusBadRequest, fmt.Errorf("checksum is only supported for files")
}
if d.Share.DisableFileViewer {
return http.StatusForbidden, fmt.Errorf("file viewing is disabled for this share")
}
checksum, err := utils.GetChecksum(d.FileInfo.RealPath, algo)
if err == errors.ErrInvalidOption {
return http.StatusBadRequest, nil
} else if err != nil {
return http.StatusInternalServerError, err
}
d.FileInfo.Checksums = map[string]string{algo: checksum}
}
return RenderJSON(w, r, d.FileInfo)
}
@@ -279,27 +300,16 @@ func resourceDeleteHandler(w http.ResponseWriter, r *http.Request, d *Context) (
// delete thumbnails
preview.DelThumbs(r.Context(), *fileInfo)
principal := quota.PrincipalForUpload(d.User, d.Share.UserID, d.Share.Hash)
sourceInfo, ok := settings.Config.Server.NameToSource[source]
if !ok {
if _, ok := settings.Config.Server.NameToSource[source]; !ok {
return http.StatusNotFound, fmt.Errorf("source %s not found", source)
}
var deleteBytes int64
if fileInfo.Type == "directory" {
deleteBytes, _ = state.IndexedPathBytes(source, fileInfo.Path, true)
} else {
deleteBytes = fileInfo.Size
}
adjustments := prepareDeleteQuotaAdjustments(d, source, fullIndexPath, fileInfo)
err = files.DeleteFiles(source, userScope, fullIndexPath, fileInfo.Type == "directory")
if err != nil {
return ErrToStatus(err), err
}
if deleteBytes > 0 {
if adjErr := state.ApplyAccountedUsageDelta(principal, source, sourceInfo.Path, fileInfo.Path, d.Share.Hash, -deleteBytes); adjErr != nil {
logger.Warningf("quota adjust after delete failed: %v", adjErr)
}
}
applyDeleteQuotaAdjustments(adjustments)
activity.RecordDelete(r, toActor(d), source, path)
return http.StatusOK, nil
@@ -326,11 +336,14 @@ func publicDeleteHandler(w http.ResponseWriter, r *http.Request, d *Context) (in
shareRelPath := strings.TrimPrefix(d.Share.Path, userScope)
fullIndexPath := utils.JoinScopedIndexPath(userScope, d.IndexPath)
boundIndexPath := utils.JoinScopedIndexPath(userScope, shareRelPath)
sourceName := d.Share.GetSourceName()
adjustments := prepareDeleteQuotaAdjustments(d, sourceName, fullIndexPath, fileInfo)
err = files.DeleteFiles(d.Share.SourcePath, boundIndexPath, fullIndexPath, fileInfo.Type == "directory")
if err != nil {
logger.Errorf("public delete handler: error deleting resource with error %v", err)
return http.StatusInternalServerError, fmt.Errorf("an error occured while deleting the resource")
}
applyDeleteQuotaAdjustments(adjustments)
preview.DelThumbs(r.Context(), *fileInfo)
activity.RecordDelete(r, toActor(d), d.Share.GetSourceName(), fileInfo.Path)
return http.StatusOK, nil
@@ -497,7 +510,7 @@ func ResourceBulkDeleteHandler(w http.ResponseWriter, r *http.Request, d *Contex
boundIndexPath := utils.JoinScopedIndexPath(userScope, withoutUserScope)
fullIndexPath := utils.JoinScopedIndexPath(userScope, indexPath)
// Delete the file/directory
adjustments := prepareDeleteQuotaAdjustments(d, sourceName, fullIndexPath, fileInfo)
err = files.DeleteFiles(sourceName, boundIndexPath, fullIndexPath, fileInfo.Type == "directory")
if err != nil {
logger.Errorf("resource bulk delete handler: error deleting file/directory: %v", err)
@@ -508,6 +521,7 @@ func ResourceBulkDeleteHandler(w http.ResponseWriter, r *http.Request, d *Contex
})
continue
}
applyDeleteQuotaAdjustments(adjustments)
// Delete thumbnails
preview.DelThumbs(r.Context(), *fileInfo)
activityPath = fileInfo.Path
@@ -577,6 +591,7 @@ func ResourceBulkDeleteHandler(w http.ResponseWriter, r *http.Request, d *Contex
continue
}
fullIndexPath := utils.JoinScopedIndexPath(userScope, sanitizedPath)
adjustments := prepareDeleteQuotaAdjustments(d, item.Source, fullIndexPath, fileInfo)
err = files.DeleteFiles(item.Source, userScope, fullIndexPath, fileInfo.Type == "directory")
if err != nil {
response.Failed = append(response.Failed, BulkDeleteItem{
@@ -586,6 +601,7 @@ func ResourceBulkDeleteHandler(w http.ResponseWriter, r *http.Request, d *Contex
})
continue
}
applyDeleteQuotaAdjustments(adjustments)
preview.DelThumbs(r.Context(), *fileInfo)
}
// Success — echo the client path in the response
@@ -798,6 +814,9 @@ func ResourcePostHandler(w http.ResponseWriter, r *http.Request, d *Context) (in
if !state.AccessPermitted(idx.Path, utils.IndexPathFromNormalized(fullIndexPath, true), filePermUser.Username) {
return http.StatusForbidden, fmt.Errorf("access denied to path %s", path)
}
if !state.AccessPermittedAtRealPath(idx, filePermUser.Username, realPath, access.IsDirAtRealPath(realPath)) {
return http.StatusForbidden, fmt.Errorf("access denied to path %s", path)
}
// Check for file/folder conflicts before creation
if stat, statErr := os.Stat(realPath); statErr == nil {
@@ -1534,6 +1553,15 @@ func ResourcePatchHandler(w http.ResponseWriter, r *http.Request, d *Context) (i
response.Failed = append(response.Failed, moveCopyWithClientPaths(item, clientFromPath, clientToPath))
continue
}
if !state.AccessPermittedAtRealPath(srcIdx, d.User.Username, realSrc, isSrcDir) {
item.Message = "access denied to source path"
if d.Share.Hash != "" {
response.Failed = append(response.Failed, MoveCopyItem{Message: item.Message})
continue
}
response.Failed = append(response.Failed, moveCopyWithClientPaths(item, clientFromPath, clientToPath))
continue
}
// Resolve the destination: copy must reject escaping symlink leaves; move/rename
// must replace an existing symlink entry without following it.
@@ -1554,10 +1582,28 @@ func ResourcePatchHandler(w http.ResponseWriter, r *http.Request, d *Context) (i
response.Failed = append(response.Failed, moveCopyWithClientPaths(item, clientFromPath, clientToPath))
continue
}
if !state.AccessPermittedAtRealPath(dstIdx, d.User.Username, realDest, access.IsDirAtRealPath(realDest)) {
item.Message = "access denied to destination path"
if d.Share.Hash != "" {
response.Failed = append(response.Failed, MoveCopyItem{Message: item.Message})
continue
}
response.Failed = append(response.Failed, moveCopyWithClientPaths(item, clientFromPath, clientToPath))
continue
}
// Auto-rename if requested
if req.Rename {
realDest = addVersionSuffix(realDest)
if !state.AccessPermittedAtRealPath(dstIdx, d.User.Username, realDest, isSrcDir) {
item.Message = "access denied to destination path"
if d.Share.Hash != "" {
response.Failed = append(response.Failed, MoveCopyItem{Message: item.Message})
continue
}
response.Failed = append(response.Failed, moveCopyWithClientPaths(item, clientFromPath, clientToPath))
continue
}
}
// Validate move/rename operation to prevent circular references
@@ -0,0 +1,186 @@
package web
import (
"bytes"
"encoding/json"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"testing"
"github.com/gtsteffaniak/filebrowser/backend/internal/adapters/fs/fileutils"
"github.com/gtsteffaniak/filebrowser/backend/internal/database/quota"
dbsql "github.com/gtsteffaniak/filebrowser/backend/internal/database/sql"
"github.com/gtsteffaniak/filebrowser/backend/internal/database/users"
"github.com/gtsteffaniak/filebrowser/backend/internal/preview"
"github.com/gtsteffaniak/filebrowser/backend/internal/state"
"github.com/gtsteffaniak/filebrowser/backend/internal/utils"
"github.com/gtsteffaniak/filebrowser/backend/pkg/indexing"
"github.com/gtsteffaniak/filebrowser/backend/pkg/settings"
)
// Use a database-backed index because deletion refreshes and removes metadata.
func setupDeleteQuotaTest(t *testing.T) string {
t.Helper()
sourcePath := setupPutQuotaTest(t)
settings.Config.Http.BaseURL = "/"
fileutils.SetFsPermissions(0644, 0755)
oldDB := indexing.GetIndexDB()
settings.Config.Server.CacheDir = t.TempDir()
if err := os.MkdirAll(filepath.Join(settings.Config.Server.CacheDir, "sql"), 0755); err != nil {
t.Fatal(err)
}
db, _, err := dbsql.NewIndexDB("delete_quota", "OFF", 1000, 32, false)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() {
indexing.SetIndexDBForTesting(oldDB)
if err := db.Close(); err != nil {
t.Error(err)
}
})
indexing.SetIndexDBForTesting(db)
source := settings.Config.Server.NameToSource["Downloads"]
source.Config.UseLogicalSize = true
source.Config.ResolvedRules.IndexingDisabled = true
indexing.Initialize(source, true, false)
indexing.GetIndex("Downloads").Config.ResolvedRules.IndexingDisabled = false
return sourcePath
}
// seedNestedQuotaFixture writes files under scope, indexes them, and creates accounted
// quotas on public/nested (5 bytes used) and sibling (7 bytes used).
func seedNestedQuotaFixture(t *testing.T, sourcePath, scope string, user *users.User) (nested, sibling *quota.FolderQuota) {
t.Helper()
for path, content := range map[string]string{"public/direct": "123", "public/nested/file": "12345", "sibling/file": "1234567"} {
full := filepath.Join(sourcePath, scope, path)
if err := os.MkdirAll(filepath.Dir(full), 0755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(full, []byte(content), 0644); err != nil {
t.Fatal(err)
}
}
if err := indexing.GetIndex("Downloads").RefreshDirectory("/", true); err != nil {
t.Fatal(err)
}
nested, err := state.CreateFolderQuota(sourcePath, utils.JoinPathAsUnix(scope, "/public/nested"), user.ID, 100, "accounted")
if err != nil {
t.Fatal(err)
}
sibling, err = state.CreateFolderQuota(sourcePath, utils.JoinPathAsUnix(scope, "/sibling"), user.ID, 100, "accounted")
if err != nil {
t.Fatal(err)
}
for path, size := range map[string]int64{"/public/nested/file": 5, "/sibling/file": 7} {
if err = state.ApplyAccountedUsageDelta(user, "Downloads", sourcePath, utils.JoinPathAsUnix(scope, path), "", size); err != nil {
t.Fatal(err)
}
}
return nested, sibling
}
// The API path is relative to the user's scope while quota paths and the index are
// source-relative, so the scoped case checks that the delete joins the scope.
func TestResourceDeleteNestedQuota(t *testing.T) {
for _, scope := range []string{"/", "/scoped"} {
t.Run(scope, func(t *testing.T) {
sourcePath := setupDeleteQuotaTest(t)
if preview.GetService() == nil {
if err := preview.StartPreviewGenerator(1, t.TempDir()); err != nil {
t.Fatal(err)
}
}
user, err := state.GetUserByUsername("admin")
if err != nil {
t.Fatal(err)
}
user.BackendScopes[0].Scope = scope
nested, sibling := seedNestedQuotaFixture(t, sourcePath, scope, &user)
req := httptest.NewRequest(http.MethodDelete, "/api/resources?source=Downloads&path=/public", nil)
status, err := resourceDeleteHandler(httptest.NewRecorder(), req, &requestContext{User: &user})
if err != nil || status != http.StatusOK {
t.Fatalf("delete: status=%d err=%v", status, err)
}
if _, err := os.Stat(filepath.Join(sourcePath, scope, "public")); !os.IsNotExist(err) {
t.Fatalf("ancestor still exists: %v", err)
}
if used, _ := state.GetQuotaCounterSnapshot(nested.ID); used != 0 {
t.Fatalf("nested usage=%d, want 0", used)
}
if used, _ := state.GetQuotaCounterSnapshot(sibling.ID); used != 7 {
t.Fatalf("sibling usage=%d, want 7", used)
}
})
}
}
func TestResourceBulkDeleteNestedQuota(t *testing.T) {
for _, scope := range []string{"/", "/scoped"} {
t.Run(scope, func(t *testing.T) {
sourcePath := setupDeleteQuotaTest(t)
if preview.GetService() == nil {
if err := preview.StartPreviewGenerator(1, t.TempDir()); err != nil {
t.Fatal(err)
}
}
user, err := state.GetUserByUsername("admin")
if err != nil {
t.Fatal(err)
}
user.BackendScopes[0].Scope = scope
nested, sibling := seedNestedQuotaFixture(t, sourcePath, scope, &user)
body, err := json.Marshal([]BulkDeleteItem{{Source: "Downloads", Path: "/public"}})
if err != nil {
t.Fatal(err)
}
req := httptest.NewRequest(http.MethodDelete, "/api/resources/bulk", bytes.NewReader(body))
req.Header.Set("Content-Type", "application/json")
rec := httptest.NewRecorder()
status, err := ResourceBulkDeleteHandler(rec, req, &requestContext{User: &user})
if err != nil || status != http.StatusOK {
t.Fatalf("bulk delete: status=%d err=%v body=%s", status, err, rec.Body.String())
}
if _, err := os.Stat(filepath.Join(sourcePath, scope, "public")); !os.IsNotExist(err) {
t.Fatalf("ancestor still exists: %v", err)
}
if used, _ := state.GetQuotaCounterSnapshot(nested.ID); used != 0 {
t.Fatalf("nested usage=%d, want 0", used)
}
if used, _ := state.GetQuotaCounterSnapshot(sibling.ID); used != 7 {
t.Fatalf("sibling usage=%d, want 7", used)
}
})
}
}
func TestResourceBulkDeletePartialFailureKeepsQuota(t *testing.T) {
sourcePath := setupDeleteQuotaTest(t)
user, err := state.GetUserByUsername("admin")
if err != nil {
t.Fatal(err)
}
nested, sibling := seedNestedQuotaFixture(t, sourcePath, "/", &user)
body, err := json.Marshal([]BulkDeleteItem{
{Source: "Downloads", Path: "/no-such-path"},
{Source: "Downloads", Path: "/public"},
})
if err != nil {
t.Fatal(err)
}
req := httptest.NewRequest(http.MethodDelete, "/api/resources/bulk", bytes.NewReader(body))
req.Header.Set("Content-Type", "application/json")
rec := httptest.NewRecorder()
status, err := ResourceBulkDeleteHandler(rec, req, &requestContext{User: &user})
if err != nil || status != http.StatusMultiStatus {
t.Fatalf("bulk delete: status=%d err=%v", status, err)
}
if used, _ := state.GetQuotaCounterSnapshot(nested.ID); used != 0 {
t.Fatalf("nested usage=%d, want 0 after successful item", used)
}
if used, _ := state.GetQuotaCounterSnapshot(sibling.ID); used != 7 {
t.Fatalf("sibling usage=%d, want 7", used)
}
}
@@ -0,0 +1,60 @@
package web
import (
"encoding/json"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"
"github.com/gtsteffaniak/filebrowser/backend/internal/state"
"github.com/gtsteffaniak/filebrowser/backend/internal/utils"
)
func TestResourcePatchHandler_CopyRename_DeniesDeniedVersionSuffix(t *testing.T) {
_, sourceRoot, sourceName, username := setupExtractAccessPolicyTest(t)
destDir := filepath.Join(sourceRoot, "dest")
if err := os.MkdirAll(destDir, 0o755); err != nil {
t.Fatal(err)
}
src := filepath.Join(sourceRoot, "src.txt")
if err := os.WriteFile(src, []byte("src"), 0o644); err != nil {
t.Fatal(err)
}
existing := filepath.Join(destDir, "report.txt")
if err := os.WriteFile(existing, []byte("existing"), 0o644); err != nil {
t.Fatal(err)
}
denied := filepath.Join(destDir, "report(1).txt")
if err := state.DenyUser(sourceRoot, utils.IndexPathFromNormalized("/dest/report(1).txt", false), username); err != nil {
t.Fatal(err)
}
user, err := state.GetUserByUsername(username)
if err != nil {
t.Fatal(err)
}
body := `{"action":"copy","rename":true,"items":[{"fromSource":"` + sourceName +
`","fromPath":"/src.txt","toSource":"` + sourceName + `","toPath":"/dest/report.txt"}]}`
req := httptest.NewRequest(http.MethodPatch, "/api/resources", strings.NewReader(body))
rec := httptest.NewRecorder()
status, handlerErr := ResourcePatchHandler(rec, req, &Context{User: &user})
if handlerErr != nil {
t.Fatalf("err=%v", handlerErr)
}
if status != http.StatusInternalServerError && status != http.StatusOK {
t.Fatalf("status=%d body=%s", status, rec.Body.String())
}
var resp MoveCopyResponse
if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil {
t.Fatal(err)
}
if len(resp.Failed) != 1 || !strings.Contains(resp.Failed[0].Message, "access denied") {
t.Fatalf("failed=%+v", resp.Failed)
}
if _, err := os.Stat(denied); err == nil {
t.Fatal("denied destination file should not exist")
}
}
@@ -17,6 +17,7 @@ import (
func setupPutQuotaTest(t *testing.T) string {
t.Helper()
lockWebGlobals(t)
downloadsPath := filepath.Join(t.TempDir(), "downloads")
dbPath := filepath.Join(t.TempDir(), "quota-http-test.sqlite")
@@ -0,0 +1,82 @@
package web
import (
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"testing"
"github.com/gtsteffaniak/filebrowser/backend/internal/preview"
"github.com/gtsteffaniak/filebrowser/backend/internal/state"
)
func TestResourcePostHandler_RejectsEscapingParentSymlinkMkdir(t *testing.T) {
downloadsPath := setupPutHTTPTest(t)
outside := filepath.Join(filepath.Dir(downloadsPath), "outside-mkdir")
if err := os.MkdirAll(outside, 0755); err != nil {
t.Fatal(err)
}
if err := os.Symlink(outside, filepath.Join(downloadsPath, "evil")); err != nil {
t.Fatal(err)
}
adminUser, err := state.GetUserByUsername("admin")
if err != nil {
t.Fatal(err)
}
req := httptest.NewRequest(http.MethodPost, "/api/resources?source=Downloads&path=/evil/newdir&isDir=true", nil)
rec := httptest.NewRecorder()
status, handlerErr := ResourcePostHandler(rec, req, &requestContext{User: &adminUser})
if status != http.StatusForbidden {
t.Fatalf("status=%d err=%v", status, handlerErr)
}
if handlerErr == nil {
t.Fatal("expected mkdir to be forbidden")
}
if _, err := os.Stat(filepath.Join(outside, "newdir")); !os.IsNotExist(err) {
t.Fatal("mkdir escaped outside the source via symlink parent")
}
}
func TestResourceDeleteHandler_RejectsEscapingIntermediate(t *testing.T) {
downloadsPath := setupPutHTTPTest(t)
if preview.GetService() == nil {
if err := preview.StartPreviewGenerator(1, t.TempDir()); err != nil {
t.Fatal(err)
}
}
outsideDir := filepath.Join(filepath.Dir(downloadsPath), "outside-delete")
if err := os.MkdirAll(outsideDir, 0755); err != nil {
t.Fatal(err)
}
victim := filepath.Join(outsideDir, "victim.txt")
if err := os.WriteFile(victim, []byte("keep"), 0644); err != nil {
t.Fatal(err)
}
if err := os.Symlink(outsideDir, filepath.Join(downloadsPath, "evil")); err != nil {
t.Fatal(err)
}
adminUser, err := state.GetUserByUsername("admin")
if err != nil {
t.Fatal(err)
}
req := httptest.NewRequest(http.MethodDelete, "/api/resources?source=Downloads&path=/evil/victim.txt", nil)
rec := httptest.NewRecorder()
status, handlerErr := resourceDeleteHandler(rec, req, &requestContext{User: &adminUser})
if status != http.StatusForbidden {
t.Fatalf("status=%d err=%v", status, handlerErr)
}
if handlerErr == nil {
t.Fatal("expected delete to be forbidden")
}
got, err := os.ReadFile(victim)
if err != nil || string(got) != "keep" {
t.Fatalf("delete escaped: err=%v content=%q", err, got)
}
}
+7 -4
View File
@@ -16,8 +16,8 @@ import (
_ "net/http/pprof"
"github.com/coreos/go-systemd/v22/activation"
"github.com/gtsteffaniak/filebrowser/backend/internal/adapters/fs/fileutils"
"github.com/gtsteffaniak/filebrowser/backend/internal/adapters/fs/files"
"github.com/gtsteffaniak/filebrowser/backend/internal/adapters/fs/fileutils"
"github.com/gtsteffaniak/filebrowser/backend/internal/auth"
"github.com/gtsteffaniak/filebrowser/backend/internal/events"
"github.com/gtsteffaniak/filebrowser/backend/internal/state"
@@ -40,6 +40,8 @@ var (
func initRuntime(deps Deps, assets fs.FS) {
runtimeDeps = deps
assetFs = assets
// Make the generated YAML's `secret:` markers available to the settings redactor.
initSettingsSecretMarkers()
}
// Embed the files in the frontend/dist directory
@@ -91,8 +93,9 @@ func StartHttp(ctx context.Context, deps Deps, shutdownComplete chan struct{}) {
configureHTTPRouter(router, api, publicRoutes, publicApi)
srv := &http.Server{
Addr: settings.HTTPListenAddr(settings.Config.Http.ListenAddress, settings.Config.Http.Port),
Handler: muxWithMiddleware(router),
Addr: settings.HTTPListenAddr(settings.Config.Http.ListenAddress, settings.Config.Http.Port),
Handler: muxWithMiddleware(router),
ConnContext: settings.AttachConnContext,
}
listenAddress := settings.Config.Http.ListenAddress
go func() {
@@ -103,7 +106,7 @@ func StartHttp(ctx context.Context, deps Deps, shutdownComplete chan struct{}) {
if err = os.Remove(settings.Config.Http.Socket); err != nil && !os.IsNotExist(err) {
logger.Fatalf("Could not remove existing socket: %v", err)
}
listener, err = net.Listen("unix", settings.Config.Http.Socket)
listener, err = settings.ListenUnixFilesystem(settings.Config.Http.Socket)
if err != nil {
logger.Fatalf("Could not listen on unix socket: %v", err)
}
+40 -17
View File
@@ -33,6 +33,19 @@ func readSettingsPatchBody(w http.ResponseWriter, r *http.Request) ([]byte, erro
return patchJSON, nil
}
// initSettingsSecretMarkers points the settings package at the generated YAML shipped in the
// running binary's asset filesystem, so it can discover the `secret:` comment markers that
// decide which config fields are redacted. The Go sources are absent in a deployed binary, so
// this is the marker source that matters in production.
func initSettingsSecretMarkers() {
if settings.SecretHintYaml != "" {
return
}
if embedded, err := readEmbeddedConfigYaml(); err == nil {
settings.SecretHintYaml = embedded
}
}
// settingsGetHandler retrieves the current system settings.
// @Summary Get system settings
// @Description Returns the current configuration settings for signup, user directories, rules, frontend.
@@ -44,6 +57,9 @@ func readSettingsPatchBody(w http.ResponseWriter, r *http.Request) ([]byte, erro
// @Router /api/settings [get]
func settingsGetHandler(w http.ResponseWriter, r *http.Request, d *Context) (int, error) {
property := r.URL.Query().Get("property")
if property == "auth" {
return RenderJSON(w, r, settings.RedactAuthSecrets(settings.Config.Auth))
}
if property != "" {
// get property by name
switch property {
@@ -51,8 +67,6 @@ func settingsGetHandler(w http.ResponseWriter, r *http.Request, d *Context) (int
return RenderJSON(w, r, state.GetDefaultUserDefaults())
case "frontend":
return RenderJSON(w, r, settings.Config.Frontend)
case "auth":
return RenderJSON(w, r, settings.Config.Auth)
case "server":
return RenderJSON(w, r, settings.Config.Server)
case "sources":
@@ -61,7 +75,26 @@ func settingsGetHandler(w http.ResponseWriter, r *http.Request, d *Context) (int
return http.StatusNotFound, nil
}
}
return RenderJSON(w, r, &settings.Config)
// Redact secrets (auth.key, totpSecret, adminPassword, OIDC clientSecret, LDAP/recaptcha
// secrets, ...) so the admin response never echoes plaintext credentials (GHSA-892f-9h5p-mfc8).
// RedactedSettingsCopy returns a copy: the live config keeps the real secrets, so auth and
// the OIDC/LDAP/OnlyOffice integrations keep working after this request.
return RenderJSON(w, r, settings.RedactedSettingsCopy(&settings.Config))
}
// readEmbeddedConfigYaml returns the config.generated.yaml shipped in the binary's asset
// filesystem. It is the comment/secret-marker source for both the YAML export and the settings
// JSON redactor.
func readEmbeddedConfigYaml() (string, error) {
path := "config.generated.yaml"
if settings.Env.EmbeddedFs {
path = "embed/config.generated.yaml"
}
content, err := fs.ReadFile(assetFs, path)
if err != nil {
return "", err
}
return string(content), nil
}
// settingsConfigHandler returns the current system settings as YAML.
@@ -85,22 +118,12 @@ func settingsConfigHandler(w http.ResponseWriter, r *http.Request, d *Context) (
var err error
var yamlConfig string
// Try to read the generated YAML file for comments (should always exist)
var embeddedYaml []byte
var readErr error
if settings.Env.EmbeddedFs {
embeddedYaml, readErr = fs.ReadFile(assetFs, "embed/config.generated.yaml")
} else {
embeddedYaml, readErr = fs.ReadFile(assetFs, "config.generated.yaml")
if readErr != nil {
return http.StatusInternalServerError, fmt.Errorf("error reading generated YAML: %v", readErr)
}
}
// Comment/secret-marker source, shared with settingsGetHandler's redactor.
embeddedYaml, _ := readEmbeddedConfigYaml()
// If we successfully read the generated YAML, use it as the comment source
if readErr == nil && len(embeddedYaml) > 0 {
yamlConfig, err = settings.GenerateConfigYamlWithEmbedded(&settings.Config, showComments, showFull, false, string(embeddedYaml))
if embeddedYaml != "" {
yamlConfig, err = settings.GenerateConfigYamlWithEmbedded(&settings.Config, showComments, showFull, false, embeddedYaml)
if err != nil {
return http.StatusInternalServerError, fmt.Errorf("error generating YAML: %v", err)
}
@@ -0,0 +1,124 @@
package web
import (
"encoding/json"
"net/http"
"net/http/httptest"
"strings"
"testing"
"github.com/gtsteffaniak/filebrowser/backend/internal/database/users"
"github.com/gtsteffaniak/filebrowser/backend/pkg/settings"
)
const settingsRedactSentinel = "SENTINEL-SETTINGS-SECRET"
// TestSettingsGetHandler_redactsSecrets is the end-to-end regression guard for
// GHSA-892f-9h5p-mfc8: GET /api/settings (and ?property=auth) must not return cleartext secrets.
func TestSettingsGetHandler_redactsSecrets(t *testing.T) {
setupTestEnv(t)
prev := settings.Config
t.Cleanup(func() { settings.Config = prev })
settings.Config.Auth.Key = settingsRedactSentinel
settings.Config.Auth.TotpSecret = settingsRedactSentinel
settings.Config.Auth.Methods.PasswordAuth.AdminPassword = settingsRedactSentinel
settings.Config.Auth.Methods.PasswordAuth.Recaptcha.Secret = settingsRedactSentinel
settings.Config.Auth.Methods.OidcAuth.ClientSecret = settingsRedactSentinel
settings.Config.Auth.Methods.LdapAuth.UserPassword = settingsRedactSentinel
settings.Config.Auth.Methods.JwtAuth.Secret = settingsRedactSentinel
settings.Config.Integrations.OnlyOffice.Secret = settingsRedactSentinel
admin := &users.User{
FrontendUser: users.FrontendUser{
Username: "settings_redact_admin",
Permissions: users.Permissions{Admin: true},
},
}
for _, path := range []string{"/api/settings", "/api/settings?property=auth"} {
t.Run(path, func(t *testing.T) {
req := httptest.NewRequest(http.MethodGet, path, nil)
rec := httptest.NewRecorder()
status, err := settingsGetHandler(rec, req, &Context{User: admin})
if status != http.StatusOK {
t.Fatalf("status=%d err=%v want 200", status, err)
}
body := rec.Body.String()
if strings.Contains(body, settingsRedactSentinel) {
t.Fatalf("response for %s leaked a plaintext secret: %s", path, body)
}
if !json.Valid([]byte(body)) {
t.Fatalf("response for %s is not valid JSON: %s", path, body)
}
})
}
}
// TestSettingsGetHandler_doesNotMutateLiveConfig is the regression guard for the live-config
// mutation: the full settings request must redact only its response, never settings.Config,
// otherwise JWT signing and the integrations break until the process restarts.
func TestSettingsGetHandler_doesNotMutateLiveConfig(t *testing.T) {
setupTestEnv(t)
prev := settings.Config
t.Cleanup(func() { settings.Config = prev })
settings.Config.Auth.Key = settingsRedactSentinel
admin := &users.User{
FrontendUser: users.FrontendUser{
Username: "settings_redact_admin3",
Permissions: users.Permissions{Admin: true},
},
}
req := httptest.NewRequest(http.MethodGet, "/api/settings", nil)
rec := httptest.NewRecorder()
status, err := settingsGetHandler(rec, req, &Context{User: admin})
if status != http.StatusOK {
t.Fatalf("status=%d err=%v want 200", status, err)
}
if strings.Contains(rec.Body.String(), settingsRedactSentinel) {
t.Fatal("response leaked a plaintext secret")
}
if settings.Config.Auth.Key != settingsRedactSentinel {
t.Fatalf("live config auth key was mutated by the settings response: %q", settings.Config.Auth.Key)
}
}
// TestSettingsGetHandler_redactsAuthKeyOnlyWherePresent confirms non-secret auth fields still
// round-trip so the response stays usable for the UI.
func TestSettingsGetHandler_keepsNonSecretAuthFields(t *testing.T) {
setupTestEnv(t)
prev := settings.Config
t.Cleanup(func() { settings.Config = prev })
settings.Config.Auth.Key = settingsRedactSentinel
settings.Config.Auth.TokenExpirationHours = 7
admin := &users.User{
FrontendUser: users.FrontendUser{
Username: "settings_redact_admin2",
Permissions: users.Permissions{Admin: true},
},
}
req := httptest.NewRequest(http.MethodGet, "/api/settings?property=auth", nil)
rec := httptest.NewRecorder()
status, err := settingsGetHandler(rec, req, &Context{User: admin})
if status != http.StatusOK {
t.Fatalf("status=%d err=%v want 200", status, err)
}
var got settings.Auth
if err := json.Unmarshal(rec.Body.Bytes(), &got); err != nil {
t.Fatalf("unmarshal auth: %v", err)
}
if got.TokenExpirationHours != 7 {
t.Fatalf("tokenExpirationHours=%d want 7", got.TokenExpirationHours)
}
if got.Key == settingsRedactSentinel {
t.Fatal("auth key was returned in cleartext")
}
}
+12 -3
View File
@@ -8,6 +8,7 @@ import (
"mime"
"net/http"
"os"
"path"
"path/filepath"
"strconv"
"strings"
@@ -72,13 +73,15 @@ func normalizeViewGrantPath(p string) string {
// shareRelativeDisplayName returns the client-facing filename for a path within a share.
// Single-file shares use file=/ at the share root; the real name comes from the share path.
// Both paths are slash-separated, so use path.Base: on Windows filepath.Base("/")
// returns "\", which would skip the share-name fallback below.
func shareRelativeDisplayName(d *Context, shareRelativePath string) string {
name := filepath.Base(normalizeViewGrantPath(shareRelativePath))
name := path.Base(normalizeViewGrantPath(shareRelativePath))
if name != "" && name != "." && name != "/" {
return name
}
if d != nil && d.Share.Hash != "" {
if shareName := filepath.Base(d.Share.Path); shareName != "" && shareName != "." && shareName != "/" {
if shareName := path.Base(normalizeViewGrantPath(d.Share.Path)); shareName != "" && shareName != "." && shareName != "/" {
return shareName
}
}
@@ -484,10 +487,16 @@ func ServeSingleFile(w http.ResponseWriter, r *http.Request, d *Context, source
bound = userScope
}
realPath, _, err := idx.GetRealPathScoped(bound, scopedFilePath)
realPath, isDir, err := idx.GetRealPathScoped(bound, scopedFilePath)
if err != nil {
return realPathErrStatus(err), err
}
if st, statErr := os.Stat(realPath); statErr == nil {
isDir = st.IsDir()
}
if !state.AccessPermittedAtRealPath(idx, permUser, realPath, isDir) {
return http.StatusForbidden, fmt.Errorf("access denied to path %s", scopedFilePath)
}
isOnlyOffice := IsOnlyOfficeCompatibleFile(displayFileName) && settings.Config.Integrations.OnlyOffice.Url != ""
var documentId string
+2 -2
View File
@@ -197,7 +197,7 @@ func TestValidateViewGrantShareBinding(t *testing.T) {
}
t.Cleanup(func() { settings.Config.Server.SourceMap = nil })
d := &requestContext{
User: testUserWithView(1, "srv"),
User: testUserWithView(1, "srv"),
Share: share.Share{
ShareColumns: share.ShareColumns{Hash: "abc123"},
SourcePath: "/srv",
@@ -215,7 +215,7 @@ func TestValidateViewGrantShareBinding(t *testing.T) {
t.Fatalf("ValidateViewGrant: %v", err)
}
wrongShare := &requestContext{
User: testUserWithView(1, "srv"),
User: testUserWithView(1, "srv"),
Share: share.Share{
ShareColumns: share.ShareColumns{Hash: "other"},
SourcePath: "/srv",
+15
View File
@@ -0,0 +1,15 @@
package web
import (
"sync"
"testing"
)
// webGlobalTestMu serializes tests that mutate process-wide settings.Config or state.
var webGlobalTestMu sync.Mutex
func lockWebGlobals(t testing.TB) {
t.Helper()
webGlobalTestMu.Lock()
t.Cleanup(webGlobalTestMu.Unlock)
}
+72 -24
View File
@@ -34,6 +34,29 @@ func createTokenAuthUser(t *testing.T, username string, perms users.Permissions)
return &reloaded
}
// registerApiToken mints a named API token for user, stores its metadata, and
// registers its hash so auth can resolve it. configure, when non-nil, may adjust
// the metadata before it is persisted.
func registerApiToken(t *testing.T, user *users.User, name string, caps users.Permissions, minimal bool, configure func(*users.AuthToken)) string {
t.Helper()
tokenString, tokenMeta, err := auth.MakeSignedTokenAPI(user, name, time.Hour, caps, minimal)
if err != nil {
t.Fatal(err)
}
tokenMeta.Name = name
tokenMeta.Token = tokenString
if configure != nil {
configure(&tokenMeta)
}
if err = state.AddUserToken(user.Username, tokenMeta); err != nil {
t.Fatal(err)
}
if err = state.AddApiToken(tokenString, user.ID); err != nil {
t.Fatal(err)
}
return tokenString
}
func cookieRequest(t *testing.T, handler http.HandlerFunc, token string) *httptest.ResponseRecorder {
t.Helper()
rec := httptest.NewRecorder()
@@ -67,6 +90,53 @@ func TestApiTokenWithoutMetadataRejected(t *testing.T) {
}
}
func TestApiTokenMinimalKeepsOwnerPermissions(t *testing.T) {
setupTestEnv(t)
originalAuthKey := settings.Config.Auth.Key
settings.Config.Auth.Key = "key"
t.Cleanup(func() { settings.Config.Auth.Key = originalAuthKey })
user := createTokenAuthUser(t, "minimal-admin", users.Permissions{Admin: true, Api: true})
tokenString := registerApiToken(t, user, "minimal-key", users.Permissions{}, true, nil)
rec := cookieRequest(t, withAdmin(mockHandler), tokenString)
if rec.Code != http.StatusOK {
t.Fatalf("minimal API token: got status %d, want %d", rec.Code, http.StatusOK)
}
}
func TestApiTokenCustomizedZeroPermissionsForbiddenOnAdmin(t *testing.T) {
setupTestEnv(t)
originalAuthKey := settings.Config.Auth.Key
settings.Config.Auth.Key = "key"
t.Cleanup(func() { settings.Config.Auth.Key = originalAuthKey })
user := createTokenAuthUser(t, "custom-zero-admin", users.Permissions{Admin: true, Api: true})
tokenString := registerApiToken(t, user, "zero-key", users.Permissions{}, false, nil)
rec := cookieRequest(t, withAdmin(mockHandler), tokenString)
if rec.Code != http.StatusForbidden {
t.Fatalf("customized zero-cap API token: got status %d, want %d", rec.Code, http.StatusForbidden)
}
}
func TestApiTokenLegacyEmptyCapsTreatedAsMinimal(t *testing.T) {
setupTestEnv(t)
originalAuthKey := settings.Config.Auth.Key
settings.Config.Auth.Key = "key"
t.Cleanup(func() { settings.Config.Auth.Key = originalAuthKey })
user := createTokenAuthUser(t, "legacy-minimal-admin", users.Permissions{Admin: true, Api: true})
tokenString := registerApiToken(t, user, "legacy-key", users.Permissions{}, true, func(meta *users.AuthToken) {
meta.Minimal = nil
})
rec := cookieRequest(t, withAdmin(mockHandler), tokenString)
if rec.Code != http.StatusOK {
t.Fatalf("legacy empty-cap API token: got status %d, want %d", rec.Code, http.StatusOK)
}
}
// A named API token with stored caps must have those caps applied at auth time.
func TestApiTokenCapsAppliedFromStoredMetadata(t *testing.T) {
setupTestEnv(t)
@@ -77,18 +147,7 @@ func TestApiTokenCapsAppliedFromStoredMetadata(t *testing.T) {
user := createTokenAuthUser(t, "capped-admin", users.Permissions{Admin: true, Api: true})
// The token cap excludes admin, so the resolved user must lose admin.
tokenString, tokenMeta, err := auth.MakeSignedTokenAPI(user, "capped-key", time.Hour, users.Permissions{Share: true}, false)
if err != nil {
t.Fatal(err)
}
tokenMeta.Name = "capped-key"
tokenMeta.Token = tokenString
if err := state.AddUserToken(user.Username, tokenMeta); err != nil {
t.Fatal(err)
}
if err := state.AddApiToken(tokenString, user.ID); err != nil {
t.Fatal(err)
}
tokenString := registerApiToken(t, user, "capped-key", users.Permissions{Share: true}, false, nil)
rec := cookieRequest(t, withAdmin(mockHandler), tokenString)
if rec.Code != http.StatusForbidden {
@@ -228,18 +287,7 @@ func TestApiTokenBasicAuthRoundTrip(t *testing.T) {
t.Cleanup(func() { settings.Config.Auth.Key = originalAuthKey })
user := createTokenAuthUser(t, "webdav-user", users.Permissions{Api: true})
tokenString, tokenMeta, err := auth.MakeSignedTokenAPI(user, "obsidian-sync", time.Hour, users.Permissions{}, true)
if err != nil {
t.Fatal(err)
}
tokenMeta.Name = "obsidian-sync"
tokenMeta.Token = tokenString
if err = state.AddUserToken(user.Username, tokenMeta); err != nil {
t.Fatal(err)
}
if err = state.AddApiToken(tokenString, user.ID); err != nil {
t.Fatal(err)
}
tokenString := registerApiToken(t, user, "obsidian-sync", users.Permissions{}, true, nil)
basicRequest := func(token string) *httptest.ResponseRecorder {
rec := httptest.NewRecorder()
+6 -1
View File
@@ -176,6 +176,8 @@ func usersPostHandler(w http.ResponseWriter, r *http.Request, d *Context) (int,
}
// Parse the JSON into the UserRequest struct
var req UserRequest
// Seed with defaults so the decode below only overrides fields the request sent.
state.PrefillUserProfileDefaults(&req.User)
if err = json.Unmarshal(body, &req); err != nil {
return http.StatusBadRequest, err
}
@@ -199,7 +201,7 @@ func usersPostHandler(w http.ResponseWriter, r *http.Request, d *Context) (int,
return status, err
}
plaintextPassword := req.User.Password
err = state.CreateUser(&req.User, plaintextPassword)
err = state.CreateUserKeepingProfile(&req.User, plaintextPassword)
if err != nil {
return http.StatusInternalServerError, err
}
@@ -406,6 +408,9 @@ func userPatchHandler(w http.ResponseWriter, r *http.Request, d *Context) (int,
if err2 != nil {
return http.StatusInternalServerError, err2
}
if !d.User.Permissions.Admin && uValue.LockPassword && state.FieldListIncludes(req.Which, "password") {
return http.StatusForbidden, fmt.Errorf("password changes are disabled for this user")
}
req.User.ID = uValue.ID
req.User.Username = uValue.Username
if state.FieldListIncludes(req.Which, "otpEnabled") && !req.User.OtpEnabled {
@@ -0,0 +1,85 @@
package web
import (
"bytes"
"net/http"
"net/http/httptest"
"testing"
"github.com/gtsteffaniak/filebrowser/backend/internal/database/users"
"github.com/gtsteffaniak/filebrowser/backend/internal/state"
)
func TestUserPatchLockPassword(t *testing.T) {
setupTestEnv(t)
admin := &users.User{
ID: 1,
FrontendUser: users.FrontendUser{
Username: "admin",
LoginMethod: users.LoginMethodOidc,
Permissions: users.Permissions{Admin: true},
},
}
if err := state.CreateUser(admin, ""); err != nil {
t.Fatal(err)
}
admin.Permissions = users.Permissions{Admin: true}
if err := state.UpdateUser(admin, "", "permissions"); err != nil {
t.Fatal(err)
}
locked := &users.User{
FrontendUser: users.FrontendUser{
Username: "locked",
LoginMethod: users.LoginMethodPassword,
},
}
if err := state.CreateUser(locked, "oldpassword"); err != nil {
t.Fatal(err)
}
locked.LockPassword = true
if err := state.UpdateUser(locked, "", "lockPassword"); err != nil {
t.Fatal(err)
}
locked, err := getUser(t, "locked")
if err != nil {
t.Fatal(err)
}
if !locked.LockPassword {
t.Fatal("lockPassword was not stored")
}
oldHash := locked.Password
patch := func(actor *users.User) int {
body := []byte(`{"which":["password"],"data":{"password":"newpassword"}}`)
req := httptest.NewRequest(http.MethodPatch, "/api/users?username=locked", bytes.NewReader(body))
req.Header.Set("X-Password", "oldpassword")
status, _ := userPatchHandler(httptest.NewRecorder(), req, &Context{User: actor})
return status
}
if status := patch(locked); status != http.StatusForbidden {
t.Fatalf("self password change with lockPassword: status=%d, want %d", status, http.StatusForbidden)
}
after, err := getUser(t, "locked")
if err != nil {
t.Fatal(err)
}
if after.Password != oldHash {
t.Fatal("password changed despite lockPassword")
}
if status := patch(admin); status != http.StatusNoContent {
t.Fatalf("admin password reset of locked user: status=%d, want %d", status, http.StatusNoContent)
}
}
func getUser(t *testing.T, username string) (*users.User, error) {
t.Helper()
u, err := state.GetUserByUsername(username)
if err != nil {
return nil, err
}
return &u, nil
}
@@ -0,0 +1,73 @@
package web
import (
"bytes"
"net/http"
"net/http/httptest"
"testing"
"github.com/gtsteffaniak/filebrowser/backend/internal/state"
"github.com/gtsteffaniak/filebrowser/backend/pkg/settings"
)
// Omitted permissions and omitted profile keys come from userDefaults; a partially
// specified fileLoading object only overrides the keys it contains.
func TestUsersPostOmittedFieldsFallBackToDefaults(t *testing.T) {
setupTestEnv(t)
defaults := settings.Config.UserDefaults
defaults.UI.Locale = "de"
defaults.Account.Permissions.Share = true
defaults.FileLoading.MaxConcurrent = 7
defaults.FileLoading.UploadChunkSize = 9
state.SetEffectiveUserDefaultsForTest(t, defaults)
admin := createPostTestAdmin(t)
body := []byte(`{"which":[],"data":{"username":"partial","password":"secret","loginMethod":"password",
"fileLoading":{"maxConcurrentUpload":3}}}`)
req := httptest.NewRequest(http.MethodPost, "/api/users", bytes.NewReader(body))
status, err := usersPostHandler(httptest.NewRecorder(), req, &Context{User: admin})
if err != nil || status != http.StatusCreated {
t.Fatalf("usersPostHandler() status=%d err=%v", status, err)
}
got, err := state.GetUserByUsername("partial")
if err != nil {
t.Fatal(err)
}
if got.Locale != "de" {
t.Errorf("locale = %q, want default de", got.Locale)
}
if !got.Permissions.Share {
t.Errorf("share = false, want true from default permissions")
}
if got.FileLoading.MaxConcurrent != 3 {
t.Errorf("maxConcurrentUpload = %d, want requested 3", got.FileLoading.MaxConcurrent)
}
if got.FileLoading.UploadChunkSize != 9 {
t.Errorf("uploadChunkSizeMb = %d, want default 9", got.FileLoading.UploadChunkSize)
}
}
// A create request must not mutate the shared in-memory userDefaults (json.Unmarshal
// reuses slice backing arrays).
func TestUsersPostDoesNotMutateUserDefaults(t *testing.T) {
setupTestEnv(t)
defaults := settings.Config.UserDefaults
defaults.Listing.NewFileTemplate = []string{"default.txt"}
state.SetEffectiveUserDefaultsForTest(t, defaults)
admin := createPostTestAdmin(t)
body := []byte(`{"which":[],"data":{"username":"nomutate","password":"secret","loginMethod":"password",
"newFileTemplate":["custom.txt"]}}`)
req := httptest.NewRequest(http.MethodPost, "/api/users", bytes.NewReader(body))
status, err := usersPostHandler(httptest.NewRecorder(), req, &Context{User: admin})
if err != nil || status != http.StatusCreated {
t.Fatalf("usersPostHandler() status=%d err=%v", status, err)
}
if got := state.EffectiveUserDefaults().Listing.NewFileTemplate; len(got) != 1 || got[0] != "default.txt" {
t.Errorf("userDefaults newFileTemplate = %v, want [default.txt]", got)
}
}
@@ -0,0 +1,140 @@
package web
import (
"bytes"
"net/http"
"net/http/httptest"
"testing"
"github.com/gtsteffaniak/filebrowser/backend/internal/database/users"
"github.com/gtsteffaniak/filebrowser/backend/internal/state"
"github.com/gtsteffaniak/filebrowser/backend/pkg/settings"
)
func createPostTestAdmin(t *testing.T) *users.User {
t.Helper()
admin := &users.User{
ID: 1,
FrontendUser: users.FrontendUser{
Username: "admin",
LoginMethod: users.LoginMethodOidc,
Permissions: users.Permissions{Admin: true},
},
}
if err := state.CreateUser(admin, ""); err != nil {
t.Fatal(err)
}
return admin
}
// Issue #3322: profile/account values in POST /api/users must not be replaced by userDefaults.
func TestUsersPostKeepsRequestedProfileValues(t *testing.T) {
setupTestEnv(t)
defaults := settings.Config.UserDefaults
defaults.UI.Locale = "en"
defaults.UI.ThemeColor = "var(--blue)"
darkTrue := true
defaults.UI.DarkMode = &darkTrue
defaults.Account.LockPassword = false
defaults.Account.DisableSettings = false
defaults.Account.ShowAdvancedProfile = false
defaults.Listing.PromptRightCloseButton = false
defaults.Listing.NewFileTemplate = []string{""}
defaults.Listing.ShowHidden = true
state.SetEffectiveUserDefaultsForTest(t, defaults)
admin := createPostTestAdmin(t)
body := []byte(`{"which":[],"data":{
"username":"profiletest","password":"secret","loginMethod":"password",
"locale":"zhCN","themeColor":"var(--green)","darkMode":false,
"lockPassword":true,"disableSettings":true,"showAdvancedProfile":true,
"promptRightCloseButton":true,"newFileTemplate":["custom.txt"],
"preview":{"image":false},
"permissions":{"admin":false,"api":true,"share":true,"realtime":false},
"scopes":[{"name":"/","scope":"/","permissions":{"view":true,"download":true,"modify":true,"create":true,"delete":true}}]
}}`)
req := httptest.NewRequest(http.MethodPost, "/api/users", bytes.NewReader(body))
rec := httptest.NewRecorder()
status, err := usersPostHandler(rec, req, &Context{User: admin})
if err != nil || status != http.StatusCreated {
t.Fatalf("usersPostHandler() status=%d err=%v", status, err)
}
got, err := state.GetUserByUsername("profiletest")
if err != nil {
t.Fatal(err)
}
if got.Locale != "zhCN" {
t.Errorf("locale = %q, want zhCN", got.Locale)
}
if got.ThemeColor != "var(--green)" {
t.Errorf("themeColor = %q, want var(--green)", got.ThemeColor)
}
if got.DarkMode {
t.Errorf("darkMode = true, want false")
}
if !got.LockPassword {
t.Errorf("lockPassword = false, want true")
}
if !got.DisableSettings {
t.Errorf("disableSettings = false, want true")
}
if !got.ShowAdvancedProfile {
t.Errorf("showAdvancedProfile = false, want true")
}
if !got.PromptRightCloseButton {
t.Errorf("promptRightCloseButton = false, want true")
}
if len(got.NewFileTemplate) != 1 || got.NewFileTemplate[0] != "custom.txt" {
t.Errorf("newFileTemplate = %v, want [custom.txt]", got.NewFileTemplate)
}
if got.Preview.Image {
t.Errorf("preview.image = true, want false")
}
// Omitted fields still come from userDefaults.
if !got.ShowHidden {
t.Errorf("showHidden = false, want true from userDefaults")
}
if !got.Preview.Video {
t.Errorf("preview.video = false, want true from userDefaults")
}
wantPerms := users.Permissions{Api: true, Share: true}
if got.Permissions.Admin != wantPerms.Admin || got.Permissions.Api != wantPerms.Api ||
got.Permissions.Share != wantPerms.Share || got.Permissions.Realtime != wantPerms.Realtime {
t.Errorf("permissions = %+v, want %+v", got.Permissions, wantPerms)
}
}
// Enforced userDefaults still win over request values on create.
func TestUsersPostEnforcedDefaultsWinOverRequest(t *testing.T) {
setupTestEnv(t)
defaults := settings.Config.UserDefaults
defaults.UI.Locale = "en"
state.SetEffectiveUserDefaultsForTest(t, defaults)
if err := state.PatchUserDefaultsEnforced([]byte(`{"ui":{"locale":true}}`)); err != nil {
t.Fatal(err)
}
admin := createPostTestAdmin(t)
body := []byte(`{"which":[],"data":{"username":"enforced","password":"secret","loginMethod":"password","locale":"zhCN","lockPassword":true}}`)
req := httptest.NewRequest(http.MethodPost, "/api/users", bytes.NewReader(body))
rec := httptest.NewRecorder()
status, err := usersPostHandler(rec, req, &Context{User: admin})
if err != nil || status != http.StatusCreated {
t.Fatalf("usersPostHandler() status=%d err=%v", status, err)
}
got, err := state.GetUserByUsername("enforced")
if err != nil {
t.Fatal(err)
}
if got.Locale != "en" {
t.Errorf("locale = %q, want enforced en", got.Locale)
}
if !got.LockPassword {
t.Errorf("lockPassword = false, want true (not enforced)")
}
}
+52 -9
View File
@@ -19,6 +19,7 @@ import (
"github.com/gtsteffaniak/filebrowser/backend/internal/activity"
"github.com/gtsteffaniak/filebrowser/backend/internal/adapters/fs/files"
"github.com/gtsteffaniak/filebrowser/backend/internal/adapters/fs/fileutils"
"github.com/gtsteffaniak/filebrowser/backend/internal/database/access"
activitydb "github.com/gtsteffaniak/filebrowser/backend/internal/database/activity"
"github.com/gtsteffaniak/filebrowser/backend/internal/database/users"
commonerrors "github.com/gtsteffaniak/filebrowser/backend/internal/errors"
@@ -237,6 +238,35 @@ func (ffs *filteredFileSystem) checkScopePath(requestPath string, forRemoval boo
return err
}
// checkResolvedAccess enforces per-path DENY rules on the symlink-resolved target.
func (ffs *filteredFileSystem) checkResolvedAccess(requestPath string, forRemoval bool) error {
idx := indexing.GetIndex(ffs.source)
if idx == nil {
return fmt.Errorf("source not found")
}
userScope, err := ffs.user.GetScopeForSourceName(ffs.source)
if err != nil {
return err
}
fullIndexPath := utils.JoinScopedIndexPath(userScope, strings.TrimPrefix(requestPath, "/"))
var realPath string
if forRemoval {
realPath, err = idx.GetRealPathForRemoval(userScope, fullIndexPath)
} else {
realPath, err = idx.GetRealPathForMutation(userScope, fullIndexPath)
}
if err != nil {
if errors.Is(err, commonerrors.ErrPathEscapesScope) {
return os.ErrPermission
}
return err
}
if !state.AccessPermittedAtRealPath(idx, ffs.user.Username, realPath, access.IsDirAtRealPath(realPath)) {
return os.ErrPermission
}
return nil
}
func (ffs *filteredFileSystem) requireCreatePermissionForNewFile(requestPath string) error {
idx := indexing.GetIndex(ffs.source)
if idx == nil {
@@ -412,6 +442,9 @@ func (ffs *filteredFileSystem) Mkdir(ctx context.Context, name string, perm os.F
logger.Debugf("Mkdir: scope check denied for %s: %v", name, err)
return err
}
if err := ffs.checkResolvedAccess(name, false); err != nil {
return err
}
if err := ffs.fs.Mkdir(ctx, name, perm); err != nil {
return err
@@ -450,6 +483,9 @@ func (ffs *filteredFileSystem) OpenFile(ctx context.Context, requestPath string,
logger.Debugf("OpenFile: scope check denied for %s: %v", requestPath, err)
return nil, err
}
if err := ffs.checkResolvedAccess(requestPath, false); err != nil {
return nil, err
}
if (flag&os.O_CREATE) != 0 && !ffs.sourceFilePerms().Create {
if err := ffs.requireCreatePermissionForNewFile(requestPath); err != nil {
@@ -462,6 +498,9 @@ func (ffs *filteredFileSystem) OpenFile(ctx context.Context, requestPath string,
logger.Debugf("OpenFile: read denied by scope for %s", requestPath)
return nil, os.ErrPermission
}
if err := ffs.checkResolvedAccess(requestPath, false); err != nil {
return nil, err
}
}
var desiredMtime time.Time
@@ -573,7 +612,7 @@ func (ffs *filteredFileSystem) RemoveAll(ctx context.Context, requestPath string
if err := ffs.checkScopePath(requestPath, true); err != nil {
logger.Debugf("RemoveAll: scope check denied for %s: %v", requestPath, err)
return err
}
}
// The API delete path frees the deleted bytes from the quota counters. Do the same
// here: a WebDAV DELETE, and the MOVE over an existing destination that the WebDAV
// library implements as RemoveAll + Rename, otherwise leave the old bytes charged
@@ -587,24 +626,28 @@ func (ffs *filteredFileSystem) RemoveAll(ctx context.Context, requestPath string
}
var deleteBytes int64
var isDir bool
if info, err := ffs.fs.Stat(ctx, requestPath); err != nil {
logger.Debugf("RemoveAll: cannot stat %s before deleting, quota not adjusted: %v", requestPath, err)
} else if info.IsDir() {
isDir = true
deleteBytes, _ = state.IndexedPathBytes(ffs.source, indexPath, true)
} else {
deleteBytes = info.Size()
}
}
// Collect the adjustments before deleting: a directory delete also has to release
// folder quotas nested under it, and their sizes come from the index.
var adjustments state.DeleteAdjustments
if sourceInfo, ok := settings.Config.Server.NameToSource[ffs.source]; ok {
adjustments = state.PrepareAccountedDelete(ffs.user, ffs.source, sourceInfo.Path, indexPath, "", isDir, deleteBytes)
}
if err := ffs.fs.RemoveAll(ctx, requestPath); err != nil {
return err
}
if deleteBytes > 0 {
if sourceInfo, ok := settings.Config.Server.NameToSource[ffs.source]; ok {
if adjErr := state.ApplyAccountedUsageDelta(ffs.user, ffs.source, sourceInfo.Path, indexPath, "", -deleteBytes); adjErr != nil {
logger.Warningf("quota adjust after webdav delete failed: %v", adjErr)
}
}
if err := state.ApplyAccountedDelete(adjustments); err != nil {
logger.Warningf("quota adjust after webdav delete failed: %v", err)
}
activity.RecordWebDAVUser(ffs.httpReq, ffs.user, activitydb.Entry{
EventType: activitydb.EventDelete,
+61
View File
@@ -1,6 +1,7 @@
package web
import (
"context"
"fmt"
"io"
"net/http"
@@ -12,6 +13,8 @@ import (
"testing"
"time"
"golang.org/x/net/webdav"
"github.com/gtsteffaniak/filebrowser/backend/internal/adapters/fs/files"
"github.com/gtsteffaniak/filebrowser/backend/internal/app"
databasequota "github.com/gtsteffaniak/filebrowser/backend/internal/database/quota"
@@ -27,6 +30,7 @@ import (
// setupWebDAVTestEnv sets up the test environment with multiple sources and users
func setupWebDAVTestEnv(t *testing.T) (string, string) {
lockWebGlobals(t)
// Create temp directories for test sources
tempDir := t.TempDir()
source1Path := filepath.Join(tempDir, "source1")
@@ -1148,6 +1152,63 @@ func TestWebDAV_OPTIONS(t *testing.T) {
})
}
type failedDeleteFileSystem struct{ webdav.FileSystem }
func (failedDeleteFileSystem) RemoveAll(context.Context, string) error { return os.ErrPermission }
func TestWebDAVDeleteNestedQuota(t *testing.T) {
for _, scope := range []string{"/", "/scoped"} {
t.Run(scope, func(t *testing.T) {
sourcePath := setupDeleteQuotaTest(t)
user, err := state.GetUserByUsername("admin")
if err != nil {
t.Fatal(err)
}
user.BackendScopes[0].Scope = scope
nested, sibling := seedNestedQuotaFixture(t, sourcePath, scope, &user)
ffs := &filteredFileSystem{fs: failedDeleteFileSystem{webdav.Dir(filepath.Join(sourcePath, scope))}, source: "Downloads", user: &user}
if err = ffs.RemoveAll(context.Background(), "/public"); err == nil {
t.Fatal("expected forbidden deletion")
}
ffs.filePerms = users.SourceFilePermissions{Delete: true}
if err = ffs.RemoveAll(context.Background(), "/public"); err == nil {
t.Fatal("expected underlying deletion failure")
}
if used, _ := state.GetQuotaCounterSnapshot(nested.ID); used != 5 {
t.Fatalf("failed delete changed usage: %d", used)
}
req := httptest.NewRequest(http.MethodDelete, "/dav/Downloads/public", nil)
req.SetPathValue("source", "Downloads")
req.SetPathValue("path", "/public")
rec := httptest.NewRecorder()
_, err = webDAVHandler(rec, req, &requestContext{User: &user})
if err != nil || rec.Code != http.StatusNoContent {
t.Fatalf("delete: code=%d err=%v body=%s", rec.Code, err, rec.Body.String())
}
if _, err := os.Stat(filepath.Join(sourcePath, scope, "public")); !os.IsNotExist(err) {
t.Fatalf("ancestor still exists: %v", err)
}
if used, _ := state.GetQuotaCounterSnapshot(nested.ID); used != 0 {
t.Fatalf("nested usage=%d, want 0", used)
}
if used, _ := state.GetQuotaCounterSnapshot(sibling.ID); used != 7 {
t.Fatalf("sibling usage=%d, want 7", used)
}
req = httptest.NewRequest(http.MethodDelete, "/dav/Downloads/sibling/file", nil)
req.SetPathValue("source", "Downloads")
req.SetPathValue("path", "/sibling/file")
rec = httptest.NewRecorder()
if _, err := webDAVHandler(rec, req, &requestContext{User: &user}); err != nil || rec.Code != http.StatusNoContent {
t.Fatalf("file delete: code=%d err=%v", rec.Code, err)
}
if used, _ := state.GetQuotaCounterSnapshot(sibling.ID); used != 0 {
t.Fatalf("file delete usage=%d, want 0", used)
}
})
}
}
// A WebDAV DELETE must free the deleted bytes from the quota counters, the way the
// API delete path does, or the old bytes stay charged until the next recount.
func TestWebDAV_DeleteFreesQuota(t *testing.T) {
+45
View File
@@ -0,0 +1,45 @@
package indexing
import "sync"
// pathCoalescer runs background work at most once per key at a time. A job submitted while
// one for the same key is running replaces any job already queued behind it; the latest
// queued job runs when the current one finishes. Concurrent requests for the same folder
// therefore hold at most one running and one queued goroutine instead of one per request,
// and the newest data is still persisted.
type pathCoalescer struct {
mu sync.Mutex
pending map[string]*func()
}
// submit runs fn in the background, coalesced with other work for key.
func (c *pathCoalescer) submit(key string, fn func()) {
c.mu.Lock()
if c.pending == nil {
c.pending = make(map[string]*func())
}
if slot, running := c.pending[key]; running {
*slot = fn
c.mu.Unlock()
return
}
slot := new(func())
c.pending[key] = slot
c.mu.Unlock()
go func() {
for {
fn()
c.mu.Lock()
next := *slot
if next == nil {
delete(c.pending, key)
c.mu.Unlock()
return
}
*slot = nil
c.mu.Unlock()
fn = next
}
}()
}
+68
View File
@@ -0,0 +1,68 @@
package indexing
import (
"sync"
"sync/atomic"
"testing"
"time"
)
func TestPathCoalescerRunsLatestAndBoundsConcurrency(t *testing.T) {
var c pathCoalescer
release := make(chan struct{})
started := make(chan struct{})
var running, maxRunning, runs atomic.Int32
var last atomic.Int32
var wg sync.WaitGroup
job := func(id int32) func() {
return func() {
n := running.Add(1)
for {
m := maxRunning.Load()
if n <= m || maxRunning.CompareAndSwap(m, n) {
break
}
}
if id == 0 {
close(started)
<-release
}
last.Store(id)
runs.Add(1)
running.Add(-1)
wg.Done()
}
}
wg.Add(1)
c.submit("/a/", job(0))
<-started
for i := int32(1); i <= 5; i++ {
c.submit("/a/", job(i))
}
wg.Add(1) // only the latest queued job (5) runs after the first
close(release)
wg.Wait()
if got := runs.Load(); got != 2 {
t.Fatalf("runs = %d, want 2 (running job + latest queued)", got)
}
if got := last.Load(); got != 5 {
t.Fatalf("last job = %d, want 5", got)
}
if got := maxRunning.Load(); got != 1 {
t.Fatalf("max concurrent = %d, want 1", got)
}
// The final job's wg.Done runs before the coalescer drops the key, so poll briefly.
var left int
for deadline := time.Now().Add(2 * time.Second); ; {
c.mu.Lock()
left = len(c.pending)
c.mu.Unlock()
if left == 0 || time.Now().After(deadline) {
break
}
time.Sleep(time.Millisecond)
}
if left != 0 {
t.Fatalf("pending keys left = %d, want 0", left)
}
}
+35 -9
View File
@@ -3,6 +3,8 @@
package indexing
import (
"errors"
"io/fs"
"os"
"runtime"
"sync"
@@ -106,35 +108,52 @@ const dirStatParallelThreshold = 256
// slow filesystems do not overwhelm the kernel or I/O stack.
const dirStatMaxWorkers = 16
// statEntry is statDirEntry behind a seam so tests can inject per-entry stat failures (EACCES, EIO)
// that are hard to produce on a real filesystem.
var statEntry = statDirEntry
// readDirFileInfos lists dir as FileInfos, statting each name relative to the open directory fd.
//
// An entry whose stat fails with a permission or not-exist error is skipped (see skipEntry), so one
// unreadable junction does not fail the whole folder. Any other error fails the listing. If every name
// was skipped as not-exist, the listing fails instead of returning an empty success.
func readDirFileInfos(dir *os.File) ([]os.FileInfo, error) {
names, err := dir.Readdirnames(-1)
if err != nil {
return nil, err
}
dirPath := dir.Name()
dirFd := int(dir.Fd())
useStatx := statxDirListingAvailable(dirFd)
if len(names) < dirStatParallelThreshold {
infos := make([]os.FileInfo, 0, len(names))
vanished := 0
for _, name := range names {
fi, err := statDirEntry(dirFd, name, useStatx)
fi, err := statEntry(dirFd, name, useStatx)
if err != nil {
if os.IsNotExist(err) {
continue
if !skipEntry(dirPath, name, err) {
return nil, err
}
return nil, err
if errors.Is(err, fs.ErrNotExist) {
vanished++
}
continue
}
infos = append(infos, fi)
}
if err := checkNotAllVanished(dirPath, len(names), vanished); err != nil {
return nil, err
}
return infos, nil
}
return statDirEntriesParallel(dirFd, names, useStatx)
return statDirEntriesParallel(dirPath, dirFd, names, useStatx)
}
// statDirEntriesParallel stats each name relative to dirFd across a bounded
// worker pool. Entry order matches names; entries that vanished are skipped.
func statDirEntriesParallel(dirFd int, names []string, useStatx bool) ([]os.FileInfo, error) {
func statDirEntriesParallel(dirPath string, dirFd int, names []string, useStatx bool) ([]os.FileInfo, error) {
workers := runtime.GOMAXPROCS(0) * 2
if workers > dirStatMaxWorkers {
workers = dirStatMaxWorkers
@@ -145,6 +164,7 @@ func statDirEntriesParallel(dirFd int, names []string, useStatx bool) ([]os.File
results := make([]os.FileInfo, len(names))
var next atomic.Int64
var vanished atomic.Int64
var errMu sync.Mutex
var firstErr error
recordErr := func(err error) {
@@ -173,12 +193,15 @@ func statDirEntriesParallel(dirFd int, names []string, useStatx bool) ([]os.File
if i >= len(names) {
return
}
fi, err := statDirEntry(dirFd, names[i], useStatx)
fi, err := statEntry(dirFd, names[i], useStatx)
switch {
case err == nil:
results[i] = fi
case os.IsNotExist(err):
// Entry disappeared between readdir and stat; leave nil.
case skipEntry(dirPath, names[i], err):
// Left nil: unreadable or vanished between readdir and stat.
if errors.Is(err, fs.ErrNotExist) {
vanished.Add(1)
}
default:
recordErr(err)
return
@@ -191,6 +214,9 @@ func statDirEntriesParallel(dirFd int, names []string, useStatx bool) ([]os.File
if firstErr != nil {
return nil, firstErr
}
if err := checkNotAllVanished(dirPath, len(names), int(vanished.Load())); err != nil {
return nil, err
}
infos := make([]os.FileInfo, 0, len(names))
for _, fi := range results {
if fi != nil {
@@ -3,9 +3,12 @@
package indexing
import (
"errors"
"fmt"
"io/fs"
"os"
"path/filepath"
"syscall"
"testing"
"time"
@@ -260,3 +263,259 @@ func TestReadDirFileInfos_Parallel(t *testing.T) {
}
}
}
// writeNumberedFiles creates count files in dir and returns their names.
func writeNumberedFiles(t *testing.T, dir string, count int) []string {
t.Helper()
names := make([]string, 0, count)
for i := 0; i < count; i++ {
name := fmt.Sprintf("file_%04d", i)
if err := os.WriteFile(filepath.Join(dir, name), []byte{byte(i % 251)}, 0o644); err != nil {
t.Fatal(err)
}
names = append(names, name)
}
return names
}
func openTestDir(t *testing.T, dir string) *os.File {
t.Helper()
f, err := os.Open(dir)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { f.Close() })
return f
}
// injectStatErr makes the stat of entry bad fail with err, on both the serial and parallel paths.
func injectStatErr(t *testing.T, bad string, err error) {
t.Helper()
orig := statEntry
t.Cleanup(func() { statEntry = orig })
statEntry = func(dirFd int, name string, useStatx bool) (os.FileInfo, error) {
if name == bad {
return nil, &fs.PathError{Op: "fstatat", Path: name, Err: err}
}
return orig(dirFd, name, useStatx)
}
}
// injectAllStatErr makes every entry's stat fail with err.
func injectAllStatErr(t *testing.T, err error) {
t.Helper()
orig := statEntry
t.Cleanup(func() { statEntry = orig })
statEntry = func(dirFd int, name string, useStatx bool) (os.FileInfo, error) {
return nil, &fs.PathError{Op: "fstatat", Path: name, Err: err}
}
}
// listingSizes covers the serial path (below dirStatParallelThreshold) and the parallel path.
var listingSizes = []int{3, dirStatParallelThreshold + 44}
func TestReadDirFileInfos_SkipsPermissionDeniedEntry(t *testing.T) {
for _, count := range listingSizes {
t.Run(fmt.Sprintf("entries=%d", count), func(t *testing.T) {
dir := t.TempDir()
names := writeNumberedFiles(t, dir, count)
injectStatErr(t, names[0], syscall.EACCES)
infos, err := readDirFileInfos(openTestDir(t, dir))
if err != nil {
t.Fatalf("one EACCES entry failed the listing: %v", err)
}
if len(infos) != count-1 {
t.Fatalf("listed %d entries, want %d", len(infos), count-1)
}
for _, fi := range infos {
if fi.Name() == names[0] {
t.Fatal("unreadable entry was listed")
}
}
})
}
}
func TestReadDirFileInfos_SkipsVanishedEntry(t *testing.T) {
for _, count := range listingSizes {
t.Run(fmt.Sprintf("entries=%d", count), func(t *testing.T) {
dir := t.TempDir()
names := writeNumberedFiles(t, dir, count)
injectStatErr(t, names[0], syscall.ENOENT)
infos, err := readDirFileInfos(openTestDir(t, dir))
if err != nil {
t.Fatalf("one vanished entry failed the listing: %v", err)
}
if len(infos) != count-1 {
t.Fatalf("listed %d entries, want %d", len(infos), count-1)
}
})
}
}
// EIO on a network mount must fail the listing; skipping it would let the scanner prune files that
// still exist.
func TestReadDirFileInfos_IOErrorFailsListing(t *testing.T) {
for _, count := range listingSizes {
t.Run(fmt.Sprintf("entries=%d", count), func(t *testing.T) {
dir := t.TempDir()
names := writeNumberedFiles(t, dir, count)
injectStatErr(t, names[count/2], syscall.EIO)
if _, err := readDirFileInfos(openTestDir(t, dir)); !errors.Is(err, syscall.EIO) {
t.Fatalf("got %v, want EIO to fail the listing", err)
}
})
}
}
func TestReadDirFileInfos_StaleHandleFailsListing(t *testing.T) {
dir := t.TempDir()
names := writeNumberedFiles(t, dir, 3)
injectStatErr(t, names[1], syscall.ESTALE)
if _, err := readDirFileInfos(openTestDir(t, dir)); !errors.Is(err, syscall.ESTALE) {
t.Fatalf("got %v, want ESTALE to fail the listing", err)
}
}
// If every entry vanished, the directory was removed or renamed mid-scan. An empty success would make
// the scanner prune all of its children.
func TestReadDirFileInfos_AllEntriesVanishedFails(t *testing.T) {
for _, count := range listingSizes {
t.Run(fmt.Sprintf("entries=%d", count), func(t *testing.T) {
dir := t.TempDir()
writeNumberedFiles(t, dir, count)
injectAllStatErr(t, syscall.ENOENT)
f := openTestDir(t, dir)
// The directory itself is renamed away mid-scan; the open handle still lists it.
if err := os.Rename(dir, dir+".moved"); err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = os.RemoveAll(dir + ".moved") })
if _, err := readDirFileInfos(f); !errors.Is(err, fs.ErrNotExist) {
t.Fatalf("got %v, want a not-exist error for an all-vanished listing", err)
}
})
}
}
// Every entry vanishing from a directory that still exists (short-lived temp files) is a valid
// empty listing, not a not-exist error for the directory.
func TestReadDirFileInfos_AllEntriesVanishedDirPresentIsEmpty(t *testing.T) {
dir := t.TempDir()
writeNumberedFiles(t, dir, 1)
injectAllStatErr(t, syscall.ENOENT)
infos, err := readDirFileInfos(openTestDir(t, dir))
if err != nil {
t.Fatalf("got %v, want an empty success", err)
}
if len(infos) != 0 {
t.Fatalf("listed %d entries, want 0", len(infos))
}
}
func TestReadDirFileInfos_EmptyDirIsNotAnError(t *testing.T) {
infos, err := readDirFileInfos(openTestDir(t, t.TempDir()))
if err != nil || len(infos) != 0 {
t.Fatalf("empty dir: got %d entries, err %v", len(infos), err)
}
}
// Real EACCES, no seam: a directory that is readable but not searchable lists its names, while each
// per-entry stat fails with EACCES. Root ignores directory permissions, so skip there.
func TestReadDirFileInfos_RealPermissionDenied(t *testing.T) {
if os.Geteuid() == 0 {
t.Skip("root ignores directory permissions")
}
dir := t.TempDir()
writeNumberedFiles(t, dir, 2)
if err := os.Chmod(dir, 0o444); err != nil {
t.Fatal(err)
}
// Registered after TempDir's cleanup, so it runs first and the directory can be removed.
t.Cleanup(func() { _ = os.Chmod(dir, 0o755) })
infos, err := readDirFileInfos(openTestDir(t, dir))
if err != nil {
t.Fatalf("unsearchable directory failed the listing: %v", err)
}
if len(infos) != 0 {
t.Fatalf("listed %d entries, want 0", len(infos))
}
}
func TestCheckNotAllVanished(t *testing.T) {
if err := checkNotAllVanished("/d", 0, 0); err != nil {
t.Errorf("empty listing: got %v, want nil", err)
}
if err := checkNotAllVanished("/d", 3, 2); err != nil {
t.Errorf("partial vanish: got %v, want nil", err)
}
gone := filepath.Join(t.TempDir(), "gone")
if err := checkNotAllVanished(gone, 3, 3); !errors.Is(err, fs.ErrNotExist) {
t.Errorf("all vanished, directory gone: got %v, want not-exist", err)
}
// A one-entry directory whose only entry was deleted mid-scan still exists: empty success.
if err := checkNotAllVanished(t.TempDir(), 1, 1); err != nil {
t.Errorf("all vanished, directory present: got %v, want nil", err)
}
// A stat failure other than not-exist (here ENOTDIR) must fail the listing, not report empty.
file := filepath.Join(t.TempDir(), "file")
if err := os.WriteFile(file, nil, 0o600); err != nil {
t.Fatal(err)
}
if err := checkNotAllVanished(filepath.Join(file, "child"), 1, 1); err == nil {
t.Error("all vanished, stat error: got nil, want error")
}
}
func TestSkipEntry_Classification(t *testing.T) {
dir := t.TempDir()
if !skipEntry(dir, "perm", syscall.EACCES) {
t.Error("EACCES should skip")
}
if !skipEntry(dir, "gone", syscall.ENOENT) {
t.Error("ENOENT should skip")
}
if skipEntry(dir, "io", syscall.EIO) {
t.Error("EIO must not skip")
}
if skipEntry(dir, "stale", syscall.ESTALE) {
t.Error("ESTALE must not skip")
}
// A vanished entry is not recorded as unreadable.
unreadableMu.Lock()
_, recorded := unreadableWarned[filepath.Join(dir, "gone")]
unreadableMu.Unlock()
if recorded {
t.Error("ENOENT must not be recorded in the warned set")
}
}
func TestSkipEntry_WarnedSetIsBounded(t *testing.T) {
origMax := maxUnreadableWarned
t.Cleanup(func() {
maxUnreadableWarned = origMax
unreadableMu.Lock()
clear(unreadableWarned)
unreadableMu.Unlock()
})
maxUnreadableWarned = 2
dir := t.TempDir()
for _, name := range []string{"a", "b", "c"} {
skipEntry(dir, name, syscall.EACCES)
}
unreadableMu.Lock()
defer unreadableMu.Unlock()
if len(unreadableWarned) > 2 {
t.Fatalf("warned set holds %d paths, bound is 2", len(unreadableWarned))
}
if _, ok := unreadableWarned[filepath.Join(dir, "c")]; !ok {
t.Error("latest path should be recorded after the set was cleared")
}
}
+75 -2
View File
@@ -2,8 +2,81 @@
package indexing
import "os"
import (
"errors"
"io"
"io/fs"
"os"
"path/filepath"
)
// readDirFileInfos lists dir as FileInfos. Each entry is stat'ed with DirEntry.Info(), which on Windows
// reuses the directory read data and on Unix lstats by path. An entry skipped by skipEntry (permission
// or not-exist) is left out; any other error fails the listing. If every name was skipped as not-exist
// (e.g. the directory was renamed mid-scan), the listing fails instead of returning an empty success.
func readDirFileInfos(dir *os.File) ([]os.FileInfo, error) {
return dir.Readdir(-1)
dirPath := dir.Name()
entries, err := dir.ReadDir(-1)
if err != nil {
// On Unix, ReadDir lstats an entry itself when the filesystem reports an unknown d_type, and
// stops at the first failure. Only that per-entry lstat error (permission or not-exist) falls
// back to listing by name; errors reading the directory itself are returned.
var pe *fs.PathError
if !errors.As(err, &pe) || pe.Op != "lstat" ||
!(errors.Is(err, fs.ErrPermission) || errors.Is(err, fs.ErrNotExist)) {
return nil, err
}
return readDirByName(dir, err)
}
infos := make([]os.FileInfo, 0, len(entries))
vanished := 0
for _, e := range entries {
fi, err := e.Info()
if err != nil {
if !skipEntry(dirPath, e.Name(), err) {
return nil, err
}
if errors.Is(err, fs.ErrNotExist) {
vanished++
}
continue
}
infos = append(infos, fi)
}
if err := checkNotAllVanished(dirPath, len(entries), vanished); err != nil {
return nil, err
}
return infos, nil
}
// readDirByName re-reads dir by name and lstats each entry, skipping those skipEntry allows. origErr is
// the ReadDir error returned if the directory cannot be re-read.
func readDirByName(dir *os.File, origErr error) ([]os.FileInfo, error) {
if _, err := dir.Seek(0, io.SeekStart); err != nil {
return nil, origErr
}
names, err := dir.Readdirnames(-1)
if err != nil {
return nil, origErr
}
dirPath := dir.Name()
infos := make([]os.FileInfo, 0, len(names))
vanished := 0
for _, name := range names {
fi, err := os.Lstat(filepath.Join(dirPath, name))
if err != nil {
if !skipEntry(dirPath, name, err) {
return nil, err
}
if errors.Is(err, fs.ErrNotExist) {
vanished++
}
continue
}
infos = append(infos, fi)
}
if err := checkNotAllVanished(dirPath, len(names), vanished); err != nil {
return nil, err
}
return infos, nil
}
+81
View File
@@ -0,0 +1,81 @@
package indexing
import (
"errors"
"io/fs"
"os"
"sync"
"github.com/gtsteffaniak/go-logger/logger"
)
// Skip policy for per-entry stat errors while listing a directory.
//
// One entry that cannot be stat'ed should not fail the whole listing. On a CIFS or SMB mount a Windows
// junction can point at a target the share user cannot read, and the whole folder would otherwise
// disappear from the UI and the index. Like `ls -l`, we skip just that entry.
//
// Only permission and not-exist errors skip an entry. Any other error (EIO, ESTALE, timeouts) still
// fails the listing: the scanner prunes entries it does not see, so skipping on a flaky mount would
// delete index rows for files that still exist.
//
// A skipped entry is treated as absent by the scanner: it is pruned from the index and its size is
// left out of the folder total, the same as `ls` shows it as unreadable.
// maxUnreadableWarned bounds the set of paths already warned about. Routine scans revisit the same
// entries forever, so the set is cleared when it fills rather than growing without limit.
var maxUnreadableWarned = 10000
var (
unreadableMu sync.Mutex
unreadableWarned = map[string]struct{}{}
)
// skipEntry reports whether a stat error for dirPath/name should skip that entry rather than fail the
// listing. Permission errors are logged once per path (then at debug level), so a persistently
// unreadable entry does not flood the log on every scan.
func skipEntry(dirPath, name string, err error) bool {
if errors.Is(err, fs.ErrNotExist) {
return true
}
if !errors.Is(err, fs.ErrPermission) {
return false
}
path := dirPath + "/" + name
unreadableMu.Lock()
_, seen := unreadableWarned[path]
if !seen {
if len(unreadableWarned) >= maxUnreadableWarned {
clear(unreadableWarned)
}
unreadableWarned[path] = struct{}{}
}
unreadableMu.Unlock()
if seen {
logger.Debugf("skipping unreadable entry %s: %v", path, err)
} else {
logger.Warningf("skipping unreadable entry %s: %v", path, err)
}
return true
}
// checkNotAllVanished fails a listing in which every one of total names was skipped as not-exist.
// That is what a directory renamed or removed mid-scan looks like when entries are stat'ed by path.
// Returning an empty success there would make the scanner prune all of its children.
// When dirPath itself still exists, its entries really were deleted (e.g. short-lived temp
// files), so the empty listing is correct and is returned as a success.
func checkNotAllVanished(dirPath string, total, vanished int) error {
if total == 0 || vanished != total {
return nil
}
_, err := os.Stat(dirPath)
if err == nil {
return nil
}
if !errors.Is(err, fs.ErrNotExist) {
// Existence is unknown (I/O or permission error): fail the scan so indexed
// children are preserved instead of pruned by an empty "successful" listing.
return err
}
return &fs.PathError{Op: "readdir", Path: dirPath, Err: fs.ErrNotExist}
}
+129 -15
View File
@@ -8,6 +8,7 @@ import (
"runtime"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/gtsteffaniak/filebrowser/backend/internal/adapters/fs/fileutils"
@@ -23,8 +24,9 @@ import (
)
var (
RealPathCache = cache.NewCache[string](48*time.Hour, 72*time.Hour)
IsDirCache = cache.NewCache[bool](48*time.Hour, 72*time.Hour)
RealPathCache = cache.NewCache[string](48*time.Hour, 72*time.Hour)
IsDirCache = cache.NewCache[bool](48*time.Hour, 72*time.Hour)
realPathGeneration atomic.Uint64
)
// getDiskUsage returns the actual disk space used by a file in bytes
@@ -119,6 +121,7 @@ type Index struct {
previousNumFiles uint64 // Track previous NumFiles to use when scan in progress (computed value is 0)
scanners map[string]*Scanner // path -> scanner
mock bool
bgWork pathCoalescer // per-path background persistence and refreshes
mu sync.RWMutex
activeScannerPath string
schedulerStop chan struct{}
@@ -137,6 +140,9 @@ type Index struct {
// Adaptive scheduler: shared slot map (UTC unix seconds -> scanners).
scheduleSlotsMu sync.Mutex
scheduleSlots map[int64][]*Scanner
// startupScanPending requests one bounded serial scan pass on startup when the
// persisted folder-size map could not be restored (quick scans would be blind).
startupScanPending bool
}
var (
@@ -640,10 +646,27 @@ func (idx *Index) getDirInfoFromContext(ctx *PathContext, isViewable, isIndexabl
if err != nil {
return nil, err
}
idx.persistDirectoryListing(ctx.IndexPath, indexableInfo, opts)
// Persist index rows and folder sizes in the background so API listing
// latency stays independent of index DB write throughput. Coalesced per path so a slow
// DB cannot accumulate one goroutine per request.
indexPath := ctx.IndexPath
idx.bgWork.submit("persist|"+indexPath, func() {
idx.persistDirectoryListing(indexPath, indexableInfo, opts)
})
return dirFileInfo, nil
}
// RefreshDirectoryAsync runs RefreshDirectory(indexPath, false) in the background, coalesced
// per path: while a refresh for indexPath runs, further requests queue at most one more
// refresh instead of starting parallel directory reads and index DB writes.
func (idx *Index) RefreshDirectoryAsync(indexPath string) {
idx.bgWork.submit("refresh|"+indexPath, func() {
if err := idx.RefreshDirectory(indexPath, false); err != nil {
logger.Debugf("Failed to refresh directory for folder size update: %v", err)
}
})
}
// persistDirectoryListing updates index rows and the in-memory folder size from
// an already-built directory listing, avoiding the second directory walk that
// RefreshDirectory performs. Listings without extended attributes carry
@@ -653,12 +676,7 @@ func (idx *Index) persistDirectoryListing(indexPath string, indexableInfo *itemi
return
}
idx.UpdateMetadata(indexableInfo, nil, true)
newSize := uint64(indexableInfo.Size)
if previousSize, exists := idx.GetFolderSize(indexPath); exists {
idx.updateFolderSizeAndParents(indexPath, newSize, previousSize, true)
} else {
idx.SetFolderSize(indexPath, newSize)
}
idx.updateFolderSizeDelta(indexPath, uint64(indexableInfo.Size))
}
// getBasicDirInfo returns just directory metadata without children
@@ -710,8 +728,14 @@ func (idx *Index) indexDirectory(indexPath string, opts Options, scanner *Scanne
idx.UpdateMetadata(dirFileInfo, scanner, !opts.Recursive)
// Always store the calculated directory size in the in-memory map
// This ensures parent directories are updated after copy/move operations
idx.SetFolderSize(indexPath, uint64(dirFileInfo.Size))
// This ensures parent directories are updated after copy/move operations.
// API-driven refreshes (no scanner) also propagate the delta to ancestors so
// parent folder sizes reflect changes below the refreshed directory.
if scanner == nil {
idx.updateFolderSizeDelta(indexPath, uint64(dirFileInfo.Size))
} else {
idx.SetFolderSize(indexPath, uint64(dirFileInfo.Size))
}
return dirFileInfo.Size, dirFileInfo.HasPreview, nil
}
@@ -1199,6 +1223,7 @@ func (idx *Index) getRealPathInternal(boundIndexPath string, enforceScope bool,
if ok && cached != "" {
return cached, isDir, nil
}
generation := realPathGeneration.Load()
absolutePath, err := filepath.Abs(joinedPath)
if err != nil {
return absolutePath, false, fmt.Errorf("could not get real path: %v, %s", joinedPath, err)
@@ -1216,9 +1241,77 @@ func (idx *Index) getRealPathInternal(boundIndexPath string, enforceScope bool,
}
RealPathCache.Set(cacheKey, realPath)
IsDirCache.Set(cacheKey+":isdir", isDir)
if realPathGeneration.Load() != generation {
RealPathCache.Delete(cacheKey)
IsDirCache.Delete(cacheKey + ":isdir")
}
return realPath, isDir, nil
}
// realPathCacheJoinedPath builds the filesystem join used in RealPathCache keys.
func realPathCacheJoinedPath(sourceRoot, indexPath string) string {
seg := utils.CleanIndexPathSegment(indexPath)
if seg == "" {
return sourceRoot
}
return filepath.Join(sourceRoot, seg)
}
func realPathCacheIndexAncestors(indexPath string) []string {
rel := utils.CleanIndexPathSegment(indexPath)
if rel == "" {
return []string{"/"}
}
parts := strings.Split(rel, "/")
ancestors := []string{"/"}
for i := range parts {
ancestors = append(ancestors, "/"+strings.Join(parts[:i+1], "/"))
}
return ancestors
}
// ClearRealPathCaches drops every cached symlink resolution. The cache library has no
// prefix delete, so subtree invalidation uses a full clear; entries repopulate lazily.
func ClearRealPathCaches() {
realPathGeneration.Add(1)
RealPathCache.ClearAll()
IsDirCache.ClearAll()
}
// InvalidateRealPathCache drops cached symlink resolutions for indexPath and its
// ancestors. scopeBounds should include any caller scope prefixes used with
// GetRealPathScoped (e.g. user or share path).
func (idx *Index) InvalidateRealPathCache(indexPath string, scopeBounds ...string) {
realPathGeneration.Add(1)
boundSet := map[string]struct{}{"/": {}, "": {}}
for _, b := range scopeBounds {
b = strings.TrimRight(strings.TrimSpace(b), "/")
if b == "" || b == "/" {
boundSet["/"] = struct{}{}
boundSet[""] = struct{}{}
continue
}
if !strings.HasPrefix(b, "/") {
b = "/" + b
}
boundSet[b] = struct{}{}
}
bounds := make([]string, 0, len(boundSet))
for b := range boundSet {
bounds = append(bounds, b)
}
for _, ip := range realPathCacheIndexAncestors(indexPath) {
joined := realPathCacheJoinedPath(idx.Path, ip)
for _, enforceScope := range []bool{false, true} {
for _, bound := range bounds {
key := fmt.Sprintf("realpath|scoped=%t|bound=%q|path=%q", enforceScope, bound, joined)
RealPathCache.Delete(key)
IsDirCache.Delete(key + ":isdir")
}
}
}
}
// GetRealPathForMutation resolves an index path for a mutating filesystem
// operation (create/overwrite/mkdir/move-into). When the leaf exists it is fully
// resolved; otherwise the deepest existing ancestor is resolved and the missing
@@ -1363,8 +1456,29 @@ func (idx *Index) RefreshDirectory(indexPath string, recursive bool) error {
return err
}
// updateFolderSizeDelta atomically records a folder's newly computed size and
// propagates the delta to all ancestors in the in-memory folderSizes map.
func (idx *Index) updateFolderSizeDelta(indexPath string, newSize uint64) {
idx.folderSizesMu.Lock()
defer idx.folderSizesMu.Unlock()
previousSize, exists := idx.folderSizes[indexPath]
if !exists {
idx.folderSizes[indexPath] = newSize
idx.folderSizesUnsynced[indexPath] = struct{}{}
return
}
idx.updateFolderSizeAndParentsLocked(indexPath, newSize, previousSize, true)
}
// updateFolderSizeAndParents updates a folder size and propagates the change to all parents
func (idx *Index) updateFolderSizeAndParents(path string, newSize uint64, previousSize uint64, updateTarget bool) {
idx.folderSizesMu.Lock()
defer idx.folderSizesMu.Unlock()
idx.updateFolderSizeAndParentsLocked(path, newSize, previousSize, updateTarget)
}
// updateFolderSizeAndParentsLocked is updateFolderSizeAndParents with folderSizesMu already held.
func (idx *Index) updateFolderSizeAndParentsLocked(path string, newSize uint64, previousSize uint64, updateTarget bool) {
// Calculate the delta
var sizeDelta int64
if newSize >= previousSize {
@@ -1377,9 +1491,6 @@ func (idx *Index) updateFolderSizeAndParents(path string, newSize uint64, previo
return // No change, nothing to propagate
}
idx.folderSizesMu.Lock()
defer idx.folderSizesMu.Unlock()
// Optionally update the target folder itself
if updateTarget {
idx.folderSizes[path] = newSize
@@ -1549,7 +1660,10 @@ func setFilePreviewFlags(fileInfo *iteminfo.ItemInfo, realPath string) {
}
switch simpleType {
case "image":
if fileInfo.Size > iteminfo.LargeFileSizeThreshold {
// Camera RAW previews are the embedded JPEG, which is read without decoding the RAW data,
// so large RAW files (medium format, uncompressed) don't need the size limit.
isRaw := iteminfo.IsRawImage(strings.ToLower(filepath.Ext(fileInfo.Name)))
if fileInfo.Size > iteminfo.LargeFileSizeThreshold && !isRaw {
fileInfo.HasPreview = false
return
}
+4
View File
@@ -631,6 +631,10 @@ func (s *Scanner) checkFolderModtime(folderPath string) (bool, error) {
// Update metadata (will be batched)
s.idx.UpdateMetadata(dirInfo, s, true)
// Propagate the freshly computed folder size to ancestors so parent
// directories reflect changes below this directory.
s.idx.updateFolderSizeDelta(folderPath, uint64(dirInfo.Size))
// Track this directory as having been updated (for file purge logic)
s.idx.mu.Lock()
if s.idx.scanUpdatedPaths == nil {
+28 -7
View File
@@ -5,9 +5,9 @@ import (
"strconv"
"time"
"github.com/gtsteffaniak/filebrowser/backend/internal/utils"
indexingdb "github.com/gtsteffaniak/filebrowser/backend/internal/database/dbindex"
"github.com/gtsteffaniak/filebrowser/backend/internal/events"
"github.com/gtsteffaniak/filebrowser/backend/internal/utils"
"github.com/gtsteffaniak/go-logger/logger"
)
@@ -244,24 +244,32 @@ func (idx *Index) SendSourceUpdateEvent() error {
// restoreFolderSizes loads the persisted directory sizes into the in-memory folderSizes map.
// Quick scans iterate that map to decide which folders to stat, so leaving it empty after a
// restart would make them blind until the next full scan.
func (idx *Index) restoreFolderSizes() {
// restart would make them blind until the next full scan. Returns false when nothing could
// be restored (load failure or no persisted directory rows).
func (idx *Index) restoreFolderSizes() bool {
if idx.db == nil {
return
return false
}
sizes, err := idx.db.LoadFolderSizes(idx.Name)
if err != nil {
logger.Errorf("[%s] Failed to load persisted folder sizes: %v", idx.Name, err)
return
return false
}
idx.folderSizesMu.Lock()
idx.folderSizes = sizes
idx.folderSizesMu.Unlock()
return len(sizes) > 0
}
// setupMultiScanner creates and starts the multi-scanner system
// isNewDb: if true, skip loading persisted complexity values (database is new or recreated)
func (idx *Index) setupMultiScanner(isNewDb bool) {
idx.setupMultiScannerEx(isNewDb, true)
}
// setupMultiScannerEx is setupMultiScanner with the scheduler goroutine made optional —
// startSched=false lets tests inspect restored state without racing a running scheduler.
func (idx *Index) setupMultiScannerEx(isNewDb bool, startSched bool) {
// Load persisted index-level stats from Bolt (complexity, numDirs/Files, used/usedAlt/total, etc.)
// Scanner-level stats below use a second GetByPath only for the Scanners map.
if err := idx.Load(); err != nil {
@@ -269,8 +277,9 @@ func (idx *Index) setupMultiScanner(isNewDb bool) {
}
// Quick scans only stat the directories present in folderSizes, which is in-memory only.
// Restoring it keeps change detection working after a restart without a full re-read.
sizesRestored := false
if !isNewDb {
idx.restoreFolderSizes()
sizesRestored = idx.restoreFolderSizes()
}
idx.mu.Lock()
@@ -313,6 +322,16 @@ func (idx *Index) setupMultiScanner(isNewDb bool) {
// Discover existing top-level directories (root scanner will create scanners for new ones dynamically)
topLevelDirs := rootScanner.getTopLevelDirs()
// Quick scans stat only the directories present in folderSizes. When the index
// was previously built but no folder sizes could be restored while directories
// exist, quick scans would run blind — and their unchanged-modtime path would
// write zero sizes back — so run one bounded serial pass to rebuild the map.
if !isNewDb && !sizesRestored && len(topLevelDirs) > 0 {
idx.mu.Lock()
idx.startupScanPending = true
idx.mu.Unlock()
}
// Create child scanner for each top-level directory
for _, dirPath := range topLevelDirs {
childScanner := idx.createScanner(dirPath)
@@ -341,7 +360,9 @@ func (idx *Index) setupMultiScanner(isNewDb bool) {
}
idx.restoreScannerNextRuns()
go idx.runIndexScheduler()
if startSched {
go idx.runIndexScheduler()
}
logger.Debugf("Created %d scanners for [%v] (1 root + %d children)", len(topLevelDirs)+1, idx.Name, len(topLevelDirs))
}
@@ -120,6 +120,12 @@ func TestFolderSizesRestoredOnStartup(t *testing.T) {
if size != 4242 {
t.Fatalf("restored size for /sub/ = %d, want 4242", size)
}
idx.mu.RLock()
pending := idx.startupScanPending
idx.mu.RUnlock()
if pending {
t.Fatal("restored folder sizes must not trigger the startup bootstrap pass")
}
}
// fullScanCounter decides quick vs full for the next run; dropping it on restart means every
@@ -161,6 +167,86 @@ func TestFullScanCounterSurvivesRestart(t *testing.T) {
}
}
// When persisted folder sizes cannot be restored but the source has directories, quick scans
// would stat nothing (they iterate folderSizes) — so a bounded serial pass must be scheduled.
func TestMissingFolderSizesTriggersBootstrap(t *testing.T) {
idx := newRestoreTestIndex(t, "missing_sizes", []string{"sub"}, persistedScanners("/", "/sub/"))
idx.setupMultiScannerEx(false, false)
idx.mu.RLock()
pending := idx.startupScanPending
idx.mu.RUnlock()
if !pending {
t.Fatal("unrestored folder sizes with existing dirs must schedule a bootstrap pass")
}
if !idx.needsAdaptiveBootstrap() {
t.Fatal("startupScanPending must make the adaptive scheduler run the serial pass")
}
}
// A scanner that missed a scheduled run while the process was down must run a full scan first:
// in-place file edits update file mtimes but not parent directory mtimes, so quick scans
// cannot detect them.
func TestOverdueScannerRunsFullScan(t *testing.T) {
idx := newRestoreTestIndex(t, "overdue", nil, nil)
s := &Scanner{
scanPath: "/",
idx: idx,
currentSchedule: 0, // fastest tier: 5m — a lastScanned an hour ago is overdue
lastScanned: time.Now().Add(-time.Hour),
fullScanCounter: 2,
}
idx.scanners = map[string]*Scanner{"/": s}
idx.restoreScannerNextRuns()
var counter int
s.withStatsRLock(func() { counter = s.fullScanCounter })
if counter != 0 {
t.Fatalf("overdue scanner fullScanCounter = %d, want 0 (full scan first)", counter)
}
}
// Serial bootstrap must reset fullScanCounter so the pass runs full scans and can rebuild
// folderSizes; a restored counter in the quick-scan range would stat nothing.
func TestSerialBootstrapPassResetsFullScanCounter(t *testing.T) {
idx := newRestoreTestIndex(t, "serial_reset", nil, persistedScanners("/"))
// No background scheduler: the test drives runSerialScanPass itself, and a running scheduler
// would scan the same scanner concurrently (data race on its stats).
idx.setupMultiScannerEx(false, false)
idx.mu.Lock()
idx.startupScanPending = true
scanner := idx.scanners["/"]
idx.mu.Unlock()
scanner.withStatsLock(func() {
scanner.fullScanCounter = 2
})
idx.runSerialScanPass()
var counter int
scanner.withStatsRLock(func() { counter = scanner.fullScanCounter })
if counter != 1 {
t.Fatalf("after bootstrap full scan fullScanCounter = %d, want 1", counter)
}
}
// A scanner still within its schedule window keeps its persisted quick/full cadence.
func TestOnScheduleScannerKeepsScanCounter(t *testing.T) {
idx := newRestoreTestIndex(t, "on_schedule", nil, nil)
s := &Scanner{
scanPath: "/",
idx: idx,
currentSchedule: 0,
lastScanned: time.Now(),
fullScanCounter: 2,
}
idx.scanners = map[string]*Scanner{"/": s}
idx.restoreScannerNextRuns()
var counter int
s.withStatsRLock(func() { counter = s.fullScanCounter })
if counter != 2 {
t.Fatalf("on-schedule scanner fullScanCounter = %d, want 2", counter)
}
}
// An index with no persisted scan times must still do the serial pass, or a new or empty index
// would never be scanned.
func TestEmptyIndexStillBootstrapsSerialPass(t *testing.T) {
@@ -33,6 +33,14 @@ func computeNextSlotTime(lastScanned, ref time.Time, interval time.Duration) tim
return alignCeilToInterval(minNext, interval)
}
// scannerMissedAlignedSlot reports whether at least one aligned scan slot after
// lastScanned has passed before now (e.g. process was down across a due time).
func scannerMissedAlignedSlot(lastScanned, now time.Time, tier int) bool {
d := scanScheduleDuration(tier)
due := computeNextSlotTime(lastScanned, lastScanned, d)
return due.Before(now)
}
func (idx *Index) removeScannerFromSlotLocked(s *Scanner) {
sec := s.calendarSlotSec
if sec == 0 {
@@ -145,6 +153,13 @@ func (idx *Index) restoreScannerNextRuns() {
d := scanScheduleDuration(tier)
next = computeNextSlotTime(s.lastScanned, now, d)
s.nextRun = next
if scannerMissedAlignedSlot(s.lastScanned, now, tier) {
// This scanner missed at least one aligned slot while the process
// was down. In-place file edits update file mtimes but not their
// parent directory mtime, so quick scans cannot see them — force
// the first run after restart to be a full scan.
s.fullScanCounter = 0
}
})
if !next.IsZero() {
idx.registerScannerNextRun(s, next)
@@ -197,6 +212,9 @@ func (idx *Index) pruneGoneChildScanners() {
func (idx *Index) needsAdaptiveBootstrap() bool {
idx.mu.RLock()
defer idx.mu.RUnlock()
if idx.startupScanPending {
return true
}
for _, s := range idx.scanners {
if s.lastScanned.IsZero() {
return true
@@ -231,6 +249,10 @@ func (idx *Index) runSerialScanPass() {
s.removeSelf()
continue
}
// Folder sizes are not restored in this pass, so run a full scan.
s.withStatsLock(func() {
s.fullScanCounter = 0
})
s.executeScan()
}
}
@@ -239,6 +261,9 @@ func (idx *Index) schedulerAdaptiveTick() {
idx.pruneGoneChildScanners()
if idx.needsAdaptiveBootstrap() {
logger.Debugf("[%s] scheduler: adaptive bootstrap (serial pass all scanners)", idx.Name)
idx.mu.Lock()
idx.startupScanPending = false
idx.mu.Unlock()
idx.runSerialScanPass()
return
}
@@ -53,6 +53,18 @@ func TestScheduleTierBoundsForComplexity(t *testing.T) {
}
}
func TestScannerMissedAlignedSlot(t *testing.T) {
loc := time.UTC
last := time.Date(2026, 4, 14, 10, 1, 0, 0, loc)
now := time.Date(2026, 4, 14, 10, 7, 0, 0, loc)
if scannerMissedAlignedSlot(last, now, 0) {
t.Fatal("next aligned slot is still in the future; should not count as missed")
}
if !scannerMissedAlignedSlot(last, time.Date(2026, 4, 14, 10, 11, 0, 0, loc), 0) {
t.Fatal("aligned slot 10:10 should be treated as missed at 10:11")
}
}
func TestComputeNextSlotTimeHourUsesHourGrid(t *testing.T) {
loc := time.UTC
last := time.Date(2026, 4, 14, 12, 0, 0, 0, loc)
+34
View File
@@ -0,0 +1,34 @@
package indexing
import (
"testing"
"github.com/gtsteffaniak/filebrowser/backend/pkg/indexing/iteminfo"
)
func TestSetFilePreviewFlagsLargeImages(t *testing.T) {
large := iteminfo.LargeFileSizeThreshold + 1
tests := []struct {
name string
file string
typ string
size int64
want bool
}{
{"normal image under the limit", "a.jpg", "image/jpeg", 5 << 20, true},
{"normal image over the limit", "a.jpg", "image/jpeg", int64(large), false},
{"RAW under the limit", "a.raf", "image/x-fuji-raf", 26 << 20, true},
{"large RAW keeps its preview, the embedded JPEG is read without decoding", "a.RAF", "image/x-fuji-raf", int64(large), true},
{"large DNG keeps its preview", "a.dng", "image/x-adobe-dng", int64(large), true},
{"empty file has no preview", "a.raf", "image/x-fuji-raf", 0, false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
info := &iteminfo.ItemInfo{Name: tt.file, Type: tt.typ, Size: tt.size}
setFilePreviewFlags(info, "/nonexistent/"+tt.file)
if info.HasPreview != tt.want {
t.Fatalf("HasPreview = %v, want %v", info.HasPreview, tt.want)
}
})
}
}
@@ -2,6 +2,7 @@ package indexing
import (
"errors"
"fmt"
"os"
"path/filepath"
"testing"
@@ -163,6 +164,66 @@ func TestGetRealPathScoped_CacheKeyDoesNotCollideWithUnscoped(t *testing.T) {
}
}
func TestRealPathCacheGeneration_BumpsOnInvalidateAndClear(t *testing.T) {
idx := symlinkTestIndex(t.TempDir())
gen := realPathGeneration.Load()
idx.InvalidateRealPathCache("/any")
if realPathGeneration.Load() != gen+1 {
t.Fatalf("InvalidateRealPathCache: gen=%d want %d", realPathGeneration.Load(), gen+1)
}
gen = realPathGeneration.Load()
ClearRealPathCaches()
if realPathGeneration.Load() != gen+1 {
t.Fatalf("ClearRealPathCaches: gen=%d want %d", realPathGeneration.Load(), gen+1)
}
}
func TestClearRealPathCaches_ClearsWarmEntries(t *testing.T) {
sourceRoot, userScope := setupSymlinkEscapeFixture(t)
idx := symlinkTestIndex(sourceRoot)
userDir := filepath.Join(sourceRoot, "user1")
if err := os.WriteFile(filepath.Join(userDir, "foo"), []byte("x"), 0o644); err != nil {
t.Fatal(err)
}
if _, _, err := idx.GetRealPathScoped(userScope, "/user1/foo"); err != nil {
t.Fatalf("warm cache: %v", err)
}
joined := filepath.Join(sourceRoot, "user1", "foo")
key := fmt.Sprintf("realpath|scoped=%t|bound=%q|path=%q", true, userScope, joined)
if _, ok := RealPathCache.Get(key); !ok {
t.Fatalf("expected cache key %q", key)
}
ClearRealPathCaches()
if _, ok := RealPathCache.Get(key); ok {
t.Fatal("ClearRealPathCaches should drop warm entries")
}
}
func TestInvalidateRealPathCache_ClearsGetRealPathEntry(t *testing.T) {
sourceRoot, userScope := setupSymlinkEscapeFixture(t)
idx := symlinkTestIndex(sourceRoot)
userDir := filepath.Join(sourceRoot, "user1")
if err := os.WriteFile(filepath.Join(userDir, "foo"), []byte("x"), 0o644); err != nil {
t.Fatal(err)
}
if _, _, err := idx.GetRealPath("/user1/foo"); err != nil {
t.Fatalf("warm cache: %v", err)
}
joined := realPathCacheJoinedPath(sourceRoot, "/user1/foo")
key := fmt.Sprintf("realpath|scoped=%t|bound=%q|path=%q", false, "/", joined)
if _, ok := RealPathCache.Get(key); !ok {
t.Fatalf("expected cache key %q", key)
}
idx.InvalidateRealPathCache("/user1/foo", userScope)
if _, ok := RealPathCache.Get(key); ok {
t.Fatal("InvalidateRealPathCache should drop unscoped cache entry")
}
}
func TestGetRealPathForMutation_ExistingInScope(t *testing.T) {
sourceRoot, userScope := setupSymlinkEscapeFixture(t)
idx := symlinkTestIndex(sourceRoot)
+2 -2
View File
@@ -135,7 +135,7 @@ type ProxyAuthConfig struct {
type Recaptcha struct {
Host string `json:"host" validate:"required"` // google recaptcha host, for example: https://www.google.com/recaptcha/api.js
Key string `json:"key" validate:"required"` // v2 site key
Secret string `json:"secret" validate:"required"` // v2 secret key
Secret string `json:"secret" validate:"required"` // secret: v2 secret key
}
// ValidateRecaptcha disables recaptcha at startup if the configured host isn't a valid https URL
@@ -168,7 +168,7 @@ type LdapConfig struct {
Server string `json:"server"` // scheme://host:port of the LDAP server (e.g. ldap://localhost:389)
BaseDN string `json:"baseDN"` // LDAP search base DN (e.g. dc=ldap,dc=goauthentik,dc=io)
UserDN string `json:"userDN"` // Bind DN for service account (e.g. cn=admin,ou=users,dc=ldap,dc=goauthentik,dc=io)
UserPassword string `json:"userPassword"` // Bind password for service account
UserPassword string `json:"userPassword"` // secret: bind password for the LDAP service account
UserFilter string `json:"userFilter"` // Search filter for finding user by username. Default (&(cn=%s)(objectClass=user)); override e.g. (email=%s) or (sAMAccountName=%s) for other directories.
Port int `json:"-"` // derived from server
Scheme string `json:"-"` // derived from server
+20 -5
View File
@@ -113,7 +113,22 @@ func warnHttpProxyConfig() {
return
}
if Config.Http.BaseURL != "/" && !Config.Http.TrustProxyHeaders {
logger.Warning(`http.baseURL is not "/" but http.trustProxyHeaders is false. Behind a reverse proxy on a subpath, set trustProxyHeaders: true so cookies, client IP, OIDC redirects, and URLs resolve correctly. See https://filebrowserquantum.com/en/docs/configuration/http/#trustproxyheaders`)
logger.Warning(`http.baseURL is not "/" but http.trustProxyHeaders is false. Behind a reverse proxy on a subpath, set trustProxyHeaders: true so cookies, OIDC redirects, and URLs resolve correctly, and list the proxy addresses in http.trustedProxies so the client IP resolves too. See https://filebrowserquantum.com/en/docs/configuration/http/#trustproxyheaders`)
}
if len(Config.Http.TrustedProxies) > 0 && !Config.Http.TrustProxyHeaders {
logger.Warning("http.trustedProxies is set but http.trustProxyHeaders is false; trusted proxy entries are ignored until trustProxyHeaders is enabled")
}
// Check the parsed list: blank entries are accepted by validation but yield no prefixes.
if Config.Http.TrustProxyHeaders && len(TrustedProxyPrefixes()) == 0 && Config.Http.Socket == "" {
logger.Warning("http.trustProxyHeaders is true but http.trustedProxies is empty; X-Forwarded-For and X-Real-IP are ignored, so client IPs (rate limits, activity logs) resolve to the proxy address. Add your reverse proxy addresses to http.trustedProxies.")
}
if Config.Http.TrustProxyHeaders && Config.Http.Socket != "" && !strings.HasPrefix(Config.Http.Socket, "@") &&
len(Config.Http.UnixSocketTrustedUIDs) == 0 && len(Config.Http.UnixSocketTrustedGIDs) == 0 {
if Config.Http.UnixSocketTrustAllLocalPeers {
logger.Warning("http.unixSocketTrustAllLocalPeers is true: any local process that can connect to the filesystem socket may present X-Forwarded-For and X-Real-IP. Prefer http.unixSocketTrustedUIDs and/or http.unixSocketTrustedGIDs so only peers with allowed Linux credentials can set forwarded headers.")
} else {
logger.Warning("http.trustProxyHeaders is true on a filesystem Unix socket, but http.unixSocketTrustedUIDs and http.unixSocketTrustedGIDs are empty; forwarded client headers from socket peers are ignored. List allowed peer UIDs and/or GIDs (Linux SO_PEERCRED), or set http.unixSocketTrustAllLocalPeers: true only if you accept legacy trust-all behavior.")
}
}
if !Env.IsDevMode {
if u := Config.Http.ExternalUrl; u != "" && strings.HasPrefix(strings.ToLower(u), "http://") {
@@ -382,7 +397,6 @@ func isGoTest() bool {
return flag.Lookup("test.v") != nil
}
func setupFFmpegIntegration() {
n := Config.Server.NumImageProcessors
if n < 1 {
@@ -980,9 +994,10 @@ func SetDefaults(generate bool) Settings {
Locale: "en",
},
FileLoading: users.FileLoading{
MaxConcurrent: 10,
UploadChunkSize: 10, // 10MB
DownloadChunkSize: 0, // 0MB, default to no chunking
MaxConcurrent: 10,
UploadChunkSize: 10, // 10MB
DownloadChunkSize: 0, // 0MB, default to no chunking
UploadStallTimeoutSeconds: 10, // pause an upload after 10 seconds without progress
},
Account: UserDefaultsAccount{
Permissions: UserDefaultsAccountPermissions{
+4 -3
View File
@@ -71,9 +71,10 @@ type UserDefaultsUIEnforcement struct {
}
type UserDefaultsFileLoadingEnforcement struct {
MaxConcurrent bool `json:"maxConcurrentUpload,omitempty"`
UploadChunkSize bool `json:"uploadChunkSizeMb,omitempty"`
DownloadChunkSize bool `json:"downloadChunkSizeMb,omitempty"`
MaxConcurrent bool `json:"maxConcurrentUpload,omitempty"`
UploadChunkSize bool `json:"uploadChunkSizeMb,omitempty"`
DownloadChunkSize bool `json:"downloadChunkSizeMb,omitempty"`
UploadStallTimeoutSeconds bool `json:"uploadStallTimeoutSeconds,omitempty"`
}
type UserDefaultsAccountEnforcement struct {
+282 -12
View File
@@ -1041,6 +1041,11 @@ func GenerateConfigYamlWithEmbedded(config *Settings, showComments bool, showFul
deprecated = make(DeprecatedFieldsMap)
}
// The embedded YAML omits deprecated fields entirely, so the marker map has no entry for a
// deprecated field that is also a secret (e.g. the top-level auth.adminUsername/adminPassword).
// Union in the field-name fallback so those are redacted too.
identifySecretFieldsByReflection(reflect.ValueOf(config), "Settings", secrets)
var node *yaml.Node
if showFull {
@@ -1081,6 +1086,282 @@ func GenerateConfigYamlWithEmbedded(config *Settings, showComments bool, showFul
return yamlOutput, nil
}
// redactPlaceholder replaces secret values in API responses without revealing the original.
const redactPlaceholder = "**hidden**"
// RedactedSettingsCopy returns a deep copy of the given config with every secret-marked field
// replaced by the redaction placeholder. The live config is never modified, so callers can
// render the result to an API response without disabling authentication (a mutated
// settings.Config would break JWT signing/verification and the OIDC/LDAP/OnlyOffice integrations
// until restart). See GHSA-892f-9h5p-mfc8.
func RedactedSettingsCopy(conf *Settings) *Settings {
if conf == nil {
return nil
}
cpVal := deepCopyValue(reflect.ValueOf(conf))
cp, ok := cpVal.Interface().(*Settings)
if !ok {
return nil
}
secrets := buildSecretFieldsFor(conf)
redactSecretsValue(reflect.ValueOf(cp), "Settings", secrets)
return cp
}
// RedactAuthSecrets returns a copy of the auth section with secret fields replaced by the
// redaction placeholder, leaving the live config untouched.
func RedactAuthSecrets(auth Auth) Auth {
cp := auth
secrets := buildSecretFieldsFor(&Settings{Auth: Config.Auth})
redactSecretsValue(reflect.ValueOf(&cp), "Auth", secrets)
return cp
}
// deepCopyValue returns a deep copy of v, preserving the exact field values (including unexported
// fields) rather than round-tripping through JSON, which would run custom unmarshalers and change
// semantics (e.g. users.Permissions.UnmarshalJSON sets Configured).
func deepCopyValue(v reflect.Value) reflect.Value {
switch v.Kind() {
case reflect.Pointer:
if v.IsNil() {
return v
}
cp := reflect.New(v.Type().Elem())
cp.Elem().Set(deepCopyValue(v.Elem()))
return cp
case reflect.Interface:
if v.IsNil() {
return v
}
cp := deepCopyValue(v.Elem())
out := reflect.New(v.Type()).Elem()
out.Set(cp)
return out
case reflect.Struct:
out := reflect.New(v.Type()).Elem()
t := v.Type()
for i := 0; i < t.NumField(); i++ {
dst := out.Field(i)
if dst.CanSet() {
dst.Set(deepCopyValue(v.Field(i)))
}
}
return out
case reflect.Slice:
if v.IsNil() {
return v
}
out := reflect.MakeSlice(v.Type(), v.Len(), v.Len())
for i := 0; i < v.Len(); i++ {
out.Index(i).Set(deepCopyValue(v.Index(i)))
}
return out
case reflect.Array:
out := reflect.New(v.Type()).Elem()
for i := 0; i < v.Len(); i++ {
out.Index(i).Set(deepCopyValue(v.Index(i)))
}
return out
case reflect.Map:
if v.IsNil() {
return v
}
out := reflect.MakeMapWithSize(v.Type(), v.Len())
iter := v.MapRange()
for iter.Next() {
out.SetMapIndex(deepCopyValue(iter.Key()), deepCopyValue(iter.Value()))
}
return out
default:
return v
}
}
// SecretFieldNames are struct field names treated as secret regardless of comment markers.
// Used both for YAML redaction and for redacting GET /api/settings responses.
var SecretFieldNames = map[string]bool{
"Key": true,
"AdminUsername": true,
"AdminPassword": true,
"TotpSecret": true,
"ClientID": true,
"ClientSecret": true,
"Secret": true,
"UserPassword": true,
"Password": true,
"ApiKey": true,
"APIKey": true,
"BearerToken": true,
"PrivateKey": true,
}
// SecretHintYaml holds the embedded config.generated.yaml text used to discover `secret:`
// comment markers at runtime. The web layer sets it from the running binary's asset
// filesystem; the settings sources are not present in a deployed binary.
var SecretHintYaml string
// ActiveSecretFields returns the struct type -> field name -> is secret mapping for the running
// config. It combines the explicit `secret:` comment markers (from the embedded YAML when
// available, otherwise the Go sources) with the SecretFieldNames reflection fallback, so a new
// field named ApiKey/Secret/etc. is caught even without a marker.
func ActiveSecretFields() SecretFieldsMap {
return buildSecretFieldsFor(&Config)
}
func buildSecretFieldsFor(conf *Settings) SecretFieldsMap {
var secrets SecretFieldsMap
// Prefer the embedded YAML: it is the only marker source available in a deployed binary,
// and the generator fixes the comments for every field real users configure.
if SecretHintYaml != "" {
if _, s, _, err := CollectCommentsFromEmbeddedYaml(SecretHintYaml); err == nil && hasAnySecret(s) {
secrets = s
}
}
if !hasAnySecret(secrets) {
// Fall back to the Go sources (repo layout / tests). CollectCommentsAndSecrets also
// unions in the reflection fallback, so this path never under-reports.
// Try each candidate directory until one parses with markers, so a missing or
// unparsable first candidate does not skip a valid later one.
for _, srcPath := range settingsSourcePaths() {
if !isSettingsSourceDir(srcPath) {
continue
}
if _, s, _, err := CollectCommentsAndSecrets(srcPath); err == nil && hasAnySecret(s) {
secrets = s
break
}
}
}
if secrets == nil {
secrets = make(SecretFieldsMap)
}
// Always apply the reflection fallback so the map is complete even when only the embedded
// YAML was consulted (it covers the deprecated-but-secret auth.adminUsername/adminPassword
// fields, which the filtered generated YAML omits entirely).
identifySecretFieldsByReflection(reflect.ValueOf(conf), "Settings", secrets)
return secrets
}
// redactSecretsValue walks a settings value and replaces every field marked secret with the
// redaction placeholder, preserving the original structure for the JSON response.
func redactSecretsValue(v reflect.Value, typeName string, secrets SecretFieldsMap) {
switch v.Kind() {
case reflect.Pointer, reflect.Interface:
if v.IsNil() {
return
}
redactSecretsValue(v.Elem(), typeName, secrets)
case reflect.Slice, reflect.Array:
for i := 0; i < v.Len(); i++ {
redactSecretsValue(v.Index(i), typeName, secrets)
}
case reflect.Map:
for _, key := range v.MapKeys() {
redactSecretsValue(v.MapIndex(key), typeName, secrets)
}
case reflect.Struct:
t := v.Type()
for i := 0; i < t.NumField(); i++ {
field := t.Field(i)
if field.PkgPath != "" {
continue
}
fieldValue := v.Field(i)
if !fieldValue.CanSet() {
continue
}
fieldTypeName := fieldTypeNameFor(field)
switch {
case isSecretField(typeName, field.Name, secrets):
redactStringValue(fieldValue)
case field.Anonymous && jsonTagIsInline(field):
// encoding/json flattens embedded inline structs into the parent object.
redactSecretsValue(fieldValue, fieldTypeName, secrets)
case fieldValue.Kind() == reflect.Struct:
redactSecretsValue(fieldValue, fieldTypeName, secrets)
case fieldValue.Kind() == reflect.Pointer && !fieldValue.IsNil() && fieldValue.Elem().Kind() == reflect.Struct:
redactSecretsValue(fieldValue, fieldTypeName, secrets)
case fieldValue.Kind() == reflect.Slice && fieldValue.Type().Elem().Kind() == reflect.Struct:
redactSecretsValue(fieldValue, fieldTypeName, secrets)
case fieldValue.Kind() == reflect.Map && fieldValue.Type().Elem().Kind() == reflect.Struct:
redactSecretsValue(fieldValue, fieldTypeName, secrets)
}
}
}
}
func isSecretField(typeName, fieldName string, secrets SecretFieldsMap) bool {
if secrets != nil && secrets[typeName][fieldName] {
return true
}
return SecretFieldNames[fieldName]
}
func fieldTypeNameFor(field reflect.StructField) string {
t := field.Type
if t.Kind() == reflect.Pointer {
t = t.Elem()
}
if t.Kind() == reflect.Slice || t.Kind() == reflect.Array || t.Kind() == reflect.Map {
t = t.Elem()
}
if t.Kind() == reflect.Pointer {
t = t.Elem()
}
return t.Name()
}
func jsonTagIsInline(field reflect.StructField) bool {
tag := field.Tag.Get("json")
return strings.Contains(tag, "inline") || tag == ""
}
// redactStringValue replaces a string-ish secret field's value with the placeholder. Non-empty
// values are replaced; already-empty values are left empty so an unset secret does not appear set.
func redactStringValue(v reflect.Value) {
if v.Kind() == reflect.Pointer {
if v.IsNil() {
return
}
v = v.Elem()
}
if v.Kind() != reflect.String || v.String() == "" {
return
}
v.SetString(redactPlaceholder)
}
// settingsSourcePaths returns candidate directories holding the settings Go sources, used as a
// fallback when the embedded YAML is unavailable during secret detection.
func settingsSourcePaths() []string {
return []string{".", "pkg/settings", "../pkg/settings", "../../pkg/settings"}
}
// hasAnySecret reports whether the map holds at least one field marked secret. Parsers create
// an entry for every struct they see, so a non-empty map alone does not mean markers were found.
func hasAnySecret(secrets SecretFieldsMap) bool {
for _, fields := range secrets {
for _, secret := range fields {
if secret {
return true
}
}
}
return false
}
// isSettingsSourceDir reports whether dir looks like the settings package sources, so an
// unrelated Go package earlier in the candidate list (e.g. the working directory) is skipped.
func isSettingsSourceDir(dir string) bool {
info, err := os.Stat(filepath.Join(dir, "generator.go"))
if err != nil || info.IsDir() {
return false
}
info, err = os.Stat(filepath.Join(dir, "structs.go"))
return err == nil && !info.IsDir()
}
// identifySecretFieldsByReflection identifies secret fields by known field names
func identifySecretFieldsByReflection(v reflect.Value, typeName string, secrets SecretFieldsMap) {
if v.Kind() == reflect.Pointer {
@@ -1099,20 +1380,9 @@ func identifySecretFieldsByReflection(v reflect.Value, typeName string, secrets
secrets[typeName] = make(map[string]bool)
}
// Known secret field names
secretFields := map[string]bool{
"Key": true,
"AdminUsername": true,
"AdminPassword": true,
"TotpSecret": true,
"ClientID": true,
"ClientSecret": true,
"Secret": true,
}
for i := 0; i < t.NumField(); i++ {
field := t.Field(i)
if secretFields[field.Name] {
if SecretFieldNames[field.Name] {
secrets[typeName][field.Name] = true
}
+18
View File
@@ -3,7 +3,9 @@ package settings
import (
"fmt"
"net"
"net/netip"
"strconv"
"strings"
)
// HTTPListenAddr returns a TCP listen address suitable for net.Listen.
@@ -20,5 +22,21 @@ func ValidateHttpConfig(h Http) error {
if h.Socket != "" && (h.TLSCert != "" || h.TLSKey != "") {
return fmt.Errorf("http.socket cannot be used with tlsCert or tlsKey")
}
for _, entry := range h.TrustedProxies {
entry = strings.TrimSpace(entry)
if entry == "" {
continue
}
if prefix, err := netip.ParsePrefix(entry); err != nil {
if _, addrErr := netip.ParseAddr(entry); addrErr != nil {
return fmt.Errorf("http.trustedProxies entry %q is not a valid IP address or CIDR range", entry)
}
} else if _, err := normalizeTrustedProxyPrefix(prefix); err != nil {
return fmt.Errorf("http.trustedProxies entry %q: %v", entry, err)
}
}
if _, err := parseSocketFileMode(h.SocketFileMode); err != nil {
return err
}
return nil
}
+318
View File
@@ -0,0 +1,318 @@
package settings
import (
"encoding/json"
"os"
"path/filepath"
"reflect"
"strings"
"testing"
)
// secretSentinelValues are deliberately unique so a surviving sentinel in the marshaled
// output is unambiguously a leak.
func setSecretSentinels(t *testing.T) {
t.Helper()
prev := Config
t.Cleanup(func() { Config = prev })
Config.Auth.Key = "SENTINEL-AUTH-KEY"
Config.Auth.TotpSecret = "SENTINEL-TOTP-SECRET"
Config.Auth.AdminUsername = "SENTINEL-LEGACY-ADMIN-USER"
Config.Auth.AdminPassword = "SENTINEL-LEGACY-ADMIN-PW"
Config.Auth.Methods.PasswordAuth.AdminUsername = "SENTINEL-ADMIN-USER"
Config.Auth.Methods.PasswordAuth.AdminPassword = "SENTINEL-ADMIN-PW"
Config.Auth.Methods.PasswordAuth.Recaptcha.Secret = "SENTINEL-RECAPTCHA-SECRET"
Config.Auth.Methods.OidcAuth.ClientID = "SENTINEL-OIDC-CLIENT-ID"
Config.Auth.Methods.OidcAuth.ClientSecret = "SENTINEL-OIDC-CLIENT-SECRET"
Config.Auth.Methods.LdapAuth.UserPassword = "SENTINEL-LDAP-PASSWORD"
Config.Auth.Methods.JwtAuth.Secret = "SENTINEL-JWT-SECRET"
Config.Integrations.OnlyOffice.Secret = "SENTINEL-OFFICE-SECRET"
}
// TestRedactedSettingsCopy_marshalsWithoutPlaintextSecrets is the regression guard for
// GHSA-892f-9h5p-mfc8: GET /api/settings returned cleartext secrets.
func TestRedactedSettingsCopy_marshalsWithoutPlaintextSecrets(t *testing.T) {
setSecretSentinels(t)
out, err := json.Marshal(RedactedSettingsCopy(&Config))
if err != nil {
t.Fatalf("marshal: %v", err)
}
if strings.Contains(string(out), "SENTINEL-") {
t.Fatalf("redacted settings still contain a plaintext secret: %s", out)
}
// The placeholder must be present so an operator can tell a secret is configured.
if !strings.Contains(string(out), redactPlaceholder) {
t.Fatalf("expected redaction placeholder %q in output", redactPlaceholder)
}
}
// TestRedactedSettingsCopy_doesNotMutateLiveConfig is the regression guard for the live-config
// mutation that broke JWT signing and the integrations after the first GET /api/settings.
func TestRedactedSettingsCopy_doesNotMutateLiveConfig(t *testing.T) {
setSecretSentinels(t)
_ = RedactedSettingsCopy(&Config)
if Config.Auth.Key != "SENTINEL-AUTH-KEY" {
t.Fatalf("live config auth key was mutated: %q", Config.Auth.Key)
}
if Config.Auth.TotpSecret != "SENTINEL-TOTP-SECRET" {
t.Fatalf("live config totp secret was mutated: %q", Config.Auth.TotpSecret)
}
if Config.Auth.Methods.PasswordAuth.Recaptcha.Secret != "SENTINEL-RECAPTCHA-SECRET" {
t.Fatalf("live recaptcha secret was mutated: %q", Config.Auth.Methods.PasswordAuth.Recaptcha.Secret)
}
if Config.Auth.Methods.LdapAuth.UserPassword != "SENTINEL-LDAP-PASSWORD" {
t.Fatalf("live LDAP password was mutated: %q", Config.Auth.Methods.LdapAuth.UserPassword)
}
// The deprecated top-level password must survive too, since redaction of the copy must not
// alias back into the source struct.
if Config.Auth.AdminPassword != "SENTINEL-LEGACY-ADMIN-PW" {
t.Fatalf("live legacy admin password was mutated: %q", Config.Auth.AdminPassword)
}
}
// TestRedactedSettingsCopy_isFaithfulDeepCopy proves the reflection copy reproduces the whole
// config graph (every field type) before redaction, so no field is silently dropped.
func TestRedactedSettingsCopy_isFaithfulDeepCopy(t *testing.T) {
prev := Config
t.Cleanup(func() { Config = prev })
Config = SetDefaults(false)
Config.Integrations.OnlyOffice.Secret = "" // keep secrets empty so redaction is a no-op
Config.Auth.Key = ""
Config.Auth.TotpSecret = ""
Config.Auth.AdminPassword = ""
Config.Auth.AdminUsername = ""
Config.Auth.Methods.PasswordAuth.AdminPassword = ""
Config.Auth.Methods.PasswordAuth.AdminUsername = ""
Config.Auth.Methods.OidcAuth.ClientID = ""
Config.Auth.Methods.OidcAuth.ClientSecret = ""
Config.Auth.Methods.LdapAuth.UserPassword = ""
Config.Auth.Methods.JwtAuth.Secret = ""
Config.Auth.Methods.PasswordAuth.Recaptcha.Secret = ""
Config.Integrations.Media.Convert.ImagePreview[HEICImagePreview] = boolPtr(false)
Config.Server.Sources = []*Source{{Path: "/srv", Name: "srv"}}
cp := RedactedSettingsCopy(&Config)
if cp == &Config {
t.Fatal("expected a new Settings, got the same pointer")
}
if !reflect.DeepEqual(cp, &Config) {
t.Fatal("deep copy does not match the source config")
}
}
// TestRedactedSettingsCopy_isDeepCopy verifies nested maps/slices are copied, not aliased, so a
// caller cannot corrupt the live config by mutating the returned copy.
func TestRedactedSettingsCopy_isDeepCopy(t *testing.T) {
prev := Config
t.Cleanup(func() { Config = prev })
Config = SetDefaults(false)
Config.Server.Sources = []*Source{{Path: "/srv", Name: "original"}}
Config.Frontend.Styling.CustomThemes = map[string]CustomTheme{"a": {Description: "original"}}
Config.Server.SourceMap = map[string]*Source{"/srv": {Path: "/srv", Name: "original"}}
cp := RedactedSettingsCopy(&Config)
cp.Server.Sources[0].Name = "changed"
cp.Server.SourceMap["/srv"].Name = "changed"
cp.Frontend.Styling.CustomThemes["a"] = CustomTheme{Description: "changed"}
if Config.Server.Sources[0].Name != "original" {
t.Fatalf("source slice aliased the live config: %q", Config.Server.Sources[0].Name)
}
if Config.Server.SourceMap["/srv"].Name != "original" {
t.Fatalf("source map aliased the live config: %q", Config.Server.SourceMap["/srv"].Name)
}
if Config.Frontend.Styling.CustomThemes["a"].Description != "original" {
t.Fatalf("custom themes map aliased the live config: %q", Config.Frontend.Styling.CustomThemes["a"].Description)
}
}
func TestRedactedSettingsCopy_preservesNonSecretValues(t *testing.T) {
setSecretSentinels(t)
Config.Frontend.Name = "SENTINEL-NOT-A-SECRET"
Config.Http.Port = 8123
Config.Server.SearchResultsLimit = 4242
out, err := json.Marshal(RedactedSettingsCopy(&Config))
if err != nil {
t.Fatalf("marshal: %v", err)
}
var got Settings
if err := json.Unmarshal(out, &got); err != nil {
t.Fatalf("unmarshal: %v", err)
}
if got.Frontend.Name != "SENTINEL-NOT-A-SECRET" {
t.Fatalf("frontend name was redacted: %q", got.Frontend.Name)
}
if got.Http.Port != 8123 {
t.Fatalf("http port was altered: %d", got.Http.Port)
}
if got.Server.SearchResultsLimit != 4242 {
t.Fatalf("search limit was altered: %d", got.Server.SearchResultsLimit)
}
// A non-secret field in a struct that also holds secrets must survive: the OIDC/recaptcha
// containers are the easiest place to over-redact.
if got.Auth.Methods.OidcAuth.ClientID != redactPlaceholder {
t.Fatalf("expected OIDC clientId redacted, got %q", got.Auth.Methods.OidcAuth.ClientID)
}
if got.Auth.Methods.PasswordAuth.Recaptcha.Host == redactPlaceholder {
t.Fatal("recaptcha host must not be redacted (it is a public URL)")
}
}
// TestRedactedSettingsCopy_emptySecretsStayEmpty ensures an unset secret does not appear to be set.
func TestRedactedSettingsCopy_emptySecretsStayEmpty(t *testing.T) {
prev := Config
t.Cleanup(func() { Config = prev })
Config = SetDefaults(false)
Config.Auth.Key = ""
Config.Auth.TotpSecret = ""
out, err := json.Marshal(RedactedSettingsCopy(&Config))
if err != nil {
t.Fatalf("marshal: %v", err)
}
var got Settings
if err := json.Unmarshal(out, &got); err != nil {
t.Fatalf("unmarshal: %v", err)
}
if got.Auth.Key != "" || got.Auth.TotpSecret != "" {
t.Fatalf("empty secrets should stay empty, got key=%q totp=%q", got.Auth.Key, got.Auth.TotpSecret)
}
}
// TestRedactAuthSecrets_doesNotMutateConfig verifies the by-value helper leaves the live config alone.
func TestRedactAuthSecrets_doesNotMutateConfig(t *testing.T) {
setSecretSentinels(t)
redacted := RedactAuthSecrets(Config.Auth)
if redacted.Key != redactPlaceholder {
t.Fatalf("expected redacted key, got %q", redacted.Key)
}
if Config.Auth.Key != "SENTINEL-AUTH-KEY" {
t.Fatalf("live config auth key was mutated: %q", Config.Auth.Key)
}
}
// TestSecretMarkers_drive_yaml_redaction guards the marker gaps fixed for GHSA-892f-9h5p-mfc8:
// the YAML export redacts recaptcha.secret and ldap.userPassword, which previously leaked
// because their Go comments lacked the `secret:` marker the generator looks for.
func TestSecretMarkers_drive_yaml_redaction(t *testing.T) {
embedded, err := os.ReadFile("../../../frontend/public/config.generated.yaml")
if err != nil {
t.Fatalf("read generated yaml: %v", err)
}
prev := Config
t.Cleanup(func() { Config = prev })
Config = SetDefaults(false)
Config.Auth.Key = "SENTINEL-AUTH-KEY"
Config.Auth.TotpSecret = "SENTINEL-TOTP"
Config.Auth.AdminPassword = "SENTINEL-LEGACY-ADMIN-PW"
Config.Auth.AdminUsername = "SENTINEL-LEGACY-ADMIN-USER"
Config.Auth.Methods.PasswordAuth.AdminPassword = "SENTINEL-NESTED-ADMIN-PW"
Config.Auth.Methods.PasswordAuth.Recaptcha.Secret = "SENTINEL-RECAPTCHA"
Config.Auth.Methods.OidcAuth.ClientSecret = "SENTINEL-OIDC"
Config.Auth.Methods.LdapAuth.UserPassword = "SENTINEL-LDAP"
Config.Auth.Methods.JwtAuth.Secret = "SENTINEL-JWT"
Config.Integrations.OnlyOffice.Secret = "SENTINEL-OFFICE"
out, err := GenerateConfigYamlWithEmbedded(&Config, false, true, false, string(embedded))
if err != nil {
t.Fatalf("generate yaml: %v", err)
}
for _, sentinel := range []string{
"SENTINEL-AUTH-KEY",
"SENTINEL-TOTP",
"SENTINEL-LEGACY-ADMIN-PW",
"SENTINEL-LEGACY-ADMIN-USER",
"SENTINEL-NESTED-ADMIN-PW",
"SENTINEL-RECAPTCHA",
"SENTINEL-OIDC",
"SENTINEL-LDAP",
"SENTINEL-JWT",
"SENTINEL-OFFICE",
} {
if strings.Contains(out, sentinel) {
t.Errorf("YAML export leaked %s", sentinel)
}
}
}
// TestTestSecretFieldNames_covers_yaml_marker_gaps guards the two fields whose Go comments lacked
// the "secret:" marker (recaptcha secret, LDAP bind password), plus the deprecated top-level
// auth fields that the filtered generated YAML omits entirely.
func TestSecretFieldNames_covers_yaml_marker_gaps(t *testing.T) {
secrets := ActiveSecretFields()
for _, tc := range []struct{ typ, field string }{
{"Auth", "Key"},
{"Auth", "TotpSecret"},
{"Auth", "AdminPassword"},
{"PasswordAuthConfig", "AdminPassword"},
{"Recaptcha", "Secret"},
{"OidcConfig", "ClientSecret"},
{"JwtAuthConfig", "Secret"},
{"LdapConfig", "UserPassword"},
{"OnlyOffice", "Secret"},
} {
if !secrets[tc.typ][tc.field] {
t.Errorf("%s.%s is not marked secret", tc.typ, tc.field)
}
}
}
// TestActiveSecretFields_fromEmbeddedYaml exercises the deployed-binary path, where the
// secret markers come from the embedded config.generated.yaml rather than the Go sources.
func TestActiveSecretFields_fromEmbeddedYaml(t *testing.T) {
data, err := os.ReadFile("../../../frontend/public/config.generated.yaml")
if err != nil {
t.Skipf("generated YAML not available: %v", err)
}
// Assert the YAML parser output directly: ActiveSecretFields also unions in the reflection
// fallback (SecretFieldNames), which would mask broken marker detection.
_, secrets, _, err := CollectCommentsFromEmbeddedYaml(string(data))
if err != nil {
t.Fatalf("parse embedded yaml: %v", err)
}
for _, tc := range []struct{ typ, field string }{
{"Auth", "Key"},
{"Recaptcha", "Secret"},
{"OidcConfig", "ClientSecret"},
{"LdapConfig", "UserPassword"},
{"OnlyOffice", "Secret"},
{"PasswordAuthConfig", "AdminPassword"},
} {
if !secrets[tc.typ][tc.field] {
t.Errorf("%s.%s is not marked secret via embedded YAML", tc.typ, tc.field)
}
}
}
func TestHasAnySecret_ignoresTypesWithoutMarkers(t *testing.T) {
if hasAnySecret(SecretFieldsMap{"Context": {}, "Other": {"Field": false}}) {
t.Fatal("per-type entries without secret markers must not count")
}
if !hasAnySecret(SecretFieldsMap{"Auth": {"Key": true}}) {
t.Fatal("a marked field must count")
}
}
func TestIsSettingsSourceDir_skipsUnrelatedPackage(t *testing.T) {
unrelated := t.TempDir()
if err := os.WriteFile(filepath.Join(unrelated, "http.go"), []byte("package web\n\n// Context x\ntype Context struct{ Token string }\n"), 0o600); err != nil {
t.Fatal(err)
}
if isSettingsSourceDir(unrelated) {
t.Fatal("unrelated package must not be treated as the settings sources")
}
if !isSettingsSourceDir(".") {
t.Fatal("the settings package directory must be recognized")
}
}
+28 -16
View File
@@ -25,17 +25,28 @@ type Settings struct {
}
type Http struct {
Socket string `json:"socket"` // socket to listen on - eg. /var/run/filebrowser.sock
TLSKey string `json:"tlsKey"` // path to TLS key
TLSCert string `json:"tlsCert"` // path to TLS cert
Port int `json:"port"` // port to listen on
ListenAddress string `json:"listen"` // address to listen on (default: 0.0.0.0)
BaseURL string `json:"baseURL"` // base URL for the server, the subpath that the server is running on.
ExternalUrl string `json:"externalUrl"` // used by share links if set (eg. http://mydomain.com)
InternalUrl string `json:"internalUrl"` // used by integrations if set, this is the base domain that an integration service will use to communicate with filebrowser (eg. http://localhost:8080)
DisableWebDAV bool `json:"disableWebDAV"` // disable webdav support (default: false)
TrustProxyHeaders bool `json:"trustProxyHeaders"` // honor X-Forwarded-* and X-Real-IP from a reverse proxy (default: false)
DisableRateLimit bool `json:"disableRateLimit"` // turns off built-in auth route rate limiting and failed-login lockout (default false).
Socket string `json:"socket"` // socket to listen on - eg. /var/run/filebrowser.sock
TLSKey string `json:"tlsKey"` // path to TLS key
TLSCert string `json:"tlsCert"` // path to TLS cert
Port int `json:"port"` // port to listen on
ListenAddress string `json:"listen"` // address to listen on (default: 0.0.0.0)
BaseURL string `json:"baseURL"` // base URL for the server, the subpath that the server is running on.
ExternalUrl string `json:"externalUrl"` // used by share links if set (eg. http://mydomain.com)
InternalUrl string `json:"internalUrl"` // used by integrations if set, this is the base domain that an integration service will use to communicate with filebrowser (eg. http://localhost:8080)
DisableWebDAV bool `json:"disableWebDAV"` // disable webdav support (default: false)
TrustProxyHeaders bool `json:"trustProxyHeaders"` // honor X-Forwarded-* and X-Real-IP from a reverse proxy (default: false)
TrustedProxies []string `json:"trustedProxies"` // IPs/CIDRs of trusted proxies; forwarded client headers are only honored when the socket peer matches one of these (default: empty = forwarded headers ignored)
// UnixSocketTrustedUIDs and UnixSocketTrustedGIDs list peer credentials (Linux SO_PEERCRED) allowed to
// present X-Forwarded-* on a filesystem http.socket listener. When both are empty, peers are not trusted
// unless UnixSocketTrustAllLocalPeers is true (legacy; startup logs a warning). A GID allowlist trusts
// every peer with that GID, not only a single reverse-proxy process. Abstract sockets (@name) never trust
// forwarded headers.
UnixSocketTrustedUIDs []int `json:"unixSocketTrustedUIDs"`
UnixSocketTrustedGIDs []int `json:"unixSocketTrustedGIDs"`
UnixSocketTrustAllLocalPeers bool `json:"unixSocketTrustAllLocalPeers"` // default false: set true to trust forwarded headers from any local filesystem socket peer when allowlists are empty
SocketFileMode string `json:"socketFileMode"` // octal mode for filesystem http.socket (default "0660"); ignored for abstract sockets
SocketGroup string `json:"socketGroup"` // optional group name or numeric gid for the socket file (filesystem sockets only)
DisableRateLimit bool `json:"disableRateLimit"` // turns off built-in auth route rate limiting and failed-login lockout (default false).
}
type Environment struct {
@@ -70,11 +81,12 @@ type Server struct {
DisableTypeDetectionByHeader bool `json:"disableTypeDetectionByHeader"` // disable type detection by header, useful if filesystem is slow.
Logging []LogConfig `json:"logging" yaml:"logging"`
Sources []*Source `json:"sources" validate:"required,dive"`
CacheDir string `json:"cacheDir"` // path to the cache directory, used for thumbnails and other cached files
CacheDirCleanup bool `json:"cacheDirCleanup"` // whether to automatically cleanup the cache directory. Note: docker must also mount a persistent volume to persist the cache (default: false)
MaxArchiveSizeGB int64 `json:"maxArchiveSize"` // maximum archive/unarchive size in GB. 0 means no limit. (default: 20)
Filesystem Filesystem `json:"filesystem"` // filesystem settings
IndexSqlConfig IndexSqlConfig `json:"indexSqlConfig"` // Index database SQL configuration
CacheDir string `json:"cacheDir"` // path to the cache directory, used for thumbnails and other cached files
CacheDirCleanup bool `json:"cacheDirCleanup"` // whether to automatically cleanup the cache directory. Note: docker must also mount a persistent volume to persist the cache (default: false)
CacheDirMaxSizeMB int64 `json:"cacheDirMaxSizeMB"` // maximum size in MB of the preview cache (cacheDir/diskcache). Past it, the least recently used files are deleted. 0 means no limit (default: 0)
MaxArchiveSizeGB int64 `json:"maxArchiveSize"` // maximum archive/unarchive size in GB. 0 means no limit. (default: 20)
Filesystem Filesystem `json:"filesystem"` // filesystem settings
IndexSqlConfig IndexSqlConfig `json:"indexSqlConfig"` // Index database SQL configuration
// not exposed to config
SourceMap map[string]*Source `json:"-" validate:"omitempty"` // uses realpath as key
NameToSource map[string]*Source `json:"-" validate:"omitempty"` // uses name as key
+200
View File
@@ -0,0 +1,200 @@
package settings
import (
"fmt"
"net"
"net/http"
"net/netip"
"strings"
"sync"
)
const ipv4MappedPrefixBits = 96
// normalizeTrustedProxyPrefix converts IPv4-mapped IPv6 prefixes to IPv4 and
// masks the result. Mapped prefixes must include the full ::ffff:0:0/96 anchor.
func normalizeTrustedProxyPrefix(prefix netip.Prefix) (netip.Prefix, error) {
if prefix.Addr().Is4In6() {
if prefix.Bits() < ipv4MappedPrefixBits {
return netip.Prefix{}, fmt.Errorf("IPv4-mapped prefix must be at least /%d", ipv4MappedPrefixBits)
}
v4Bits := prefix.Bits() - ipv4MappedPrefixBits
if v4Bits > 32 {
v4Bits = 32
}
return netip.PrefixFrom(prefix.Addr().Unmap(), v4Bits).Masked(), nil
}
return prefix.Masked(), nil
}
var (
trustedProxiesMu sync.Mutex
trustedProxiesKey string
trustedProxiesCache []netip.Prefix
)
// TrustedProxyPrefixes parses Config.Http.TrustedProxies into CIDR prefixes,
// caching the result until the configured list changes. Entries may be CIDR
// ranges or bare IPs; unparsable entries are skipped (ValidateHttpConfig
// rejects them at startup).
func TrustedProxyPrefixes() []netip.Prefix {
key := strings.Join(Config.Http.TrustedProxies, "\x1e")
trustedProxiesMu.Lock()
defer trustedProxiesMu.Unlock()
if key == trustedProxiesKey {
return trustedProxiesCache
}
var prefixes []netip.Prefix
for _, entry := range Config.Http.TrustedProxies {
entry = strings.TrimSpace(entry)
if entry == "" {
continue
}
if prefix, err := netip.ParsePrefix(entry); err == nil {
normalized, normErr := normalizeTrustedProxyPrefix(prefix)
if normErr != nil {
continue
}
prefixes = append(prefixes, normalized)
continue
}
if addr, err := netip.ParseAddr(entry); err == nil {
if addr.Is4In6() {
addr = addr.Unmap()
}
prefixes = append(prefixes, netip.PrefixFrom(addr, addr.BitLen()))
}
}
trustedProxiesKey = key
trustedProxiesCache = prefixes
return prefixes
}
// parseHopAddr parses an IP (optionally with a port, as in host:port) into a
// canonical address. IPv4-mapped IPv6 addresses are unmapped to plain IPv4 so
// they compare correctly against IPv4 prefixes.
func parseHopAddr(hop string) (netip.Addr, bool) {
hop = strings.TrimSpace(hop)
addr, err := netip.ParseAddr(hop)
if err != nil {
host, _, splitErr := net.SplitHostPort(hop)
if splitErr != nil {
return netip.Addr{}, false
}
addr, err = netip.ParseAddr(host)
if err != nil {
return netip.Addr{}, false
}
}
return addr.Unmap(), true
}
// addrMatchesPrefixes reports whether addr falls inside any of the prefixes.
func addrMatchesPrefixes(addr netip.Addr, prefixes []netip.Prefix) bool {
for _, prefix := range prefixes {
if prefix.Contains(addr) {
return true
}
}
return false
}
// ipMatchesPrefixes reports whether ip falls inside any of the given prefixes.
// Entries that are not valid IPs (optionally with a port) never match.
func ipMatchesPrefixes(ip string, prefixes []netip.Prefix) bool {
addr, ok := parseHopAddr(ip)
if !ok {
return false
}
return addrMatchesPrefixes(addr, prefixes)
}
// IsTrustedProxyPeer reports whether the HTTP connection peer may present trusted
// proxy headers. Forwarded client headers must only be honored when this returns
// true; otherwise any client that can reach the listener directly could spoof its address.
//
// When http.socket is set the server listens only on that Unix socket. Abstract
// sockets ("@name") have no filesystem permissions and their peers are never trusted.
// On filesystem sockets, when either allowlist is set, only peers whose Linux SO_PEERCRED
// UID or GID matches may present forwarded headers. When both allowlists are empty,
// peers are trusted only if http.unixSocketTrustAllLocalPeers is true (legacy opt-in).
func IsTrustedProxyPeer(r *http.Request) bool {
if r == nil {
return false
}
if socket := Config.Http.Socket; socket != "" {
if strings.HasPrefix(socket, "@") {
return false
}
if len(Config.Http.UnixSocketTrustedUIDs) == 0 && len(Config.Http.UnixSocketTrustedGIDs) == 0 {
return Config.Http.UnixSocketTrustAllLocalPeers
}
cred, ok := UnixPeerFromContext(r.Context())
if !ok {
return false
}
return unixPeerAllowed(cred.UID, cred.GID)
}
return ipMatchesPrefixes(r.RemoteAddr, TrustedProxyPrefixes())
}
// canonicalHopIP returns the bare, unmapped IP for a forwarded hop, dropping
// any source port. It returns "" when the hop is not a valid IP.
func canonicalHopIP(hop string) string {
addr, ok := parseHopAddr(hop)
if !ok {
return ""
}
return addr.String()
}
// TrustedProxyClientIP resolves the originating client IP from an
// X-Forwarded-For header value.
//
// When http.trustedProxies is empty, the rightmost entry is returned: that is
// the peer recorded by the single trusted reverse proxy. Over TCP, callers only
// reach this after IsTrustedProxyPeer, which needs a configured prefix, so the
// empty case applies to Unix-socket listeners (http.socket) and direct callers.
//
// When trustedProxies is configured, the chain is walked right-to-left past
// every entry that belongs to a trusted proxy; the first untrusted entry is
// the client. This keeps client identity correct in multi-hop deployments
// (e.g. CDN -> reverse proxy -> filebrowser), where the rightmost entry would
// otherwise be the edge proxy shared by all clients.
func TrustedProxyClientIP(xff string) string {
parts := strings.Split(xff, ",")
last := -1
for i := len(parts) - 1; i >= 0; i-- {
if strings.TrimSpace(parts[i]) != "" {
last = i
break
}
}
if last < 0 {
return ""
}
prefixes := TrustedProxyPrefixes()
if len(prefixes) == 0 {
return canonicalHopIP(parts[last])
}
for i := last; i >= 0; i-- {
ip := strings.TrimSpace(parts[i])
if ip == "" {
continue
}
addr, ok := parseHopAddr(ip)
if !ok {
return ""
}
if !addrMatchesPrefixes(addr, prefixes) {
return addr.String()
}
}
// Every entry is a trusted proxy; the leftmost is closest to the client.
for _, part := range parts {
if ip := strings.TrimSpace(part); ip != "" {
return canonicalHopIP(ip)
}
}
return ""
}
@@ -0,0 +1,191 @@
package settings
import (
"context"
"net/http"
"net/netip"
"testing"
)
func resetTrustedProxyCache() {
trustedProxiesMu.Lock()
trustedProxiesKey = ""
trustedProxiesCache = nil
trustedProxiesMu.Unlock()
}
func TestNormalizeTrustedProxyPrefixIPv4Mapped(t *testing.T) {
prefix, err := netip.ParsePrefix("::ffff:198.51.100.0/120")
if err != nil {
t.Fatal(err)
}
got, err := normalizeTrustedProxyPrefix(prefix)
if err != nil {
t.Fatalf("normalize: %v", err)
}
if got.Addr().String() != "198.51.100.0" || got.Bits() != 24 {
t.Fatalf("got %v, want 198.51.100.0/24", got)
}
}
func TestNormalizeTrustedProxyPrefixRejectsShortMapped(t *testing.T) {
prefix, err := netip.ParsePrefix("::ffff:198.51.100.0/80")
if err != nil {
t.Fatal(err)
}
_, err = normalizeTrustedProxyPrefix(prefix)
if err == nil {
t.Fatal("expected error for mapped prefix shorter than /96")
}
}
func TestTrustedProxyPrefixesMatchesPlainIPv4Client(t *testing.T) {
resetTrustedProxyCache()
Config.Http.TrustedProxies = []string{"::ffff:198.51.100.0/120"}
t.Cleanup(func() {
Config.Http.TrustedProxies = nil
resetTrustedProxyCache()
})
prefixes := TrustedProxyPrefixes()
if len(prefixes) != 1 {
t.Fatalf("prefixes len = %d, want 1", len(prefixes))
}
if !ipMatchesPrefixes("198.51.100.7", prefixes) {
t.Fatal("plain IPv4 client should match normalized mapped prefix")
}
}
func TestTrustedProxyClientIPRejectsMalformedHop(t *testing.T) {
resetTrustedProxyCache()
Config.Http.TrustedProxies = []string{"198.51.100.7"}
t.Cleanup(func() {
Config.Http.TrustedProxies = nil
resetTrustedProxyCache()
})
got := TrustedProxyClientIP("203.0.113.50, not-an-ip, 198.51.100.7")
if got != "" {
t.Fatalf("malformed hop: got %q, want empty string", got)
}
}
func TestIPMatchesPrefixesUnmapsMappedHop(t *testing.T) {
resetTrustedProxyCache()
Config.Http.TrustedProxies = []string{"198.51.100.0/24"}
t.Cleanup(func() {
Config.Http.TrustedProxies = nil
resetTrustedProxyCache()
})
if !ipMatchesPrefixes("::ffff:198.51.100.7", TrustedProxyPrefixes()) {
t.Fatal("IPv4-mapped hop should match plain IPv4 prefix")
}
}
func TestTrustedProxyClientIPStripsHopPorts(t *testing.T) {
resetTrustedProxyCache()
Config.Http.TrustedProxies = []string{"198.51.100.0/24"}
t.Cleanup(func() {
Config.Http.TrustedProxies = nil
resetTrustedProxyCache()
})
got := TrustedProxyClientIP("203.0.113.50:40000, 198.51.100.7:5555")
if got != "203.0.113.50" {
t.Fatalf("got %q, want 203.0.113.50 without port", got)
}
}
func TestTrustedProxyClientIPCanonicalizesWithoutPrefixes(t *testing.T) {
resetTrustedProxyCache()
Config.Http.TrustedProxies = nil
t.Cleanup(resetTrustedProxyCache)
if got := TrustedProxyClientIP("203.0.113.50, 198.51.100.7:5555"); got != "198.51.100.7" {
t.Fatalf("got %q, want 198.51.100.7", got)
}
}
func TestIsTrustedProxyPeer(t *testing.T) {
resetTrustedProxyCache()
Config.Http.TrustedProxies = nil
t.Cleanup(func() {
Config.Http.TrustedProxies = nil
resetTrustedProxyCache()
})
if IsTrustedProxyPeer(&http.Request{RemoteAddr: "10.0.0.1:1234"}) {
t.Fatal("no trusted proxies configured: peer must not be trusted")
}
Config.Http.TrustedProxies = []string{"10.0.0.0/8"}
resetTrustedProxyCache()
if !IsTrustedProxyPeer(&http.Request{RemoteAddr: "10.0.0.1:1234"}) {
t.Fatal("peer in configured range should be trusted")
}
if !IsTrustedProxyPeer(&http.Request{RemoteAddr: "10.0.0.1"}) {
t.Fatal("bare IP peer in configured range should be trusted")
}
if IsTrustedProxyPeer(&http.Request{RemoteAddr: "192.0.2.1:1234"}) {
t.Fatal("peer outside configured range must not be trusted")
}
}
func TestIsTrustedProxyPeerUnixSocket(t *testing.T) {
resetTrustedProxyCache()
Config.Http.TrustedProxies = nil
Config.Http.Socket = "/tmp/filebrowser.sock"
t.Cleanup(func() {
Config.Http.Socket = ""
Config.Http.UnixSocketTrustedUIDs = nil
Config.Http.UnixSocketTrustedGIDs = nil
Config.Http.UnixSocketTrustAllLocalPeers = false
resetTrustedProxyCache()
})
if IsTrustedProxyPeer(&http.Request{RemoteAddr: "@"}) {
t.Fatal("filesystem unix socket with empty allowlists must not trust forwarded headers without opt-in")
}
Config.Http.UnixSocketTrustAllLocalPeers = true
if !IsTrustedProxyPeer(&http.Request{RemoteAddr: "@"}) {
t.Fatal("unixSocketTrustAllLocalPeers should restore legacy trust-all behavior")
}
Config.Http.UnixSocketTrustAllLocalPeers = false
Config.Http.UnixSocketTrustedUIDs = []int{33}
reqNoCred := &http.Request{RemoteAddr: "@"}
if IsTrustedProxyPeer(reqNoCred) {
t.Fatal("with allowlists set, peer without credentials must not be trusted")
}
req := (&http.Request{}).WithContext(ContextWithUnixPeerForTest(context.Background(), 33, 1000))
if !IsTrustedProxyPeer(req) {
t.Fatal("peer with allowed UID should be trusted")
}
reqBad := (&http.Request{}).WithContext(ContextWithUnixPeerForTest(context.Background(), 34, 1000))
if IsTrustedProxyPeer(reqBad) {
t.Fatal("peer with disallowed UID must not be trusted")
}
if got := TrustedProxyClientIP("203.0.113.50"); got != "203.0.113.50" {
t.Fatalf("got %q, want 203.0.113.50", got)
}
}
func TestIsTrustedProxyPeerAbstractSocketNotTrusted(t *testing.T) {
resetTrustedProxyCache()
Config.Http.TrustedProxies = nil
Config.Http.Socket = "@filebrowser"
t.Cleanup(func() {
Config.Http.Socket = ""
resetTrustedProxyCache()
})
if IsTrustedProxyPeer(&http.Request{RemoteAddr: "@"}) {
t.Fatal("abstract socket peers must not be trusted")
}
}
func TestValidateHttpConfigRejectsShortMappedPrefix(t *testing.T) {
err := ValidateHttpConfig(Http{TrustedProxies: []string{"::ffff:198.51.100.0/80"}})
if err == nil {
t.Fatal("expected validation error for short IPv4-mapped prefix")
}
}
+37
View File
@@ -0,0 +1,37 @@
//go:build linux
package settings
import (
"context"
"net"
"golang.org/x/sys/unix"
)
func peerCredentialFromConn(c net.Conn) (UnixPeerCredential, bool) {
uc, ok := c.(*net.UnixConn)
if !ok {
return UnixPeerCredential{}, false
}
raw, err := uc.SyscallConn()
if err != nil {
return UnixPeerCredential{}, false
}
var ucred *unix.Ucred
var ctlErr error
ctlErr = raw.Control(func(fd uintptr) {
ucred, ctlErr = unix.GetsockoptUcred(int(fd), unix.SOL_SOCKET, unix.SO_PEERCRED)
})
if ctlErr != nil || ucred == nil {
return UnixPeerCredential{}, false
}
return UnixPeerCredential{UID: int(ucred.Uid), GID: int(ucred.Gid), Ok: true}, true
}
func attachUnixPeerContext(ctx context.Context, c net.Conn) context.Context {
if cred, ok := peerCredentialFromConn(c); ok {
ctx = context.WithValue(ctx, unixPeerCredKey{}, cred)
}
return ctx
}
Loaded 100 of 198 files, more files were not shown because too many files have changed in this diff. Show more