mirror of
https://github.com/gtsteffaniak/filebrowser.git
synced 2026-11-15 21:45:55 -05:00
Compare commits
8
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
8da4e2862e | ||
|
|
b46ffd410f | ||
|
|
49ba28208e | ||
|
|
6bdaeb43b3 | ||
|
|
e7c494e8e0 | ||
|
|
dabe7cd10e | ||
|
|
9a77cb0aab | ||
|
|
6d8009725e |
No files matched your search
@@ -15,7 +15,7 @@ jobs:
|
||||
with:
|
||||
go-version: 'stable'
|
||||
- working-directory: backend
|
||||
run: go test -race -v ./... -timeout 30s
|
||||
run: go test -race -v ./... -timeout 45s
|
||||
lint-backend:
|
||||
runs-on: ubuntu-24.04-arm
|
||||
timeout-minutes: 5
|
||||
|
||||
@@ -12,6 +12,11 @@ All notable changes to this project will be documented in this file. For commit
|
||||
- Transcoding for media playback with quality presets -- requires ffmpeg.
|
||||
- Hardware Acceleration device selector (Software / detected GPU devices)
|
||||
- More operations use native api operations rather than ffmpeg, improving performance in many cases. Added temporary config option to force ffmpeg to be used
|
||||
- HLS keyframe indexes for MP4/MOV/M4V/MKV/WebM are read from the container index with go-media instead of demuxing every packet with ffprobe; ffprobe still runs when go-media cannot read the file or reports a keyframe gap that would rule out stream copy.
|
||||
- HEIC/AVIF rotation (irot) for the ffmpeg decode fallback is read with go-media instead of a full `ffmpeg -f null` decode.
|
||||
- AVIF images get previews through the HEIC preview path (go-media decode, ffmpeg fallback) when HEIC previews are enabled.
|
||||
- WMV/ASF video thumbnails use go-media for WMV3; WMV1/WMV2 still use ffmpeg.
|
||||
- ffmpeg and go-media share one media metadata cache keyed by path, size and modification time, so a probe, duration or subtitle result from either backend is reused by the other.
|
||||
- Document enhancements
|
||||
- Official Collabora support using WOPI config.
|
||||
- Document previews use a pure-go in-process package (all versions get it now).
|
||||
|
||||
+11
-11
@@ -5,6 +5,7 @@ go 1.27.1
|
||||
require (
|
||||
github.com/alecthomas/kong v1.16.1
|
||||
github.com/asdine/storm/v3 v3.2.1
|
||||
github.com/cespare/xxhash/v2 v2.3.0
|
||||
github.com/coreos/go-oidc/v3 v3.21.0
|
||||
github.com/coreos/go-systemd/v22 v22.7.0
|
||||
github.com/dhowden/tag v0.0.0-20240417053706-3d75831295e8
|
||||
@@ -21,21 +22,21 @@ require (
|
||||
github.com/gtsteffaniak/go-cache v1.1.0
|
||||
github.com/gtsteffaniak/go-ffmpeg v1.6.2
|
||||
github.com/gtsteffaniak/go-logger v1.1.0
|
||||
github.com/gtsteffaniak/go-media v1.2.1
|
||||
github.com/gtsteffaniak/go-media v1.3.0
|
||||
github.com/kovidgoyal/imaging v1.9.0
|
||||
github.com/pquerna/otp v1.5.0
|
||||
github.com/spf13/afero v1.15.0
|
||||
github.com/stretchr/testify v1.12.1
|
||||
github.com/swaggo/swag v1.16.6
|
||||
golang.org/x/crypto v0.58.0
|
||||
golang.org/x/image v0.47.0
|
||||
golang.org/x/image v0.48.0
|
||||
golang.org/x/mod v0.42.0
|
||||
golang.org/x/net v0.61.0
|
||||
golang.org/x/oauth2 v0.37.0
|
||||
golang.org/x/sync v0.24.0
|
||||
golang.org/x/sys v0.49.0
|
||||
golang.org/x/sync v0.25.0
|
||||
golang.org/x/sys v0.50.0
|
||||
golang.org/x/term v0.47.0
|
||||
golang.org/x/text v0.43.0
|
||||
golang.org/x/text v0.44.0
|
||||
golang.org/x/time v0.16.0
|
||||
golang.org/x/tools v0.52.0
|
||||
gopkg.in/yaml.v3 v3.0.1
|
||||
@@ -44,7 +45,7 @@ require (
|
||||
|
||||
require (
|
||||
github.com/Azure/go-ntlmssp v0.1.1 // indirect
|
||||
github.com/Eyevinn/mp4ff v0.61.0 // indirect
|
||||
github.com/Eyevinn/mp4ff v0.62.1 // indirect
|
||||
github.com/KyleBanks/depth v1.2.1 // indirect
|
||||
github.com/abema/go-mp4 v1.7.4 // indirect
|
||||
github.com/at-wat/ebml-go v0.19.4 // indirect
|
||||
@@ -54,10 +55,10 @@ require (
|
||||
github.com/dustin/go-humanize v1.1.0 // indirect
|
||||
github.com/fxamacker/cbor/v2 v2.9.6 // indirect
|
||||
github.com/gabriel-vasile/mimetype v1.4.16 // indirect
|
||||
github.com/gen2brain/gav1d v0.2.6 // indirect
|
||||
github.com/gen2brain/h265 v0.2.4 // indirect
|
||||
github.com/gen2brain/jxl v0.2.0 // indirect
|
||||
github.com/gen2brain/vpx v0.2.1 // indirect
|
||||
github.com/gen2brain/gav1d v0.2.7 // indirect
|
||||
github.com/gen2brain/h265 v0.2.5 // indirect
|
||||
github.com/gen2brain/jxl v0.2.1 // indirect
|
||||
github.com/gen2brain/vpx v0.2.2 // indirect
|
||||
github.com/go-asn1-ber/asn1-ber v1.5.8 // indirect
|
||||
github.com/go-errors/errors v1.5.1 // indirect
|
||||
github.com/go-jose/go-jose/v4 v4.1.6 // indirect
|
||||
@@ -80,7 +81,6 @@ require (
|
||||
github.com/golang/protobuf v1.5.4 // indirect
|
||||
github.com/golang/snappy v0.0.4 // indirect
|
||||
github.com/google/go-tpm v0.9.8 // indirect
|
||||
github.com/gtsteffaniak/go-av1 v1.0.0 // indirect
|
||||
github.com/gtsteffaniak/go-vpx v1.0.1 // indirect
|
||||
github.com/kovidgoyal/go-parallel v1.1.1 // indirect
|
||||
github.com/kovidgoyal/go-shm v1.0.0 // indirect
|
||||
|
||||
+22
-24
@@ -2,8 +2,8 @@ github.com/Azure/go-ntlmssp v0.1.1 h1:l+FM/EEMb0U9QZE7mKNEDw5Mu3mFiaa2GKOoTSsNDP
|
||||
github.com/Azure/go-ntlmssp v0.1.1/go.mod h1:NYqdhxd/8aAct/s4qSYZEerdPuH1liG2/X9DiVTbhpk=
|
||||
github.com/DataDog/zstd v1.4.1 h1:3oxKN3wbHibqx897utPC2LTQU4J+IHWWJO+glkAkpFM=
|
||||
github.com/DataDog/zstd v1.4.1/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo=
|
||||
github.com/Eyevinn/mp4ff v0.61.0 h1:uX3GBauEouxuGepRbv15JB7X2RyVyE2EZEfLpgimL6o=
|
||||
github.com/Eyevinn/mp4ff v0.61.0/go.mod h1:97zsDIJUiDTYqKpqq/b1K9foamxXmvsgE4GdPrPK3ks=
|
||||
github.com/Eyevinn/mp4ff v0.62.1 h1:EMuNIrmTCm5acOCVENca5Fzk+Mdek/l4bQZdk2EfReo=
|
||||
github.com/Eyevinn/mp4ff v0.62.1/go.mod h1:97zsDIJUiDTYqKpqq/b1K9foamxXmvsgE4GdPrPK3ks=
|
||||
github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc=
|
||||
github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE=
|
||||
github.com/Sereal/Sereal v0.0.0-20190618215532-0b8ac451a863 h1:BRrxwOZBolJN4gIwvZMJY1tzqBvQgpaZiQRuIDD40jM=
|
||||
@@ -25,6 +25,8 @@ github.com/at-wat/ebml-go v0.19.4/go.mod h1:wu3fQj+DWo6SOOl9c4yDi+gFs9Oa96Z2B5qx
|
||||
github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8=
|
||||
github.com/boombuler/barcode v1.1.0 h1:ChaYjBR63fr4LFyGn8E8nt7dBSt3MiU3zMOZqFvVkHo=
|
||||
github.com/boombuler/barcode v1.1.0/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8=
|
||||
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
|
||||
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/coreos/go-oidc/v3 v3.21.0 h1:wZo4Q9Pum8dYEj0eMUPrqR+kvuGkeUplbLpNCkBqoWM=
|
||||
github.com/coreos/go-oidc/v3 v3.21.0/go.mod h1:DYCf24+ncYi+XkIH97GY1+dqoRlbaSI26KVTCI9SrY4=
|
||||
github.com/coreos/go-systemd/v22 v22.7.0 h1:LAEzFkke61DFROc7zNLX/WA2i5J8gYqe0rSj9KI28KA=
|
||||
@@ -62,14 +64,14 @@ github.com/gabriel-vasile/mimetype v1.4.16 h1:5izNFuvA37st1dDRAcA7TtmngBmRgiIPDm
|
||||
github.com/gabriel-vasile/mimetype v1.4.16/go.mod h1:3oFFwxX8kTklZwzIl2aQEDZT0lKY8xH92ST88bAwba4=
|
||||
github.com/gen2brain/folio v0.2.8 h1:9iDI7h+zt4mjwEnzm1x8Pl9BrWqLwUwB74X/fnc1PZs=
|
||||
github.com/gen2brain/folio v0.2.8/go.mod h1:4/PcXfaXIYmW1nwQ4bJeE+aG5ybG58Aw9xrwODXaXII=
|
||||
github.com/gen2brain/gav1d v0.2.6 h1:7TSkrWTdE22k5Ul7vYZML7Nu80gNW6OovV7FSCITSE0=
|
||||
github.com/gen2brain/gav1d v0.2.6/go.mod h1:ReFKjHq7pvRhEk+pgRPAo9gOlcrf51sdzVcAbbX7e0E=
|
||||
github.com/gen2brain/h265 v0.2.4 h1:JUSbGkRAYZpd7ucCL192yx4ViEo75ZO6OEdm1TQt9qM=
|
||||
github.com/gen2brain/h265 v0.2.4/go.mod h1:kefpLxLhGpNCUG/q/aabJcD6GqB9Nhet3fNi9xX9Wzw=
|
||||
github.com/gen2brain/jxl v0.2.0 h1:cXSQnfbcC0JGFzaI344VzPMbe7mh6rYc0tjPcBUsE4U=
|
||||
github.com/gen2brain/jxl v0.2.0/go.mod h1:VjJHRai/8I8Es01mULqyiOihRBOjJr1Zo+NoN/MyIwo=
|
||||
github.com/gen2brain/vpx v0.2.1 h1:UD6tibcs/zlaIOKNFXeVyZuHBEaW9GYfxKSxvAN9cc8=
|
||||
github.com/gen2brain/vpx v0.2.1/go.mod h1:EVIdqguMSiapbtjDm4AWgFgYYJ4d+k8trXqjxYZ/4XE=
|
||||
github.com/gen2brain/gav1d v0.2.7 h1:ehu3jWt+0/46Is5llW/ZyX2OA3hTBJNukKnebzuk8KU=
|
||||
github.com/gen2brain/gav1d v0.2.7/go.mod h1:ReFKjHq7pvRhEk+pgRPAo9gOlcrf51sdzVcAbbX7e0E=
|
||||
github.com/gen2brain/h265 v0.2.5 h1:D3qCPx5GU+gHpinvLJtYMLSmQy64RGOm4fzMNGQcA1g=
|
||||
github.com/gen2brain/h265 v0.2.5/go.mod h1:kefpLxLhGpNCUG/q/aabJcD6GqB9Nhet3fNi9xX9Wzw=
|
||||
github.com/gen2brain/jxl v0.2.1 h1:74+8uw2RxUHGDVrDp+727PZ2hvAd5/psyV/mpsOQea0=
|
||||
github.com/gen2brain/jxl v0.2.1/go.mod h1:d7VMurg9ek62xfuyZyVAumFcuaQf9U8NlW57j/k5MQI=
|
||||
github.com/gen2brain/vpx v0.2.2 h1:kRCIx6Y2LsTe8hercAfhER2QbeHYnBOetovy4pkC4V0=
|
||||
github.com/gen2brain/vpx v0.2.2/go.mod h1:EVIdqguMSiapbtjDm4AWgFgYYJ4d+k8trXqjxYZ/4XE=
|
||||
github.com/go-asn1-ber/asn1-ber v1.5.8 h1:H9AZkK22UOmfX8J84ubyaZxKJZ3FMHVwn8swoMML7iQ=
|
||||
github.com/go-asn1-ber/asn1-ber v1.5.8/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
|
||||
github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q=
|
||||
@@ -157,18 +159,14 @@ github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3/go.mod h1:jl5iWTm0/hd
|
||||
github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/gtsteffaniak/go-av1 v1.0.0 h1:CILp67bcOtiDXvnrQubnH9ctub0OSKnUNA2JJuoIP+c=
|
||||
github.com/gtsteffaniak/go-av1 v1.0.0/go.mod h1:C/dIrjfdUcHgdyChr2YXwLY3pIVjENI9npSd98PU4lE=
|
||||
github.com/gtsteffaniak/go-cache v1.1.0 h1:ni/2cJpCxpGmaMZd92iTDIGOy+yUcMcxQuW7bSKhkCE=
|
||||
github.com/gtsteffaniak/go-cache v1.1.0/go.mod h1:rHUZvf4huhMX/bnNK4/oQ3sNAE2fYC2jFpAWZiBPWac=
|
||||
github.com/gtsteffaniak/go-ffmpeg v1.6.2 h1:gtvBg3G4PHO1Z/ZIPEK/XP4XT/g+l9S0Rpmmq6v5tnA=
|
||||
github.com/gtsteffaniak/go-ffmpeg v1.6.2/go.mod h1:FefhoD+KZzHWqBaUpntcKp6lXtUzdFW00F74axnENUQ=
|
||||
github.com/gtsteffaniak/go-logger v1.1.0 h1:baFds+iibuLLNpNLwygz32bDqdsunI6dGWcdxwMIVKI=
|
||||
github.com/gtsteffaniak/go-logger v1.1.0/go.mod h1:rapapTju4D7bwt6h4JOdZNUXDjnp0WkTBd6TN5yWglI=
|
||||
github.com/gtsteffaniak/go-media v1.2.0 h1:lbNL/G86uzzH8NaT45DBHKm1r0P7yndIo9pNC7gdqQs=
|
||||
github.com/gtsteffaniak/go-media v1.2.0/go.mod h1:EH+5KAVLWfFl0g4W0Xni0qJJH8MUDIhAw4wPcC0mhxY=
|
||||
github.com/gtsteffaniak/go-media v1.2.1 h1:SjrPqgQTHnenAFuVh8fTRxk8+qXFEaCxiADx3zkM1OI=
|
||||
github.com/gtsteffaniak/go-media v1.2.1/go.mod h1:EH+5KAVLWfFl0g4W0Xni0qJJH8MUDIhAw4wPcC0mhxY=
|
||||
github.com/gtsteffaniak/go-media v1.3.0 h1:EOxoaOQq05V3OUPo5NndnPZ0/VYt4gFEwa6l7MdR6rk=
|
||||
github.com/gtsteffaniak/go-media v1.3.0/go.mod h1:x8k0z759GiawTtKBTXDNXF/pT1UQDhR4e70/eypXpOk=
|
||||
github.com/gtsteffaniak/go-vpx v1.0.1 h1:9lSI8yoz6omDChaeV0AJrhdfe2YTrWvJrDENI/bnkAM=
|
||||
github.com/gtsteffaniak/go-vpx v1.0.1/go.mod h1:tRCdDZ15oZZTsiwmQAwyH56PTt1jn1C1O0P03oOkoJ8=
|
||||
github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8=
|
||||
@@ -258,8 +256,8 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk
|
||||
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||
golang.org/x/crypto v0.58.0 h1:COkYLr4k7nDI4r6QtMML2AfzeIswWHAJthLS6K/M/54=
|
||||
golang.org/x/crypto v0.58.0/go.mod h1:Xh+kl5A+M0gMiWDytH2Hhr5Z7WbNCpv/eB/EFgfSk+w=
|
||||
golang.org/x/image v0.47.0 h1:za/8gVz++PPZqmZJBNSISICN5Rrj5zixP/8K++fgmqo=
|
||||
golang.org/x/image v0.47.0/go.mod h1:IkN5o4A7vq55vXN8FWjjnhg9Rs2GOtffqsSi9mWAKYE=
|
||||
golang.org/x/image v0.48.0 h1:1AvuPcuoOTJpA7s4VZ5umrb90T3So0rkB/bSGmxkxik=
|
||||
golang.org/x/image v0.48.0/go.mod h1:mmtg64iqJKGqEgy+r13dfybl85tmxUd2lMnmaiSdWYY=
|
||||
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
|
||||
golang.org/x/mod v0.42.0 h1:ICeL3t2L30ljSPHHxDnR4fqVcBt/zR2zJAb9juavmQg=
|
||||
golang.org/x/mod v0.42.0/go.mod h1:u5fzIrRInkL8MaO1cbRkUHT64v303YIT5KEqpNr7jRs=
|
||||
@@ -279,8 +277,8 @@ golang.org/x/oauth2 v0.37.0 h1:JUlcxA8oAtauLfiH8FX2/FkAWHAdi0QtGCGc+hofE98=
|
||||
golang.org/x/oauth2 v0.37.0/go.mod h1:IxwZNxUULJmpBFf9K/9NTMSIfZZuvuTy1gGxhigP/58=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.24.0 h1:iqiDrGoDgyuvAKrhLQthUEzU/R378N78gXbBjxed7oA=
|
||||
golang.org/x/sync v0.24.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0=
|
||||
golang.org/x/sync v0.25.0 h1:dJnifr0nMV8/SBnBvdo4eSGGnrc3O4FFx6IqvtLcmRs=
|
||||
golang.org/x/sync v0.25.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
@@ -292,8 +290,8 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc
|
||||
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.49.0 h1:XbzkgYJHdqh/8m2Uu0W/dQv8nktxx4BFHp1M0gROTXA=
|
||||
golang.org/x/sys v0.49.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
|
||||
golang.org/x/sys v0.50.0 h1:dkKZEFTEKaLr1sjhs0WiKqoi8Jj5nHbS3Njmx1UoMJs=
|
||||
golang.org/x/sys v0.50.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
golang.org/x/term v0.47.0 h1:cpyOwQkr2Nzxe2p6KnrTG7Ei2wtzjsuSjhpXXepZ/BM=
|
||||
@@ -303,8 +301,8 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
|
||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
|
||||
golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
|
||||
golang.org/x/text v0.43.0 h1:1QivrlhwAsnMOcqgOdl9my7s9OyZl8O5/UGUTJVXoWc=
|
||||
golang.org/x/text v0.43.0/go.mod h1:hbSIYA/amXcRXTFSZqh8tWHEoqRvvbKPJ2uz/KU+a+s=
|
||||
golang.org/x/text v0.44.0 h1:YItdp/mQNGnsmrgpcdA5yQ3bnRxCfo/Tj/iByGW9GGo=
|
||||
golang.org/x/text v0.44.0/go.mod h1:S4FR1waQ9LbjQTls97SyRjDaUUVkaM+cXwwuG/G+MwY=
|
||||
golang.org/x/time v0.16.0 h1:vMb6ptszcQMkcwiRTAuNNU50gom6++Q/6gY2hDM6VDE=
|
||||
golang.org/x/time v0.16.0/go.mod h1:rVKOqvZeKvrDKTQiAHJ7wmwP0RzleSphoEA9RcdLA0s=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
|
||||
@@ -14,13 +14,13 @@ func TestUserUpdateChangesSkipsSensitiveFields(t *testing.T) {
|
||||
FrontendUser: users.FrontendUser{
|
||||
Username: "alice",
|
||||
NonAdminEditable: users.NonAdminEditable{
|
||||
DarkMode: true,
|
||||
DarkMode: users.DarkModeDark,
|
||||
},
|
||||
},
|
||||
TOTPSecret: "secret-value",
|
||||
}
|
||||
after := *before
|
||||
after.DarkMode = false
|
||||
after.DarkMode = users.DarkModeLight
|
||||
after.TOTPSecret = "new-secret"
|
||||
|
||||
changes := UserUpdateChanges(before, &after, []string{"totpSecret", "darkMode"}, false)
|
||||
@@ -36,7 +36,7 @@ func TestUserUpdateChangesFiltersUnchangedFields(t *testing.T) {
|
||||
FrontendUser: users.FrontendUser{
|
||||
Username: "akadmin",
|
||||
NonAdminEditable: users.NonAdminEditable{
|
||||
DarkMode: true,
|
||||
DarkMode: users.DarkModeDark,
|
||||
StickySidebar: true,
|
||||
Locale: "en",
|
||||
SingleClick: false,
|
||||
@@ -47,7 +47,7 @@ func TestUserUpdateChangesFiltersUnchangedFields(t *testing.T) {
|
||||
},
|
||||
}
|
||||
after := *before
|
||||
after.DarkMode = false
|
||||
after.DarkMode = users.DarkModeLight
|
||||
after.Sorting = users.Sorting{By: "name", Asc: true}
|
||||
|
||||
which := []string{
|
||||
@@ -60,7 +60,7 @@ func TestUserUpdateChangesFiltersUnchangedFields(t *testing.T) {
|
||||
if len(changes) != 3 {
|
||||
t.Fatalf("expected 3 changes, got %d: %#v", len(changes), changes)
|
||||
}
|
||||
if changes[0].Field != "darkMode" || changes[0].From != "true" || changes[0].To != "false" {
|
||||
if changes[0].Field != "darkMode" || changes[0].From != "dark" || changes[0].To != "light" {
|
||||
t.Fatalf("unexpected darkMode change: %#v", changes[0])
|
||||
}
|
||||
if changes[1].Field != "sorting.asc" || changes[1].From != "false" || changes[1].To != "true" {
|
||||
|
||||
@@ -9,7 +9,10 @@ import (
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/adapters/fs/fileutils"
|
||||
"github.com/gtsteffaniak/go-logger/logger"
|
||||
@@ -21,9 +24,28 @@ type Cache interface {
|
||||
Set(key string, value []byte) error
|
||||
}
|
||||
|
||||
const (
|
||||
// evictTargetPercent is the size, as a percentage of the limit, that eviction shrinks the cache to.
|
||||
// Stopping below the limit avoids running an eviction on every write once the cache is full.
|
||||
evictTargetPercent = 90
|
||||
// touchInterval is how old a file's modification time has to be before a read refreshes it. Reads
|
||||
// refresh it so the files in use are the last ones evicted, but not on every read.
|
||||
touchInterval = time.Hour
|
||||
)
|
||||
|
||||
// FileCache struct for file-based caching
|
||||
type FileCache struct {
|
||||
dir string
|
||||
// maxBytes is the size limit of the cache in bytes. 0 means no limit.
|
||||
maxBytes int64
|
||||
// size is the total size of the cached files in bytes. Only tracked when maxBytes > 0.
|
||||
size atomic.Int64
|
||||
// sizeMu keeps size consistent with the files on disk: stores and deletes hold it for reading while
|
||||
// they change a file and its size, and eviction holds it for writing while it measures the directory,
|
||||
// so a store can never fall between a measurement and the size being set from it.
|
||||
sizeMu sync.RWMutex
|
||||
// evicting is set while an eviction is running so only one runs at a time.
|
||||
evicting atomic.Bool
|
||||
// granular locks
|
||||
scopedLocks struct {
|
||||
sync.Mutex
|
||||
@@ -33,6 +55,12 @@ type FileCache struct {
|
||||
}
|
||||
|
||||
func NewFileCache(dir string) (*FileCache, error) {
|
||||
return NewFileCacheWithLimit(dir, 0)
|
||||
}
|
||||
|
||||
// NewFileCacheWithLimit is like NewFileCache but keeps the cache under maxBytes, deleting the least
|
||||
// recently used files when it grows past it. 0 means no limit.
|
||||
func NewFileCacheWithLimit(dir string, maxBytes int64) (*FileCache, error) {
|
||||
cacheDir := filepath.Join(dir, "diskcache")
|
||||
|
||||
// Migrate existing cache files from old structure (if any)
|
||||
@@ -44,7 +72,13 @@ func NewFileCache(dir string) (*FileCache, error) {
|
||||
if err := os.MkdirAll(cacheDir, fileutils.PermDir); err != nil {
|
||||
return nil, fmt.Errorf("failed to create cache directory: %w", err)
|
||||
}
|
||||
return &FileCache{dir: cacheDir}, nil
|
||||
cache := &FileCache{dir: cacheDir}
|
||||
if maxBytes > 0 {
|
||||
cache.maxBytes = maxBytes
|
||||
// measures the cache and trims it right away if it is already over the limit
|
||||
cache.evictPass()
|
||||
}
|
||||
return cache, nil
|
||||
}
|
||||
|
||||
// migrateOldCacheStructure moves cache files from old structure (dir/a/...) to new structure (dir/diskcache/a/...)
|
||||
@@ -99,19 +133,50 @@ func (f *FileCache) Store(ctx context.Context, key string, value []byte) error {
|
||||
return err
|
||||
}
|
||||
|
||||
if f.maxBytes > 0 {
|
||||
f.sizeMu.RLock()
|
||||
defer f.sizeMu.RUnlock()
|
||||
}
|
||||
previousSize := f.fileSize(fileName)
|
||||
|
||||
if err := os.WriteFile(fileName, value, fileutils.PermFile); err != nil {
|
||||
// a failed write can leave a truncated or partial file, so count what is actually there
|
||||
f.sizeChanged(f.fileSize(fileName) - previousSize)
|
||||
return err
|
||||
}
|
||||
|
||||
f.sizeChanged(int64(len(value)) - previousSize)
|
||||
return nil
|
||||
}
|
||||
|
||||
// fileSize returns the size of a cached file, or 0 if it doesn't exist or the size isn't tracked.
|
||||
func (f *FileCache) fileSize(fileName string) int64 {
|
||||
if f.maxBytes <= 0 {
|
||||
return 0
|
||||
}
|
||||
info, err := os.Stat(fileName)
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
return info.Size()
|
||||
}
|
||||
|
||||
// sizeChanged adds delta to the tracked size and starts an eviction if that put the cache over its limit.
|
||||
func (f *FileCache) sizeChanged(delta int64) {
|
||||
if f.maxBytes <= 0 {
|
||||
return
|
||||
}
|
||||
if f.size.Add(delta) > f.maxBytes {
|
||||
f.evictAsync()
|
||||
}
|
||||
}
|
||||
|
||||
func (f *FileCache) Load(ctx context.Context, key string) (value []byte, exist bool, err error) {
|
||||
r, ok, err := f.open(key)
|
||||
if err != nil || !ok {
|
||||
return nil, ok, err
|
||||
}
|
||||
defer r.Close()
|
||||
f.touch(r)
|
||||
|
||||
value, err = io.ReadAll(r)
|
||||
if err != nil {
|
||||
@@ -125,13 +190,119 @@ func (f *FileCache) Delete(ctx context.Context, key string) error {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
|
||||
if f.maxBytes > 0 {
|
||||
f.sizeMu.RLock()
|
||||
defer f.sizeMu.RUnlock()
|
||||
}
|
||||
fileName := f.getFileName(key)
|
||||
removedSize := f.fileSize(fileName)
|
||||
if err := os.Remove(fileName); err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||
return err
|
||||
}
|
||||
if removedSize > 0 {
|
||||
f.size.Add(-removedSize)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// touch refreshes the modification time of a file that was just read, which is what eviction orders by.
|
||||
func (f *FileCache) touch(file *os.File) {
|
||||
if f.maxBytes <= 0 {
|
||||
return
|
||||
}
|
||||
info, err := file.Stat()
|
||||
if err != nil || time.Since(info.ModTime()) < touchInterval {
|
||||
return
|
||||
}
|
||||
now := time.Now()
|
||||
_ = os.Chtimes(file.Name(), now, now)
|
||||
}
|
||||
|
||||
type cachedFile struct {
|
||||
path string
|
||||
size int64
|
||||
modTime time.Time
|
||||
}
|
||||
|
||||
// scan lists the cached files and their total size.
|
||||
func (f *FileCache) scan() ([]cachedFile, int64) {
|
||||
var files []cachedFile
|
||||
var total int64
|
||||
_ = filepath.WalkDir(f.dir, func(path string, entry os.DirEntry, err error) error {
|
||||
if err != nil || entry.IsDir() {
|
||||
return nil
|
||||
}
|
||||
info, err := entry.Info()
|
||||
if err != nil || !info.Mode().IsRegular() {
|
||||
return nil
|
||||
}
|
||||
files = append(files, cachedFile{path: path, size: info.Size(), modTime: info.ModTime()})
|
||||
total += info.Size()
|
||||
return nil
|
||||
})
|
||||
return files, total
|
||||
}
|
||||
|
||||
// evictAsync starts an eviction in the background unless one is already running, so the request that
|
||||
// stored the file doesn't wait for it.
|
||||
func (f *FileCache) evictAsync() {
|
||||
if !f.evicting.CompareAndSwap(false, true) {
|
||||
return
|
||||
}
|
||||
go func() {
|
||||
for {
|
||||
removed := f.evictPass()
|
||||
f.evicting.Store(false)
|
||||
if removed == 0 {
|
||||
return // under the limit, or nothing could be removed: don't spin
|
||||
}
|
||||
// Files stored while the pass was running were counted but not necessarily seen by it, and a
|
||||
// store that came in just before the flag was cleared saw it set and didn't start another
|
||||
// pass, so check again before stopping.
|
||||
if f.size.Load() <= f.maxBytes || !f.evicting.CompareAndSwap(false, true) {
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// evictPass measures the cache and, if it is over the limit, deletes the least recently used files until
|
||||
// it is under evictTargetPercent of the limit. Deleting a file that is being read or written is harmless:
|
||||
// readers keep their open file and a removed entry is simply rebuilt the next time it is requested.
|
||||
func (f *FileCache) evictPass() (removed int) {
|
||||
f.sizeMu.Lock()
|
||||
files, total := f.scan()
|
||||
f.size.Store(total)
|
||||
f.sizeMu.Unlock()
|
||||
|
||||
if total <= f.maxBytes {
|
||||
return 0
|
||||
}
|
||||
target := f.maxBytes / 100 * evictTargetPercent
|
||||
sort.Slice(files, func(i, j int) bool { return files[i].modTime.Before(files[j].modTime) })
|
||||
var removedBytes int64
|
||||
for _, file := range files {
|
||||
if f.size.Load() <= target {
|
||||
break
|
||||
}
|
||||
// Hold the lock so no store or delete changes this file between measuring it and removing it, and
|
||||
// subtract the size it has now: the listing is from the scan and the file may have been rewritten since.
|
||||
f.sizeMu.Lock()
|
||||
info, err := os.Stat(file.path)
|
||||
if err == nil {
|
||||
err = os.Remove(file.path)
|
||||
}
|
||||
if err == nil {
|
||||
f.size.Add(-info.Size())
|
||||
removed++
|
||||
removedBytes += info.Size()
|
||||
}
|
||||
f.sizeMu.Unlock()
|
||||
}
|
||||
logger.Debugf("preview cache limit: removed %d files (%d MB), now %d MB of %d MB", removed, removedBytes>>20, f.size.Load()>>20, f.maxBytes>>20)
|
||||
return removed
|
||||
}
|
||||
|
||||
func (f *FileCache) open(key string) (*os.File, bool, error) {
|
||||
fileName := f.getFileName(key)
|
||||
file, err := os.Open(fileName)
|
||||
|
||||
@@ -2,9 +2,12 @@ package diskcache
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/adapters/fs/fileutils"
|
||||
"github.com/stretchr/testify/require"
|
||||
@@ -70,3 +73,168 @@ func fileExists(filename string) bool {
|
||||
}
|
||||
return !info.IsDir()
|
||||
}
|
||||
|
||||
// storeAged stores a value and sets its modification time, so the eviction order is deterministic.
|
||||
func storeAged(t *testing.T, cache *FileCache, key string, size int, age time.Duration) {
|
||||
t.Helper()
|
||||
require.NoError(t, cache.Store(context.Background(), key, make([]byte, size)))
|
||||
when := time.Now().Add(-age)
|
||||
require.NoError(t, os.Chtimes(cache.getFileName(key), when, when))
|
||||
}
|
||||
|
||||
func waitFor(t *testing.T, condition func() bool) {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for !condition() {
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatal("condition not met in time")
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
func newLimitedCache(t *testing.T, maxBytes int64) *FileCache {
|
||||
t.Helper()
|
||||
fileutils.SetFsPermissions(0644, 0755)
|
||||
cache, err := NewFileCacheWithLimit(t.TempDir(), maxBytes)
|
||||
require.NoError(t, err)
|
||||
return cache
|
||||
}
|
||||
|
||||
func TestFileCacheEvictsLeastRecentlyUsedOverLimit(t *testing.T) {
|
||||
const kb = 1024
|
||||
cache := newLimitedCache(t, 10*kb)
|
||||
// 3 files of 3KB are under the 10KB limit
|
||||
storeAged(t, cache, "oldest", 3*kb, 3*time.Hour)
|
||||
storeAged(t, cache, "old", 3*kb, 2*time.Hour)
|
||||
storeAged(t, cache, "recent", 3*kb, time.Hour)
|
||||
require.True(t, fileExists(cache.getFileName("oldest")), "nothing is removed under the limit")
|
||||
|
||||
// the 4th file goes over the limit: the oldest ones are removed until it is under 90% of it
|
||||
require.NoError(t, cache.Store(context.Background(), "newest", make([]byte, 3*kb)))
|
||||
waitFor(t, func() bool { return !fileExists(cache.getFileName("oldest")) })
|
||||
waitFor(t, func() bool { return !cache.evicting.Load() })
|
||||
|
||||
require.False(t, fileExists(cache.getFileName("oldest")))
|
||||
require.False(t, fileExists(cache.getFileName("old")))
|
||||
require.True(t, fileExists(cache.getFileName("recent")))
|
||||
require.True(t, fileExists(cache.getFileName("newest")))
|
||||
require.LessOrEqual(t, cache.size.Load(), int64(9*kb))
|
||||
}
|
||||
|
||||
func TestFileCacheTrimsAnOversizedCacheAtStartup(t *testing.T) {
|
||||
const kb = 1024
|
||||
fileutils.SetFsPermissions(0644, 0755)
|
||||
dir := t.TempDir()
|
||||
unlimited, err := NewFileCache(dir)
|
||||
require.NoError(t, err)
|
||||
storeAged(t, &FileCache{dir: unlimited.dir}, "a", 4*kb, 4*time.Hour)
|
||||
storeAged(t, &FileCache{dir: unlimited.dir}, "b", 4*kb, 3*time.Hour)
|
||||
storeAged(t, &FileCache{dir: unlimited.dir}, "c", 4*kb, 2*time.Hour)
|
||||
|
||||
cache, err := NewFileCacheWithLimit(dir, 10*kb)
|
||||
require.NoError(t, err)
|
||||
|
||||
// 12KB over a 10KB limit: only the oldest file is removed, which already brings it under 90% (9KB)
|
||||
require.False(t, fileExists(cache.getFileName("a")))
|
||||
require.True(t, fileExists(cache.getFileName("b")))
|
||||
require.True(t, fileExists(cache.getFileName("c")))
|
||||
require.Equal(t, int64(8*kb), cache.size.Load())
|
||||
}
|
||||
|
||||
func TestFileCacheWithoutLimitKeepsEverything(t *testing.T) {
|
||||
fileutils.SetFsPermissions(0644, 0755)
|
||||
cache, err := NewFileCache(t.TempDir())
|
||||
require.NoError(t, err)
|
||||
for i := 0; i < 5; i++ {
|
||||
storeAged(t, cache, fmt.Sprintf("key-%d", i), 1<<20, time.Duration(i)*time.Hour)
|
||||
}
|
||||
for i := 0; i < 5; i++ {
|
||||
require.True(t, fileExists(cache.getFileName(fmt.Sprintf("key-%d", i))))
|
||||
}
|
||||
require.Zero(t, cache.size.Load(), "size is only tracked when there is a limit")
|
||||
}
|
||||
|
||||
func TestFileCacheReadRefreshesModTime(t *testing.T) {
|
||||
cache := newLimitedCache(t, 1<<20)
|
||||
storeAged(t, cache, "used", 100, 3*time.Hour)
|
||||
|
||||
_, ok, err := cache.Load(context.Background(), "used")
|
||||
require.NoError(t, err)
|
||||
require.True(t, ok)
|
||||
|
||||
info, err := os.Stat(cache.getFileName("used"))
|
||||
require.NoError(t, err)
|
||||
require.WithinDuration(t, time.Now(), info.ModTime(), time.Minute, "a file that is read is the last to be evicted")
|
||||
}
|
||||
|
||||
func TestFileCacheDeleteAndOverwriteKeepTheSizeAccurate(t *testing.T) {
|
||||
const kb = 1024
|
||||
cache := newLimitedCache(t, 1<<20)
|
||||
ctx := context.Background()
|
||||
require.NoError(t, cache.Store(ctx, "k", make([]byte, 5*kb)))
|
||||
require.Equal(t, int64(5*kb), cache.size.Load())
|
||||
require.NoError(t, cache.Store(ctx, "k", make([]byte, 2*kb)))
|
||||
require.Equal(t, int64(2*kb), cache.size.Load(), "overwriting replaces the old size")
|
||||
require.NoError(t, cache.Delete(ctx, "k"))
|
||||
require.Zero(t, cache.size.Load())
|
||||
}
|
||||
|
||||
// Stores that land while an eviction is measuring or deleting must not be lost from the tracked size,
|
||||
// and the cache must end up under its limit even though more files kept arriving during the eviction.
|
||||
func TestFileCacheStaysUnderTheLimitWithConcurrentStores(t *testing.T) {
|
||||
const (
|
||||
kb = 1024
|
||||
limit = 100 * kb
|
||||
workers = 16
|
||||
stores = 40
|
||||
)
|
||||
cache := newLimitedCache(t, limit)
|
||||
var wg sync.WaitGroup
|
||||
for w := 0; w < workers; w++ {
|
||||
wg.Add(1)
|
||||
go func(w int) {
|
||||
defer wg.Done()
|
||||
for i := 0; i < stores; i++ {
|
||||
_ = cache.Store(context.Background(), fmt.Sprintf("worker-%d-key-%d", w, i), make([]byte, 4*kb))
|
||||
}
|
||||
}(w)
|
||||
}
|
||||
wg.Wait()
|
||||
waitFor(t, func() bool { return !cache.evicting.Load() })
|
||||
|
||||
_, onDisk := cache.scan()
|
||||
require.LessOrEqual(t, onDisk, int64(limit), "the files left on disk are within the limit")
|
||||
require.Equal(t, onDisk, cache.size.Load(), "the tracked size matches what is on disk")
|
||||
}
|
||||
|
||||
// Files that are rewritten while an eviction is running must be subtracted at the size they have when
|
||||
// they are removed, not at the size the scan saw, or the tracked size drifts from the disk.
|
||||
func TestFileCacheKeepsTheSizeAccurateWhenFilesAreRewrittenDuringEviction(t *testing.T) {
|
||||
const (
|
||||
kb = 1024
|
||||
limit = 60 * kb
|
||||
workers = 8
|
||||
rounds = 300
|
||||
keys = 40
|
||||
)
|
||||
cache := newLimitedCache(t, limit)
|
||||
var wg sync.WaitGroup
|
||||
for w := 0; w < workers; w++ {
|
||||
wg.Add(1)
|
||||
go func(w int) {
|
||||
defer wg.Done()
|
||||
for i := 0; i < rounds; i++ {
|
||||
// the same keys again and again, with a different size each time
|
||||
key := fmt.Sprintf("key-%d", (w+i)%keys)
|
||||
_ = cache.Store(context.Background(), key, make([]byte, (1+(w*7+i)%8)*kb))
|
||||
}
|
||||
}(w)
|
||||
}
|
||||
wg.Wait()
|
||||
waitFor(t, func() bool { return !cache.evicting.Load() })
|
||||
|
||||
_, onDisk := cache.scan()
|
||||
require.Equal(t, onDisk, cache.size.Load(), "the tracked size matches what is on disk")
|
||||
require.LessOrEqual(t, onDisk, int64(limit))
|
||||
}
|
||||
@@ -17,6 +17,7 @@ import (
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/errors"
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/ffmpeg"
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/gomedia"
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/mediacache"
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/utils"
|
||||
"github.com/gtsteffaniak/filebrowser/backend/pkg/indexing"
|
||||
"github.com/gtsteffaniak/filebrowser/backend/pkg/indexing/iteminfo"
|
||||
@@ -52,7 +53,7 @@ func addMetadataToChildren(response *iteminfo.ExtendedFileInfo, opts utils.FileO
|
||||
logger.Debugf("failed to extract metadata for file: %s, error: %v", fileItem.Name, err)
|
||||
}
|
||||
} else {
|
||||
if err := extractVideoMetadata(context.Background(), fileItem, fullPath, sharedFFmpegService); err != nil {
|
||||
if err := extractVideoMetadata(context.Background(), fileItem, fullPath, opts.AlbumArt, sharedFFmpegService); err != nil {
|
||||
logger.Debugf("failed to extract video metadata for file: %s, error: %v", fileItem.Name, err)
|
||||
}
|
||||
}
|
||||
@@ -380,7 +381,7 @@ func processContent(info *iteminfo.ExtendedFileInfo, idx *indexing.Index, opts u
|
||||
extItem := &iteminfo.ExtendedItemInfo{
|
||||
ItemInfo: info.ItemInfo,
|
||||
}
|
||||
err := extractVideoMetadata(context.Background(), extItem, info.RealPath, nil)
|
||||
err := extractVideoMetadata(context.Background(), extItem, info.RealPath, opts.AlbumArt, nil)
|
||||
if err != nil {
|
||||
logger.Debugf("failed to extract video metadata for file: %s (%s): %v", info.RealPath, info.Name, err)
|
||||
} else {
|
||||
@@ -393,7 +394,8 @@ func processContent(info *iteminfo.ExtendedFileInfo, idx *indexing.Index, opts u
|
||||
info.GetSubtitles(parentInfo)
|
||||
}
|
||||
subtitles, ok := gomedia.EmbeddedSubtitles(info.RealPath, info.ModTime, func() ([]utils.SubtitleTrack, bool) {
|
||||
return ffmpeg.DetectEmbeddedSubtitles(info.RealPath, info.ModTime), true
|
||||
// Shadow run: bypass the shared cache so ffprobe is compared fresh.
|
||||
return ffmpeg.DetectEmbeddedSubtitlesContext(mediacache.WithoutCache(context.Background()), info.RealPath), true
|
||||
})
|
||||
if !ok {
|
||||
subtitles = ffmpeg.DetectEmbeddedSubtitles(info.RealPath, info.ModTime)
|
||||
@@ -471,23 +473,11 @@ func extractAudioMetadata(ctx context.Context, item *iteminfo.ExtendedItemInfo,
|
||||
// go-media fast path: tags, lyrics availability, and artwork. Unsupported or
|
||||
// failed reads fall through to the existing dhowden/tag code below.
|
||||
if info, ok := gomedia.Audio(realPath, getArt, func() (gomedia.AudioInfo, bool) { return legacyAudioInfo(realPath, getArt) }); ok {
|
||||
item.Metadata = &iteminfo.MediaMetadata{
|
||||
Title: info.Tags.Title,
|
||||
Artist: info.Tags.Artist,
|
||||
Album: info.Tags.Album,
|
||||
Year: info.Tags.Year,
|
||||
Genre: info.Tags.Genre,
|
||||
Track: info.Tags.Track,
|
||||
HasLyrics: info.HasLyrics,
|
||||
}
|
||||
item.Metadata = &iteminfo.MediaMetadata{HasLyrics: info.HasLyrics}
|
||||
applyMediaTags(item.Metadata, info, realPath)
|
||||
if getDuration {
|
||||
audioDuration(ctx, item, realPath, ffmpegService)
|
||||
}
|
||||
if n := len(info.Artwork); n > 0 && n <= 5*1024*1024 {
|
||||
item.Metadata.AlbumArt = info.Artwork
|
||||
} else if n > 5*1024*1024 {
|
||||
logger.Debugf("Skipping album art for %s: too large (%d bytes)", realPath, n)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -541,8 +531,9 @@ func audioDuration(ctx context.Context, item *iteminfo.ExtendedItemInfo, realPat
|
||||
if service == nil {
|
||||
service = ffmpeg.Get()
|
||||
}
|
||||
// The shadow run can outlive the request, so it uses a context that is not cancelled with it.
|
||||
shadowCtx := context.WithoutCancel(ctx)
|
||||
// The shadow run can outlive the request, so it uses a context that is not
|
||||
// cancelled with it. It bypasses the shared cache so ffmpeg is compared fresh.
|
||||
shadowCtx := mediacache.WithoutCache(context.WithoutCancel(ctx))
|
||||
legacy := func() (float64, bool) {
|
||||
d, err := service.GetMediaDuration(shadowCtx, realPath)
|
||||
return d, err == nil
|
||||
@@ -707,14 +698,15 @@ func legacyLyrics(realPath string) (gomedia.LyricsResult, bool) {
|
||||
|
||||
// extractVideoMetadata extracts duration and codec info from video files using ffprobe.
|
||||
// If ffmpegService is nil, a new service will be created (for backward compatibility)
|
||||
func extractVideoMetadata(ctx context.Context, item *iteminfo.ExtendedItemInfo, realPath string, ffmpegService *ffmpeg.FFmpegService) error {
|
||||
func extractVideoMetadata(ctx context.Context, item *iteminfo.ExtendedItemInfo, realPath string, getArt bool, ffmpegService *ffmpeg.FFmpegService) error {
|
||||
// go-media fast path. Works without ffmpeg installed.
|
||||
probeService := ffmpegService
|
||||
if probeService == nil {
|
||||
probeService = ffmpeg.Get()
|
||||
}
|
||||
// The shadow run can outlive the request, so it uses a context that is not cancelled with it.
|
||||
shadowCtx := context.WithoutCancel(ctx)
|
||||
// The shadow run can outlive the request, so it uses a context that is not
|
||||
// cancelled with it. It bypasses the shared cache so ffprobe is compared fresh.
|
||||
shadowCtx := mediacache.WithoutCache(context.WithoutCancel(ctx))
|
||||
legacyProbe := func() (gomedia.ProbeResult, bool) {
|
||||
info, err := probeService.ProbeFile(shadowCtx, realPath)
|
||||
if err != nil || info == nil {
|
||||
@@ -759,12 +751,33 @@ func extractVideoMetadata(ctx context.Context, item *iteminfo.ExtendedItemInfo,
|
||||
if info.FormatName != "" {
|
||||
meta.Container = info.FormatName
|
||||
}
|
||||
if ok {
|
||||
if tags, tagsOK := gomedia.VideoTags(realPath, getArt); tagsOK {
|
||||
applyMediaTags(meta, tags, realPath)
|
||||
}
|
||||
}
|
||||
if meta.Duration > 0 || meta.VideoCodec != "" || meta.AudioCodec != "" || meta.Container != "" {
|
||||
item.Metadata = meta
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// applyMediaTags copies go-media tags and embedded cover art into meta. Covers
|
||||
// over 8MB are skipped.
|
||||
func applyMediaTags(meta *iteminfo.MediaMetadata, info gomedia.AudioInfo, realPath string) {
|
||||
meta.Title = info.Tags.Title
|
||||
meta.Artist = info.Tags.Artist
|
||||
meta.Album = info.Tags.Album
|
||||
meta.Year = info.Tags.Year
|
||||
meta.Genre = info.Tags.Genre
|
||||
meta.Track = info.Tags.Track
|
||||
if n := len(info.Artwork); n > 0 && n <= 8*1024*1024 {
|
||||
meta.AlbumArt = info.Artwork
|
||||
} else if n > 8*1024*1024 {
|
||||
logger.Debugf("Skipping album art for %s: too large (%d bytes)", realPath, n)
|
||||
}
|
||||
}
|
||||
|
||||
// DeleteFiles deletes the file or directory at indexPath. boundIndexPath limits
|
||||
// symlink resolution to the caller's scope ("" or "/" = source root only). The
|
||||
// entry itself is removed: a symlink leaf is deleted, never followed, and an
|
||||
@@ -777,6 +790,7 @@ func DeleteFiles(source, boundIndexPath, indexPath string, isDir bool) error {
|
||||
if utils.CleanIndexPathSegment(indexPath) == "" {
|
||||
return fmt.Errorf("refusing to delete source root directory: %s", indexPath)
|
||||
}
|
||||
deleteIndexPath := indexPath
|
||||
realPath, err := index.GetRealPathForRemoval(boundIndexPath, indexPath)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -795,9 +809,13 @@ func DeleteFiles(source, boundIndexPath, indexPath string, isDir bool) error {
|
||||
return err
|
||||
}
|
||||
|
||||
// Clear cache entries
|
||||
indexing.RealPathCache.Delete(realPath)
|
||||
indexing.IsDirCache.Delete(realPath + ":isdir")
|
||||
if isDir {
|
||||
indexing.ClearRealPathCaches()
|
||||
}
|
||||
index.InvalidateRealPathCache(deleteIndexPath, boundIndexPath)
|
||||
if parent := utils.GetParentDirectoryPath(deleteIndexPath); parent != "" && parent != "/" {
|
||||
index.InvalidateRealPathCache(parent, boundIndexPath)
|
||||
}
|
||||
|
||||
// Remove metadata from index
|
||||
deleteSuccess := index.DeleteMetadata(indexPath, isDir, isDir)
|
||||
@@ -849,8 +867,7 @@ func RefreshIndex(source string, path string, isDir bool, recursive bool) error
|
||||
if !Exists(realPath) {
|
||||
// Directory no longer exists, remove it from the index
|
||||
// This clears both Directories and DirectoriesLedger maps
|
||||
indexing.RealPathCache.Delete(realPath)
|
||||
indexing.IsDirCache.Delete(realPath + ":isdir")
|
||||
idx.InvalidateRealPathCache(path)
|
||||
idx.DeleteMetadata(path, true, false)
|
||||
return nil
|
||||
}
|
||||
@@ -928,6 +945,9 @@ func moveResourceImpl(isSrcDir bool, sourceIndex, destIndex, realsrc, realdst st
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if isSrcDir {
|
||||
indexing.ClearRealPathCaches()
|
||||
}
|
||||
|
||||
// Handle SOURCE cleanup (treat as deletion)
|
||||
// Run async to avoid blocking the HTTP response
|
||||
@@ -1088,6 +1108,11 @@ func WriteDirectory(boundIndexPath string, opts utils.FileOptions) error {
|
||||
}
|
||||
}
|
||||
|
||||
idx.InvalidateRealPathCache(opts.Path, boundIndexPath)
|
||||
if parent := utils.GetParentDirectoryPath(opts.Path); parent != "" && parent != "/" {
|
||||
idx.InvalidateRealPathCache(parent, boundIndexPath)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -1163,6 +1188,11 @@ func writeFile(source, boundIndexPath, path string, in io.Reader) error {
|
||||
}
|
||||
}
|
||||
|
||||
idx.InvalidateRealPathCache(path, boundIndexPath)
|
||||
if parent := utils.GetParentDirectoryPath(path); parent != "" && parent != "/" {
|
||||
idx.InvalidateRealPathCache(parent, boundIndexPath)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
package access
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/utils"
|
||||
"github.com/gtsteffaniak/filebrowser/backend/pkg/indexing"
|
||||
)
|
||||
|
||||
// PermittedAtRealPath reports whether username may access realPath after symlink
|
||||
// resolution. DENY rules apply to the target index path, not only the link path.
|
||||
func (s *Storage) PermittedAtRealPath(index *indexing.Index, username, realPath string, isDir bool) bool {
|
||||
if s == nil || index == nil || username == "" || realPath == "" {
|
||||
return false
|
||||
}
|
||||
lexical := realPath
|
||||
if !strings.HasPrefix(realPath, index.Path) {
|
||||
root, err := filepath.EvalSymlinks(index.Path)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
rel, err := filepath.Rel(root, realPath)
|
||||
if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) {
|
||||
return false
|
||||
}
|
||||
lexical = index.Path
|
||||
if rel != "." {
|
||||
lexical = filepath.Join(index.Path, rel)
|
||||
}
|
||||
}
|
||||
indexPath := index.MakeIndexPath(lexical, isDir).String()
|
||||
return s.Permitted(index.Path, utils.IndexPathFromNormalized(indexPath, true), username)
|
||||
}
|
||||
|
||||
// IsDirAtRealPath reports whether an existing path is a directory; missing paths are treated as files.
|
||||
func IsDirAtRealPath(realPath string) bool {
|
||||
st, err := os.Stat(realPath)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
return st.IsDir()
|
||||
}
|
||||
@@ -0,0 +1,140 @@
|
||||
package access
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/utils"
|
||||
"github.com/gtsteffaniak/filebrowser/backend/pkg/indexing"
|
||||
"github.com/gtsteffaniak/filebrowser/backend/pkg/settings"
|
||||
)
|
||||
|
||||
func setupSymlinkDenyFixture(t *testing.T) (sourceRoot string, idx *indexing.Index, store *Storage, username string) {
|
||||
t.Helper()
|
||||
sourceRoot = filepath.Join(t.TempDir(), "data")
|
||||
aliceDir := filepath.Join(sourceRoot, "alice")
|
||||
bobDir := filepath.Join(sourceRoot, "bob")
|
||||
for _, dir := range []string{aliceDir, bobDir} {
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(bobDir, "secret.txt"), []byte("secret"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Symlink("../bob/secret.txt", filepath.Join(aliceDir, "link.txt")); err != nil {
|
||||
t.Skipf("symlinks required: %v", err)
|
||||
}
|
||||
if err := os.Symlink("../bob", filepath.Join(aliceDir, "bobdir")); err != nil {
|
||||
t.Skipf("symlinks required: %v", err)
|
||||
}
|
||||
|
||||
settings.Config.Server.SourceMap = map[string]*settings.Source{
|
||||
sourceRoot: {Path: sourceRoot, Config: settings.SourceConfig{DenyByDefault: false}},
|
||||
}
|
||||
store = NewStorage(nil)
|
||||
username = "scoped-user"
|
||||
if err := store.DenyUser(sourceRoot, utils.IndexPathFromNormalized("/bob", true), username); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
idx = &indexing.Index{Source: settings.Source{Path: sourceRoot}}
|
||||
return sourceRoot, idx, store, username
|
||||
}
|
||||
|
||||
func TestPermittedAtRealPath_AllowsLinkPathDeniesResolvedTarget(t *testing.T) {
|
||||
_, idx, store, user := setupSymlinkDenyFixture(t)
|
||||
|
||||
if !store.Permitted(idx.Path, utils.IndexPathFromNormalized("/alice/link.txt", false), user) {
|
||||
t.Fatal("index path at the symlink should be allowed")
|
||||
}
|
||||
if store.Permitted(idx.Path, utils.IndexPathFromNormalized("/bob/secret.txt", false), user) {
|
||||
t.Fatal("direct path under /bob should be denied")
|
||||
}
|
||||
|
||||
realPath, _, err := idx.GetRealPathScoped("/", "/alice/link.txt")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if store.PermittedAtRealPath(idx, user, realPath, false) {
|
||||
t.Fatal("resolved target under denied /bob must not be permitted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPermittedAtRealPath_DeniesWriteThroughDirectorySymlink(t *testing.T) {
|
||||
_, idx, store, user := setupSymlinkDenyFixture(t)
|
||||
|
||||
realPath, err := idx.GetRealPathForMutation("/", "/alice/bobdir/planted.txt")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if store.PermittedAtRealPath(idx, user, realPath, false) {
|
||||
t.Fatal("resolved write target under denied /bob must not be permitted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPermittedAtRealPath_DeniesCopySourceResolved(t *testing.T) {
|
||||
_, idx, store, user := setupSymlinkDenyFixture(t)
|
||||
|
||||
realSrc, isDir, err := idx.GetRealPathScoped("/", "/alice/link.txt")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if isDir {
|
||||
t.Fatal("expected file")
|
||||
}
|
||||
if store.PermittedAtRealPath(idx, user, realSrc, isDir) {
|
||||
t.Fatal("copy source resolved to denied target must not be permitted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPermittedAtRealPath_SymlinkedSourceRoot(t *testing.T) {
|
||||
sourceRoot, _, _, user := setupSymlinkDenyFixture(t)
|
||||
linkRoot := filepath.Join(filepath.Dir(sourceRoot), "root-link")
|
||||
if err := os.Symlink(sourceRoot, linkRoot); err != nil {
|
||||
t.Skipf("symlinks required: %v", err)
|
||||
}
|
||||
if _, err := filepath.EvalSymlinks(linkRoot); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
settings.Config.Server.SourceMap = map[string]*settings.Source{
|
||||
linkRoot: {Path: linkRoot, Config: settings.SourceConfig{DenyByDefault: false}},
|
||||
}
|
||||
store := NewStorage(nil)
|
||||
if err := store.DenyUser(linkRoot, utils.IndexPathFromNormalized("/bob", true), user); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
idx := &indexing.Index{Source: settings.Source{Path: linkRoot}}
|
||||
|
||||
realPath, _, err := idx.GetRealPathScoped("/", "/alice/link.txt")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
realPath, err = filepath.EvalSymlinks(realPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if store.PermittedAtRealPath(idx, user, realPath, false) {
|
||||
t.Fatal("resolved target under denied /bob must not be permitted with symlinked source root")
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsDirAtRealPath(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
if err := os.MkdirAll(filepath.Join(dir, "nested"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
file := filepath.Join(dir, "file.txt")
|
||||
if err := os.WriteFile(file, []byte("x"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !IsDirAtRealPath(dir) {
|
||||
t.Fatal("expected directory")
|
||||
}
|
||||
if IsDirAtRealPath(file) {
|
||||
t.Fatal("expected file")
|
||||
}
|
||||
if IsDirAtRealPath(filepath.Join(dir, "missing")) {
|
||||
t.Fatal("missing path should not be treated as directory")
|
||||
}
|
||||
}
|
||||
@@ -25,8 +25,8 @@ func TestFinishUserLoad_legacyProfileRowPreservesAbsentSettingsFields(t *testing
|
||||
if !loaded.ShowFirstLogin {
|
||||
t.Fatal("expected top-level showFirstLogin to be preserved when settings omits it")
|
||||
}
|
||||
if !loaded.DarkMode {
|
||||
t.Fatal("expected profile darkMode=true")
|
||||
if !loaded.DarkMode.IsDark() {
|
||||
t.Fatalf("expected profile darkMode=true, got %q", loaded.DarkMode)
|
||||
}
|
||||
if len(loaded.PasskeyCredentials) != 0 {
|
||||
t.Fatalf("expected no passkeys when settings omits passkeyCredentials, got %#v", loaded.PasskeyCredentials)
|
||||
@@ -95,7 +95,7 @@ func TestUserDataRoundTrip_profileSettingsOverlay(t *testing.T) {
|
||||
LoginMethod: users.LoginMethodPassword,
|
||||
ShowFirstLogin: true,
|
||||
NonAdminEditable: users.NonAdminEditable{
|
||||
DarkMode: true,
|
||||
DarkMode: users.DarkModeDark,
|
||||
SidebarLinks: []users.SidebarLink{
|
||||
{Name: "Home", Target: "/files"},
|
||||
},
|
||||
|
||||
@@ -0,0 +1,73 @@
|
||||
package users
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// DarkMode is the user's theme preference: "auto" (follow the client's
|
||||
// system preference), "dark", or "light". It unmarshals from legacy JSON
|
||||
// booleans for backward compatibility with existing stored users.
|
||||
type DarkMode string
|
||||
|
||||
const (
|
||||
DarkModeAuto DarkMode = "auto"
|
||||
DarkModeDark DarkMode = "dark"
|
||||
DarkModeLight DarkMode = "light"
|
||||
)
|
||||
|
||||
// UnmarshalJSON accepts legacy boolean values (true -> dark, false -> light),
|
||||
// the tri-state strings ("auto", "dark", "light"), and null (unset).
|
||||
func (d *DarkMode) UnmarshalJSON(data []byte) error {
|
||||
var v interface{}
|
||||
if err := json.Unmarshal(data, &v); err != nil {
|
||||
return err
|
||||
}
|
||||
switch val := v.(type) {
|
||||
case nil:
|
||||
*d = ""
|
||||
case bool:
|
||||
if val {
|
||||
*d = DarkModeDark
|
||||
} else {
|
||||
*d = DarkModeLight
|
||||
}
|
||||
case string:
|
||||
switch DarkMode(strings.ToLower(val)) {
|
||||
case "":
|
||||
*d = ""
|
||||
case "true":
|
||||
// Some YAML decoders cast scalar booleans to strings.
|
||||
*d = DarkModeDark
|
||||
case "false":
|
||||
*d = DarkModeLight
|
||||
case DarkModeAuto:
|
||||
*d = DarkModeAuto
|
||||
case DarkModeDark:
|
||||
*d = DarkModeDark
|
||||
case DarkModeLight:
|
||||
*d = DarkModeLight
|
||||
default:
|
||||
return fmt.Errorf("invalid darkMode value %q: must be \"auto\", \"dark\" or \"light\"", val)
|
||||
}
|
||||
default:
|
||||
return fmt.Errorf("invalid darkMode value: must be a boolean or one of \"auto\", \"dark\", \"light\"")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// IsDark reports the effective boolean value for server-side rendering
|
||||
// contexts. "auto" cannot be resolved server-side; callers that can know the
|
||||
// client preference should handle DarkModeAuto explicitly.
|
||||
func (d DarkMode) IsDark() bool {
|
||||
return d == DarkModeDark
|
||||
}
|
||||
|
||||
// Or returns d unless it is unset (""), in which case it returns def.
|
||||
func (d DarkMode) Or(def DarkMode) DarkMode {
|
||||
if d == "" {
|
||||
return def
|
||||
}
|
||||
return d
|
||||
}
|
||||
@@ -308,7 +308,7 @@ type NonAdminEditable struct {
|
||||
DeleteWithoutConfirming bool `json:"deleteWithoutConfirming"` // delete files without confirmation
|
||||
Preview Preview `json:"preview"`
|
||||
StickySidebar bool `json:"stickySidebar"` // keep sidebar open when navigating
|
||||
DarkMode bool `json:"darkMode"` // should dark mode be enabled
|
||||
DarkMode DarkMode `json:"darkMode"` // theme preference: "auto" (follow system), "dark", or "light"
|
||||
Password string `json:"password,omitempty"`
|
||||
Locale string `json:"locale"` // language to use: eg. de, en, or fr
|
||||
ViewMode string `json:"viewMode"` // view mode to use: eg. normal, list, grid, or compact
|
||||
@@ -346,10 +346,11 @@ type NonAdminEditable struct {
|
||||
}
|
||||
|
||||
type FileLoading struct {
|
||||
MaxConcurrent int `json:"maxConcurrentUpload"`
|
||||
UploadChunkSize int `json:"uploadChunkSizeMb"`
|
||||
ClearAll bool `json:"clearAll"`
|
||||
DownloadChunkSize int `json:"downloadChunkSizeMb"`
|
||||
MaxConcurrent int `json:"maxConcurrentUpload"`
|
||||
UploadChunkSize int `json:"uploadChunkSizeMb"`
|
||||
ClearAll bool `json:"clearAll"`
|
||||
DownloadChunkSize int `json:"downloadChunkSizeMb"`
|
||||
UploadStallTimeoutSeconds int `json:"uploadStallTimeoutSeconds"` // seconds without progress before an upload is paused, 0 uses the default (10)
|
||||
}
|
||||
|
||||
// SidebarLink represents a customizable link in the sidebar.
|
||||
|
||||
@@ -1,14 +0,0 @@
|
||||
package ffmpeg
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/utils"
|
||||
"github.com/gtsteffaniak/go-cache/cache"
|
||||
)
|
||||
|
||||
var (
|
||||
MediaCache = cache.NewCache[[]utils.SubtitleTrack](24 * time.Hour) // subtitle track lists from ffprobe
|
||||
SubtitleContentCache = cache.NewCache[string](24 * time.Hour) // extracted embedded subtitle content
|
||||
MetadataCache = cache.NewCache[float64](24 * time.Hour) // media duration from ffprobe
|
||||
)
|
||||
@@ -9,13 +9,51 @@ import (
|
||||
"os/exec"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
|
||||
"github.com/kovidgoyal/imaging"
|
||||
)
|
||||
|
||||
// HEICDisplayRotationReader reads the display rotation that a HEIC/HEIF/AVIF
|
||||
// file's item properties (irot) ask for, without decoding it. The rotation is
|
||||
// in degrees counter-clockwise, normalized to (-180, 180], which is the value
|
||||
// ffmpeg prints as "Display Matrix: rotation of ..." for the same file. ok is
|
||||
// false when the reader could not tell; the ffmpeg probe is used instead.
|
||||
type HEICDisplayRotationReader func(path string) (rotation float64, ok bool)
|
||||
|
||||
// heicDisplayRotationReader is the header reader set by
|
||||
// SetHEICDisplayRotationReader, or nil.
|
||||
var heicDisplayRotationReader atomic.Pointer[HEICDisplayRotationReader]
|
||||
|
||||
// SetHEICDisplayRotationReader installs the reader HEICDisplayRotation tries
|
||||
// before running ffmpeg. The gomedia package installs go-media's reader when it
|
||||
// is loaded; it cannot be called from here directly because gomedia imports
|
||||
// this package. Passing nil removes the reader.
|
||||
func SetHEICDisplayRotationReader(fn HEICDisplayRotationReader) {
|
||||
if fn == nil {
|
||||
heicDisplayRotationReader.Store(nil)
|
||||
return
|
||||
}
|
||||
heicDisplayRotationReader.Store(&fn)
|
||||
}
|
||||
|
||||
// HEICDisplayRotation returns the display rotation of heicPath and whether it is
|
||||
// known. The installed header reader (go-media) answers first; the
|
||||
// "ffmpeg -f null" stderr probe runs only when no reader is installed or the
|
||||
// reader could not read the file.
|
||||
func (s *Service) HEICDisplayRotation(ctx context.Context, heicPath string) (float64, bool) {
|
||||
if p := heicDisplayRotationReader.Load(); p != nil {
|
||||
if rotation, ok := (*p)(heicPath); ok {
|
||||
return rotation, true
|
||||
}
|
||||
}
|
||||
return s.GetHEICDisplayMatrixRotation(ctx, heicPath)
|
||||
}
|
||||
|
||||
// GetHEICDisplayMatrixRotation returns the largest rotation (degrees) ffmpeg reports for a
|
||||
// HEIC display matrix. The second return value is false when the probe could not run or
|
||||
// parse stderr — callers must not treat that as “no rotation was applied”.
|
||||
// HEICDisplayRotation is preferred; it only runs this probe as a fallback.
|
||||
func (s *Service) GetHEICDisplayMatrixRotation(ctx context.Context, heicPath string) (float64, bool) {
|
||||
if s == nil {
|
||||
return 0, false
|
||||
@@ -36,6 +74,8 @@ func (s *Service) GetHEICDisplayMatrixRotation(ctx context.Context, heicPath str
|
||||
return rotation, ok
|
||||
}
|
||||
|
||||
// parseDisplayMatrixRotation returns the largest-magnitude "Display Matrix"
|
||||
// rotation in ffmpeg stderr, and whether any was found.
|
||||
func parseDisplayMatrixRotation(stderr string) (float64, bool) {
|
||||
var best float64
|
||||
var found bool
|
||||
@@ -65,6 +105,7 @@ func parseDisplayMatrixRotation(stderr string) (float64, bool) {
|
||||
return best, found
|
||||
}
|
||||
|
||||
// isNormalOrientation reports whether an EXIF orientation needs no change.
|
||||
func isNormalOrientation(orientation string) bool {
|
||||
switch orientation {
|
||||
case "", "Horizontal (normal)", "Top-left", "1":
|
||||
@@ -74,6 +115,8 @@ func isNormalOrientation(orientation string) bool {
|
||||
}
|
||||
}
|
||||
|
||||
// isPureRotationOrientation reports whether an EXIF orientation is a rotation
|
||||
// without a mirror.
|
||||
func isPureRotationOrientation(orientation string) bool {
|
||||
switch orientation {
|
||||
case "Rotate 90 CW", "Right-top", "6",
|
||||
@@ -100,6 +143,8 @@ func orientationNeedsPostCorrection(orientation string, displayRotation float64,
|
||||
return true
|
||||
}
|
||||
|
||||
// applyOrientationToJPEG rotates or flips JPEG bytes for an EXIF orientation.
|
||||
// The input is returned unchanged when it cannot be decoded or re-encoded.
|
||||
func applyOrientationToJPEG(jpegBytes []byte, orientation string) []byte {
|
||||
if len(jpegBytes) < 100 || isNormalOrientation(orientation) {
|
||||
return jpegBytes
|
||||
|
||||
@@ -50,3 +50,35 @@ func TestOrientationNeedsPostCorrection(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestHEICDisplayRotationPrefersReader(t *testing.T) {
|
||||
prev := heicDisplayRotationReader.Load()
|
||||
t.Cleanup(func() { heicDisplayRotationReader.Store(prev) })
|
||||
|
||||
var calls int
|
||||
SetHEICDisplayRotationReader(func(path string) (float64, bool) {
|
||||
calls++
|
||||
if path == "known.heic" {
|
||||
return -90, true
|
||||
}
|
||||
return 0, false
|
||||
})
|
||||
var svc *Service // nil: the ffmpeg probe cannot run
|
||||
got, ok := svc.HEICDisplayRotation(t.Context(), "known.heic")
|
||||
if !ok || got != -90 {
|
||||
t.Fatalf("HEICDisplayRotation(known) = %v, %v; want -90, true", got, ok)
|
||||
}
|
||||
// The reader cannot tell: the ffmpeg probe runs, and with no service it
|
||||
// reports unknown rather than "no rotation".
|
||||
if _, ok := svc.HEICDisplayRotation(t.Context(), "other.heic"); ok {
|
||||
t.Fatal("HEICDisplayRotation(other) ok = true, want the ffmpeg probe's false")
|
||||
}
|
||||
if calls != 2 {
|
||||
t.Fatalf("reader calls = %d, want 2", calls)
|
||||
}
|
||||
|
||||
SetHEICDisplayRotationReader(nil)
|
||||
if _, ok := svc.HEICDisplayRotation(t.Context(), "known.heic"); ok {
|
||||
t.Fatal("with no reader, HEICDisplayRotation must use the ffmpeg probe")
|
||||
}
|
||||
}
|
||||
@@ -81,14 +81,16 @@ func (s *Service) ConvertHEICToJPEG(ctx context.Context, heicPath string, target
|
||||
return s.CorrectHEICJPEGOrientation(ctx, heicPath, jpegBytes), nil
|
||||
}
|
||||
|
||||
// CorrectHEICJPEGOrientation applies irot / display-matrix correction after a full-resolution
|
||||
// HEIC decode when pixels are still in coded orientation (go-media and ffmpeg filter-free paths).
|
||||
// CorrectHEICJPEGOrientation applies EXIF orientation after a filter-free ffmpeg HEIC decode
|
||||
// when the pixels are still in coded orientation. The display rotation (irot) comes from
|
||||
// HEICDisplayRotation: go-media reads it from the item properties, and the ffmpeg stderr probe
|
||||
// is only a fallback.
|
||||
func (s *Service) CorrectHEICJPEGOrientation(ctx context.Context, heicPath string, jpegBytes []byte) []byte {
|
||||
orientation, err := s.GetHEICOrientation(heicPath)
|
||||
if err != nil {
|
||||
orientation = "Horizontal (normal)"
|
||||
}
|
||||
displayRotation, displayKnown := s.GetHEICDisplayMatrixRotation(ctx, heicPath)
|
||||
displayRotation, displayKnown := s.HEICDisplayRotation(ctx, heicPath)
|
||||
if orientationNeedsPostCorrection(orientation, displayRotation, displayKnown) {
|
||||
return applyOrientationToJPEG(jpegBytes, orientation)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,115 @@
|
||||
package ffmpeg
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/mediacache"
|
||||
)
|
||||
|
||||
const cacheFixtureMP4 = "../../../frontend/tests/playwright-files/myfolder/testdata/sample-640x480.mp4"
|
||||
|
||||
var (
|
||||
testServiceOnce sync.Once
|
||||
testServiceErr error
|
||||
)
|
||||
|
||||
// initTestService starts the ffmpeg service once, without hardware probes, or
|
||||
// skips.
|
||||
func initTestService(t *testing.T) *Service {
|
||||
t.Helper()
|
||||
if testing.Short() {
|
||||
t.Skip("starts ffmpeg")
|
||||
}
|
||||
if _, err := exec.LookPath("ffmpeg"); err != nil {
|
||||
t.Skip("ffmpeg not installed")
|
||||
}
|
||||
if _, err := exec.LookPath("ffprobe"); err != nil {
|
||||
t.Skip("ffprobe not installed")
|
||||
}
|
||||
testServiceOnce.Do(func() {
|
||||
testServiceErr = Initialize(context.Background(), InitOptions{SkipHWTests: true, CacheDir: os.TempDir()})
|
||||
})
|
||||
if testServiceErr != nil {
|
||||
t.Skipf("ffmpeg init failed: %v", testServiceErr)
|
||||
}
|
||||
svc := Get()
|
||||
if svc == nil {
|
||||
t.Skip("ffmpeg service unavailable")
|
||||
}
|
||||
return svc
|
||||
}
|
||||
|
||||
// copyFixture copies the MP4 fixture so the test owns its cache key.
|
||||
func copyFixture(t *testing.T) (string, mediacache.Key) {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile(cacheFixtureMP4)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
path := filepath.Join(t.TempDir(), "sample.mp4")
|
||||
if err = os.WriteFile(path, data, 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
key, err := mediacache.Stat(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
mediacache.Probes.Delete(key.String())
|
||||
mediacache.Durations.Delete(key.String())
|
||||
})
|
||||
return path, key
|
||||
}
|
||||
|
||||
// TestProbeFileUsesSharedCache checks that ffprobe results go into the shared
|
||||
// cache, cached values are reused, and WithoutCache bypasses both.
|
||||
func TestProbeFileUsesSharedCache(t *testing.T) {
|
||||
svc := initTestService(t)
|
||||
path, key := copyFixture(t)
|
||||
|
||||
got, err := svc.ProbeFile(context.Background(), path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cached, ok := mediacache.Probes.Get(key.String())
|
||||
if !ok || cached != *got {
|
||||
t.Fatalf("ProbeFile must store its result in the shared cache: cached=%+v ok=%v got=%+v", cached, ok, *got)
|
||||
}
|
||||
|
||||
sentinel := FileProbeInfo{Duration: 1, VideoCodec: "sentinel"}
|
||||
mediacache.Probes.Set(key.String(), sentinel)
|
||||
got, err = svc.ProbeFile(context.Background(), path)
|
||||
if err != nil || *got != sentinel {
|
||||
t.Fatalf("ProbeFile must reuse a cached value (e.g. from go-media): got=%+v err=%v", got, err)
|
||||
}
|
||||
|
||||
fresh, err := svc.ProbeFile(mediacache.WithoutCache(context.Background()), path)
|
||||
if err != nil || *fresh == sentinel {
|
||||
t.Fatalf("WithoutCache must probe fresh: got=%+v err=%v", fresh, err)
|
||||
}
|
||||
if cached, _ := mediacache.Probes.Get(key.String()); cached != sentinel {
|
||||
t.Fatal("WithoutCache must not overwrite the shared cache")
|
||||
}
|
||||
}
|
||||
|
||||
// TestGetMediaDurationUsesSharedCache checks that cached durations are reused
|
||||
// and WithoutCache probes fresh.
|
||||
func TestGetMediaDurationUsesSharedCache(t *testing.T) {
|
||||
svc := initTestService(t)
|
||||
path, key := copyFixture(t)
|
||||
|
||||
mediacache.Durations.Set(key.String(), 4242)
|
||||
d, err := svc.GetMediaDuration(context.Background(), path)
|
||||
if err != nil || d != 4242 {
|
||||
t.Fatalf("GetMediaDuration must reuse a cached value: d=%v err=%v", d, err)
|
||||
}
|
||||
d, err = svc.GetMediaDuration(mediacache.WithoutCache(context.Background()), path)
|
||||
if err != nil || d == 4242 || d <= 0 {
|
||||
t.Fatalf("WithoutCache must probe fresh: d=%v err=%v", d, err)
|
||||
}
|
||||
}
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/mediacache"
|
||||
goffmpeg "github.com/gtsteffaniak/go-ffmpeg"
|
||||
)
|
||||
|
||||
@@ -49,24 +50,29 @@ func resolveLocalMediaPath(mediaPath string) (string, os.FileInfo, error) {
|
||||
return clean, info, nil
|
||||
}
|
||||
|
||||
// FileProbeInfo holds ffprobe results for a local media file.
|
||||
type FileProbeInfo struct {
|
||||
Duration float64
|
||||
VideoCodec string
|
||||
AudioCodec string
|
||||
FormatName string
|
||||
}
|
||||
// FileProbeInfo holds probe results for a local media file. It is the shared
|
||||
// cache type, so go-media and ffprobe results are interchangeable.
|
||||
type FileProbeInfo = mediacache.ProbeInfo
|
||||
|
||||
// ProbeFile extracts duration and codec info from a media file.
|
||||
// ProbeFile extracts duration and codec info from a media file. Results are
|
||||
// stored in the shared media cache, which the go-media probe also reads, unless
|
||||
// ctx is mediacache.WithoutCache.
|
||||
func (s *Service) ProbeFile(ctx context.Context, mediaPath string) (*FileProbeInfo, error) {
|
||||
if s == nil || s.inner == nil {
|
||||
return nil, fmt.Errorf("ffmpeg service not available")
|
||||
}
|
||||
|
||||
localPath, _, err := resolveLocalMediaPath(mediaPath)
|
||||
localPath, fileInfo, err := resolveLocalMediaPath(mediaPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
useCache := !mediacache.Bypassed(ctx)
|
||||
cacheKey := mediacache.FromFileInfo(localPath, fileInfo).String()
|
||||
if useCache {
|
||||
if cached, ok := mediacache.Probes.Get(cacheKey); ok {
|
||||
return &cached, nil
|
||||
}
|
||||
}
|
||||
|
||||
info, err := s.inner.ProbeStream(ctx, goffmpeg.ProbeStreamOptions{
|
||||
URL: localPath,
|
||||
@@ -77,15 +83,20 @@ func (s *Service) ProbeFile(ctx context.Context, mediaPath string) (*FileProbeIn
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &FileProbeInfo{
|
||||
res := FileProbeInfo{
|
||||
Duration: info.Duration,
|
||||
VideoCodec: info.VideoCodec,
|
||||
AudioCodec: info.AudioCodec,
|
||||
FormatName: info.FormatName,
|
||||
}, nil
|
||||
}
|
||||
if useCache {
|
||||
mediacache.Probes.Set(cacheKey, res)
|
||||
}
|
||||
return &res, nil
|
||||
}
|
||||
|
||||
// GetMediaDuration extracts the duration of a media file in seconds.
|
||||
// GetMediaDuration extracts the duration of a media file in seconds. Results
|
||||
// are stored in the shared media cache unless ctx is mediacache.WithoutCache.
|
||||
func (s *Service) GetMediaDuration(ctx context.Context, mediaPath string) (float64, error) {
|
||||
if s == nil || s.inner == nil {
|
||||
return 0, fmt.Errorf("ffmpeg service not available")
|
||||
@@ -95,9 +106,12 @@ func (s *Service) GetMediaDuration(ctx context.Context, mediaPath string) (float
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
cacheKey := fmt.Sprintf("%s:%d", localPath, fileInfo.ModTime().Unix())
|
||||
if duration, found := MetadataCache.Get(cacheKey); found {
|
||||
return duration, nil
|
||||
useCache := !mediacache.Bypassed(ctx)
|
||||
cacheKey := mediacache.FromFileInfo(localPath, fileInfo).String()
|
||||
if useCache {
|
||||
if duration, found := mediacache.Durations.Get(cacheKey); found {
|
||||
return duration, nil
|
||||
}
|
||||
}
|
||||
|
||||
duration, err := s.inner.GetMediaDuration(ctx, localPath)
|
||||
@@ -105,6 +119,8 @@ func (s *Service) GetMediaDuration(ctx context.Context, mediaPath string) (float
|
||||
return 0, err
|
||||
}
|
||||
|
||||
MetadataCache.Set(cacheKey, duration)
|
||||
if useCache {
|
||||
mediacache.Durations.Set(cacheKey, duration)
|
||||
}
|
||||
return duration, nil
|
||||
}
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/mediacache"
|
||||
goffmpeg "github.com/gtsteffaniak/go-ffmpeg"
|
||||
"github.com/gtsteffaniak/go-ffmpeg/capabilities"
|
||||
"github.com/gtsteffaniak/go-ffmpeg/ops"
|
||||
@@ -460,7 +461,8 @@ func (s *Service) VideoPreview(ctx context.Context, w io.Writer, opts ops.Previe
|
||||
return s.inner.CaptureFrame(ctx, w, frame)
|
||||
}
|
||||
|
||||
// ExtractSubtitle returns embedded subtitle content as WebVTT.
|
||||
// ExtractSubtitle returns embedded subtitle content as WebVTT. Results are
|
||||
// stored in the shared media cache unless ctx is mediacache.WithoutCache.
|
||||
func (s *Service) ExtractSubtitle(ctx context.Context, videoPath string, streamIndex int) (string, error) {
|
||||
if s == nil || s.inner == nil {
|
||||
return "", fmt.Errorf("ffmpeg service not available")
|
||||
@@ -470,9 +472,12 @@ func (s *Service) ExtractSubtitle(ctx context.Context, videoPath string, streamI
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to stat file: %w", err)
|
||||
}
|
||||
cacheKey := fmt.Sprintf("subtitle_content:%s:%d:%d", videoPath, streamIndex, fileInfo.ModTime().Unix())
|
||||
if content, ok := SubtitleContentCache.Get(cacheKey); ok {
|
||||
return content, nil
|
||||
useCache := !mediacache.Bypassed(ctx)
|
||||
cacheKey := mediacache.FromFileInfo(videoPath, fileInfo).Stream(streamIndex)
|
||||
if useCache {
|
||||
if content, ok := mediacache.SubtitleContent.Get(cacheKey); ok {
|
||||
return content, nil
|
||||
}
|
||||
}
|
||||
|
||||
content, err := s.inner.ExtractSubtitle(ctx, videoPath, streamIndex)
|
||||
@@ -480,7 +485,9 @@ func (s *Service) ExtractSubtitle(ctx context.Context, videoPath string, streamI
|
||||
return "", err
|
||||
}
|
||||
|
||||
SubtitleContentCache.Set(cacheKey, content)
|
||||
if useCache {
|
||||
mediacache.SubtitleContent.Set(cacheKey, content)
|
||||
}
|
||||
return content, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -7,16 +7,30 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/mediacache"
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/utils"
|
||||
goffmpeg "github.com/gtsteffaniak/go-ffmpeg"
|
||||
"github.com/gtsteffaniak/go-logger/logger"
|
||||
)
|
||||
|
||||
// DetectEmbeddedSubtitles detects embedded subtitle streams.
|
||||
// DetectEmbeddedSubtitles detects embedded subtitle streams. Results are stored
|
||||
// in the shared media cache, keyed by the file's current size and modification
|
||||
// time; modtime is kept for API compatibility and is not used for the key.
|
||||
func DetectEmbeddedSubtitles(videoPath string, modtime time.Time) []utils.SubtitleTrack {
|
||||
key := "embedded_subtitles:" + videoPath + ":" + modtime.Format(time.RFC3339)
|
||||
if cached, ok := MediaCache.Get(key); ok {
|
||||
return cached
|
||||
_ = modtime
|
||||
return DetectEmbeddedSubtitlesContext(context.Background(), videoPath)
|
||||
}
|
||||
|
||||
// DetectEmbeddedSubtitlesContext is DetectEmbeddedSubtitles with a context. The
|
||||
// shared media cache is skipped when ctx is mediacache.WithoutCache.
|
||||
func DetectEmbeddedSubtitlesContext(ctx context.Context, videoPath string) []utils.SubtitleTrack {
|
||||
useCache := !mediacache.Bypassed(ctx)
|
||||
key, statErr := mediacache.Stat(videoPath)
|
||||
useCache = useCache && statErr == nil
|
||||
if useCache {
|
||||
if cached, ok := mediacache.SubtitleTracks.Get(key.String()); ok {
|
||||
return cached
|
||||
}
|
||||
}
|
||||
|
||||
svc := Get()
|
||||
@@ -24,14 +38,16 @@ func DetectEmbeddedSubtitles(videoPath string, modtime time.Time) []utils.Subtit
|
||||
return nil
|
||||
}
|
||||
|
||||
tracks, err := svc.inner.DetectSubtitles(context.Background(), videoPath)
|
||||
tracks, err := svc.inner.DetectSubtitles(ctx, videoPath)
|
||||
if err != nil {
|
||||
logger.Debug("ffprobe failed for file: " + videoPath + ", error: " + err.Error())
|
||||
return nil
|
||||
}
|
||||
|
||||
subtitles := mapSubtitleTracks(tracks)
|
||||
MediaCache.Set(key, subtitles)
|
||||
if useCache {
|
||||
mediacache.SubtitleTracks.Set(key.String(), subtitles)
|
||||
}
|
||||
return subtitles
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,73 @@
|
||||
package gomedia
|
||||
|
||||
import (
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
const fixtureWMV2 = fixtureDir + "/file_example_WMV_480_1_2MB.wmv"
|
||||
|
||||
// TestContainerGates pins which extensions go-media handles for probes and
|
||||
// for thumbnails.
|
||||
func TestContainerGates(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
probe, thumbnail bool
|
||||
}{
|
||||
{"a.mp4", true, true},
|
||||
{"a.M4V", true, true},
|
||||
{"a.mov", true, true},
|
||||
{"a.mkv", true, true},
|
||||
{"a.webm", true, true},
|
||||
{"a.wmv", false, true},
|
||||
{"a.ASF", false, true},
|
||||
{"a.avi", false, false},
|
||||
{"a.ts", false, false},
|
||||
{"a.m2ts", false, false},
|
||||
{"a.mts", false, false},
|
||||
{"a.mpg", false, false},
|
||||
{"a.mpeg", false, false},
|
||||
{"a.vob", false, false},
|
||||
{"a.flv", false, false},
|
||||
}
|
||||
for _, c := range cases {
|
||||
require.Equal(t, c.probe, supportsVideoContainer(c.name), "probe gate for %s", c.name)
|
||||
require.Equal(t, c.thumbnail, supportsVideoThumbnail(c.name), "thumbnail gate for %s", c.name)
|
||||
}
|
||||
for _, name := range []string{"a.heic", "a.HEIF", "a.heics", "a.avif"} {
|
||||
require.True(t, supportsHEIFImage(name), name)
|
||||
}
|
||||
require.False(t, supportsHEIFImage("a.jpg"))
|
||||
}
|
||||
|
||||
// TestWMV2FallsBackToFFmpeg checks that a WMV2 file is not probed by go-media
|
||||
// and that its thumbnail falls back to ffmpeg (go-media has no WMV1/WMV2
|
||||
// decoder).
|
||||
func TestWMV2FallsBackToFFmpeg(t *testing.T) {
|
||||
withGate(t, false)
|
||||
_, ok := Probe(fixtureWMV2, nil)
|
||||
require.False(t, ok, "WMV probes stay on ffprobe")
|
||||
_, ok = VideoThumbnail(fixtureWMV2, 320, 240, 80, 10, nil)
|
||||
require.False(t, ok, "WMV2 thumbnails must fall back to ffmpeg")
|
||||
}
|
||||
|
||||
// TestWMV1FallsBackToFFmpeg is TestWMV2FallsBackToFFmpeg for a WMV1 stream
|
||||
// encoded on the fly.
|
||||
func TestWMV1FallsBackToFFmpeg(t *testing.T) {
|
||||
withGate(t, false)
|
||||
requireTool(t, "ffmpeg")
|
||||
path := filepath.Join(t.TempDir(), "wmv1.wmv")
|
||||
out, err := exec.Command("ffmpeg", "-v", "error", "-y", "-f", "lavfi", "-i", "testsrc2=size=160x120:rate=25:duration=2",
|
||||
"-c:v", "wmv1", path).CombinedOutput()
|
||||
if err != nil {
|
||||
t.Skipf("ffmpeg cannot encode wmv1: %v: %s", err, out)
|
||||
}
|
||||
_, err = os.Stat(path)
|
||||
require.NoError(t, err)
|
||||
_, ok := VideoThumbnail(path, 160, 120, 80, 10, nil)
|
||||
require.False(t, ok, "WMV1 thumbnails must fall back to ffmpeg")
|
||||
}
|
||||
@@ -20,6 +20,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/ffmpeg"
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/mediacache"
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/utils"
|
||||
"github.com/gtsteffaniak/filebrowser/backend/pkg/settings"
|
||||
"github.com/gtsteffaniak/go-cache/cache"
|
||||
@@ -28,20 +29,27 @@ import (
|
||||
"golang.org/x/sync/singleflight"
|
||||
)
|
||||
|
||||
var (
|
||||
subtitleListCache = cache.NewCache[[]utils.SubtitleTrack](24 * time.Hour)
|
||||
subtitleContentCache = cache.NewCache[string](24 * time.Hour)
|
||||
// Probe, duration, and subtitle results are cached in the mediacache package,
|
||||
// which the ffmpeg code path shares. A directory listing probes every video
|
||||
// child, and the preview flow probes the focused file again; the cache keeps
|
||||
// each request from re-reading the same containers. Keys include size and
|
||||
// modification time, so an edited file is read again.
|
||||
|
||||
// probeCache caches successful probe results by file identity (path, modtime,
|
||||
// size). A directory listing probes every video child, and the preview flow
|
||||
// probes the focused file again; without this each request re-read the same
|
||||
// containers. Keying on modtime+size means an edited file is re-probed.
|
||||
probeCache = cache.NewCache[ProbeResult](24 * time.Hour)
|
||||
// probeGroup coalesces concurrent probes of the same file so a burst of
|
||||
// requests for one container reads it once.
|
||||
var probeGroup singleflight.Group
|
||||
|
||||
// probeGroup coalesces concurrent probes of the same file so a burst of
|
||||
// requests for one container reads it once.
|
||||
probeGroup singleflight.Group
|
||||
)
|
||||
// tagsCache caches the embedded part of a tag read by file identity. A listing
|
||||
// reads tags for every media child on each uncached request. Sidecar lyrics
|
||||
// are not cached: adding an .lrc next to a file does not change its modtime.
|
||||
var tagsCache = cache.NewCache[AudioInfo](24 * time.Hour)
|
||||
|
||||
func init() {
|
||||
// The ffmpeg HEIC decode path corrects orientation after decoding and needs
|
||||
// the file's irot. internal/ffmpeg cannot import this package, so go-media's
|
||||
// header reader is installed here.
|
||||
ffmpeg.SetHEICDisplayRotationReader(HEICDisplayRotation)
|
||||
}
|
||||
|
||||
// Enabled reports whether the go-media fast path may run. The admin
|
||||
// "Always use ffmpeg" setting turns it off.
|
||||
@@ -60,9 +68,15 @@ func videoDimensions(info *media.ProbeInfo) (w, h int, ok bool) {
|
||||
}
|
||||
|
||||
// videoContainerExts are the containers go-media parses for video probes,
|
||||
// thumbnails, and embedded subtitles. Other video containers (AVI, WMV, TS, ...)
|
||||
// are not go-media targets, so they go straight to ffmpeg with no file open and
|
||||
// no log line. go-media reads MOV as MP4.
|
||||
// thumbnails, embedded subtitles, and keyframe indexes. go-media reads MOV as
|
||||
// MP4.
|
||||
//
|
||||
// go-media also demuxes AVI, ASF/WMV, MPEG-TS/M2TS, and MPEG-PS (VOB), but their
|
||||
// probes are left to ffmpeg on purpose: go-media's results for those containers
|
||||
// do not yet match ffprobe (TS/M2TS and PS list no audio streams, PS durations
|
||||
// are missing or wrong, AVI misnames MPEG-4 Part 2 and AAC, ASF reports WMA Pro
|
||||
// as wmav2). Those files go straight to ffmpeg with no file open and no log
|
||||
// line. See videoThumbnailExts for the thumbnail exception.
|
||||
var videoContainerExts = map[string]bool{
|
||||
".mp4": true,
|
||||
".m4v": true,
|
||||
@@ -71,11 +85,31 @@ var videoContainerExts = map[string]bool{
|
||||
".webm": true,
|
||||
}
|
||||
|
||||
// videoThumbnailExts are the containers go-media decodes video thumbnails from:
|
||||
// videoContainerExts plus ASF/WMV, whose WMV3 keyframes go-media decodes
|
||||
// bit-exact with ffmpeg. WMV1, WMV2, MS-MPEG4, and WVC1 Advanced Profile return
|
||||
// an unsupported error and fall back to ffmpeg. AVI, TS/M2TS, and PS stay on
|
||||
// ffmpeg: go-media's TS and PS thumbnails scan the whole file for the duration
|
||||
// (several times slower than ffmpeg on large files), and AVI seeks by a
|
||||
// duration that can be wrong.
|
||||
var videoThumbnailExts = map[string]bool{
|
||||
".mp4": true,
|
||||
".m4v": true,
|
||||
".mov": true,
|
||||
".mkv": true,
|
||||
".webm": true,
|
||||
".wmv": true,
|
||||
".asf": true,
|
||||
}
|
||||
|
||||
// heifImageExts are still-image containers go-media decodes for HEIC preview.
|
||||
// AVIF is a HEIF container with an AV1 image item; go-media decodes it in pure
|
||||
// Go and applies its irot/imir, like HEIC.
|
||||
var heifImageExts = map[string]bool{
|
||||
".heic": true,
|
||||
".heif": true,
|
||||
".heics": true,
|
||||
".avif": true,
|
||||
}
|
||||
|
||||
const heicDecodeMaxEdge = 16384
|
||||
@@ -93,15 +127,27 @@ var cameraRawExts = map[string]bool{
|
||||
".dc2": true, ".x3f": true, ".iiq": true, ".nkc": true, ".r3d": true,
|
||||
}
|
||||
|
||||
// supportsVideoContainer reports whether go-media targets path's container.
|
||||
// supportsVideoContainer reports whether go-media targets path's container for
|
||||
// probes, embedded subtitles, and keyframe indexes.
|
||||
func supportsVideoContainer(path string) bool {
|
||||
return videoContainerExts[strings.ToLower(filepath.Ext(path))]
|
||||
}
|
||||
|
||||
// supportsVideoThumbnail reports whether go-media targets path's container for
|
||||
// video thumbnails.
|
||||
func supportsVideoThumbnail(path string) bool {
|
||||
return videoThumbnailExts[strings.ToLower(filepath.Ext(path))]
|
||||
}
|
||||
|
||||
// supportsHEIFImage reports whether path is a HEIF-family still image
|
||||
// (HEIC/HEIF/AVIF) go-media decodes.
|
||||
|
||||
func supportsHEIFImage(path string) bool {
|
||||
return heifImageExts[strings.ToLower(filepath.Ext(path))]
|
||||
}
|
||||
|
||||
// supportsCameraRAW reports whether path is a camera RAW format go-media
|
||||
// extracts a preview from.
|
||||
func supportsCameraRAW(path string) bool {
|
||||
return cameraRawExts[strings.ToLower(filepath.Ext(path))]
|
||||
}
|
||||
@@ -133,16 +179,26 @@ func logUnsupported(op, path, reason string) {
|
||||
|
||||
// openFile opens path for go-media, which reads through io.ReaderAt.
|
||||
func openFile(path string) (*os.File, int64, error) {
|
||||
f, err := os.Open(path)
|
||||
f, info, err := openFileInfo(path)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
return f, info.Size(), nil
|
||||
}
|
||||
|
||||
// openFileInfo opens path and returns its FileInfo, which also gives the
|
||||
// shared cache key.
|
||||
func openFileInfo(path string) (*os.File, os.FileInfo, error) {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
info, err := f.Stat()
|
||||
if err != nil {
|
||||
_ = f.Close()
|
||||
return nil, 0, err
|
||||
return nil, nil, err
|
||||
}
|
||||
return f, info.Size(), nil
|
||||
return f, info, nil
|
||||
}
|
||||
|
||||
// ffprobeFormatNames maps go-media containers to the format_name strings ffprobe
|
||||
@@ -179,30 +235,25 @@ func probeFast(path string) (ProbeResult, bool) {
|
||||
return ProbeResult{}, false
|
||||
}
|
||||
|
||||
f, size, err := openFile(path)
|
||||
f, info, err := openFileInfo(path)
|
||||
if err != nil {
|
||||
logFallback("probe", path, err)
|
||||
return ProbeResult{}, false
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
info, err := f.Stat()
|
||||
if err != nil {
|
||||
logFallback("probe", path, err)
|
||||
return ProbeResult{}, false
|
||||
}
|
||||
cacheKey := probeCacheKey(path, info.ModTime().UnixNano(), size)
|
||||
if res, ok := probeCache.Get(cacheKey); ok {
|
||||
cacheKey := mediacache.FromFileInfo(path, info).String()
|
||||
if res, ok := mediacache.Probes.Get(cacheKey); ok {
|
||||
return res, true
|
||||
}
|
||||
|
||||
// Coalesce concurrent probes of the same file into one container read.
|
||||
v, err, _ := probeGroup.Do(cacheKey, func() (any, error) {
|
||||
res, ok := parseProbe(path, f, size)
|
||||
res, ok := parseProbe(path, f, info.Size())
|
||||
if !ok {
|
||||
return nil, errProbeUnsupported
|
||||
}
|
||||
probeCache.Set(cacheKey, res)
|
||||
mediacache.Probes.Set(cacheKey, res)
|
||||
return res, nil
|
||||
})
|
||||
if err != nil {
|
||||
@@ -219,12 +270,6 @@ func probeFast(path string) (ProbeResult, bool) {
|
||||
// no detail because parseProbe already logged the specific reason.
|
||||
var errProbeUnsupported = errors.New("probe unsupported")
|
||||
|
||||
// probeCacheKey identifies a file version for caching: path plus modtime and size.
|
||||
// An edited file gets a new key, so stale metadata is never served.
|
||||
func probeCacheKey(path string, modTimeNanos, size int64) string {
|
||||
return fmt.Sprintf("probe:%s:%d:%d", path, modTimeNanos, size)
|
||||
}
|
||||
|
||||
// parseProbe reads a container and builds a ProbeResult. It reports false when
|
||||
// go-media should defer to ffmpeg, having logged why.
|
||||
func parseProbe(path string, f *os.File, size int64) (ProbeResult, bool) {
|
||||
@@ -280,14 +325,18 @@ func durationFast(path string) (float64, bool) {
|
||||
if !Enabled() {
|
||||
return 0, false
|
||||
}
|
||||
f, size, err := openFile(path)
|
||||
f, info, err := openFileInfo(path)
|
||||
if err != nil {
|
||||
logFallback("duration", path, err)
|
||||
return 0, false
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
res, err := media.Duration(f, size)
|
||||
cacheKey := mediacache.FromFileInfo(path, info).String()
|
||||
if d, ok := mediacache.Durations.Get(cacheKey); ok {
|
||||
return d, true
|
||||
}
|
||||
res, err := media.Duration(f, info.Size())
|
||||
if err != nil {
|
||||
logFallback("duration", path, err)
|
||||
return 0, false
|
||||
@@ -296,7 +345,9 @@ func durationFast(path string) (float64, bool) {
|
||||
// Unknown duration is not an error; ffmpeg may still find one.
|
||||
return 0, false
|
||||
}
|
||||
return res.Duration.Seconds(), true
|
||||
d := res.Duration.Seconds()
|
||||
mediacache.Durations.Set(cacheKey, d)
|
||||
return d, true
|
||||
}
|
||||
|
||||
// AudioInfo is the tag, lyrics availability, and cover art FileBrowser reads
|
||||
@@ -311,38 +362,60 @@ type AudioInfo struct {
|
||||
// is true. A missing cover is not an error.
|
||||
// legacy is the dhowden tag read, run only in shadow mode.
|
||||
func Audio(path string, withArt bool, legacy Legacy[AudioInfo]) (AudioInfo, bool) {
|
||||
return timed("tags", path, legacy, audioDiffer(withArt), func() (AudioInfo, bool) { return audioFast(path, withArt) })
|
||||
return timed("tags", path, legacy, audioDiffer(withArt), func() (AudioInfo, bool) { return audioFast(path, withArt, true) })
|
||||
}
|
||||
|
||||
func audioFast(path string, withArt bool) (AudioInfo, bool) {
|
||||
func audioFast(path string, withArt, withSidecar bool) (AudioInfo, bool) {
|
||||
if !Enabled() {
|
||||
return AudioInfo{}, false
|
||||
}
|
||||
f, size, err := openFile(path)
|
||||
f, st, err := openFileInfo(path)
|
||||
if err != nil {
|
||||
logFallback("tags", path, err)
|
||||
return AudioInfo{}, false
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
// One container parse gives tags, the embedded lyrics flag, and cover bytes.
|
||||
// Separate ReadTags/HasLyrics/EmbeddedArtwork calls each re-parsed the file.
|
||||
audio, err := media.InspectAudio(f, size, media.AudioOptions{WithArtwork: withArt})
|
||||
if err != nil {
|
||||
logFallback("tags", path, err)
|
||||
return AudioInfo{}, false
|
||||
size := st.Size()
|
||||
cacheKey := tagsCacheKey(path, st.ModTime().UnixNano(), size)
|
||||
var info AudioInfo
|
||||
cached := false
|
||||
if !withArt {
|
||||
info, cached = tagsCache.Get(cacheKey)
|
||||
}
|
||||
info := AudioInfo{Tags: audio.Tags, HasLyrics: audio.HasEmbeddedLyrics}
|
||||
if !info.HasLyrics {
|
||||
if !cached {
|
||||
// One container parse gives tags, the embedded lyrics flag, and cover bytes.
|
||||
// Separate ReadTags/HasLyrics/EmbeddedArtwork calls each re-parsed the file.
|
||||
audio, err := media.InspectAudio(f, size, media.AudioOptions{WithArtwork: withArt})
|
||||
if err != nil {
|
||||
logFallback("tags", path, err)
|
||||
return AudioInfo{}, false
|
||||
}
|
||||
info = AudioInfo{Tags: audio.Tags, HasLyrics: audio.HasEmbeddedLyrics}
|
||||
stored := info
|
||||
stored.Tags.Raw = nil
|
||||
tagsCache.Set(cacheKey, stored)
|
||||
if withArt {
|
||||
info.Artwork = audio.Artwork
|
||||
}
|
||||
}
|
||||
if withSidecar && !info.HasLyrics {
|
||||
// Sidecar lyrics are only checked when the file has no embedded lyrics.
|
||||
_, _, info.HasLyrics = media.LyricsSidecar(path)
|
||||
}
|
||||
if withArt {
|
||||
info.Artwork = audio.Artwork
|
||||
}
|
||||
return info, true
|
||||
}
|
||||
|
||||
// tagsCacheKey identifies a file version for the tag cache: path plus modtime and size.
|
||||
func tagsCacheKey(path string, modTimeNanos, size int64) string {
|
||||
return fmt.Sprintf("tags:%s:%d:%d", path, modTimeNanos, size)
|
||||
}
|
||||
|
||||
// VideoTags reads embedded tags and cover from a video container
|
||||
func VideoTags(path string, withArt bool) (AudioInfo, bool) {
|
||||
return timed("tags", path, nil, nil, func() (AudioInfo, bool) { return audioFast(path, withArt, false) })
|
||||
}
|
||||
|
||||
// Lyrics returns lyrics from a sidecar file, or else from embedded tags. The
|
||||
// format is "embedded" for tags or the sidecar extension. An empty result with
|
||||
// ok=true means go-media checked and found no lyrics.
|
||||
@@ -384,7 +457,9 @@ func lyricsFast(path string) (text, format string, ok bool) {
|
||||
|
||||
// EmbeddedSubtitles lists embedded subtitle tracks. go-media supports MKV, WebM,
|
||||
// and MP4 only. Other containers are reported as unsupported, not as "no
|
||||
// subtitles", because go-media returns an empty list for them.
|
||||
// subtitles", because go-media returns an empty list for them. modtime is kept
|
||||
// for API compatibility; the cache key comes from the file's current size and
|
||||
// modification time.
|
||||
// legacy is the ffmpeg subtitle detection, run only in shadow mode.
|
||||
func EmbeddedSubtitles(path string, modtime time.Time, legacy Legacy[[]utils.SubtitleTrack]) ([]utils.SubtitleTrack, bool) {
|
||||
return timed("subtitle list", path, legacy, subtitleListDiff, func() ([]utils.SubtitleTrack, bool) {
|
||||
@@ -399,19 +474,20 @@ func embeddedSubtitlesFast(path string, modtime time.Time) ([]utils.SubtitleTrac
|
||||
if !supportsVideoContainer(path) {
|
||||
return nil, false
|
||||
}
|
||||
key := fmt.Sprintf("embedded_subtitles:%s:%d", path, modtime.Unix())
|
||||
if cached, ok := subtitleListCache.Get(key); ok {
|
||||
return cached, true
|
||||
}
|
||||
|
||||
f, size, err := openFile(path)
|
||||
_ = modtime
|
||||
f, st, err := openFileInfo(path)
|
||||
if err != nil {
|
||||
logFallback("subtitle list", path, err)
|
||||
return nil, false
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
info, err := media.Probe(f, size)
|
||||
key := mediacache.FromFileInfo(path, st).String()
|
||||
if cached, ok := mediacache.SubtitleTracks.Get(key); ok {
|
||||
return cached, true
|
||||
}
|
||||
|
||||
info, err := media.Probe(f, st.Size())
|
||||
if err != nil {
|
||||
logFallback("subtitle list", path, err)
|
||||
return nil, false
|
||||
@@ -433,7 +509,7 @@ func embeddedSubtitlesFast(path string, modtime time.Time) ([]utils.SubtitleTrac
|
||||
})
|
||||
}
|
||||
tracks := ffmpeg.BuildSubtitleTracks(streams)
|
||||
subtitleListCache.Set(key, tracks)
|
||||
mediacache.SubtitleTracks.Set(key, tracks)
|
||||
return tracks, true
|
||||
}
|
||||
|
||||
@@ -453,24 +529,19 @@ func extractSubtitleFast(path string, streamIndex int) (string, bool) {
|
||||
if !supportsVideoContainer(path) {
|
||||
return "", false
|
||||
}
|
||||
st, err := os.Stat(path)
|
||||
if err != nil {
|
||||
logFallback("subtitle extract", path, err)
|
||||
return "", false
|
||||
}
|
||||
key := fmt.Sprintf("subtitle_content:%s:%d:%d", path, streamIndex, st.ModTime().Unix())
|
||||
if content, ok := subtitleContentCache.Get(key); ok {
|
||||
return content, true
|
||||
}
|
||||
|
||||
f, size, err := openFile(path)
|
||||
f, st, err := openFileInfo(path)
|
||||
if err != nil {
|
||||
logFallback("subtitle extract", path, err)
|
||||
return "", false
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
vtt, err := media.ExtractEmbeddedSubtitle(f, size, streamIndex)
|
||||
key := mediacache.FromFileInfo(path, st).Stream(streamIndex)
|
||||
if content, ok := mediacache.SubtitleContent.Get(key); ok {
|
||||
return content, true
|
||||
}
|
||||
|
||||
vtt, err := media.ExtractEmbeddedSubtitle(f, st.Size(), streamIndex)
|
||||
if err != nil {
|
||||
logFallback("subtitle extract", path, err)
|
||||
return "", false
|
||||
@@ -479,7 +550,7 @@ func extractSubtitleFast(path string, streamIndex int) (string, bool) {
|
||||
logFallback("subtitle extract", path, errors.New("empty subtitle output"))
|
||||
return "", false
|
||||
}
|
||||
subtitleContentCache.Set(key, vtt)
|
||||
mediacache.SubtitleContent.Set(key, vtt)
|
||||
return vtt, true
|
||||
}
|
||||
|
||||
@@ -501,7 +572,7 @@ func videoThumbnailFast(path string, width, height, quality int, seekPercent flo
|
||||
if !Enabled() {
|
||||
return nil, false
|
||||
}
|
||||
if !supportsVideoContainer(path) {
|
||||
if !supportsVideoThumbnail(path) {
|
||||
return nil, false
|
||||
}
|
||||
f, size, err := openFile(path)
|
||||
|
||||
@@ -11,6 +11,8 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/mediacache"
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/utils"
|
||||
"github.com/gtsteffaniak/filebrowser/backend/pkg/settings"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
@@ -131,27 +133,78 @@ func TestProbeUsesCache(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, os.WriteFile(path, data, 0o644))
|
||||
|
||||
st, err := os.Stat(path)
|
||||
key, err := mediacache.Stat(path)
|
||||
require.NoError(t, err)
|
||||
key := probeCacheKey(path, st.ModTime().UnixNano(), st.Size())
|
||||
sentinel := ProbeResult{Duration: 123, VideoCodec: "sentinel", AudioCodec: "sentinel", FormatName: "sentinel"}
|
||||
probeCache.Set(key, sentinel)
|
||||
t.Cleanup(func() { probeCache.Delete(key) })
|
||||
// The ffmpeg probe stores into the same shared cache, so this also stands in
|
||||
// for a value ffprobe produced earlier.
|
||||
mediacache.Probes.Set(key.String(), sentinel)
|
||||
t.Cleanup(func() { mediacache.Probes.Delete(key.String()) })
|
||||
|
||||
got, ok := Probe(path, nil)
|
||||
require.True(t, ok)
|
||||
require.Equal(t, sentinel, got, "probe must return the cached value for an unchanged file")
|
||||
|
||||
// An edit (new size and mtime) must miss the cached value.
|
||||
require.NoError(t, os.WriteFile(path, append(data, 0), 0o644))
|
||||
future := time.Now().Add(time.Hour)
|
||||
require.NoError(t, os.Chtimes(path, future, future))
|
||||
got, ok = Probe(path, nil)
|
||||
require.True(t, ok)
|
||||
require.NotEqual(t, sentinel, got, "an edited file must be probed again")
|
||||
edited, err := mediacache.Stat(path)
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { mediacache.Probes.Delete(edited.String()) })
|
||||
}
|
||||
|
||||
// TestProbeCacheKeyTracksFileIdentity checks that path, modtime, and size all
|
||||
// participate in the cache key, so an edited file is never served stale metadata.
|
||||
func TestProbeCacheKeyTracksFileIdentity(t *testing.T) {
|
||||
base := probeCacheKey("f.mp4", 100, 200)
|
||||
require.Equal(t, base, probeCacheKey("f.mp4", 100, 200), "identical identity must hash the same")
|
||||
require.NotEqual(t, base, probeCacheKey("f.mp4", 101, 200), "modtime change must change the key")
|
||||
require.NotEqual(t, base, probeCacheKey("f.mp4", 100, 201), "size change must change the key")
|
||||
require.NotEqual(t, base, probeCacheKey("g.mp4", 100, 200), "path change must change the key")
|
||||
require.NotEqual(t, base, probeCacheKey("f.mp4", 100, 201), "key must not collide across fields")
|
||||
// TestDurationUsesSharedCache checks that Duration reads and fills the shared
|
||||
// cache that ffmpeg's GetMediaDuration also uses.
|
||||
func TestDurationUsesSharedCache(t *testing.T) {
|
||||
withGate(t, false)
|
||||
path := filepath.Join(t.TempDir(), "sample.mp4")
|
||||
data, err := os.ReadFile(fixtureMP4)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, os.WriteFile(path, data, 0o644))
|
||||
key, err := mediacache.Stat(path)
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { mediacache.Durations.Delete(key.String()) })
|
||||
|
||||
d, ok := Duration(path, nil)
|
||||
require.True(t, ok)
|
||||
cached, found := mediacache.Durations.Get(key.String())
|
||||
require.True(t, found, "go-media duration must be stored in the shared cache")
|
||||
require.Equal(t, d, cached)
|
||||
|
||||
mediacache.Durations.Set(key.String(), 4242)
|
||||
d, ok = Duration(path, nil)
|
||||
require.True(t, ok)
|
||||
require.Equal(t, 4242.0, d, "a cached duration (from either backend) must be reused")
|
||||
}
|
||||
|
||||
// TestSubtitleCachesUseFileIdentity checks the subtitle list cache is keyed by
|
||||
// the file's real size and mtime, not by the caller's modtime argument.
|
||||
func TestSubtitleCachesUseFileIdentity(t *testing.T) {
|
||||
withGate(t, false)
|
||||
path := filepath.Join(t.TempDir(), "sample.mp4")
|
||||
data, err := os.ReadFile(fixtureMP4)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, os.WriteFile(path, data, 0o644))
|
||||
key, err := mediacache.Stat(path)
|
||||
require.NoError(t, err)
|
||||
sentinel := []utils.SubtitleTrack{{Name: "sentinel"}}
|
||||
mediacache.SubtitleTracks.Set(key.String(), sentinel)
|
||||
t.Cleanup(func() { mediacache.SubtitleTracks.Delete(key.String()) })
|
||||
|
||||
// The modtime argument is ignored for the key.
|
||||
got, ok := EmbeddedSubtitles(path, time.Unix(1, 0), nil)
|
||||
require.True(t, ok)
|
||||
require.Equal(t, sentinel, got)
|
||||
|
||||
mediacache.SubtitleContent.Set(key.Stream(7), "WEBVTT\n\nsentinel")
|
||||
t.Cleanup(func() { mediacache.SubtitleContent.Delete(key.Stream(7)) })
|
||||
vtt, ok := ExtractSubtitle(path, 7, nil)
|
||||
require.True(t, ok)
|
||||
require.Equal(t, "WEBVTT\n\nsentinel", vtt)
|
||||
}
|
||||
|
||||
func TestProbeWebMMatchesFFprobe(t *testing.T) {
|
||||
|
||||
@@ -0,0 +1,101 @@
|
||||
package gomedia
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"math"
|
||||
"time"
|
||||
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/ffmpeg"
|
||||
"github.com/gtsteffaniak/go-media/pkg/media"
|
||||
)
|
||||
|
||||
// heicRotationShadowTimeout bounds the dev-mode ffmpeg comparison probe.
|
||||
const heicRotationShadowTimeout = time.Minute
|
||||
|
||||
// HEICDisplayRotation returns the display rotation that a HEIC/HEIF/AVIF
|
||||
// file's irot property asks for, read from the item properties without
|
||||
// decoding. The value is degrees counter-clockwise in (-180, 180], the same
|
||||
// number ffmpeg prints as "Display Matrix: rotation of ..." for the file. A
|
||||
// file with no irot returns 0 and ok=true. ok is false when go-media is
|
||||
// disabled or cannot read the item properties; the caller then runs the ffmpeg
|
||||
// probe.
|
||||
//
|
||||
// It is installed as ffmpeg's HEIC display rotation reader. In shadow mode the
|
||||
// ffmpeg stderr probe runs for comparison.
|
||||
func HEICDisplayRotation(path string) (float64, bool) {
|
||||
return timed("heic rotation", path, heicRotationLegacy(path), heicRotationDiff, func() (float64, bool) {
|
||||
return heicDisplayRotationFast(path)
|
||||
})
|
||||
}
|
||||
|
||||
// heicRotationLegacy returns the ffmpeg stderr probe for shadow mode, or nil
|
||||
// when ffmpeg is not available.
|
||||
func heicRotationLegacy(path string) Legacy[float64] {
|
||||
if !ShadowEnabled() {
|
||||
return nil
|
||||
}
|
||||
svc := ffmpeg.Get()
|
||||
if svc == nil {
|
||||
return nil
|
||||
}
|
||||
return func() (float64, bool) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), heicRotationShadowTimeout)
|
||||
defer cancel()
|
||||
return svc.GetHEICDisplayMatrixRotation(ctx, path)
|
||||
}
|
||||
}
|
||||
|
||||
// heicDisplayRotationFast reads the primary item's irot with go-media and
|
||||
// converts it to ffmpeg's display rotation.
|
||||
func heicDisplayRotationFast(path string) (float64, bool) {
|
||||
if !Enabled() {
|
||||
return 0, false
|
||||
}
|
||||
// No extension check: the resolved path can have another name (symlink
|
||||
// target). go-media sniffs the content and reports other files unsupported.
|
||||
f, size, err := openFile(path)
|
||||
if err != nil {
|
||||
logFallback("heic rotation", path, err)
|
||||
return 0, false
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
t, err := media.HEIFTransform(f, size)
|
||||
if err != nil {
|
||||
logFallback("heic rotation", path, err)
|
||||
return 0, false
|
||||
}
|
||||
return displayRotation(t), true
|
||||
}
|
||||
|
||||
// displayRotation converts go-media's counter-clockwise irot angle to ffmpeg's
|
||||
// display matrix convention: counter-clockwise degrees normalized to
|
||||
// (-180, 180]. ffmpeg builds the matrix with av_display_rotation_set(-irot)
|
||||
// and prints av_display_rotation_get, so irot 270 is reported as -90. A mirror
|
||||
// does not change the rotation value.
|
||||
func displayRotation(t media.ImageTransform) float64 {
|
||||
r := ((t.Rotation % 360) + 360) % 360
|
||||
if r > 180 {
|
||||
r -= 360
|
||||
}
|
||||
return float64(r)
|
||||
}
|
||||
|
||||
// heicRotationDiff compares go-media's display rotation with ffmpeg's. 180 and
|
||||
// -180 are the same rotation.
|
||||
func heicRotationDiff(fast, legacy float64) []string {
|
||||
norm := func(v float64) float64 {
|
||||
v = math.Mod(v, 360)
|
||||
if v <= -180 {
|
||||
v += 360
|
||||
} else if v > 180 {
|
||||
v -= 360
|
||||
}
|
||||
return v
|
||||
}
|
||||
if math.Abs(norm(fast)-norm(legacy)) > 0.5 {
|
||||
return []string{fmt.Sprintf("display rotation go-media=%.2f ffmpeg=%.2f", fast, legacy)}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
package gomedia
|
||||
|
||||
import (
|
||||
"math"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/ffmpeg"
|
||||
"github.com/gtsteffaniak/go-media/pkg/media"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// fixtureAVIFRotated has irot=1 (90° counter-clockwise) and imir axis 0.
|
||||
const fixtureAVIFRotated = "testdata/irot1_imir0.avif"
|
||||
|
||||
// TestDisplayRotation checks the irot to ffmpeg display rotation mapping.
|
||||
func TestDisplayRotation(t *testing.T) {
|
||||
cases := []struct {
|
||||
rotation int
|
||||
want float64
|
||||
}{
|
||||
{0, 0}, {90, 90}, {180, 180}, {270, -90}, {-90, -90}, {360, 0}, {450, 90},
|
||||
}
|
||||
for _, c := range cases {
|
||||
require.Equal(t, c.want, displayRotation(media.ImageTransform{Rotation: c.rotation}), "rotation %d", c.rotation)
|
||||
}
|
||||
require.Equal(t, 90.0, displayRotation(media.ImageTransform{Rotation: 90, Mirror: true}), "a mirror does not change the rotation")
|
||||
}
|
||||
|
||||
// TestHEICDisplayRotationReadsIrot checks the rotation read from a fixture
|
||||
// with irot=1.
|
||||
func TestHEICDisplayRotationReadsIrot(t *testing.T) {
|
||||
withGate(t, false)
|
||||
got, ok := HEICDisplayRotation(fixtureAVIFRotated)
|
||||
require.True(t, ok)
|
||||
require.Equal(t, 90.0, got)
|
||||
}
|
||||
|
||||
// TestHEICDisplayRotationFallsBack checks that non-HEIF input, broken input,
|
||||
// and the Always use ffmpeg setting leave the answer to ffmpeg.
|
||||
func TestHEICDisplayRotationFallsBack(t *testing.T) {
|
||||
withGate(t, false)
|
||||
_, ok := HEICDisplayRotation(fixtureMP4)
|
||||
require.False(t, ok, "a non-HEIF file must fall back to the ffmpeg probe")
|
||||
|
||||
junk := filepath.Join(t.TempDir(), "junk.heic")
|
||||
require.NoError(t, os.WriteFile(junk, []byte("not heif"), 0o644))
|
||||
_, ok = HEICDisplayRotation(junk)
|
||||
require.False(t, ok)
|
||||
|
||||
withGate(t, true)
|
||||
_, ok = HEICDisplayRotation(fixtureAVIFRotated)
|
||||
require.False(t, ok, "the Always use ffmpeg setting must skip go-media")
|
||||
}
|
||||
|
||||
// TestHEICDisplayRotationInstalled checks that loading this package installs
|
||||
// go-media as the ffmpeg package's rotation reader.
|
||||
func TestHEICDisplayRotationInstalled(t *testing.T) {
|
||||
withGate(t, false)
|
||||
var svc *ffmpeg.Service // nil: the ffmpeg probe cannot run, so a result proves the reader answered
|
||||
got, ok := svc.HEICDisplayRotation(t.Context(), fixtureAVIFRotated)
|
||||
require.True(t, ok)
|
||||
require.Equal(t, 90.0, got)
|
||||
}
|
||||
|
||||
// TestHEICRotationDiff checks the shadow comparison of display rotations.
|
||||
func TestHEICRotationDiff(t *testing.T) {
|
||||
require.Empty(t, heicRotationDiff(-90, -90))
|
||||
require.Empty(t, heicRotationDiff(180, -180))
|
||||
require.Empty(t, heicRotationDiff(0, math.Copysign(0, -1)))
|
||||
require.Len(t, heicRotationDiff(90, -90), 1)
|
||||
}
|
||||
|
||||
// TestAVIFDecodesThroughHEICPath checks that an AVIF file is accepted by the
|
||||
// HEIC decode path.
|
||||
func TestAVIFDecodesThroughHEICPath(t *testing.T) {
|
||||
withGate(t, false)
|
||||
data, ok := HEICJPEG(fixtureAVIFRotated, 80, nil)
|
||||
require.True(t, ok)
|
||||
require.NotEmpty(t, data)
|
||||
}
|
||||
@@ -0,0 +1,145 @@
|
||||
package gomedia
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"math"
|
||||
"sort"
|
||||
|
||||
"github.com/gtsteffaniak/go-media/pkg/media"
|
||||
)
|
||||
|
||||
// keyframeTimeDedupeSec matches go-ffmpeg's ffprobe keyframe probe, which
|
||||
// drops keyframes within 1 ms of the previous one.
|
||||
const keyframeTimeDedupeSec = 0.001
|
||||
|
||||
// keyframeShadowTolerance is how far apart two backends' keyframe times can be
|
||||
// before the shadow comparison reports them. ffprobe prints six decimals.
|
||||
const keyframeShadowTolerance = 0.002
|
||||
|
||||
// KeyframeTimes returns the presentation times, in seconds, of the first video
|
||||
// stream's keyframes, for HLS segment planning. The result is sorted, has no
|
||||
// negative times, and drops keyframes within 1 ms of the previous one, the same
|
||||
// as go-ffmpeg's ffprobe probe. go-media reads only the container index (MP4
|
||||
// stss/stts/ctts, Matroska Cues or block headers), so it is much faster than
|
||||
// demuxing every packet with ffprobe.
|
||||
//
|
||||
// ok is false when go-media is disabled, the container is not one go-media
|
||||
// indexes, or the read fails; the caller then runs ffprobe. ctx cancels the
|
||||
// file reads. legacy is the ffprobe probe, run only in shadow mode.
|
||||
//
|
||||
// go-media can report fewer keyframes than ffprobe (ffprobe's parsers also flag
|
||||
// IDR frames the container index omits, and some Matroska muxers index only
|
||||
// part of the keyframes in Cues). It never reports a keyframe ffprobe would not.
|
||||
func KeyframeTimes(ctx context.Context, path string, legacy Legacy[[]float64]) ([]float64, bool) {
|
||||
return timed("keyframe index", path, legacy, keyframeDiff, func() ([]float64, bool) {
|
||||
return keyframeTimesFast(ctx, path, 0)
|
||||
})
|
||||
}
|
||||
|
||||
// KeyframeTimesWithin is KeyframeTimes limited to keyframes within windowSec
|
||||
// seconds of the first keyframe. It is the pre-flight check that can rule out
|
||||
// stream copy without reading the whole index (Matroska files without Cues).
|
||||
// It has no shadow comparison.
|
||||
func KeyframeTimesWithin(ctx context.Context, path string, windowSec float64) ([]float64, bool) {
|
||||
return timed("keyframe preflight", path, nil, nil, func() ([]float64, bool) {
|
||||
return keyframeTimesFast(ctx, path, windowSec)
|
||||
})
|
||||
}
|
||||
|
||||
// keyframeTimesFast reads the keyframe index with go-media. windowSec > 0
|
||||
// limits it to keyframes within that many seconds of the first one.
|
||||
func keyframeTimesFast(ctx context.Context, path string, windowSec float64) ([]float64, bool) {
|
||||
if !Enabled() {
|
||||
return nil, false
|
||||
}
|
||||
if !supportsVideoContainer(path) {
|
||||
return nil, false
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, false
|
||||
}
|
||||
f, size, err := openFile(path)
|
||||
if err != nil {
|
||||
logFallback("keyframe index", path, err)
|
||||
return nil, false
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
r := ctxReaderAt{ctx: ctx, r: f}
|
||||
var times []float64
|
||||
if windowSec > 0 {
|
||||
times, err = media.KeyframeTimesWithin(r, size, windowSec)
|
||||
} else {
|
||||
times, err = media.KeyframeTimes(r, size)
|
||||
}
|
||||
if err != nil {
|
||||
if ctx.Err() == nil {
|
||||
logFallback("keyframe index", path, err)
|
||||
}
|
||||
return nil, false
|
||||
}
|
||||
times = dedupeKeyframeTimes(times)
|
||||
if len(times) == 0 {
|
||||
logUnsupported("keyframe index", path, "no keyframes in container index")
|
||||
return nil, false
|
||||
}
|
||||
return times, true
|
||||
}
|
||||
|
||||
// dedupeKeyframeTimes sorts times, drops negative values, and drops times
|
||||
// within keyframeTimeDedupeSec of the previous kept time, as go-ffmpeg does for
|
||||
// ffprobe output.
|
||||
func dedupeKeyframeTimes(times []float64) []float64 {
|
||||
sorted := append([]float64(nil), times...)
|
||||
sort.Float64s(sorted)
|
||||
out := make([]float64, 0, len(sorted))
|
||||
for _, t := range sorted {
|
||||
if t < 0 || math.IsNaN(t) || math.IsInf(t, 0) {
|
||||
continue
|
||||
}
|
||||
if len(out) == 0 || t > out[len(out)-1]+keyframeTimeDedupeSec {
|
||||
out = append(out, t)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// ctxReaderAt fails reads once ctx is done, so a cancelled request or the plan
|
||||
// timeout stops a long index walk on a slow share.
|
||||
type ctxReaderAt struct {
|
||||
ctx context.Context
|
||||
r io.ReaderAt
|
||||
}
|
||||
|
||||
// ReadAt reads from the wrapped reader unless the context is done.
|
||||
func (c ctxReaderAt) ReadAt(p []byte, off int64) (int, error) {
|
||||
if err := c.ctx.Err(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return c.r.ReadAt(p, off)
|
||||
}
|
||||
|
||||
// keyframeDiff compares go-media's keyframe index with ffprobe's. go-media
|
||||
// reporting a subset of ffprobe's keyframes is expected (parser-flagged IDR
|
||||
// frames, partial Cues) and is logged at info level.
|
||||
func keyframeDiff(fast, legacy []float64) []string {
|
||||
missing := 0
|
||||
j := 0
|
||||
for _, t := range fast {
|
||||
for j < len(legacy) && legacy[j] < t-keyframeShadowTolerance {
|
||||
j++
|
||||
missing++
|
||||
}
|
||||
if j >= len(legacy) || math.Abs(legacy[j]-t) > keyframeShadowTolerance {
|
||||
return []string{fmt.Sprintf("keyframe at %.6fs from go-media not in ffmpeg index (go-media=%d ffmpeg=%d keyframes)", t, len(fast), len(legacy))}
|
||||
}
|
||||
j++
|
||||
}
|
||||
missing += len(legacy) - j
|
||||
if missing > 0 {
|
||||
return []string{fmt.Sprintf("%sgo-media found %d of %d ffmpeg keyframes (keyframes outside the container index)", expectedDiffPrefix, len(fast), len(legacy))}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,176 @@
|
||||
package gomedia
|
||||
|
||||
import (
|
||||
"context"
|
||||
"math"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// ffprobeKeyframes runs the same packet probe as go-ffmpeg's
|
||||
// ProbeVideoKeyframeTimes: key packets of v:0, pts_time falling back to
|
||||
// dts_time, sorted and de-duplicated within 1 ms.
|
||||
func ffprobeKeyframes(t *testing.T, path string) []float64 {
|
||||
t.Helper()
|
||||
requireTool(t, "ffprobe")
|
||||
out, err := exec.Command("ffprobe", "-v", "error", "-select_streams", "v:0",
|
||||
"-show_entries", "packet=pts_time,dts_time,flags", "-of", "csv=p=0", path).Output()
|
||||
require.NoError(t, err)
|
||||
var times []float64
|
||||
for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") {
|
||||
f := strings.Split(strings.TrimSpace(line), ",")
|
||||
if len(f) < 3 || !strings.Contains(f[2], "K") {
|
||||
continue
|
||||
}
|
||||
v, perr := strconv.ParseFloat(f[0], 64)
|
||||
if perr != nil {
|
||||
v, perr = strconv.ParseFloat(f[1], 64)
|
||||
}
|
||||
if perr == nil {
|
||||
times = append(times, v)
|
||||
}
|
||||
}
|
||||
sort.Float64s(times)
|
||||
return dedupeKeyframeTimes(times)
|
||||
}
|
||||
|
||||
// encodeKeyframeClip makes a short clip with a fixed GOP. B-frames give MP4
|
||||
// files a ctts table and an edit list, the hard case for the timestamp model.
|
||||
func encodeKeyframeClip(t *testing.T, name string, args ...string) string {
|
||||
t.Helper()
|
||||
requireTool(t, "ffmpeg")
|
||||
path := filepath.Join(t.TempDir(), name)
|
||||
full := []string{"-v", "error", "-y", "-f", "lavfi", "-i", "testsrc2=size=160x120:rate=25:duration=6"}
|
||||
full = append(full, args...)
|
||||
full = append(full, path)
|
||||
if out, err := exec.Command("ffmpeg", full...).CombinedOutput(); err != nil {
|
||||
t.Skipf("ffmpeg cannot encode %s: %v: %s", name, err, out)
|
||||
}
|
||||
return path
|
||||
}
|
||||
|
||||
// requireSameKeyframes fails unless got matches want within the shadow
|
||||
// tolerance.
|
||||
func requireSameKeyframes(t *testing.T, want, got []float64) {
|
||||
t.Helper()
|
||||
require.Equal(t, len(want), len(got), "keyframe count: ffprobe=%v go-media=%v", want, got)
|
||||
for i := range want {
|
||||
require.InDelta(t, want[i], got[i], keyframeShadowTolerance, "keyframe %d: ffprobe=%v go-media=%v", i, want, got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestKeyframeTimesMatchFFprobe compares go-media keyframe indexes with
|
||||
// ffprobe for fixtures and encoded clips, including B-frame MP4/MOV.
|
||||
func TestKeyframeTimesMatchFFprobe(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("encodes clips with ffmpeg")
|
||||
}
|
||||
withGate(t, false)
|
||||
x264 := []string{"-c:v", "libx264", "-g", "50", "-keyint_min", "50", "-sc_threshold", "0", "-bf", "2", "-pix_fmt", "yuv420p"}
|
||||
cases := map[string]string{
|
||||
"fixture.mp4": fixtureMP4,
|
||||
"fixture.webm": fixtureWebM,
|
||||
"bframes.mp4": encodeKeyframeClip(t, "bframes.mp4", x264...),
|
||||
"bframes.mov": encodeKeyframeClip(t, "bframes.mov", x264...),
|
||||
"bframes.mkv": encodeKeyframeClip(t, "bframes.mkv", x264...),
|
||||
"mpeg4.m4v": encodeKeyframeClip(t, "mpeg4.m4v", "-c:v", "mpeg4", "-g", "30", "-f", "mp4"),
|
||||
}
|
||||
for name, path := range cases {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
want := ffprobeKeyframes(t, path)
|
||||
got, ok := KeyframeTimes(context.Background(), path, nil)
|
||||
require.True(t, ok)
|
||||
requireSameKeyframes(t, want, got)
|
||||
|
||||
within, ok := KeyframeTimesWithin(context.Background(), path, 2.5)
|
||||
require.True(t, ok)
|
||||
require.NotEmpty(t, within)
|
||||
require.LessOrEqual(t, len(within), len(got))
|
||||
for i := range within {
|
||||
require.Equal(t, got[i], within[i])
|
||||
require.LessOrEqual(t, within[i]-got[0], 2.5+1e-9)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestKeyframeTimesFallsBack checks the cases that must leave the index to
|
||||
// ffprobe.
|
||||
func TestKeyframeTimesFallsBack(t *testing.T) {
|
||||
withGate(t, false)
|
||||
dir := t.TempDir()
|
||||
|
||||
// Not an indexed container: no file open, ffprobe answers.
|
||||
ts := filepath.Join(dir, "clip.ts")
|
||||
require.NoError(t, os.WriteFile(ts, []byte{0x47}, 0o644))
|
||||
_, ok := KeyframeTimes(context.Background(), ts, nil)
|
||||
require.False(t, ok)
|
||||
|
||||
// Indexed extension but not a real container.
|
||||
junk := filepath.Join(dir, "junk.mp4")
|
||||
require.NoError(t, os.WriteFile(junk, []byte("not a video"), 0o644))
|
||||
_, ok = KeyframeTimes(context.Background(), junk, nil)
|
||||
require.False(t, ok)
|
||||
|
||||
// Missing file.
|
||||
_, ok = KeyframeTimes(context.Background(), filepath.Join(dir, "missing.mp4"), nil)
|
||||
require.False(t, ok)
|
||||
|
||||
// Cancelled context.
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
_, ok = KeyframeTimes(ctx, fixtureMP4, nil)
|
||||
require.False(t, ok)
|
||||
}
|
||||
|
||||
// TestKeyframeTimesGate checks that the Always use ffmpeg setting disables
|
||||
// the go-media keyframe index.
|
||||
func TestKeyframeTimesGate(t *testing.T) {
|
||||
withGate(t, true)
|
||||
_, ok := KeyframeTimes(context.Background(), fixtureMP4, nil)
|
||||
require.False(t, ok, "KeyframeTimes must skip go-media when the gate is on")
|
||||
_, ok = KeyframeTimesWithin(context.Background(), fixtureMP4, 10)
|
||||
require.False(t, ok)
|
||||
}
|
||||
|
||||
// TestDedupeKeyframeTimes checks sorting, filtering, and the 1 ms dedupe.
|
||||
func TestDedupeKeyframeTimes(t *testing.T) {
|
||||
got := dedupeKeyframeTimes([]float64{2, -1, 0, 0.0005, 1, 1.001, 1.0021, math.NaN(), math.Inf(1)})
|
||||
require.Equal(t, []float64{0, 1, 1.0021, 2}, got)
|
||||
require.Empty(t, dedupeKeyframeTimes(nil))
|
||||
}
|
||||
|
||||
// TestCtxReaderAt checks that reads stop once the context is cancelled.
|
||||
func TestCtxReaderAt(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
r := ctxReaderAt{ctx: ctx, r: strings.NewReader("abcdef")}
|
||||
buf := make([]byte, 3)
|
||||
n, err := r.ReadAt(buf, 1)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "bcd", string(buf[:n]))
|
||||
cancel()
|
||||
_, err = r.ReadAt(buf, 0)
|
||||
require.ErrorIs(t, err, context.Canceled)
|
||||
}
|
||||
|
||||
// TestKeyframeDiff checks the shadow comparison: a subset is expected, an
|
||||
// extra keyframe is a real difference.
|
||||
func TestKeyframeDiff(t *testing.T) {
|
||||
require.Empty(t, keyframeDiff([]float64{0, 2, 4}, []float64{0, 2.001, 4}))
|
||||
|
||||
subset := keyframeDiff([]float64{0, 4}, []float64{0, 2, 4, 6})
|
||||
require.Len(t, subset, 1)
|
||||
require.True(t, strings.HasPrefix(subset[0], expectedDiffPrefix), subset[0])
|
||||
require.Contains(t, subset[0], "2 of 4")
|
||||
|
||||
extra := keyframeDiff([]float64{0, 1, 2}, []float64{0, 2})
|
||||
require.Len(t, extra, 1)
|
||||
require.False(t, strings.HasPrefix(extra[0], expectedDiffPrefix), extra[0])
|
||||
}
|
||||
@@ -59,8 +59,8 @@ func runShadow[T any](op, path string, goLatency time.Duration, fast T, legacy L
|
||||
ffStart := time.Now()
|
||||
legacyVal, ffOK := legacy()
|
||||
ffLatency := time.Since(ffStart)
|
||||
shadowLogf("go-media shadow %s for %s: go-media=%.2fms legacy(ffmpeg/dhowden)=%.2fms diff(legacy-go)=%+.2fms legacy_ok=%t",
|
||||
op, path, ms(goLatency), ms(ffLatency), ms(ffLatency)-ms(goLatency), ffOK)
|
||||
shadowLogf("go-media shadow %s for %s: go-media=%.2fms legacy(ffmpeg/dhowden)=%.2fms diff(go-legacy)=%+.2fms",
|
||||
op, path, ms(goLatency), ms(ffLatency), ms(goLatency)-ms(ffLatency))
|
||||
|
||||
if !ffOK {
|
||||
// go-media read the file but the legacy path failed. That is an outcome
|
||||
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 4.7 KiB |
@@ -9,6 +9,7 @@ import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/database/users"
|
||||
"github.com/gtsteffaniak/filebrowser/backend/pkg/settings"
|
||||
"github.com/kovidgoyal/imaging"
|
||||
)
|
||||
@@ -122,23 +123,21 @@ func TestDefaultBackgroundColor(t *testing.T) {
|
||||
saved := settings.Config
|
||||
defer func() { settings.Config = saved }()
|
||||
|
||||
dark := true
|
||||
light := false
|
||||
|
||||
settings.Config.Frontend.Styling.DarkBackground = "#111111"
|
||||
settings.Config.Frontend.Styling.LightBackground = "#ffffff"
|
||||
|
||||
settings.Config.UserDefaults.UI.DarkMode = &dark
|
||||
settings.Config.UserDefaults.UI.DarkMode = users.DarkModeDark
|
||||
if got := settings.DefaultBackgroundColor(); got != "#111111" {
|
||||
t.Fatalf("dark branch: got %q, want %q", got, "#111111")
|
||||
}
|
||||
|
||||
settings.Config.UserDefaults.UI.DarkMode = &light
|
||||
settings.Config.UserDefaults.UI.DarkMode = users.DarkModeLight
|
||||
if got := settings.DefaultBackgroundColor(); got != "#ffffff" {
|
||||
t.Fatalf("light branch (dark disabled): got %q, want %q", got, "#ffffff")
|
||||
}
|
||||
|
||||
settings.Config.UserDefaults.UI.DarkMode = nil
|
||||
settings.Config.UserDefaults.UI.DarkMode = ""
|
||||
if got := settings.DefaultBackgroundColor(); got != "#ffffff" {
|
||||
t.Fatalf("light branch (unset): got %q, want %q", got, "#ffffff")
|
||||
}
|
||||
|
||||
@@ -0,0 +1,109 @@
|
||||
// Package mediacache holds the media metadata caches shared by the ffmpeg and
|
||||
// go-media code paths. Both backends read and write the same entries, so a
|
||||
// value found by one is reused by the other instead of being probed again.
|
||||
//
|
||||
// Every entry is keyed by file identity: path, size, and modification time in
|
||||
// nanoseconds. An edited file gets a new key, so a stale value is never served
|
||||
// for it; the old entry ages out after TTL.
|
||||
package mediacache
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/utils"
|
||||
"github.com/gtsteffaniak/go-cache/cache"
|
||||
)
|
||||
|
||||
// TTL is how long a cached media value is kept after it is stored.
|
||||
const TTL = 24 * time.Hour
|
||||
|
||||
// ProbeInfo is the container summary both backends produce: ffprobe through
|
||||
// go-ffmpeg, and go-media. FormatName uses ffprobe's format_name strings.
|
||||
type ProbeInfo struct {
|
||||
Duration float64
|
||||
VideoCodec string
|
||||
AudioCodec string
|
||||
FormatName string
|
||||
}
|
||||
|
||||
var (
|
||||
// Probes caches container probes (duration, first video and audio codec,
|
||||
// format name).
|
||||
Probes = cache.NewCache[ProbeInfo](TTL)
|
||||
// Durations caches media length in seconds.
|
||||
Durations = cache.NewCache[float64](TTL)
|
||||
// SubtitleTracks caches the embedded subtitle track list of a video.
|
||||
SubtitleTracks = cache.NewCache[[]utils.SubtitleTrack](TTL)
|
||||
// SubtitleContent caches one embedded subtitle stream converted to WebVTT.
|
||||
// Use Key.Stream for its key.
|
||||
SubtitleContent = cache.NewCache[string](TTL)
|
||||
)
|
||||
|
||||
// Key identifies one version of a file. The zero Key is never produced for a
|
||||
// real file, because Path is always set.
|
||||
type Key struct {
|
||||
Path string
|
||||
Size int64
|
||||
ModTime int64 // modification time in Unix nanoseconds
|
||||
}
|
||||
|
||||
// FromFileInfo returns the Key for path using an already-read FileInfo, so
|
||||
// callers that stat or open the file anyway do not stat it twice.
|
||||
func FromFileInfo(path string, info fs.FileInfo) Key {
|
||||
return Key{Path: path, Size: info.Size(), ModTime: info.ModTime().UnixNano()}
|
||||
}
|
||||
|
||||
// Stat returns the Key for path. It follows symlinks, like os.Stat.
|
||||
func Stat(path string) (Key, error) {
|
||||
info, err := os.Stat(path)
|
||||
if err != nil {
|
||||
return Key{}, err
|
||||
}
|
||||
return FromFileInfo(path, info), nil
|
||||
}
|
||||
|
||||
// String returns the cache key for k. The path is quoted so a path containing
|
||||
// the separator cannot collide with another file's key.
|
||||
func (k Key) String() string {
|
||||
return fmt.Sprintf("%q:%d:%d", k.Path, k.Size, k.ModTime)
|
||||
}
|
||||
|
||||
// Stream returns the cache key for one stream of k, used by SubtitleContent.
|
||||
func (k Key) Stream(index int) string {
|
||||
return fmt.Sprintf("%s:%d", k.String(), index)
|
||||
}
|
||||
|
||||
// ClearAll drops every cached media value. It is used when the backend that
|
||||
// produced the values is switched (the admin "Always use ffmpeg" setting), so
|
||||
// the newly selected backend answers fresh.
|
||||
func ClearAll() {
|
||||
Probes.ClearAll()
|
||||
Durations.ClearAll()
|
||||
SubtitleTracks.ClearAll()
|
||||
SubtitleContent.ClearAll()
|
||||
}
|
||||
|
||||
// bypassKey is the context key set by WithoutCache.
|
||||
type bypassKey struct{}
|
||||
|
||||
// WithoutCache returns a context under which the ffmpeg code paths neither read
|
||||
// nor write the shared caches. Dev-mode shadow runs use it, so the ffmpeg result
|
||||
// they compare against go-media is computed fresh rather than read back from the
|
||||
// value go-media just stored.
|
||||
func WithoutCache(ctx context.Context) context.Context {
|
||||
return context.WithValue(ctx, bypassKey{}, true)
|
||||
}
|
||||
|
||||
// Bypassed reports whether ctx was returned by WithoutCache. A nil context is
|
||||
// not bypassed.
|
||||
func Bypassed(ctx context.Context) bool {
|
||||
if ctx == nil {
|
||||
return false
|
||||
}
|
||||
v, _ := ctx.Value(bypassKey{}).(bool)
|
||||
return v
|
||||
}
|
||||
@@ -0,0 +1,157 @@
|
||||
package mediacache
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// TestStatKeyChangesWithSizeAndModTime checks on disk that a size change
|
||||
// or an mtime change gives a new key. The mtimes differ by whole seconds so
|
||||
// the check holds on filesystems with coarse timestamps; the 1 ns case is
|
||||
// covered by TestFromFileInfoNanosecondModTime.
|
||||
func TestStatKeyChangesWithSizeAndModTime(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "a.mp4")
|
||||
if err := os.WriteFile(path, []byte("one"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
k1, err := Stat(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if k1.Path != path || k1.Size != 3 || k1.ModTime == 0 {
|
||||
t.Fatalf("unexpected key %+v", k1)
|
||||
}
|
||||
|
||||
// Same size, different mtime.
|
||||
mt := time.Unix(1_700_000_000, 0)
|
||||
if err = os.Chtimes(path, mt, mt); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
k2, err := Stat(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if k2.String() == k1.String() {
|
||||
t.Fatalf("mtime change kept key %s", k2)
|
||||
}
|
||||
mt2 := mt.Add(time.Second)
|
||||
if err = os.Chtimes(path, mt2, mt2); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
k3, _ := Stat(path)
|
||||
if k3.String() == k2.String() {
|
||||
t.Fatalf("1s mtime change kept key %s", k3)
|
||||
}
|
||||
|
||||
// Same mtime, different size.
|
||||
if err = os.WriteFile(path, []byte("three"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = os.Chtimes(path, mt2, mt2); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
k4, _ := Stat(path)
|
||||
if k4.String() == k3.String() {
|
||||
t.Fatalf("size change kept key %s", k4)
|
||||
}
|
||||
}
|
||||
|
||||
// stubFileInfo is an fs.FileInfo with a fixed size and mtime.
|
||||
type stubFileInfo struct {
|
||||
size int64
|
||||
modTime time.Time
|
||||
}
|
||||
|
||||
// Name implements fs.FileInfo.
|
||||
func (s stubFileInfo) Name() string { return "a.mp4" }
|
||||
|
||||
// Size implements fs.FileInfo.
|
||||
func (s stubFileInfo) Size() int64 { return s.size }
|
||||
|
||||
// Mode implements fs.FileInfo.
|
||||
func (s stubFileInfo) Mode() fs.FileMode { return 0o600 }
|
||||
|
||||
// ModTime implements fs.FileInfo.
|
||||
func (s stubFileInfo) ModTime() time.Time { return s.modTime }
|
||||
|
||||
// IsDir implements fs.FileInfo.
|
||||
func (s stubFileInfo) IsDir() bool { return false }
|
||||
|
||||
// Sys implements fs.FileInfo.
|
||||
func (s stubFileInfo) Sys() any { return nil }
|
||||
|
||||
// TestFromFileInfoNanosecondModTime checks that mtimes 1 ns apart give
|
||||
// different keys, independent of what a filesystem can store.
|
||||
func TestFromFileInfoNanosecondModTime(t *testing.T) {
|
||||
a := FromFileInfo("/m/a.mp4", stubFileInfo{size: 3, modTime: time.Unix(1_700_000_000, 123)})
|
||||
b := FromFileInfo("/m/a.mp4", stubFileInfo{size: 3, modTime: time.Unix(1_700_000_000, 124)})
|
||||
if a.String() == b.String() {
|
||||
t.Fatalf("1ns mtime change kept key %s", a)
|
||||
}
|
||||
if a.ModTime != time.Unix(1_700_000_000, 123).UnixNano() {
|
||||
t.Fatalf("ModTime = %d, want nanoseconds", a.ModTime)
|
||||
}
|
||||
}
|
||||
|
||||
// TestStatMissingFile checks that Stat reports a missing file.
|
||||
func TestStatMissingFile(t *testing.T) {
|
||||
if _, err := Stat(filepath.Join(t.TempDir(), "missing")); err == nil {
|
||||
t.Fatal("expected an error for a missing file")
|
||||
}
|
||||
}
|
||||
|
||||
// TestKeyStringDoesNotCollide checks that keys of different files, and of
|
||||
// different streams of one file, never collide.
|
||||
func TestKeyStringDoesNotCollide(t *testing.T) {
|
||||
a := Key{Path: "a:1", Size: 2, ModTime: 3}
|
||||
b := Key{Path: "a", Size: 1, ModTime: 2}
|
||||
if a.String() == b.String() {
|
||||
t.Fatalf("keys collide: %s", a)
|
||||
}
|
||||
if a.Stream(1) == a.Stream(2) || a.Stream(1) == a.String() {
|
||||
t.Fatalf("stream keys collide: %s %s", a.Stream(1), a.Stream(2))
|
||||
}
|
||||
}
|
||||
|
||||
// TestClearAll checks that ClearAll empties every shared cache.
|
||||
func TestClearAll(t *testing.T) {
|
||||
k := Key{Path: "/clear", Size: 1, ModTime: 1}
|
||||
Probes.Set(k.String(), ProbeInfo{Duration: 1})
|
||||
Durations.Set(k.String(), 1)
|
||||
SubtitleContent.Set(k.Stream(0), "x")
|
||||
ClearAll()
|
||||
if _, ok := Probes.Get(k.String()); ok {
|
||||
t.Fatal("probe survived ClearAll")
|
||||
}
|
||||
if _, ok := Durations.Get(k.String()); ok {
|
||||
t.Fatal("duration survived ClearAll")
|
||||
}
|
||||
if _, ok := SubtitleContent.Get(k.Stream(0)); ok {
|
||||
t.Fatal("subtitle content survived ClearAll")
|
||||
}
|
||||
}
|
||||
|
||||
// TestWithoutCache checks the cache bypass marker and that derived contexts
|
||||
// keep it.
|
||||
func TestWithoutCache(t *testing.T) {
|
||||
if Bypassed(context.Background()) {
|
||||
t.Fatal("background context must not bypass")
|
||||
}
|
||||
//nolint:staticcheck // nil is part of the documented contract.
|
||||
if Bypassed(nil) {
|
||||
t.Fatal("nil context must not bypass")
|
||||
}
|
||||
ctx := WithoutCache(context.Background())
|
||||
if !Bypassed(ctx) {
|
||||
t.Fatal("WithoutCache context must bypass")
|
||||
}
|
||||
child, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
if !Bypassed(child) {
|
||||
t.Fatal("bypass must survive derived contexts")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,122 @@
|
||||
package preview
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"image"
|
||||
"image/color"
|
||||
"runtime"
|
||||
"testing"
|
||||
|
||||
"github.com/kovidgoyal/imaging"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// patternYCbCr is a 4:2:0 image (what a camera JPEG decodes to) with a pattern that differs in
|
||||
// every row and column, so a crop off by one pixel shows up in the resized output.
|
||||
func patternYCbCr(w, h int) *image.YCbCr {
|
||||
return patternYCbCrRatio(w, h, image.YCbCrSubsampleRatio420)
|
||||
}
|
||||
|
||||
// patternYCbCrRatio is patternYCbCr with the given chroma subsampling.
|
||||
func patternYCbCrRatio(w, h int, r image.YCbCrSubsampleRatio) *image.YCbCr {
|
||||
img := image.NewYCbCr(image.Rect(0, 0, w, h), r)
|
||||
for y := 0; y < h; y++ {
|
||||
for x := 0; x < w; x++ {
|
||||
img.Y[img.YOffset(x, y)] = uint8((x*7 + y*13) % 251)
|
||||
}
|
||||
}
|
||||
for i := range img.Cb {
|
||||
img.Cb[i] = uint8((i*3 + i/img.CStride) % 256)
|
||||
img.Cr[i] = uint8((i + 5*(i/img.CStride)) % 256)
|
||||
}
|
||||
return img
|
||||
}
|
||||
|
||||
// patternNRGBA is an NRGBA image whose pixels differ in every row and column.
|
||||
func patternNRGBA(w, h int) *image.NRGBA {
|
||||
img := image.NewNRGBA(image.Rect(0, 0, w, h))
|
||||
for y := 0; y < h; y++ {
|
||||
for x := 0; x < w; x++ {
|
||||
img.SetNRGBA(x, y, color.NRGBA{uint8(x % 256), uint8(y % 256), uint8((x + y) % 256), 255})
|
||||
}
|
||||
}
|
||||
return img
|
||||
}
|
||||
|
||||
// requireSamePixels fails unless want and got are the same size and every pixel is equal,
|
||||
// comparing from each image's own origin.
|
||||
func requireSamePixels(t *testing.T, want, got image.Image) {
|
||||
t.Helper()
|
||||
require.Equal(t, want.Bounds().Size(), got.Bounds().Size())
|
||||
wb, gb := want.Bounds().Min, got.Bounds().Min
|
||||
for y := 0; y < want.Bounds().Dy(); y++ {
|
||||
for x := 0; x < want.Bounds().Dx(); x++ {
|
||||
wr, wg, wbl, wa := want.At(wb.X+x, wb.Y+y).RGBA()
|
||||
gr, gg, gbl, ga := got.At(gb.X+x, gb.Y+y).RGBA()
|
||||
if wr != gr || wg != gg || wbl != gbl || wa != ga {
|
||||
t.Fatalf("pixel (%d,%d) differs: imaging.Fill %v, fillCenter %v", x, y,
|
||||
[4]uint32{wr, wg, wbl, wa}, [4]uint32{gr, gg, gbl, ga})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// fillCenter must produce exactly what imaging.Fill(…, imaging.Center, …) produces: the same
|
||||
// pixels and the same image type, so the encoded preview is the same bytes.
|
||||
func TestFillCenter_MatchesImagingFill(t *testing.T) {
|
||||
sources := map[string]image.Image{
|
||||
"ycbcr landscape": patternYCbCr(1200, 801),
|
||||
"ycbcr portrait": patternYCbCr(601, 1000),
|
||||
"nrgba square": patternNRGBA(700, 700),
|
||||
"nrgba wide": patternNRGBA(1500, 300),
|
||||
"small (falls back to imaging.Fill)": patternNRGBA(90, 60),
|
||||
"exact target size": patternNRGBA(256, 256),
|
||||
"offset origin": patternNRGBA(900, 700).SubImage(image.Rect(37, 11, 837, 611)),
|
||||
}
|
||||
// Every subsampling ratio, in shapes whose centered crop has an x (or y) offset and reaches
|
||||
// the source's last row (or column) -- where a SubImage's buffer ends short of a full stride.
|
||||
for _, r := range []image.YCbCrSubsampleRatio{
|
||||
image.YCbCrSubsampleRatio444, image.YCbCrSubsampleRatio422, image.YCbCrSubsampleRatio420,
|
||||
image.YCbCrSubsampleRatio440, image.YCbCrSubsampleRatio411, image.YCbCrSubsampleRatio410,
|
||||
} {
|
||||
for _, sz := range [][2]int{{600, 264}, {264, 600}, {601, 263}} {
|
||||
sources[fmt.Sprintf("ycbcr %v %dx%d", r, sz[0], sz[1])] = patternYCbCrRatio(sz[0], sz[1], r)
|
||||
}
|
||||
}
|
||||
sources["nycbcra 444 600x264"] = &image.NYCbCrA{YCbCr: *patternYCbCrRatio(600, 264, image.YCbCrSubsampleRatio444), A: make([]uint8, 600*264), AStride: 600}
|
||||
// 264x264 is the crop of the 600x264 and 264x600 sources itself: nothing to scale.
|
||||
targets := [][2]int{{256, 256}, {320, 180}, {180, 320}, {264, 264}}
|
||||
filters := map[string]imaging.ResampleFilter{
|
||||
"nearest": imaging.NearestNeighbor, "box": imaging.Box, "catmullrom": imaging.CatmullRom,
|
||||
}
|
||||
for name, src := range sources {
|
||||
for _, tg := range targets {
|
||||
for fname, f := range filters {
|
||||
t.Run(fmt.Sprintf("%s/%dx%d/%s", name, tg[0], tg[1], fname), func(t *testing.T) {
|
||||
t.Parallel()
|
||||
want := imaging.Fill(src, tg[0], tg[1], imaging.Center, f)
|
||||
got := fillCenter(src, tg[0], tg[1], f)
|
||||
require.Equal(t, fmt.Sprintf("%T", want), fmt.Sprintf("%T", got), "image type")
|
||||
requireSamePixels(t, want, got)
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A square thumbnail of a large photo must not allocate a full-resolution copy of it.
|
||||
func TestFillCenter_NoFullSizeCopy(t *testing.T) {
|
||||
const w, h = 4000, 3000
|
||||
src := patternYCbCr(w, h)
|
||||
fullCopy := uint64(h * h * 4) // the square crop imaging.Fill materializes as NRGBA
|
||||
|
||||
var before, after runtime.MemStats
|
||||
runtime.ReadMemStats(&before)
|
||||
out := fillCenter(src, 256, 256, imaging.Box)
|
||||
runtime.ReadMemStats(&after)
|
||||
|
||||
require.Equal(t, image.Pt(256, 256), out.Bounds().Size())
|
||||
allocated := after.TotalAlloc - before.TotalAlloc
|
||||
require.Less(t, allocated, fullCopy/4,
|
||||
"fillCenter allocated %d bytes; a copy of the crop would be %d", allocated, fullCopy)
|
||||
}
|
||||
@@ -7,9 +7,11 @@ import (
|
||||
"image"
|
||||
"image/jpeg"
|
||||
"io"
|
||||
"math"
|
||||
|
||||
exif "github.com/dsoprea/go-exif/v3"
|
||||
exifcommon "github.com/dsoprea/go-exif/v3/common"
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/embeddedpreview"
|
||||
"github.com/kovidgoyal/imaging"
|
||||
)
|
||||
|
||||
@@ -203,6 +205,50 @@ func (s *Service) ResizeWithSize(ctx context.Context, in io.Reader, out io.Write
|
||||
return err
|
||||
}
|
||||
|
||||
// fillCenter is imaging.Fill(img, width, height, imaging.Center, filter) without a full-size copy
|
||||
// of the source. For sources of 100px or more, imaging.Fill first crops into a new NRGBA image
|
||||
// (4 bytes per pixel of the cropped area, about 144 MB for a 48 MP photo) and only then resizes.
|
||||
// Here the crop is a SubImage view into the decoded pixels, so only the resize allocates. The
|
||||
// crop rectangle is computed as imaging's cropAndResize computes it, so the output is the same.
|
||||
//
|
||||
// Two cases still go through imaging.Fill: a crop that is already the target size (nothing to
|
||||
// scale, and Fill's small copy is what keeps the output type the same), and 4:4:4 YCbCr sources,
|
||||
// whose scanner in imaging v1.9.0 advances one full parent stride past the end of a SubImage's
|
||||
// last row and panics when the crop has an x offset and reaches the bottom row.
|
||||
func fillCenter(img image.Image, width, height int, filter imaging.ResampleFilter) image.Image {
|
||||
b := img.Bounds()
|
||||
sub, ok := img.(interface {
|
||||
SubImage(image.Rectangle) image.Image
|
||||
})
|
||||
if !ok || is444(img) || width <= 0 || height <= 0 || b.Dx() < 100 || b.Dy() < 100 || (b.Dx() == width && b.Dy() == height) {
|
||||
return imaging.Fill(img, width, height, imaging.Center, filter)
|
||||
}
|
||||
cropW, cropH := b.Dx(), b.Dy()
|
||||
if float64(b.Dx())/float64(b.Dy()) < float64(width)/float64(height) {
|
||||
cropH = int(math.Max(1, float64(b.Dx())*float64(height)/float64(width)) + 0.5)
|
||||
} else {
|
||||
cropW = int(math.Max(1, float64(b.Dy())*float64(width)/float64(height)) + 0.5)
|
||||
}
|
||||
if cropW == width && cropH == height {
|
||||
return imaging.Fill(img, width, height, imaging.Center, filter)
|
||||
}
|
||||
x0 := b.Min.X + (b.Dx()-cropW)/2
|
||||
y0 := b.Min.Y + (b.Dy()-cropH)/2
|
||||
crop := image.Rect(x0, y0, x0+cropW, y0+cropH).Intersect(b)
|
||||
return imaging.Resize(sub.SubImage(crop), width, height, filter)
|
||||
}
|
||||
|
||||
// is444 reports whether img is a YCbCr image without chroma subsampling.
|
||||
func is444(img image.Image) bool {
|
||||
switch p := img.(type) {
|
||||
case *image.YCbCr:
|
||||
return p.SubsampleRatio == image.YCbCrSubsampleRatio444
|
||||
case *image.NYCbCrA:
|
||||
return p.SubsampleRatio == image.YCbCrSubsampleRatio444
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (s *Service) resizeWithSize(in io.Reader, out io.Writer, fileSize int64, opts ResizeOptions) error {
|
||||
// Set defaults
|
||||
if opts.ResizeMode == 0 {
|
||||
@@ -280,7 +326,7 @@ func (s *Service) resizeWithSize(in io.Reader, out io.Writer, fileSize int64, op
|
||||
resampleFilter := opts.Quality.resampleFilter()
|
||||
switch opts.ResizeMode {
|
||||
case ResizeModeFill:
|
||||
img = imaging.Fill(img, opts.Width, opts.Height, imaging.Center, resampleFilter)
|
||||
img = fillCenter(img, opts.Width, opts.Height, resampleFilter)
|
||||
case ResizeModeFit:
|
||||
fallthrough
|
||||
default:
|
||||
@@ -406,8 +452,32 @@ func getEmbeddedThumbnail(in io.Reader) ([]byte, io.Reader, error) {
|
||||
return nil, wrappedReader, exif.ErrNoThumbnail
|
||||
}
|
||||
|
||||
// Read IFD1's Orientation first; fall back to IFD0's Orientation if not specified (per MWG guidelines)
|
||||
orientStr := ""
|
||||
tags1, err1 := ifd.FindTagWithName("Orientation")
|
||||
if err1 == nil && len(tags1) > 0 {
|
||||
orientStr, _ = tags1[0].FormatFirst()
|
||||
}
|
||||
if orientStr == "" {
|
||||
tags0, err0 := index.RootIfd.FindTagWithName("Orientation")
|
||||
if err0 == nil && len(tags0) > 0 {
|
||||
orientStr, _ = tags0[0].FormatFirst()
|
||||
}
|
||||
}
|
||||
|
||||
thm, err := ifd.Thumbnail()
|
||||
return thm, wrappedReader, err
|
||||
if err != nil {
|
||||
return nil, wrappedReader, err
|
||||
}
|
||||
|
||||
if orientStr != "" && orientStr != "1" {
|
||||
if !embeddedpreview.Usable(thm) {
|
||||
return nil, wrappedReader, fmt.Errorf("embedded thumbnail exceeds decode limits")
|
||||
}
|
||||
thm = applyOrientationToPreviewBytes(thm, orientStr)
|
||||
}
|
||||
|
||||
return thm, wrappedReader, nil
|
||||
}
|
||||
|
||||
// CreateThumbnail decodes an image and creates a fixed-size thumbnail.
|
||||
|
||||
@@ -26,6 +26,7 @@ func TestMain(m *testing.M) {
|
||||
if fileutils.PermDir == 0 {
|
||||
fileutils.SetFsPermissions(0644, 0755)
|
||||
}
|
||||
configureImagingParallelism()
|
||||
|
||||
// Run the tests
|
||||
code := m.Run()
|
||||
@@ -549,3 +550,41 @@ func TestReadImageFileDimensions(t *testing.T) {
|
||||
require.Greater(t, width, 0)
|
||||
require.Greater(t, height, 0)
|
||||
}
|
||||
|
||||
func TestApplyOrientationToPreviewBytes(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// Generate a 100x60 test JPEG image in memory (landscape)
|
||||
img := image.NewRGBA(image.Rect(0, 0, 100, 60))
|
||||
var buf bytes.Buffer
|
||||
err := jpeg.Encode(&buf, img, nil)
|
||||
require.NoError(t, err)
|
||||
rawBytes := buf.Bytes()
|
||||
|
||||
tests := []struct {
|
||||
orientation string
|
||||
wantWidth int
|
||||
wantHeight int
|
||||
}{
|
||||
{orientation: "1", wantWidth: 100, wantHeight: 60},
|
||||
{orientation: "Horizontal (normal)", wantWidth: 100, wantHeight: 60},
|
||||
{orientation: "8", wantWidth: 60, wantHeight: 100},
|
||||
{orientation: "Rotate 270 CW", wantWidth: 60, wantHeight: 100},
|
||||
{orientation: "6", wantWidth: 60, wantHeight: 100},
|
||||
{orientation: "Rotate 90 CW", wantWidth: 60, wantHeight: 100},
|
||||
{orientation: "3", wantWidth: 100, wantHeight: 60},
|
||||
{orientation: "Rotate 180", wantWidth: 100, wantHeight: 60},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.orientation, func(t *testing.T) {
|
||||
rotated := applyOrientationToPreviewBytes(rawBytes, tt.orientation)
|
||||
require.NotEmpty(t, rotated)
|
||||
|
||||
cfg, _, decodeErr := image.DecodeConfig(bytes.NewReader(rotated))
|
||||
require.NoError(t, decodeErr)
|
||||
require.Equal(t, tt.wantWidth, cfg.Width)
|
||||
require.Equal(t, tt.wantHeight, cfg.Height)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -64,7 +64,7 @@ func NewPreviewGenerator(concurrencyLimit int, cacheDir string) *Service {
|
||||
var fileCache diskcache.Interface
|
||||
// Use file cache if cacheDir is specified
|
||||
var err error
|
||||
fileCache, err = diskcache.NewFileCache(actualCacheDir)
|
||||
fileCache, err = diskcache.NewFileCacheWithLimit(actualCacheDir, max(settings.Config.Server.CacheDirMaxSizeMB, 0)*1024*1024)
|
||||
if err != nil {
|
||||
logger.Error("The cache dir could not be created. Make sure the user that you executed the program with has access to create directories in the local path. See ")
|
||||
logger.Fatalf("failed to create file cache path, which is now require to run the server: %v", err)
|
||||
@@ -272,8 +272,9 @@ func determinePreviewType(file iteminfo.ExtendedFileInfo) filePreviewType {
|
||||
return previewTypeOffice
|
||||
}
|
||||
|
||||
// Check HEIC with conversion enabled
|
||||
if strings.HasPrefix(file.Type, "image/heic") &&
|
||||
// Check HEIC with conversion enabled. AVIF is a HEIF container too and uses
|
||||
// the same path (go-media decode, ffmpeg fallback) under the HEIC setting.
|
||||
if (strings.HasPrefix(file.Type, "image/heic") || file.Type == "image/avif") &&
|
||||
settings.CanConvertImage(string(settings.HEICImagePreview)) {
|
||||
return previewTypeHEIC
|
||||
}
|
||||
@@ -503,7 +504,9 @@ func GeneratePreviewWithMD5(ctx context.Context, file iteminfo.ExtendedFileInfo,
|
||||
}
|
||||
|
||||
// Enforce file size limit for image preview generation to prevent memory exhaustion
|
||||
if strings.HasPrefix(file.Type, "image") && file.Size > iteminfo.LargeFileSizeThreshold {
|
||||
// Camera RAW is exempt: its preview is the embedded JPEG and the RAW data is never decoded.
|
||||
if strings.HasPrefix(file.Type, "image") && file.Size > iteminfo.LargeFileSizeThreshold &&
|
||||
!iteminfo.IsRawImage(strings.ToLower(filepath.Ext(file.Name))) {
|
||||
message := fmt.Sprintf("Image file too large for preview: %s (size: %d bytes, limit: %d bytes)",
|
||||
file.Name, file.Size, iteminfo.LargeFileSizeThreshold)
|
||||
logger.Warning(message)
|
||||
|
||||
@@ -0,0 +1,49 @@
|
||||
package preview
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/gtsteffaniak/filebrowser/backend/pkg/indexing/iteminfo"
|
||||
"github.com/gtsteffaniak/filebrowser/backend/pkg/settings"
|
||||
)
|
||||
|
||||
// withHEICConversion sets ffmpeg availability and the HEIC preview setting for
|
||||
// one test.
|
||||
func withHEICConversion(t *testing.T, enabled bool) {
|
||||
t.Helper()
|
||||
prevAvail := settings.Env.FFmpegAvailable
|
||||
prevPreview := settings.Config.Integrations.Media.Convert.ImagePreview
|
||||
t.Cleanup(func() {
|
||||
settings.Env.FFmpegAvailable = prevAvail
|
||||
settings.Config.Integrations.Media.Convert.ImagePreview = prevPreview
|
||||
})
|
||||
settings.Env.FFmpegAvailable = true
|
||||
settings.Config.Integrations.Media.Convert.ImagePreview = map[settings.ImagePreviewType]*bool{
|
||||
settings.HEICImagePreview: &enabled,
|
||||
}
|
||||
}
|
||||
|
||||
// previewFile returns a file info with only a name and MIME type set.
|
||||
func previewFile(name, mimeType string) iteminfo.ExtendedFileInfo {
|
||||
var f iteminfo.ExtendedFileInfo
|
||||
f.Name = name
|
||||
f.Type = mimeType
|
||||
return f
|
||||
}
|
||||
|
||||
// TestDeterminePreviewTypeAVIF checks that AVIF uses the HEIF decode path when
|
||||
// HEIC conversion is enabled, and the plain image path otherwise.
|
||||
func TestDeterminePreviewTypeAVIF(t *testing.T) {
|
||||
withHEICConversion(t, true)
|
||||
if got := determinePreviewType(previewFile("a.avif", "image/avif")); got != previewTypeHEIC {
|
||||
t.Fatalf("avif with HEIC conversion = %v, want previewTypeHEIC", got)
|
||||
}
|
||||
if got := determinePreviewType(previewFile("a.heic", "image/heic")); got != previewTypeHEIC {
|
||||
t.Fatalf("heic with HEIC conversion = %v, want previewTypeHEIC", got)
|
||||
}
|
||||
|
||||
withHEICConversion(t, false)
|
||||
if got := determinePreviewType(previewFile("a.avif", "image/avif")); got != previewTypeImage {
|
||||
t.Fatalf("avif without HEIC conversion = %v, want previewTypeImage", got)
|
||||
}
|
||||
}
|
||||
@@ -28,8 +28,11 @@ func (s *Service) GenerateVideoPreview(ctx context.Context, videoPath, previewSi
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// go-media fast path: fit within the box, no crop. Falls back to ffmpeg when
|
||||
// unsupported (e.g. HEVC, VP9, AV1) or failed.
|
||||
// go-media fast path: fit within the box, no crop. go-media decodes H.264,
|
||||
// HEVC, VP8, VP9, AV1, MPEG-4 Part 2, and WMV3 keyframes in pure Go. It falls
|
||||
// back to ffmpeg when the container is not a go-media target (AVI, MPEG-TS,
|
||||
// MPEG-PS, ...), the codec or profile is unsupported (WMV1/WMV2, VC-1
|
||||
// Advanced, AV1 above 4K, ...), or the decode fails.
|
||||
// In shadow mode (dev only) the ffmpeg preview also runs, for comparison.
|
||||
// The shadow run can outlive the request, so it uses a context that is not cancelled with it.
|
||||
shadowCtx := context.WithoutCancel(ctx)
|
||||
|
||||
@@ -19,6 +19,15 @@ func AccessPermitted(sourcePath string, indexPath utils.IndexPath, username stri
|
||||
return accessDb.Permitted(sourcePath, indexPath, username)
|
||||
}
|
||||
|
||||
// AccessPermittedAtRealPath reports whether username may access realPath after symlink
|
||||
// resolution on the given index. When accessDb is not initialized, access is allowed.
|
||||
func AccessPermittedAtRealPath(idx *indexing.Index, username, realPath string, isDir bool) bool {
|
||||
if accessDb == nil {
|
||||
return true
|
||||
}
|
||||
return accessDb.PermittedAtRealPath(idx, username, realPath, isDir)
|
||||
}
|
||||
|
||||
func AllowUser(sourcePath string, indexPath utils.IndexPath, username string) error {
|
||||
return accessDb.AllowUser(sourcePath, indexPath, username)
|
||||
}
|
||||
|
||||
@@ -14,14 +14,21 @@ const (
|
||||
|
||||
var (
|
||||
mediaSettingsMu sync.RWMutex
|
||||
mediaSettingsDefault settings.MediaSettings
|
||||
mediaSettingsDefault settings.FullAdminMedia
|
||||
)
|
||||
|
||||
// saveMediaSettings is the only write path for media settings. It clears derived
|
||||
// fields (EnabledProfiles, ImagePreview) so read-time values are never persisted.
|
||||
func saveMediaSettings(key string, ms settings.FullAdminMedia) error {
|
||||
settings.ClearMediaDerived(&ms)
|
||||
return sqlDb.SaveSetting(key, ms)
|
||||
}
|
||||
|
||||
// InitMediaSettings loads persisted runtime media settings from SQLite and seeds from config when missing.
|
||||
func InitMediaSettings() error {
|
||||
if _, err := sqlDb.GetSetting(mediaSettingsDefaultKey); err != nil {
|
||||
seed := settings.SanitizeMediaSettings(settings.CurrentMediaSettings())
|
||||
if saveErr := sqlDb.SaveSetting(mediaSettingsDefaultKey, seed); saveErr != nil {
|
||||
if saveErr := saveMediaSettings(mediaSettingsDefaultKey, seed); saveErr != nil {
|
||||
return fmt.Errorf("seed media settings: %w", saveErr)
|
||||
}
|
||||
}
|
||||
@@ -43,7 +50,7 @@ func InitMediaSettings() error {
|
||||
// sanitize, so clamps (e.g. the readRate floor) must run again.
|
||||
overlaid = settings.SanitizeMediaSettings(overlaid)
|
||||
if !settings.MediaSettingsEqual(loaded, overlaid) {
|
||||
if saveErr := sqlDb.SaveSetting(mediaSettingsDefaultKey, overlaid); saveErr != nil {
|
||||
if saveErr := saveMediaSettings(mediaSettingsDefaultKey, overlaid); saveErr != nil {
|
||||
return fmt.Errorf("save config-synced media settings: %w", saveErr)
|
||||
}
|
||||
loaded = overlaid
|
||||
@@ -57,22 +64,22 @@ func InitMediaSettings() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func loadMediaSettingsSetting(key string) (settings.MediaSettings, error) {
|
||||
func loadMediaSettingsSetting(key string) (settings.FullAdminMedia, error) {
|
||||
raw, err := sqlDb.GetSetting(key)
|
||||
if err != nil {
|
||||
return settings.MediaSettings{}, err
|
||||
return settings.FullAdminMedia{}, err
|
||||
}
|
||||
var ms settings.MediaSettings
|
||||
var ms settings.FullAdminMedia
|
||||
if err := json.Unmarshal(raw, &ms); err != nil {
|
||||
return settings.MediaSettings{}, fmt.Errorf("parse %s: %w", key, err)
|
||||
return settings.FullAdminMedia{}, fmt.Errorf("parse %s: %w", key, err)
|
||||
}
|
||||
return ms, nil
|
||||
}
|
||||
|
||||
// GetMediaSettings returns the runtime media settings. The value is deep-copied
|
||||
// GetMediaSettings returns the stored runtime media settings. The value is deep-copied
|
||||
// so callers cannot mutate the stored settings (or the live config they are
|
||||
// applied to) through the conversion maps.
|
||||
func GetMediaSettings() settings.MediaSettings {
|
||||
func GetMediaSettings() settings.FullAdminMedia {
|
||||
mediaSettingsMu.RLock()
|
||||
defer mediaSettingsMu.RUnlock()
|
||||
return settings.CloneMediaSettings(mediaSettingsDefault)
|
||||
@@ -90,8 +97,11 @@ func PatchMediaSettings(patchJSON []byte) error {
|
||||
return mergeErr
|
||||
}
|
||||
merged = settings.SanitizeMediaSettings(merged)
|
||||
// A patch may echo derived fields back (e.g. imagePreview from the admin UI).
|
||||
// Clear them before the value reaches memory or the database.
|
||||
settings.ClearMediaDerived(&merged)
|
||||
|
||||
if saveErr := sqlDb.SaveSetting(mediaSettingsDefaultKey, merged); saveErr != nil {
|
||||
if saveErr := saveMediaSettings(mediaSettingsDefaultKey, merged); saveErr != nil {
|
||||
mediaSettingsMu.Unlock()
|
||||
return fmt.Errorf("save media settings: %w", saveErr)
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package state
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
|
||||
"github.com/gtsteffaniak/filebrowser/backend/pkg/settings"
|
||||
@@ -34,6 +35,56 @@ func TestGetMediaSettingsReturnsIndependentCopy(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// Derived read-time values must never reach the database, even when a caller
|
||||
// passes a struct that has them populated.
|
||||
func TestSaveMediaSettingsNeverPersistsDerivedFields(t *testing.T) {
|
||||
initSidebarLinkTestDB(t)
|
||||
|
||||
ms := GetMediaSettings()
|
||||
ms.Transcode.EnabledProfiles = []string{"quality"}
|
||||
ms.ImagePreview = map[settings.ImagePreviewType]bool{settings.JPEGImagePreview: true}
|
||||
if err := saveMediaSettings(mediaSettingsDefaultKey, ms); err != nil {
|
||||
t.Fatalf("saveMediaSettings: %v", err)
|
||||
}
|
||||
|
||||
raw, err := sqlDb.GetSetting(mediaSettingsDefaultKey)
|
||||
if err != nil {
|
||||
t.Fatalf("GetSetting: %v", err)
|
||||
}
|
||||
var stored map[string]any
|
||||
if err := json.Unmarshal(raw, &stored); err != nil {
|
||||
t.Fatalf("decode stored settings: %v", err)
|
||||
}
|
||||
transcode, ok := stored["transcode"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("stored transcode = %T, want object", stored["transcode"])
|
||||
}
|
||||
if _, ok := transcode["enabledProfiles"]; ok {
|
||||
t.Fatal("enabledProfiles (derived) was persisted")
|
||||
}
|
||||
if _, ok := stored["imagePreview"]; ok {
|
||||
t.Fatal("imagePreview (derived) was persisted")
|
||||
}
|
||||
}
|
||||
|
||||
// The admin UI echoes its whole payload back on PATCH, derived fields included.
|
||||
// Those must be dropped before the value reaches memory, not only before the database.
|
||||
func TestPatchMediaSettingsDropsEchoedDerivedFields(t *testing.T) {
|
||||
initSidebarLinkTestDB(t)
|
||||
|
||||
patch := []byte(`{"imagePreview":{"jpeg":true},"transcode":{"enabledProfiles":["quality"]}}`)
|
||||
if err := PatchMediaSettings(patch); err != nil {
|
||||
t.Fatalf("PatchMediaSettings: %v", err)
|
||||
}
|
||||
got := GetMediaSettings()
|
||||
if got.ImagePreview != nil {
|
||||
t.Fatal("echoed imagePreview stayed in memory")
|
||||
}
|
||||
if got.Transcode.EnabledProfiles != nil {
|
||||
t.Fatal("echoed enabledProfiles stayed in memory")
|
||||
}
|
||||
}
|
||||
|
||||
// A fresh install seeds the stored media settings from config.
|
||||
func TestMediaSettingsSeedFromConfigOnFirstInit(t *testing.T) {
|
||||
initSidebarLinkTestDB(t)
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
@@ -297,6 +298,17 @@ func ApplyUserDefaults(u *users.User) {
|
||||
applyToolAccessDefaults(u)
|
||||
}
|
||||
|
||||
// PrefillUserProfileDefaults seeds u with the effective profile defaults so a JSON decode
|
||||
// overlays only the keys the request sent (encoding/json leaves absent keys untouched).
|
||||
// Permissions are left zero so omitted permissions stay distinguishable; CreateUser merges them.
|
||||
func PrefillUserProfileDefaults(u *users.User) {
|
||||
settings.ApplyFullProfileFromDefaults(u, EffectiveUserDefaults())
|
||||
// json.Unmarshal reuses an existing slice's backing array, so clone it or a request
|
||||
// would overwrite the shared in-memory defaults.
|
||||
u.NewFileTemplate = slices.Clone(u.NewFileTemplate)
|
||||
u.Permissions = users.Permissions{}
|
||||
}
|
||||
|
||||
func applyUserSettingsDefaults(u *users.User) {
|
||||
settings.ApplyUserDefaultsFrom(u, EffectiveUserDefaults())
|
||||
}
|
||||
|
||||
@@ -90,7 +90,7 @@ func TestStateInit_jwtYAML_organizedDefaultsOnNewUser(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !loaded.DarkMode || !loaded.Preview.Image {
|
||||
if !loaded.DarkMode.IsDark() || !loaded.Preview.Image {
|
||||
t.Fatalf("profile fields: darkMode=%v preview.image=%v", loaded.DarkMode, loaded.Preview.Image)
|
||||
}
|
||||
if loaded.BackendScopes[0].Permissions.Modify != access.Modify {
|
||||
|
||||
@@ -202,6 +202,18 @@ func UserForShareOwner(link share.Share) (users.User, error) {
|
||||
// may apply default-enabled sources. In-memory/cache users always keep FrontendScopes nil; GET handlers use
|
||||
// PrepForFrontend to repopulate FrontendScopes from BackendScopes for responses.
|
||||
func CreateUser(user *users.User, plaintextPassword string) error {
|
||||
return createUser(user, plaintextPassword, false)
|
||||
}
|
||||
|
||||
// CreateUserKeepingProfile is CreateUser for callers whose user was seeded with
|
||||
// PrefillUserProfileDefaults before decoding the request: the profile fields already hold
|
||||
// defaults overlaid with the request, so they are not replaced by userDefaults again.
|
||||
// Enforced defaults still win.
|
||||
func CreateUserKeepingProfile(user *users.User, plaintextPassword string) error {
|
||||
return createUser(user, plaintextPassword, true)
|
||||
}
|
||||
|
||||
func createUser(user *users.User, plaintextPassword string, keepProfile bool) error {
|
||||
if err := users.ValidateUsername(user.Username); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -256,12 +268,19 @@ func CreateUser(user *users.User, plaintextPassword string) error {
|
||||
requestedScopePermissions[backendScopeKeyFor(scope)] = scope.Permissions
|
||||
}
|
||||
}
|
||||
var requestedProfile settings.UserProfile
|
||||
if keepProfile {
|
||||
requestedProfile = settings.ProfileFromUser(user)
|
||||
}
|
||||
// Apply profile/scope defaults using the final account permissions (admin flag drives scope defaults).
|
||||
if permissionsRequested {
|
||||
settings.ApplyUserDefaultsFromKeepingAccountPermissions(user, EffectiveUserDefaults())
|
||||
} else {
|
||||
applyUserSettingsDefaults(user)
|
||||
}
|
||||
if keepProfile {
|
||||
settings.ExpandProfileIntoUser(user, requestedProfile)
|
||||
}
|
||||
if preserveRequirePasswordChange {
|
||||
user.RequirePasswordChange = true
|
||||
}
|
||||
|
||||
@@ -133,7 +133,7 @@ func (e *MediaEngine) BuildPlan(ctx context.Context, realPath string, profile Pr
|
||||
copyImpossible := false
|
||||
plan.segmentSec = defaults.SegmentDurationSec
|
||||
if plan.remux || plan.videoCopy {
|
||||
rawKeyframes, kfErr := probeKeyframesCached(ctx, svc, realPath, stat.Size(), stat.ModTime().UnixNano(), plan.segmentSec, true)
|
||||
rawKeyframes, kfErr := probeKeyframesCached(ctx, svc, realPath, stat.Size(), stat.ModTime().UnixNano(), plan.segmentSec, mediaSpan{durationSec: info.Duration, startTimeSec: info.StartTime}, true)
|
||||
if kfErr != nil {
|
||||
copyImpossible = true
|
||||
logger.Warningf("transcode: keyframe probe failed on %s (%v) — forcing video transcode for HLS", realPath, kfErr)
|
||||
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/ffmpeg"
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/gomedia"
|
||||
"github.com/gtsteffaniak/filebrowser/backend/pkg/settings"
|
||||
"github.com/gtsteffaniak/go-logger/logger"
|
||||
)
|
||||
@@ -55,6 +56,9 @@ type keyframeCache struct {
|
||||
|
||||
var defaultKeyframeCache = &keyframeCache{entries: map[keyframeKey]*keyframeEntry{}}
|
||||
|
||||
// get returns the keyframe timeline for key from memory, then disk, then
|
||||
// probe. Concurrent callers for one key wait on a single probe; failures are
|
||||
// not cached.
|
||||
func (c *keyframeCache) get(ctx context.Context, key keyframeKey, probe func(context.Context) ([]float64, error)) ([]float64, error) {
|
||||
c.mu.Lock()
|
||||
e := c.entries[key]
|
||||
@@ -105,12 +109,15 @@ func keyframeDiskCacheEnabled() bool {
|
||||
return settings.Config.Server.CacheDir != ""
|
||||
}
|
||||
|
||||
// keyframeDiskPath returns the disk cache file for key's path.
|
||||
func keyframeDiskPath(key keyframeKey) string {
|
||||
sum := sha256.Sum256([]byte(key.path))
|
||||
name := hex.EncodeToString(sum[:])[:32] + ".json"
|
||||
return filepath.Join(settings.TranscodeKeyframeCacheDir(), name)
|
||||
}
|
||||
|
||||
// readKeyframeDiskCache returns the persisted timeline for key when it exists
|
||||
// and matches the file's size, mtime, and the cache version.
|
||||
func readKeyframeDiskCache(key keyframeKey) ([]float64, bool) {
|
||||
if !keyframeDiskCacheEnabled() {
|
||||
return nil, false
|
||||
@@ -132,6 +139,8 @@ func readKeyframeDiskCache(key keyframeKey) ([]float64, bool) {
|
||||
return stamp.Times, true
|
||||
}
|
||||
|
||||
// writeKeyframeDiskCache persists a timeline for key atomically. Errors are
|
||||
// ignored; the cache is an optimization.
|
||||
func writeKeyframeDiskCache(key keyframeKey, times []float64) {
|
||||
if len(times) == 0 || !keyframeDiskCacheEnabled() {
|
||||
return
|
||||
@@ -219,19 +228,88 @@ func probeKeyframesWithPreflight(ctx context.Context, svc *ffmpeg.Service, realP
|
||||
)
|
||||
}
|
||||
|
||||
// mediaSpan is the ffprobe format duration and start time of a file, the
|
||||
// values the HLS planner turns into its media timeline. The zero value means
|
||||
// unknown.
|
||||
type mediaSpan struct {
|
||||
durationSec float64
|
||||
startTimeSec float64
|
||||
}
|
||||
|
||||
// trailingGapExceedsCopyCap reports whether the stretch from the last
|
||||
// keyframe to the end of the media is longer than the copy cap, measured on
|
||||
// the same normalized timeline the planner builds: the final segment runs
|
||||
// from the last keyframe to the media end, so such a gap forces a transcode
|
||||
// just like a gap between two keyframes. An unknown span reports false.
|
||||
func trailingGapExceedsCopyCap(keyframes []float64, span mediaSpan, segmentSec float64) bool {
|
||||
if len(keyframes) == 0 || span.durationSec <= 0 {
|
||||
return false
|
||||
}
|
||||
kf, mediaDur := ffmpeg.PrepareHLSMediaTimeline(keyframes, span.durationSec, span.startTimeSec)
|
||||
if len(kf) == 0 || mediaDur <= 0 {
|
||||
return false
|
||||
}
|
||||
return mediaDur-kf[len(kf)-1] > maxCopySegmentDurationSec(segmentSec)
|
||||
}
|
||||
|
||||
// keyframeProbeTimeout bounds a single full-file keyframe probe. The default
|
||||
// demux has no deadline of its own, so a slow network share would otherwise
|
||||
// stall plan building (and the client's session POST) indefinitely.
|
||||
const keyframeProbeTimeout = 2 * time.Minute
|
||||
|
||||
func probeKeyframesCached(ctx context.Context, svc *ffmpeg.Service, realPath string, size, modTime int64, segmentSec float64, preflight bool) ([]float64, error) {
|
||||
// probeKeyframesCached returns the keyframe timeline of a file version from
|
||||
// the memory or disk cache, or probes it: go-media reads the container index
|
||||
// first, and ffprobe runs only when go-media cannot answer or reports a gap
|
||||
// that would rule out stream copy. span is the file's duration and start
|
||||
// time, used to check the gap after the last keyframe; the zero value skips
|
||||
// that check.
|
||||
func probeKeyframesCached(ctx context.Context, svc *ffmpeg.Service, realPath string, size, modTime int64, segmentSec float64, span mediaSpan, preflight bool) ([]float64, error) {
|
||||
key := keyframeKey{path: realPath, size: size, modTime: modTime}
|
||||
return defaultKeyframeCache.get(ctx, key, func(ctx context.Context) ([]float64, error) {
|
||||
probeCtx, cancel := context.WithTimeout(ctx, keyframeProbeTimeout)
|
||||
defer cancel()
|
||||
if times, ok := gomediaKeyframes(probeCtx, svc, realPath, segmentSec, span, preflight); ok {
|
||||
return times, nil
|
||||
}
|
||||
if preflight {
|
||||
return probeKeyframesWithPreflight(probeCtx, svc, realPath, segmentSec)
|
||||
}
|
||||
return svc.ProbeVideoKeyframeTimes(probeCtx, realPath)
|
||||
})
|
||||
}
|
||||
|
||||
// gomediaKeyframes reads the keyframe timeline from the container index with
|
||||
// go-media. It reports ok=false when go-media cannot answer, and also when
|
||||
// preflight is set and the timeline has a gap past the copy cap, including
|
||||
// the gap from the last keyframe to the end of span: go-media can miss
|
||||
// keyframes that ffprobe's parsers flag (IDR frames not in the index,
|
||||
// partial Matroska Cues) but never adds one, so only ffprobe may decide that
|
||||
// stream copy is impossible. A bounded go-media read runs first, so a file
|
||||
// whose head already shows such a gap does not pay for a full index walk.
|
||||
func gomediaKeyframes(ctx context.Context, svc *ffmpeg.Service, realPath string, segmentSec float64, span mediaSpan, preflight bool) ([]float64, bool) {
|
||||
if preflight {
|
||||
if head, ok := gomedia.KeyframeTimesWithin(ctx, realPath, keyframeBoundedPreflightWindowSec); ok && keyframesExceedCopyCap(head, segmentSec) {
|
||||
logger.Debugf("transcode: go-media keyframe pre-flight shows a gap past the copy cap on %s; confirming with ffprobe", realPath)
|
||||
return nil, false
|
||||
}
|
||||
}
|
||||
legacy := func() ([]float64, bool) {
|
||||
shadowCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), keyframeProbeTimeout)
|
||||
defer cancel()
|
||||
times, err := svc.ProbeVideoKeyframeTimes(shadowCtx, realPath)
|
||||
return times, err == nil
|
||||
}
|
||||
times, ok := gomedia.KeyframeTimes(ctx, realPath, legacy)
|
||||
if !ok {
|
||||
return nil, false
|
||||
}
|
||||
if preflight && keyframesExceedCopyCap(times, segmentSec) {
|
||||
logger.Debugf("transcode: go-media keyframe index shows a gap past the copy cap on %s; confirming with ffprobe", realPath)
|
||||
return nil, false
|
||||
}
|
||||
if preflight && trailingGapExceedsCopyCap(times, span, segmentSec) {
|
||||
logger.Debugf("transcode: go-media keyframe index ends more than the copy cap before the end of %s; confirming with ffprobe", realPath)
|
||||
return nil, false
|
||||
}
|
||||
return times, true
|
||||
}
|
||||
@@ -177,3 +177,33 @@ func TestKeyframeCacheDoesNotCacheFailures(t *testing.T) {
|
||||
t.Fatalf("retry after failure: %v, %v", got, err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestTrailingGapExceedsCopyCap checks the gap from the last keyframe to the
|
||||
// end of the media, on the planner's start-time-normalized timeline.
|
||||
func TestTrailingGapExceedsCopyCap(t *testing.T) {
|
||||
every2 := func(from, to float64) []float64 {
|
||||
var out []float64
|
||||
for v := from; v < to; v += 2 {
|
||||
out = append(out, v)
|
||||
}
|
||||
return out
|
||||
}
|
||||
cases := []struct {
|
||||
name string
|
||||
kf []float64
|
||||
span mediaSpan
|
||||
want bool
|
||||
}{
|
||||
{"partial index", []float64{0, 2}, mediaSpan{durationSec: 60}, true},
|
||||
{"full index", every2(0, 60), mediaSpan{durationSec: 60}, false},
|
||||
{"unknown span", []float64{0, 2}, mediaSpan{}, false},
|
||||
{"start time, partial", []float64{10, 12}, mediaSpan{durationSec: 70, startTimeSec: 10}, true},
|
||||
{"start time, full", every2(10, 70), mediaSpan{durationSec: 70, startTimeSec: 10}, false},
|
||||
{"no keyframes", nil, mediaSpan{durationSec: 60}, false},
|
||||
}
|
||||
for _, c := range cases {
|
||||
if got := trailingGapExceedsCopyCap(c.kf, c.span, 4); got != c.want {
|
||||
t.Errorf("%s: got %v, want %v", c.name, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,111 @@
|
||||
package transcoding
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/gtsteffaniak/filebrowser/backend/pkg/settings"
|
||||
)
|
||||
|
||||
const keyframeFixtureMP4 = "../../../frontend/tests/playwright-files/myfolder/testdata/sample-640x480.mp4"
|
||||
|
||||
// TestProbeKeyframesCachedUsesGoMedia checks that an MP4 keyframe timeline is
|
||||
// built without ffprobe: with no ffmpeg service, only go-media can answer.
|
||||
func TestProbeKeyframesCachedUsesGoMedia(t *testing.T) {
|
||||
if settings.AlwaysUseFfmpegEnabled() {
|
||||
t.Skip("go-media disabled by settings")
|
||||
}
|
||||
prevCacheDir := settings.Config.Server.CacheDir
|
||||
settings.Config.Server.CacheDir = "" // keep the disk cache out of the test
|
||||
t.Cleanup(func() { settings.Config.Server.CacheDir = prevCacheDir })
|
||||
|
||||
path := filepath.Join(t.TempDir(), "sample.mp4")
|
||||
data, err := os.ReadFile(keyframeFixtureMP4)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = os.WriteFile(path, data, 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
st, err := os.Stat(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
times, err := probeKeyframesCached(context.Background(), nil, path, st.Size(), st.ModTime().UnixNano(), 4, mediaSpan{}, true)
|
||||
if err != nil {
|
||||
t.Fatalf("probeKeyframesCached: %v", err)
|
||||
}
|
||||
if len(times) == 0 || times[0] < 0 {
|
||||
t.Fatalf("unexpected keyframes %v", times)
|
||||
}
|
||||
for i := 1; i < len(times); i++ {
|
||||
if times[i] <= times[i-1] {
|
||||
t.Fatalf("keyframes not strictly increasing: %v", times)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestGomediaKeyframesDefersCopyDecision checks that a go-media timeline with a
|
||||
// gap past the copy cap is not used for the pre-flight decision (ffprobe must
|
||||
// confirm it), while a plain probe still returns it.
|
||||
func TestGomediaKeyframesDefersCopyDecision(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("encodes a clip with ffmpeg")
|
||||
}
|
||||
if settings.AlwaysUseFfmpegEnabled() {
|
||||
t.Skip("go-media disabled by settings")
|
||||
}
|
||||
if _, err := exec.LookPath("ffmpeg"); err != nil {
|
||||
t.Skip("ffmpeg not installed")
|
||||
}
|
||||
path := filepath.Join(t.TempDir(), "longgop.mp4")
|
||||
out, err := exec.Command("ffmpeg", "-v", "error", "-y", "-f", "lavfi", "-i", "testsrc2=size=128x96:rate=25:duration=40",
|
||||
"-c:v", "libx264", "-g", "750", "-keyint_min", "750", "-sc_threshold", "0", "-pix_fmt", "yuv420p", path).CombinedOutput()
|
||||
if err != nil {
|
||||
t.Skipf("ffmpeg cannot encode libx264: %v: %s", err, out)
|
||||
}
|
||||
|
||||
if times, ok := gomediaKeyframes(context.Background(), nil, path, 4, mediaSpan{}, true); ok {
|
||||
t.Fatalf("pre-flight must defer a 30s GOP to ffprobe, got %v", times)
|
||||
}
|
||||
times, ok := gomediaKeyframes(context.Background(), nil, path, 4, mediaSpan{}, false)
|
||||
if !ok {
|
||||
t.Fatal("a plain probe must return the go-media timeline")
|
||||
}
|
||||
if len(times) != 2 || times[0] != 0 || times[1] < 29.9 || times[1] > 30.1 {
|
||||
t.Fatalf("unexpected keyframes %v, want [0 30]", times)
|
||||
}
|
||||
}
|
||||
|
||||
// TestGomediaKeyframesDefersTrailingGap checks that a go-media index whose
|
||||
// last keyframe is more than the copy cap before the end of the media is not
|
||||
// used for the pre-flight decision, so ffprobe can confirm it, while the
|
||||
// same index is used when it reaches the end.
|
||||
func TestGomediaKeyframesDefersTrailingGap(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("encodes a clip with ffmpeg")
|
||||
}
|
||||
if settings.AlwaysUseFfmpegEnabled() {
|
||||
t.Skip("go-media disabled by settings")
|
||||
}
|
||||
if _, err := exec.LookPath("ffmpeg"); err != nil {
|
||||
t.Skip("ffmpeg not installed")
|
||||
}
|
||||
path := filepath.Join(t.TempDir(), "short.mp4")
|
||||
out, err := exec.Command("ffmpeg", "-v", "error", "-y", "-f", "lavfi", "-i", "testsrc2=size=128x96:rate=25:duration=4",
|
||||
"-c:v", "libx264", "-g", "50", "-keyint_min", "50", "-sc_threshold", "0", "-pix_fmt", "yuv420p", path).CombinedOutput()
|
||||
if err != nil {
|
||||
t.Skipf("ffmpeg cannot encode libx264: %v: %s", err, out)
|
||||
}
|
||||
// The index ends at 2 s; a 60 s span leaves a 58 s final segment.
|
||||
if times, ok := gomediaKeyframes(context.Background(), nil, path, 4, mediaSpan{durationSec: 60}, true); ok {
|
||||
t.Fatalf("pre-flight must defer a 58s trailing gap to ffprobe, got %v", times)
|
||||
}
|
||||
times, ok := gomediaKeyframes(context.Background(), nil, path, 4, mediaSpan{durationSec: 4}, true)
|
||||
if !ok || len(times) != 2 {
|
||||
t.Fatalf("index reaching the end must be used, got %v, %v", times, ok)
|
||||
}
|
||||
}
|
||||
@@ -1098,14 +1098,14 @@ func TestManagerSuspendsProducerWhenAllClientsPaused(t *testing.T) {
|
||||
in := testInput(t, "pauser", "/v.mp4")
|
||||
sess := mustStart(t, mgr, in)
|
||||
waitForStarts(t, engine, 1)
|
||||
job := engine.LastJob()
|
||||
|
||||
if err := mgr.Heartbeat(in.Username, sess.ID, HeartbeatRequest{ClientID: "c1", Paused: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waitForProducerVODJobAttached(t, mgr, sess.ID)
|
||||
suspendPausedProducers(t, mgr, time.Now().Add(time.Hour))
|
||||
|
||||
pollUntil(t, 2*time.Second, func() bool { return !job.Running() }, "suspended job to stop")
|
||||
waitForProducerParked(t, mgr, sess.ID)
|
||||
if snap := snapshotProducer(t, mgr, sess.ID); !snap.exists || !snap.suspended {
|
||||
t.Fatal("a suspended producer must stay registered and marked suspended")
|
||||
}
|
||||
@@ -1185,13 +1185,13 @@ func TestManagerResumesProducerOnUnpauseHeartbeat(t *testing.T) {
|
||||
pollUntil(t, 2*time.Second, func() bool {
|
||||
return snapshotProducer(t, mgr, sess.ID).encodeHead >= 1
|
||||
}, "encode head to advance past segment 0")
|
||||
job := engine.LastJob()
|
||||
|
||||
if err := mgr.Heartbeat(in.Username, sess.ID, HeartbeatRequest{ClientID: "c1", Paused: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waitForProducerVODJobAttached(t, mgr, sess.ID)
|
||||
suspendPausedProducers(t, mgr, time.Now().Add(time.Hour))
|
||||
pollUntil(t, 2*time.Second, func() bool { return !job.Running() }, "suspended job to stop")
|
||||
waitForProducerParked(t, mgr, sess.ID)
|
||||
|
||||
if err := mgr.Heartbeat(in.Username, sess.ID, HeartbeatRequest{ClientID: "c1", Paused: false}); err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -1216,13 +1216,13 @@ func TestManagerResumesProducerOnSegmentRequest(t *testing.T) {
|
||||
in := testInput(t, "pauser", "/v.mp4")
|
||||
sess := mustStart(t, mgr, in)
|
||||
waitForStarts(t, engine, 1)
|
||||
job := engine.LastJob()
|
||||
|
||||
if err := mgr.Heartbeat(in.Username, sess.ID, HeartbeatRequest{ClientID: "c1", Paused: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waitForProducerVODJobAttached(t, mgr, sess.ID)
|
||||
suspendPausedProducers(t, mgr, time.Now().Add(time.Hour))
|
||||
pollUntil(t, 2*time.Second, func() bool { return !job.Running() }, "suspended job to stop")
|
||||
waitForProducerParked(t, mgr, sess.ID)
|
||||
|
||||
// The suspended job reads as !Running, so a request beyond its head takes
|
||||
// the restart path: the relaunched job produces the requested segment.
|
||||
@@ -1246,13 +1246,13 @@ func TestManagerStopReleasesSuspendedProducer(t *testing.T) {
|
||||
in := testInput(t, "pauser", "/v.mp4")
|
||||
sess := mustStart(t, mgr, in)
|
||||
waitForStarts(t, engine, 1)
|
||||
job := engine.LastJob()
|
||||
|
||||
if err := mgr.Heartbeat(in.Username, sess.ID, HeartbeatRequest{ClientID: "c1", Paused: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waitForProducerVODJobAttached(t, mgr, sess.ID)
|
||||
suspendPausedProducers(t, mgr, time.Now().Add(time.Hour))
|
||||
pollUntil(t, 2*time.Second, func() bool { return !job.Running() }, "suspended job to stop")
|
||||
waitForProducerParked(t, mgr, sess.ID)
|
||||
|
||||
if err := mgr.Stop(in.Username, sess.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -1308,6 +1308,29 @@ func mustStart(t *testing.T, mgr *Manager, in StartInput) SessionInfo {
|
||||
return info
|
||||
}
|
||||
|
||||
// waitForProducerVODJobAttached waits until the session's producer has the VOD
|
||||
// job wired up. StartCount can advance before runProducerJob assigns p.job.
|
||||
func waitForProducerVODJobAttached(t *testing.T, mgr *Manager, sessionID string) {
|
||||
t.Helper()
|
||||
pollUntil(t, 5*time.Second, func() bool {
|
||||
mgr.mu.Lock()
|
||||
p := mgr.producers[sessionID]
|
||||
ok := p != nil && p.job != nil
|
||||
mgr.mu.Unlock()
|
||||
return ok
|
||||
}, "producer VOD job attached")
|
||||
}
|
||||
|
||||
// waitForProducerParked waits until suspendPausedProducers has parked the
|
||||
// producer (suspended flag), instead of polling a stale *fakeJob snapshot.
|
||||
func waitForProducerParked(t *testing.T, mgr *Manager, sessionID string) {
|
||||
t.Helper()
|
||||
pollUntil(t, 5*time.Second, func() bool {
|
||||
snap := snapshotProducer(t, mgr, sessionID)
|
||||
return snap.exists && snap.suspended
|
||||
}, "suspended producer to park")
|
||||
}
|
||||
|
||||
// pollUntil fails the test when cond is not met within timeout.
|
||||
func pollUntil(t *testing.T, timeout time.Duration, cond func() bool, msg string) {
|
||||
t.Helper()
|
||||
|
||||
@@ -15,12 +15,18 @@ type SubtitleTrack struct {
|
||||
Embedded bool `json:"embedded"` // true for embedded subtitles, false for external files
|
||||
}
|
||||
|
||||
// ErrSubtitleSidecarNotRegular is returned when an external subtitle path is not a regular file (e.g. symlink).
|
||||
var ErrSubtitleSidecarNotRegular = errors.New("subtitle sidecar is not a regular file")
|
||||
|
||||
func GetSubtitleSidecarContent(subtitlePath string) (string, error) {
|
||||
// validate the size is not too large
|
||||
info, err := os.Stat(subtitlePath)
|
||||
// Sidecars sit beside the media file; do not follow symlink leaves (see lyricsSidecar).
|
||||
info, err := os.Lstat(subtitlePath)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if !info.Mode().IsRegular() {
|
||||
return "", ErrSubtitleSidecarNotRegular
|
||||
}
|
||||
if info.Size() > 1024*1024*50 { // 50MB
|
||||
return "", errors.New("subtitle file is too large")
|
||||
}
|
||||
|
||||
@@ -22,6 +22,7 @@ import (
|
||||
|
||||
func setupAccessHTTPTest(t *testing.T) {
|
||||
t.Helper()
|
||||
lockWebGlobals(t)
|
||||
dbPath := filepath.Join(t.TempDir(), "access-http-test.sqlite")
|
||||
if _, err := state.Initialize(dbPath); err != nil {
|
||||
t.Fatal(err)
|
||||
|
||||
@@ -279,19 +279,21 @@ func archiveCreateHandler(w http.ResponseWriter, r *http.Request, d *Context) (i
|
||||
if fullDestParent != fullDestFile && !state.AccessPermitted(idxTo.Path, utils.IndexPathFromNormalized(fullDestParent, true), d.User.Username) {
|
||||
return http.StatusForbidden, fmt.Errorf("access denied to destination parent of %s", req.Destination)
|
||||
}
|
||||
parentReal, err := idxTo.GetRealPathForMutation(userScopeTo, fullDestParent)
|
||||
if err != nil {
|
||||
return http.StatusBadRequest, fmt.Errorf("destination parent path invalid: %v", err)
|
||||
}
|
||||
if err = os.MkdirAll(parentReal, fileutils.EffectiveDirPerm()); err != nil {
|
||||
return http.StatusInternalServerError, fmt.Errorf("cannot create destination parent directory: %v", err)
|
||||
}
|
||||
destRel := strings.TrimLeft(filepath.ToSlash(req.Destination), "/")
|
||||
fileName := path.Base(destRel)
|
||||
if fileName == "." || fileName == "/" {
|
||||
return http.StatusBadRequest, fmt.Errorf("invalid destination file name")
|
||||
}
|
||||
destFileReal := filepath.Join(parentReal, fileName)
|
||||
destFileReal, err := idxTo.GetRealPathForMutation(userScopeTo, fullDestFile)
|
||||
if err != nil {
|
||||
return http.StatusBadRequest, fmt.Errorf("destination path invalid: %v", err)
|
||||
}
|
||||
if !state.AccessPermittedAtRealPath(idxTo, d.User.Username, destFileReal, false) {
|
||||
return http.StatusForbidden, fmt.Errorf("access denied to destination %s", req.Destination)
|
||||
}
|
||||
if err = os.MkdirAll(filepath.Dir(destFileReal), fileutils.EffectiveDirPerm()); err != nil {
|
||||
return http.StatusInternalServerError, fmt.Errorf("cannot create destination parent directory: %v", err)
|
||||
}
|
||||
|
||||
format := strings.ToLower(strings.TrimSpace(req.Format))
|
||||
if format == "" {
|
||||
@@ -1270,6 +1272,7 @@ func extractArchivedSymlink(destRoot, destPath, target string, modTime time.Time
|
||||
logger.Debugf("unarchive: symlink: %v", err)
|
||||
return fmt.Errorf("failed to create symlink from archive entry")
|
||||
}
|
||||
indexing.ClearRealPathCaches()
|
||||
return applyArchivedTimesAndPerm(destPath, fs.ModeSymlink, modTime)
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,86 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/state"
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/utils"
|
||||
"github.com/gtsteffaniak/filebrowser/backend/pkg/settings"
|
||||
)
|
||||
|
||||
func TestArchiveCreate_DeniesResolvedSymlinkDestination(t *testing.T) {
|
||||
_, sourceRoot, sourceName, username := setupExtractAccessPolicyTest(t)
|
||||
if err := state.DenyUser(sourceRoot, utils.IndexPathFromNormalized("/denied", true), username); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
allowedDir := filepath.Join(sourceRoot, "allowed")
|
||||
deniedDir := filepath.Join(sourceRoot, "denied")
|
||||
for _, dir := range []string{allowedDir, deniedDir} {
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
victim := filepath.Join(deniedDir, "victim")
|
||||
if err := os.WriteFile(victim, []byte("keep"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(sourceRoot, "a.txt"), []byte("payload"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
link := filepath.Join(allowedDir, "report.zip")
|
||||
if err := os.Symlink(filepath.Join("..", "denied", "victim"), link); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
user, err := state.GetUserByUsername(username)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
settings.Config.Server.SourceMap[sourceRoot].Config.DenyByDefault = false
|
||||
|
||||
body := `{"fromSource":"` + sourceName + `","paths":["/a.txt"],"destination":"/allowed/report.zip","format":"zip"}`
|
||||
req := httptest.NewRequest(http.MethodPost, "/api/resources/archive", strings.NewReader(body))
|
||||
rec := httptest.NewRecorder()
|
||||
status, handlerErr := archiveCreateHandler(rec, req, &Context{User: &user})
|
||||
if status != http.StatusForbidden || handlerErr == nil {
|
||||
t.Fatalf("status=%d err=%v", status, handlerErr)
|
||||
}
|
||||
got, err := os.ReadFile(victim)
|
||||
if err != nil || string(got) != "keep" {
|
||||
t.Fatalf("victim content=%q err=%v", got, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestArchiveCreate_AllowsNonSymlinkDestination(t *testing.T) {
|
||||
_, sourceRoot, sourceName, username := setupExtractAccessPolicyTest(t)
|
||||
dest := filepath.Join(sourceRoot, "out.zip")
|
||||
if err := os.WriteFile(filepath.Join(sourceRoot, "a.txt"), []byte("payload"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
user, err := state.GetUserByUsername(username)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
body := `{"fromSource":"` + sourceName + `","paths":["/a.txt"],"destination":"/out.zip","format":"zip"}`
|
||||
req := httptest.NewRequest(http.MethodPost, "/api/resources/archive", strings.NewReader(body))
|
||||
rec := httptest.NewRecorder()
|
||||
status, handlerErr := archiveCreateHandler(rec, req, &Context{User: &user})
|
||||
if status != http.StatusOK || handlerErr != nil {
|
||||
t.Fatalf("status=%d err=%v body=%s", status, handlerErr, rec.Body.String())
|
||||
}
|
||||
var resp map[string]string
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := os.Stat(dest); err != nil {
|
||||
t.Fatalf("expected archive at %s: %v", dest, err)
|
||||
}
|
||||
}
|
||||
@@ -23,6 +23,7 @@ import (
|
||||
|
||||
func setupExtractAccessPolicyTest(t *testing.T) (destDir, sourceRoot, sourceName, username string) {
|
||||
t.Helper()
|
||||
lockWebGlobals(t)
|
||||
tempDir := t.TempDir()
|
||||
dbPath := filepath.Join(tempDir, "access.sqlite")
|
||||
if _, err := state.Initialize(dbPath); err != nil {
|
||||
|
||||
@@ -3,11 +3,16 @@ package web
|
||||
import (
|
||||
"archive/tar"
|
||||
"archive/zip"
|
||||
"errors"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
commonerrors "github.com/gtsteffaniak/filebrowser/backend/internal/errors"
|
||||
"github.com/gtsteffaniak/filebrowser/backend/pkg/indexing"
|
||||
"github.com/gtsteffaniak/filebrowser/backend/pkg/settings"
|
||||
)
|
||||
|
||||
func TestNormalizeArchiveEntryName(t *testing.T) {
|
||||
@@ -557,3 +562,19 @@ func TestExtractZip_symlinkDestination(t *testing.T) {
|
||||
t.Fatalf("victim.txt should be unchanged: %q, %v", victimData, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestArchiveCreateDestination_RejectsEscapingSymlinkLeaf(t *testing.T) {
|
||||
t.Parallel()
|
||||
root := t.TempDir()
|
||||
outside := filepath.Join(filepath.Dir(root), "outside-archive")
|
||||
if err := os.MkdirAll(outside, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Symlink(outside, filepath.Join(root, "report.zip")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
idx := &indexing.Index{Source: settings.Source{Path: root}}
|
||||
if _, err := idx.GetRealPathForMutation("", "/report.zip"); !errors.Is(err, commonerrors.ErrPathEscapesScope) {
|
||||
t.Fatalf("expected ErrPathEscapesScope, got %v", err)
|
||||
}
|
||||
}
|
||||
@@ -25,6 +25,7 @@ import (
|
||||
|
||||
func setupShareArchiveDownloadTest(t *testing.T) (sourceRoot string, sourceName string, owner *users.User) {
|
||||
t.Helper()
|
||||
lockWebGlobals(t)
|
||||
dbPath := filepath.Join(t.TempDir(), "share-dl-test.sqlite")
|
||||
if _, err := state.Initialize(dbPath); err != nil {
|
||||
t.Fatal(err)
|
||||
|
||||
@@ -1,16 +1,13 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"crypto/md5"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
"unicode"
|
||||
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/activity"
|
||||
activitydb "github.com/gtsteffaniak/filebrowser/backend/internal/database/activity"
|
||||
@@ -39,9 +36,6 @@ const (
|
||||
// duplicateResultsCache caches duplicate search results for 15 seconds
|
||||
var duplicateResultsCache = cache.NewCache[duplicateResponse](15 * time.Second)
|
||||
|
||||
// checksumCache caches file checksums for 1 hour, keyed by source/path/modtime
|
||||
var checksumCache = cache.NewCache[string](1 * time.Hour)
|
||||
|
||||
type duplicateGroup struct {
|
||||
Size int64 `json:"size"`
|
||||
Count int `json:"count"`
|
||||
@@ -127,10 +121,12 @@ func (s *duplicateProcessingStats) shouldStop() (bool, string) {
|
||||
// identical content but different filenames)
|
||||
//
|
||||
// Performance Optimizations:
|
||||
// - Checksums are cached for 1 hour (keyed by path/size/modtime) to speed up subsequent requests
|
||||
// - Files are processed in batches to balance memory usage vs SQL query count
|
||||
// - Resource limits prevent timeouts: max 2 minutes, max 10K checksum operations
|
||||
// - Large fuzzy groups are skipped to avoid expensive checksum operations on false positives
|
||||
// - Checksums are cached for 1 hour (keyed by path/size/modtime) to speed up subsequent requests
|
||||
// The checksum budget counts only cache misses (real disk I/O); cached digests are free,
|
||||
// so a re-run proceeds past the previously-examined files instead of stalling at the limit.
|
||||
// - Files are processed in batches to balance memory usage vs SQL query count
|
||||
// - Resource limits prevent timeouts: max 2 minutes, max 10K checksum operations
|
||||
// - Large fuzzy groups are skipped to avoid expensive checksum operations on false positives
|
||||
//
|
||||
// Response includes incomplete flag if processing was stopped early due to resource limits.
|
||||
//
|
||||
@@ -170,7 +166,10 @@ func duplicatesHandler(w http.ResponseWriter, r *http.Request, d *Context) (int,
|
||||
|
||||
// Safety check: Reject duplicate search during active indexing to prevent resource contention
|
||||
// This protects against CPU/memory spikes and ensures indexing performance is not degraded
|
||||
if string(index.Status) == "indexing" {
|
||||
// Use GetStatus() (computed from active scanners and persisted lastScanned) rather than
|
||||
// the raw Status field, which is only set to "ready" by PostScan and would otherwise
|
||||
// block searches indefinitely after a restart until the first scheduled scan runs.
|
||||
if index.GetStatus() == indexing.INDEXING {
|
||||
return http.StatusServiceUnavailable, fmt.Errorf("duplicate search is not available while indexing is in progress - please try again when indexing completes")
|
||||
}
|
||||
|
||||
@@ -238,7 +237,13 @@ func duplicatesHandler(w http.ResponseWriter, r *http.Request, d *Context) (int,
|
||||
startTime: time.Now(),
|
||||
uniqueChecksums: make(map[string]bool),
|
||||
}
|
||||
duplicateGroups := findDuplicatesInIndex(index, opts, stats)
|
||||
duplicateGroups := findDuplicatesInIndex(r.Context(), index, opts, stats)
|
||||
|
||||
// The client went away — do not cache or record a truncated result. A cached
|
||||
// partial response would poison the 15s duplicateResultsCache for the next user.
|
||||
if err := r.Context().Err(); err != nil {
|
||||
return http.StatusRequestTimeout, err
|
||||
}
|
||||
|
||||
// Log resource usage for monitoring
|
||||
uniqueChecksumCount := len(stats.uniqueChecksums)
|
||||
@@ -254,8 +259,7 @@ func duplicatesHandler(w http.ResponseWriter, r *http.Request, d *Context) (int,
|
||||
Reason: stats.stopReason,
|
||||
}
|
||||
|
||||
// Cache the results before returning (even partial results)
|
||||
duplicateResultsCache.Set(cacheKey, response)
|
||||
cacheDuplicateResponse(r.Context(), cacheKey, response)
|
||||
|
||||
if stats.stopped {
|
||||
w.Header().Set("X-Search-Incomplete", "true")
|
||||
@@ -267,7 +271,7 @@ func duplicatesHandler(w http.ResponseWriter, r *http.Request, d *Context) (int,
|
||||
}
|
||||
|
||||
// findDuplicatesInIndex finds duplicates using the shared IndexDB with resource limits
|
||||
func findDuplicatesInIndex(index *indexing.Index, opts *duplicatesOptions, stats *duplicateProcessingStats) []duplicateGroup {
|
||||
func findDuplicatesInIndex(ctx context.Context, index *indexing.Index, opts *duplicatesOptions, stats *duplicateProcessingStats) []duplicateGroup {
|
||||
// Get the shared IndexDB
|
||||
indexDB := indexing.GetIndexDB()
|
||||
|
||||
@@ -290,6 +294,11 @@ func findDuplicatesInIndex(index *indexing.Index, opts *duplicatesOptions, stats
|
||||
|
||||
// Process sizes in batches
|
||||
for batchStart := 0; batchStart < len(sizes); batchStart += bulkSizeQueryLimit {
|
||||
if ctx.Err() != nil {
|
||||
stats.stopped = true
|
||||
stats.stopReason = ctx.Err().Error()
|
||||
break
|
||||
}
|
||||
// Check resource limits before processing each batch
|
||||
if shouldStop, reason := stats.shouldStop(); shouldStop {
|
||||
stats.stopped = true
|
||||
@@ -327,6 +336,11 @@ func findDuplicatesInIndex(index *indexing.Index, opts *duplicatesOptions, stats
|
||||
// Memory optimization: Delete entries from filesBySize immediately after processing
|
||||
// to allow GC to reclaim memory as we go, rather than holding all data until batch completes
|
||||
for _, size := range sizeBatch {
|
||||
if ctx.Err() != nil {
|
||||
stats.stopped = true
|
||||
stats.stopReason = ctx.Err().Error()
|
||||
break
|
||||
}
|
||||
files, ok := filesBySize[size]
|
||||
if !ok || len(files) < 2 {
|
||||
// Clear entry even if not processing to free memory
|
||||
@@ -345,8 +359,8 @@ func findDuplicatesInIndex(index *indexing.Index, opts *duplicatesOptions, stats
|
||||
continue
|
||||
}
|
||||
|
||||
// Group files by MIME type in memory (already ordered by type from SQL)
|
||||
filesByType := groupFilesByType(files)
|
||||
// Group files by MIME (+ extension when MIME is ambiguous) within this size bucket.
|
||||
filesByType := groupFilesByTypeBucket(files)
|
||||
|
||||
// Process each MIME type group
|
||||
for _, typeFiles := range filesByType {
|
||||
@@ -364,29 +378,13 @@ func findDuplicatesInIndex(index *indexing.Index, opts *duplicatesOptions, stats
|
||||
|
||||
// Use fuzzy filename matching to group files within this MIME type
|
||||
// This reduces checksum operations by grouping similar filenames
|
||||
filenameGroups := groupFilesByFilename(typeFiles, size)
|
||||
filenameGroups := groupFilesByFilename(ctx, typeFiles, size, stats)
|
||||
|
||||
totalFilesInGroups := 0
|
||||
for _, g := range filenameGroups {
|
||||
if len(g) >= 2 {
|
||||
totalFilesInGroups += len(g)
|
||||
}
|
||||
}
|
||||
|
||||
// Process each fuzzy filename group
|
||||
for _, fileGroup := range filenameGroups {
|
||||
if len(fileGroup) < 2 {
|
||||
continue
|
||||
}
|
||||
|
||||
// Skip checksumming if fuzzy group is too large - groups with many files
|
||||
// are unlikely to be true duplicates (fuzzy matching is too permissive)
|
||||
// This optimization prevents expensive checksum operations on false positives
|
||||
if len(fileGroup) > maxFuzzyGroupSize {
|
||||
logger.Debugf("[Duplicates] Skipping checksum for fuzzy group with %d files (exceeds limit of %d)", len(fileGroup), maxFuzzyGroupSize)
|
||||
continue
|
||||
}
|
||||
|
||||
// Stop if we've hit the group limit
|
||||
if len(groupsWithChecksums) >= maxGroups {
|
||||
break
|
||||
@@ -403,7 +401,7 @@ func findDuplicatesInIndex(index *indexing.Index, opts *duplicatesOptions, stats
|
||||
// Verify with checksums using 3-pass progressive verification
|
||||
// At this point, files match on: size + MIME type + fuzzy filename similarity (50%+)
|
||||
// Large fuzzy groups (>10 files) are skipped above to avoid expensive false positives
|
||||
verifiedGroups := groupFilesByChecksum(fileGroup, index, size, stats)
|
||||
verifiedGroups := groupFilesByChecksum(ctx, fileGroup, index, size, stats)
|
||||
|
||||
// Create SearchResult objects and track checksums for merging
|
||||
for _, checksumGroup := range verifiedGroups {
|
||||
@@ -483,100 +481,15 @@ func findDuplicatesInIndex(index *indexing.Index, opts *duplicatesOptions, stats
|
||||
})
|
||||
}
|
||||
|
||||
// Groups are already sorted by size (largest to smallest) from SQL query
|
||||
return duplicateGroups
|
||||
}
|
||||
|
||||
// mergeGroupsByChecksum merges groups that share any checksum values
|
||||
// This catches cases where files have identical content but were grouped separately
|
||||
func mergeGroupsByChecksum(groups []duplicateGroupWithChecksums) []duplicateGroupWithChecksums {
|
||||
if len(groups) <= 1 {
|
||||
return groups
|
||||
// cacheDuplicateResponse stores a duplicate search response when the client context is still active.
|
||||
func cacheDuplicateResponse(ctx context.Context, cacheKey string, response duplicateResponse) {
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
|
||||
// Build a map: checksum -> list of group indices that contain this checksum
|
||||
checksumToGroups := make(map[string][]int)
|
||||
for i, group := range groups {
|
||||
if group.Checksum != "" {
|
||||
checksumToGroups[group.Checksum] = append(checksumToGroups[group.Checksum], i)
|
||||
}
|
||||
}
|
||||
|
||||
// Use union-find to merge groups that share checksums
|
||||
parent := make([]int, len(groups))
|
||||
for i := range parent {
|
||||
parent[i] = i
|
||||
}
|
||||
|
||||
var find func(int) int
|
||||
find = func(x int) int {
|
||||
if parent[x] != x {
|
||||
parent[x] = find(parent[x]) // Path compression
|
||||
}
|
||||
return parent[x]
|
||||
}
|
||||
|
||||
union := func(x, y int) {
|
||||
px, py := find(x), find(y)
|
||||
if px != py {
|
||||
parent[px] = py
|
||||
}
|
||||
}
|
||||
|
||||
// Merge groups that share any checksum
|
||||
for _, groupIndices := range checksumToGroups {
|
||||
if len(groupIndices) < 2 {
|
||||
continue
|
||||
}
|
||||
// Merge all groups that share this checksum
|
||||
for i := 1; i < len(groupIndices); i++ {
|
||||
union(groupIndices[0], groupIndices[i])
|
||||
}
|
||||
}
|
||||
|
||||
// Group indices by their root parent
|
||||
groupsByRoot := make(map[int][]int)
|
||||
for i := range groups {
|
||||
root := find(i)
|
||||
groupsByRoot[root] = append(groupsByRoot[root], i)
|
||||
}
|
||||
|
||||
// Merge groups that belong to the same root
|
||||
merged := make([]duplicateGroupWithChecksums, 0, len(groupsByRoot))
|
||||
for _, indices := range groupsByRoot {
|
||||
if len(indices) == 1 {
|
||||
// No merging needed, just add the group
|
||||
merged = append(merged, groups[indices[0]])
|
||||
continue
|
||||
}
|
||||
|
||||
// Merge multiple groups
|
||||
combinedFiles := make([]*indexing.SearchResult, 0)
|
||||
var combinedSize int64
|
||||
var combinedChecksum string
|
||||
|
||||
for _, idx := range indices {
|
||||
group := groups[idx]
|
||||
combinedFiles = append(combinedFiles, group.Files...)
|
||||
if combinedChecksum == "" {
|
||||
combinedChecksum = group.Checksum
|
||||
}
|
||||
if combinedSize == 0 {
|
||||
combinedSize = group.Size
|
||||
}
|
||||
}
|
||||
|
||||
// Only include merged groups with 2+ files
|
||||
if len(combinedFiles) >= 2 {
|
||||
merged = append(merged, duplicateGroupWithChecksums{
|
||||
Size: combinedSize,
|
||||
Files: combinedFiles,
|
||||
Checksum: combinedChecksum,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return merged
|
||||
duplicateResultsCache.Set(cacheKey, response)
|
||||
}
|
||||
|
||||
func prepDuplicatesOptions(r *http.Request, d *Context) (*duplicatesOptions, error) {
|
||||
@@ -624,349 +537,6 @@ func prepDuplicatesOptions(r *http.Request, d *Context) (*duplicatesOptions, err
|
||||
}, nil
|
||||
}
|
||||
|
||||
// groupFilesByChecksum groups files by partial checksum and returns groups with their checksums
|
||||
func groupFilesByChecksum(files []*iteminfo.FileInfo, index *indexing.Index, fileSize int64, stats *duplicateProcessingStats) []checksumGroup {
|
||||
if len(files) < 2 {
|
||||
return nil
|
||||
}
|
||||
|
||||
// PASS 1: Header checksums only (first 8KB)
|
||||
// This is the fastest way to eliminate files that are definitely different
|
||||
// ONLY checksum files that already matched on filename+size
|
||||
headerGroups := make(map[string][]*iteminfo.FileInfo)
|
||||
|
||||
for _, file := range files {
|
||||
// Check if we've hit the checksum operation limit
|
||||
if stats.checksumOperations >= maxChecksumOperations {
|
||||
logger.Warningf("[Duplicates] Reached checksum operation limit (%d) in header pass", maxChecksumOperations)
|
||||
break
|
||||
}
|
||||
|
||||
// Verify size matches expected size (should match as we queried by size)
|
||||
if file.Size != fileSize {
|
||||
continue
|
||||
}
|
||||
|
||||
// Construct filesystem path for checksum computation
|
||||
// index.Path is the absolute filesystem root, file.Path is index-relative
|
||||
filePath := filepath.Join(index.Path, file.Path)
|
||||
headerChecksum, err := computeHeaderChecksum(index.Path, filePath, fileSize, file.ModTime)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
stats.checksumOperations++
|
||||
stats.uniqueChecksums[headerChecksum] = true
|
||||
headerGroups[headerChecksum] = append(headerGroups[headerChecksum], file)
|
||||
}
|
||||
|
||||
// Filter to only groups with 2+ files (candidates for duplicates)
|
||||
headerCandidates := make([][]*iteminfo.FileInfo, 0)
|
||||
filesAfterHeaderPass := 0
|
||||
for _, group := range headerGroups {
|
||||
if len(group) >= 2 {
|
||||
headerCandidates = append(headerCandidates, group)
|
||||
filesAfterHeaderPass += len(group)
|
||||
}
|
||||
}
|
||||
|
||||
if len(headerCandidates) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
// PASS 2: Middle checksums (header + middle) for header-matched groups
|
||||
// Only process files that matched on header to save disk I/O
|
||||
middleGroups := make(map[string][]*iteminfo.FileInfo)
|
||||
|
||||
for _, headerGroup := range headerCandidates {
|
||||
if len(headerGroup) < 2 {
|
||||
continue
|
||||
}
|
||||
|
||||
for _, file := range headerGroup {
|
||||
// Check if we've hit the checksum operation limit
|
||||
if stats.checksumOperations >= maxChecksumOperations {
|
||||
logger.Warningf("[Duplicates] Reached checksum operation limit (%d) in middle pass", maxChecksumOperations)
|
||||
break
|
||||
}
|
||||
|
||||
filePath := filepath.Join(index.Path, file.Path)
|
||||
middleChecksum, err := computeMiddleChecksum(index.Path, filePath, fileSize, file.ModTime)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
stats.checksumOperations++
|
||||
stats.uniqueChecksums[middleChecksum] = true
|
||||
middleGroups[middleChecksum] = append(middleGroups[middleChecksum], file)
|
||||
}
|
||||
}
|
||||
|
||||
// Final verification: Use middle checksum as final (header + middle is sufficient)
|
||||
// Files that match on header + middle but differ at end are extremely rare
|
||||
// This 2-pass system catches 99.9%+ of differences while being faster
|
||||
groups := make([]checksumGroup, 0, len(middleGroups))
|
||||
for checksum, group := range middleGroups {
|
||||
if len(group) >= 2 {
|
||||
groups = append(groups, checksumGroup{
|
||||
Files: group,
|
||||
Checksum: checksum,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return groups
|
||||
}
|
||||
|
||||
// computeHeaderChecksum calculates MD5 hash of only the first 8KB of a file
|
||||
// This is the fastest initial pass to eliminate non-matching files
|
||||
func computeHeaderChecksum(sourcePath, filePath string, size int64, modTime time.Time) (string, error) {
|
||||
// Generate cache key for header-only checksum
|
||||
cacheKey := fmt.Sprintf("%s:%s:%d:%d:header", sourcePath, filePath, size, modTime.Unix())
|
||||
|
||||
// Check cache first
|
||||
if cachedChecksum, ok := checksumCache.Get(cacheKey); ok {
|
||||
return cachedChecksum, nil
|
||||
}
|
||||
|
||||
file, err := os.Open(filePath)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
hash := md5.New()
|
||||
buf := make([]byte, 8192) // 8KB buffer
|
||||
|
||||
// Read only first 8KB (or entire file if smaller)
|
||||
n, err := io.ReadFull(file, buf)
|
||||
if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF {
|
||||
return "", err
|
||||
}
|
||||
hash.Write(buf[:n])
|
||||
|
||||
checksum := fmt.Sprintf("%x", hash.Sum(nil))
|
||||
|
||||
// Cache the result
|
||||
checksumCache.Set(cacheKey, checksum)
|
||||
|
||||
return checksum, nil
|
||||
}
|
||||
|
||||
// computeMiddleChecksum calculates MD5 hash of header + middle portion
|
||||
// Only called when header checksums match (progressive verification)
|
||||
func computeMiddleChecksum(sourcePath, filePath string, size int64, modTime time.Time) (string, error) {
|
||||
// Generate cache key for header+middle checksum
|
||||
cacheKey := fmt.Sprintf("%s:%s:%d:%d:middle", sourcePath, filePath, size, modTime.Unix())
|
||||
|
||||
// Check cache first
|
||||
if cachedChecksum, ok := checksumCache.Get(cacheKey); ok {
|
||||
return cachedChecksum, nil
|
||||
}
|
||||
|
||||
file, err := os.Open(filePath)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
hash := md5.New()
|
||||
buf := make([]byte, 8192) // 8KB buffer
|
||||
|
||||
// Read first 8KB
|
||||
n, err := io.ReadFull(file, buf)
|
||||
if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF {
|
||||
return "", err
|
||||
}
|
||||
hash.Write(buf[:n])
|
||||
|
||||
// For larger files, also read middle 8KB
|
||||
if size > 16384 { // 16KB
|
||||
middleOffset := size / 2
|
||||
if _, err := file.Seek(middleOffset, 0); err == nil {
|
||||
n, err := io.ReadFull(file, buf)
|
||||
if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF {
|
||||
return "", err
|
||||
}
|
||||
hash.Write(buf[:n])
|
||||
}
|
||||
}
|
||||
|
||||
checksum := fmt.Sprintf("%x", hash.Sum(nil))
|
||||
|
||||
// Cache the result
|
||||
checksumCache.Set(cacheKey, checksum)
|
||||
|
||||
return checksum, nil
|
||||
}
|
||||
|
||||
// groupFilesByFilename groups files by fuzzy filename matching
|
||||
// Works with iteminfo.FileInfo from the main IndexDB
|
||||
// Normalizes filenames on-the-fly for comparison
|
||||
// NOTE: Files passed in should already be filtered by MIME type at the SQL level,
|
||||
// so no need to check extensions here (extension checking is redundant with MIME type filtering)
|
||||
func groupFilesByFilename(files []*iteminfo.FileInfo, expectedSize int64) [][]*iteminfo.FileInfo {
|
||||
if len(files) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
groups := [][]*iteminfo.FileInfo{}
|
||||
used := make(map[int]bool)
|
||||
|
||||
for i := 0; i < len(files); i++ {
|
||||
if used[i] || files[i].Name == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
group := []*iteminfo.FileInfo{files[i]}
|
||||
used[i] = true
|
||||
|
||||
// Normalize filename for fuzzy matching
|
||||
filename1 := normalizeFilename(files[i].Name)
|
||||
|
||||
for j := i + 1; j < len(files); j++ {
|
||||
if used[j] || files[j].Name == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
// Check size (should match, but safe to check)
|
||||
if files[j].Size != expectedSize {
|
||||
continue
|
||||
}
|
||||
|
||||
filename2 := normalizeFilename(files[j].Name)
|
||||
|
||||
// Check if filenames are similar enough using levenshtein distance
|
||||
if filenamesSimilar(filename1, filename2) {
|
||||
group = append(group, files[j])
|
||||
used[j] = true
|
||||
}
|
||||
}
|
||||
|
||||
if len(group) >= 2 {
|
||||
groups = append(groups, group)
|
||||
}
|
||||
}
|
||||
|
||||
return groups
|
||||
}
|
||||
|
||||
// normalizeFilename converts filename to lowercase and removes special characters
|
||||
// to make fuzzy matching more effective
|
||||
func normalizeFilename(filename string) string {
|
||||
// Convert to lowercase
|
||||
filename = strings.ToLower(filename)
|
||||
|
||||
// Remove file extension for comparison
|
||||
if idx := strings.LastIndex(filename, "."); idx > 0 {
|
||||
filename = filename[:idx]
|
||||
}
|
||||
|
||||
// Remove non-alphanumeric characters except spaces and hyphens
|
||||
var result strings.Builder
|
||||
for _, r := range filename {
|
||||
if unicode.IsLetter(r) || unicode.IsDigit(r) || r == ' ' || r == '-' {
|
||||
result.WriteRune(r)
|
||||
}
|
||||
}
|
||||
|
||||
return strings.TrimSpace(result.String())
|
||||
}
|
||||
|
||||
// filenamesSimilar checks if two normalized filenames are similar enough to be considered duplicates
|
||||
func filenamesSimilar(name1, name2 string) bool {
|
||||
// Exact match
|
||||
if name1 == name2 {
|
||||
return true
|
||||
}
|
||||
|
||||
// Empty names don't match
|
||||
if len(name1) == 0 || len(name2) == 0 {
|
||||
return false
|
||||
}
|
||||
|
||||
// Calculate Levenshtein distance
|
||||
distance := levenshteinDistance(name1, name2)
|
||||
maxLen := max(len(name1), len(name2))
|
||||
|
||||
// Require at least 50% similarity (lowered to catch more duplicates with username prefixes)
|
||||
// Examples: "video.mp4" vs "username_video.mp4" should match
|
||||
similarity := 1.0 - float64(distance)/float64(maxLen)
|
||||
return similarity >= 0.5
|
||||
}
|
||||
|
||||
// levenshteinDistance calculates the edit distance between two strings
|
||||
func levenshteinDistance(s1, s2 string) int {
|
||||
if len(s1) == 0 {
|
||||
return len(s2)
|
||||
}
|
||||
if len(s2) == 0 {
|
||||
return len(s1)
|
||||
}
|
||||
|
||||
// Create matrix
|
||||
matrix := make([][]int, len(s1)+1)
|
||||
for i := range matrix {
|
||||
matrix[i] = make([]int, len(s2)+1)
|
||||
}
|
||||
|
||||
// Initialize first row and column
|
||||
for i := 0; i <= len(s1); i++ {
|
||||
matrix[i][0] = i
|
||||
}
|
||||
for j := 0; j <= len(s2); j++ {
|
||||
matrix[0][j] = j
|
||||
}
|
||||
|
||||
// Fill matrix
|
||||
for i := 1; i <= len(s1); i++ {
|
||||
for j := 1; j <= len(s2); j++ {
|
||||
cost := 0
|
||||
if s1[i-1] != s2[j-1] {
|
||||
cost = 1
|
||||
}
|
||||
|
||||
matrix[i][j] = min(
|
||||
matrix[i-1][j]+1, // deletion
|
||||
matrix[i][j-1]+1, // insertion
|
||||
matrix[i-1][j-1]+cost, // substitution
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
return matrix[len(s1)][len(s2)]
|
||||
}
|
||||
|
||||
// min returns the minimum of three integers
|
||||
func min(a, b, c int) int {
|
||||
if a < b {
|
||||
if a < c {
|
||||
return a
|
||||
}
|
||||
return c
|
||||
}
|
||||
if b < c {
|
||||
return b
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
// max returns the maximum of two integers
|
||||
func max(a, b int) int {
|
||||
if a > b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
// groupFilesByType groups files by MIME type
|
||||
// Assumes files are already sorted by type (from SQL ORDER BY)
|
||||
func groupFilesByType(files []*iteminfo.FileInfo) map[string][]*iteminfo.FileInfo {
|
||||
grouped := make(map[string][]*iteminfo.FileInfo)
|
||||
for _, file := range files {
|
||||
grouped[file.Type] = append(grouped[file.Type], file)
|
||||
}
|
||||
return grouped
|
||||
}
|
||||
|
||||
// filterFilesByPermission filters files to only include those the user is permitted to access
|
||||
// This is called early in the duplicate search process to avoid processing files the user can't see
|
||||
func filterFilesByPermission(files []*iteminfo.FileInfo, index *indexing.Index, username string) []*iteminfo.FileInfo {
|
||||
|
||||
@@ -0,0 +1,157 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/gtsteffaniak/filebrowser/backend/pkg/indexing"
|
||||
"github.com/gtsteffaniak/filebrowser/backend/pkg/indexing/iteminfo"
|
||||
)
|
||||
|
||||
type duplicateCorpus struct {
|
||||
dir string
|
||||
files []*iteminfo.FileInfo
|
||||
}
|
||||
|
||||
func buildDuplicateCorpus(b *testing.B) *duplicateCorpus {
|
||||
b.Helper()
|
||||
dir := b.TempDir()
|
||||
now := time.Now()
|
||||
|
||||
write := func(name string, payload []byte, mutateTail bool) {
|
||||
if mutateTail && len(payload) > 32 {
|
||||
dup := append([]byte(nil), payload...)
|
||||
dup[len(dup)-1] ^= 0xff
|
||||
payload = dup
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(dir, name), payload, 0o600); err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
jpegHeader := []byte{0xff, 0xd8, 0xff, 0xe0}
|
||||
pngHeader := []byte{0x89, 0x50, 0x4e, 0x47}
|
||||
pdfHeader := []byte("%PDF-1.4\n")
|
||||
mp4Header := []byte{0x00, 0x00, 0x00, 0x18, 'f', 't', 'y', 'p'}
|
||||
|
||||
mkPayload := func(header []byte, size int) []byte {
|
||||
out := make([]byte, size)
|
||||
copy(out, header)
|
||||
for i := len(header); i < len(out); i++ {
|
||||
out[i] = byte(i)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
baseJPEG := mkPayload(jpegHeader, 65536)
|
||||
basePNG := mkPayload(pngHeader, 65536)
|
||||
basePDF := mkPayload(pdfHeader, 65536)
|
||||
baseMP4 := mkPayload(mp4Header, 65536)
|
||||
|
||||
entries := []struct {
|
||||
name string
|
||||
typ string
|
||||
data []byte
|
||||
}{
|
||||
{"exact-a.jpg", "image/jpeg", baseJPEG},
|
||||
{"exact-b.jpg", "image/jpeg", baseJPEG},
|
||||
{"renamed-copy.jpeg", "image/jpeg", baseJPEG},
|
||||
{"user_exact-a.jpg", "image/jpeg", baseJPEG},
|
||||
{"png-a.png", "image/png", basePNG},
|
||||
{"png-b.png", "image/png", basePNG},
|
||||
{"doc-a.pdf", "application/pdf", basePDF},
|
||||
{"doc-b.pdf", "application/pdf", basePDF},
|
||||
{"clip-a.mp4", "video/mp4", baseMP4},
|
||||
{"clip-b.mp4", "video/mp4", baseMP4},
|
||||
{"sized-a.bin", "application/octet-stream", baseJPEG},
|
||||
{"sized-b.bin", "application/octet-stream", mkPayload(jpegHeader, 65536)},
|
||||
}
|
||||
|
||||
files := make([]*iteminfo.FileInfo, 0, len(entries)+50)
|
||||
for _, e := range entries {
|
||||
write(e.name, e.data, false)
|
||||
files = append(files, &iteminfo.FileInfo{
|
||||
ItemInfo: iteminfo.ItemInfo{Name: e.name, Size: int64(len(e.data)), ModTime: now, Type: e.typ},
|
||||
Path: "/" + e.name,
|
||||
})
|
||||
}
|
||||
write("tail-b.jpg", baseJPEG, true)
|
||||
|
||||
for i := 0; i < 12; i++ {
|
||||
name := fmt.Sprintf("fuzzy_photo_%d.jpg", i)
|
||||
write(name, baseJPEG, false)
|
||||
files = append(files, &iteminfo.FileInfo{
|
||||
ItemInfo: iteminfo.ItemInfo{Name: name, Size: int64(len(baseJPEG)), ModTime: now, Type: "image/jpeg"},
|
||||
Path: "/" + name,
|
||||
})
|
||||
}
|
||||
|
||||
for i := 0; i < 30; i++ {
|
||||
name := fmt.Sprintf("bucket_%04d.jpg", i)
|
||||
write(name, mkPayload(jpegHeader, 65536), i%7 == 0)
|
||||
files = append(files, &iteminfo.FileInfo{
|
||||
ItemInfo: iteminfo.ItemInfo{Name: name, Size: 65536, ModTime: now, Type: "image/jpeg"},
|
||||
Path: "/" + name,
|
||||
})
|
||||
}
|
||||
|
||||
return &duplicateCorpus{dir: dir, files: files}
|
||||
}
|
||||
|
||||
func BenchmarkDuplicateTypeBuckets(b *testing.B) {
|
||||
corpus := buildDuplicateCorpus(b)
|
||||
b.ResetTimer()
|
||||
b.ReportAllocs()
|
||||
for i := 0; i < b.N; i++ {
|
||||
_ = groupFilesByTypeBucket(corpus.files)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkDuplicateFilenameGroups(b *testing.B) {
|
||||
corpus := buildDuplicateCorpus(b)
|
||||
subset := corpus.files[len(corpus.files)-30:]
|
||||
ctx := context.Background()
|
||||
stats := &duplicateProcessingStats{startTime: time.Now()}
|
||||
b.ResetTimer()
|
||||
b.ReportAllocs()
|
||||
for i := 0; i < b.N; i++ {
|
||||
_ = groupFilesByFilename(ctx, subset, 65536, stats)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkDuplicateChecksumGrouping(b *testing.B) {
|
||||
corpus := buildDuplicateCorpus(b)
|
||||
pair := []*iteminfo.FileInfo{corpus.files[0], corpus.files[1]}
|
||||
idx := &indexing.Index{Path: corpus.dir}
|
||||
ctx := context.Background()
|
||||
stats := &duplicateProcessingStats{startTime: time.Now(), uniqueChecksums: map[string]bool{}}
|
||||
b.ResetTimer()
|
||||
b.ReportAllocs()
|
||||
for i := 0; i < b.N; i++ {
|
||||
_ = groupFilesByChecksum(ctx, pair, idx, pair[0].Size, stats)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkDuplicateMerge(b *testing.B) {
|
||||
mkFiles := func(n int, prefix string) []*indexing.SearchResult {
|
||||
out := make([]*indexing.SearchResult, n)
|
||||
for i := range out {
|
||||
out[i] = &indexing.SearchResult{Path: fmt.Sprintf("/%s/%d", prefix, i)}
|
||||
}
|
||||
return out
|
||||
}
|
||||
groups := []duplicateGroupWithChecksums{
|
||||
{Size: 100, Checksum: "d1", Files: mkFiles(4, "a")},
|
||||
{Size: 100, Checksum: "d1", Files: mkFiles(3, "b")},
|
||||
{Size: 200, Checksum: "d2", Files: mkFiles(2, "c")},
|
||||
}
|
||||
b.ResetTimer()
|
||||
b.ReportAllocs()
|
||||
for i := 0; i < b.N; i++ {
|
||||
_ = mergeGroupsByChecksum(groups)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/gtsteffaniak/filebrowser/backend/pkg/indexing/iteminfo"
|
||||
)
|
||||
|
||||
// typeBucketKey returns the MIME/extension bucket used inside a fixed-size group.
|
||||
// When the MIME type already implies a single on-disk format, extension is omitted
|
||||
// as a redundant pre-filter. Otherwise a normalized lowercase extension is appended.
|
||||
//
|
||||
// JPEG aliases: ".jpg", ".jpeg", and ".jpe" normalize to "jpg" so differently named
|
||||
// JPEGs stay in one bucket without splitting the MIME-implied image/jpeg group.
|
||||
func typeBucketKey(file *iteminfo.FileInfo) string {
|
||||
mime := file.Type
|
||||
if mimeImpliesExtension(mime) {
|
||||
return mime
|
||||
}
|
||||
return mime + "\x1f" + normalizedFileExtension(file.Name)
|
||||
}
|
||||
|
||||
func mimeImpliesExtension(mime string) bool {
|
||||
switch mime {
|
||||
case "image/jpeg", "image/png", "image/gif", "image/webp", "image/bmp", "image/tiff",
|
||||
"video/mp4", "video/webm", "video/quicktime", "video/x-msvideo",
|
||||
"audio/mpeg", "audio/wav", "audio/flac", "audio/ogg",
|
||||
"application/pdf", "application/zip", "application/gzip",
|
||||
"text/plain", "text/html", "text/css", "application/json":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// normalizedFileExtension returns a lowercase extension without the dot, folding JPEG aliases to "jpg".
|
||||
func normalizedFileExtension(name string) string {
|
||||
ext := strings.ToLower(strings.TrimPrefix(filepath.Ext(name), "."))
|
||||
switch ext {
|
||||
case "jpeg", "jpe":
|
||||
return "jpg"
|
||||
default:
|
||||
return ext
|
||||
}
|
||||
}
|
||||
|
||||
func groupFilesByTypeBucket(files []*iteminfo.FileInfo) map[string][]*iteminfo.FileInfo {
|
||||
grouped := make(map[string][]*iteminfo.FileInfo)
|
||||
for _, file := range files {
|
||||
key := typeBucketKey(file)
|
||||
grouped[key] = append(grouped[key], file)
|
||||
}
|
||||
return grouped
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/gtsteffaniak/filebrowser/backend/pkg/indexing/iteminfo"
|
||||
)
|
||||
|
||||
func TestTypeBucketKeyJPEGAliases(t *testing.T) {
|
||||
jpg := &iteminfo.FileInfo{ItemInfo: iteminfo.ItemInfo{Name: "a.jpg", Type: "image/jpeg"}}
|
||||
jpeg := &iteminfo.FileInfo{ItemInfo: iteminfo.ItemInfo{Name: "b.jpeg", Type: "image/jpeg"}}
|
||||
if typeBucketKey(jpg) != typeBucketKey(jpeg) {
|
||||
t.Fatalf("jpg and jpeg should share bucket: %q vs %q", typeBucketKey(jpg), typeBucketKey(jpeg))
|
||||
}
|
||||
}
|
||||
|
||||
func TestTypeBucketKeyAmbiguousMIMEUsesExtension(t *testing.T) {
|
||||
a := &iteminfo.FileInfo{ItemInfo: iteminfo.ItemInfo{Name: "a.bin", Type: "application/octet-stream"}}
|
||||
b := &iteminfo.FileInfo{ItemInfo: iteminfo.ItemInfo{Name: "b.dat", Type: "application/octet-stream"}}
|
||||
if typeBucketKey(a) == typeBucketKey(b) {
|
||||
t.Fatal("different extensions should not share bucket for ambiguous MIME")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestChecksumCacheRejectsExpiredDigest(t *testing.T) {
|
||||
key := checksumCacheKey{
|
||||
sourcePath: "/src",
|
||||
filePath: "/file-a",
|
||||
size: 4096,
|
||||
modTime: 42,
|
||||
kind: checksumKindHeader,
|
||||
}
|
||||
encoded := checksumCacheKeyString(key)
|
||||
checksumTypedCache.SetWithExp(encoded, "digest", time.Millisecond)
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
if _, ok := getChecksumFromCache(key); ok {
|
||||
t.Fatal("expired digest should not be returned")
|
||||
}
|
||||
}
|
||||
|
||||
func TestChecksumCacheMissAfterFileIdentityChange(t *testing.T) {
|
||||
modA := checksumCacheKey{
|
||||
sourcePath: "/src",
|
||||
filePath: "/file-a",
|
||||
size: 4096,
|
||||
modTime: 100,
|
||||
kind: checksumKindHeader,
|
||||
}
|
||||
modB := modA
|
||||
modB.modTime = 200
|
||||
|
||||
checksumTypedCache.Set(checksumCacheKeyString(modA), "digest-a")
|
||||
if _, ok := getChecksumFromCache(modB); ok {
|
||||
t.Fatal("modified file identity must not hit the prior cache key")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCacheDuplicateResponseSkipsCanceledContext(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
|
||||
cacheKey := "cancel-test:" + time.Now().Format(time.RFC3339Nano)
|
||||
cacheDuplicateResponse(ctx, cacheKey, duplicateResponse{
|
||||
Groups: []duplicateGroup{{Size: 1, Count: 2}},
|
||||
})
|
||||
if _, ok := duplicateResultsCache.Get(cacheKey); ok {
|
||||
t.Fatal("canceled context must not cache duplicate results")
|
||||
}
|
||||
|
||||
cacheDuplicateResponse(context.Background(), cacheKey, duplicateResponse{
|
||||
Groups: []duplicateGroup{{Size: 2, Count: 2}},
|
||||
})
|
||||
if _, ok := duplicateResultsCache.Get(cacheKey); !ok {
|
||||
t.Fatal("active context should cache duplicate results")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,264 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"hash"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/cespare/xxhash/v2"
|
||||
"github.com/gtsteffaniak/filebrowser/backend/pkg/indexing"
|
||||
"github.com/gtsteffaniak/filebrowser/backend/pkg/indexing/iteminfo"
|
||||
"github.com/gtsteffaniak/go-cache/cache"
|
||||
)
|
||||
|
||||
const (
|
||||
checksumSampleSize = 8192
|
||||
checksumSmallFile = 16384
|
||||
checksumCacheTTL = time.Hour
|
||||
)
|
||||
|
||||
type checksumSampleKind uint8
|
||||
|
||||
const (
|
||||
checksumKindHeader checksumSampleKind = iota
|
||||
checksumKindMiddle
|
||||
)
|
||||
|
||||
// checksumCacheKey identifies a cached sample digest for a file snapshot.
|
||||
type checksumCacheKey struct {
|
||||
sourcePath string
|
||||
filePath string
|
||||
size int64
|
||||
modTime int64
|
||||
kind checksumSampleKind
|
||||
}
|
||||
|
||||
var (
|
||||
// checksumTypedCache stores sample digests keyed by string-encoded checksumCacheKey.
|
||||
// The go-cache backend sweeps expired entries on an hourly ticker, so stale keys from
|
||||
// renamed/deleted/modified files are evicted without a lookup — unlike a bare sync.Map
|
||||
// which would grow without bound. Cleanup interval is the second arg (1h, matching TTL).
|
||||
checksumTypedCache = cache.NewCache[string](checksumCacheTTL, checksumCacheTTL)
|
||||
checksumBufPool = sync.Pool{
|
||||
New: func() any {
|
||||
buf := make([]byte, checksumSampleSize)
|
||||
return &buf
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
type fileSampleDigests struct {
|
||||
header string
|
||||
middle string
|
||||
}
|
||||
|
||||
func duplicateChecksumHasher() hash.Hash {
|
||||
return xxhash.New()
|
||||
}
|
||||
|
||||
func digestHex(h hash.Hash) string {
|
||||
sum := h.Sum(nil)
|
||||
dst := make([]byte, hex.EncodedLen(len(sum)))
|
||||
hex.Encode(dst, sum)
|
||||
return string(dst)
|
||||
}
|
||||
|
||||
// checksumCacheKeyString encodes a typed cache key as a single string so it can
|
||||
// be stored in the TTL-sweeping go-cache. Length prefixes keep fields unambiguous.
|
||||
func checksumCacheKeyString(key checksumCacheKey) string {
|
||||
return fmt.Sprintf("%d|%d|%d|%d|%s|%s",
|
||||
key.kind, key.size, key.modTime, len(key.sourcePath),
|
||||
key.sourcePath, key.filePath)
|
||||
}
|
||||
|
||||
func getChecksumFromCache(key checksumCacheKey) (string, bool) {
|
||||
return checksumTypedCache.Get(checksumCacheKeyString(key))
|
||||
}
|
||||
|
||||
func setChecksumCache(key checksumCacheKey, digest string) {
|
||||
checksumTypedCache.Set(checksumCacheKeyString(key), digest)
|
||||
}
|
||||
|
||||
func readFileSampleDigests(ctx context.Context, sourcePath, filePath string, size int64, modTime time.Time, stats *duplicateProcessingStats) (fileSampleDigests, error) {
|
||||
headerKey := checksumCacheKey{sourcePath: sourcePath, filePath: filePath, size: size, modTime: modTime.UnixNano(), kind: checksumKindHeader}
|
||||
middleKey := checksumCacheKey{sourcePath: sourcePath, filePath: filePath, size: size, modTime: modTime.UnixNano(), kind: checksumKindMiddle}
|
||||
|
||||
var out fileSampleDigests
|
||||
if h, ok := getChecksumFromCache(headerKey); ok {
|
||||
out.header = h
|
||||
}
|
||||
if m, ok := getChecksumFromCache(middleKey); ok {
|
||||
out.middle = m
|
||||
}
|
||||
if out.header != "" && out.middle != "" {
|
||||
return out, nil
|
||||
}
|
||||
|
||||
if ctx.Err() != nil {
|
||||
return fileSampleDigests{}, ctx.Err()
|
||||
}
|
||||
// Only count operations that actually touch disk. Cache hits are O(1) lookups
|
||||
// and must not consume the checksum budget — otherwise a second search over
|
||||
// the same files stalls at the limit without reading a single new byte.
|
||||
// The budget check lives here (post-cache) so cached digests are always served.
|
||||
if shouldStop, _ := stats.shouldStop(); shouldStop {
|
||||
return fileSampleDigests{}, fmt.Errorf("duplicate search budget exhausted")
|
||||
}
|
||||
|
||||
file, err := os.Open(filePath)
|
||||
if err != nil {
|
||||
return fileSampleDigests{}, err
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
// A disk read is happening — charge it against the checksum budget.
|
||||
stats.checksumOperations++
|
||||
|
||||
raw := checksumBufPool.Get()
|
||||
bufPtr, ok := raw.(*[]byte)
|
||||
if !ok {
|
||||
return fileSampleDigests{}, fmt.Errorf("invalid checksum buffer pool type")
|
||||
}
|
||||
buf := *bufPtr
|
||||
defer checksumBufPool.Put(bufPtr)
|
||||
|
||||
if size <= checksumSmallFile {
|
||||
h := duplicateChecksumHasher()
|
||||
remain := size
|
||||
for remain > 0 {
|
||||
if ctx.Err() != nil {
|
||||
return fileSampleDigests{}, ctx.Err()
|
||||
}
|
||||
toRead := int64(len(buf))
|
||||
if toRead > remain {
|
||||
toRead = remain
|
||||
}
|
||||
n, readErr := file.Read(buf[:toRead])
|
||||
if n > 0 {
|
||||
_, _ = h.Write(buf[:n])
|
||||
remain -= int64(n)
|
||||
}
|
||||
if readErr == io.EOF {
|
||||
break
|
||||
}
|
||||
if readErr != nil {
|
||||
return fileSampleDigests{}, readErr
|
||||
}
|
||||
if n == 0 {
|
||||
break
|
||||
}
|
||||
}
|
||||
d := digestHex(h)
|
||||
out.header = d
|
||||
out.middle = d
|
||||
} else if out.header == "" || out.middle == "" {
|
||||
n, readErr := io.ReadFull(file, buf)
|
||||
if readErr != nil && readErr != io.EOF && readErr != io.ErrUnexpectedEOF {
|
||||
return fileSampleDigests{}, readErr
|
||||
}
|
||||
|
||||
if out.header == "" {
|
||||
h := duplicateChecksumHasher()
|
||||
_, _ = h.Write(buf[:n])
|
||||
out.header = digestHex(h)
|
||||
}
|
||||
if out.middle == "" {
|
||||
middle := duplicateChecksumHasher()
|
||||
_, _ = middle.Write(buf[:n])
|
||||
middleOffset := size / 2
|
||||
if _, err := file.Seek(middleOffset, io.SeekStart); err != nil {
|
||||
return fileSampleDigests{}, err
|
||||
}
|
||||
n, readErr := io.ReadFull(file, buf)
|
||||
if readErr != nil && readErr != io.EOF && readErr != io.ErrUnexpectedEOF {
|
||||
return fileSampleDigests{}, readErr
|
||||
}
|
||||
_, _ = middle.Write(buf[:n])
|
||||
out.middle = digestHex(middle)
|
||||
}
|
||||
}
|
||||
|
||||
if out.header != "" {
|
||||
setChecksumCache(headerKey, out.header)
|
||||
}
|
||||
if out.middle != "" {
|
||||
setChecksumCache(middleKey, out.middle)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func groupFilesByChecksum(ctx context.Context, files []*iteminfo.FileInfo, index *indexing.Index, fileSize int64, stats *duplicateProcessingStats) []checksumGroup {
|
||||
if len(files) < 2 {
|
||||
return nil
|
||||
}
|
||||
|
||||
headerGroups := make(map[string][]*iteminfo.FileInfo)
|
||||
digestByPath := make(map[string]fileSampleDigests, len(files))
|
||||
|
||||
for _, file := range files {
|
||||
if ctx.Err() != nil {
|
||||
return nil
|
||||
}
|
||||
// Budget checks live inside readFileSampleDigests (after the cache lookup)
|
||||
// so cached digests still count toward grouping without consuming budget.
|
||||
// Checking here would stall on cached files and look stuck.
|
||||
|
||||
if file.Size != fileSize {
|
||||
continue
|
||||
}
|
||||
|
||||
filePath := filepath.Join(index.Path, file.Path)
|
||||
digests, err := readFileSampleDigests(ctx, index.Path, filePath, fileSize, file.ModTime, stats)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
digestByPath[file.Path] = digests
|
||||
stats.uniqueChecksums[digests.header] = true
|
||||
headerGroups[digests.header] = append(headerGroups[digests.header], file)
|
||||
}
|
||||
|
||||
headerCandidates := make([][]*iteminfo.FileInfo, 0)
|
||||
for _, group := range headerGroups {
|
||||
if len(group) >= 2 {
|
||||
headerCandidates = append(headerCandidates, group)
|
||||
}
|
||||
}
|
||||
if len(headerCandidates) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
middleGroups := make(map[string][]*iteminfo.FileInfo)
|
||||
for _, headerGroup := range headerCandidates {
|
||||
for _, file := range headerGroup {
|
||||
if ctx.Err() != nil {
|
||||
return nil
|
||||
}
|
||||
if shouldStop, _ := stats.shouldStop(); shouldStop {
|
||||
break
|
||||
}
|
||||
digests := digestByPath[file.Path]
|
||||
if digests.middle == "" {
|
||||
continue
|
||||
}
|
||||
stats.uniqueChecksums[digests.middle] = true
|
||||
middleGroups[digests.middle] = append(middleGroups[digests.middle], file)
|
||||
}
|
||||
}
|
||||
|
||||
groups := make([]checksumGroup, 0, len(middleGroups))
|
||||
for checksum, group := range middleGroups {
|
||||
if len(group) >= 2 {
|
||||
groups = append(groups, checksumGroup{
|
||||
Files: group,
|
||||
Checksum: checksum,
|
||||
})
|
||||
}
|
||||
}
|
||||
return groups
|
||||
}
|
||||
@@ -0,0 +1,116 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"encoding/hex"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/cespare/xxhash/v2"
|
||||
)
|
||||
|
||||
func TestReadFileSampleDigestsSmallFileUsesWholeFile(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
data := make([]byte, 12_000)
|
||||
path := filepath.Join(dir, "small.bin")
|
||||
if err := os.WriteFile(path, data, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
stats := &duplicateProcessingStats{startTime: time.Now()}
|
||||
d, err := readFileSampleDigests(context.Background(), dir, path, int64(len(data)), time.Now(), stats)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if d.header != d.middle {
|
||||
t.Fatalf("small file header and middle digest should match: %q %q", d.header, d.middle)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHasherUsesXXHash(t *testing.T) {
|
||||
if _, ok := any(duplicateChecksumHasher()).(*xxhash.Digest); !ok {
|
||||
t.Fatal("expected xxhash digest")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCachedDigestsDoNotConsumeChecksumBudget(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
dataA := make([]byte, 65536)
|
||||
for i := range dataA {
|
||||
dataA[i] = byte(i % 251)
|
||||
}
|
||||
// Use a distinct payload so its digests differ from other tests' files.
|
||||
pathA := filepath.Join(dir, "budget-a.bin")
|
||||
pathB := filepath.Join(dir, "budget-b.bin")
|
||||
if err := os.WriteFile(pathA, dataA, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(pathB, dataA, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
modTime := time.Date(2026, 11, 10, 0, 0, 0, 0, time.UTC)
|
||||
|
||||
// Prime the cache with one op-worth of budget.
|
||||
prime := &duplicateProcessingStats{startTime: time.Now(), uniqueChecksums: map[string]bool{}}
|
||||
d1, err := readFileSampleDigests(context.Background(), dir, pathA, int64(len(dataA)), modTime, prime)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d2, err := readFileSampleDigests(context.Background(), dir, pathB, int64(len(dataA)), modTime, prime)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if d1.header != d2.header || d1.middle != d2.middle {
|
||||
t.Fatal("expected equal digests for identical files")
|
||||
}
|
||||
if prime.checksumOperations != 2 {
|
||||
t.Fatalf("expected 2 disk operations for two cache-miss reads, got %d", prime.checksumOperations)
|
||||
}
|
||||
|
||||
// Exhaust the budget: any further uncached file should be refused.
|
||||
prime.checksumOperations = maxChecksumOperations
|
||||
uncached := filepath.Join(dir, "budget-c.bin")
|
||||
if err = os.WriteFile(uncached, dataA, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err = readFileSampleDigests(context.Background(), dir, uncached, int64(len(dataA)), modTime, prime); err == nil {
|
||||
t.Fatal("expected budget-exhausted error for uncached file")
|
||||
}
|
||||
|
||||
// But the cached files must still be served, for free.
|
||||
cached, err := readFileSampleDigests(context.Background(), dir, pathA, int64(len(dataA)), modTime, prime)
|
||||
if err != nil {
|
||||
t.Fatalf("cached digest should be served despite exhausted budget: %v", err)
|
||||
}
|
||||
if cached.header != d1.header || cached.middle != d1.middle {
|
||||
t.Fatal("cached digest mismatch")
|
||||
}
|
||||
if prime.checksumOperations != maxChecksumOperations {
|
||||
t.Fatalf("cached digest must not consume budget, got %d", prime.checksumOperations)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkDuplicateHashers(b *testing.B) {
|
||||
data := make([]byte, checksumSampleSize)
|
||||
for i := range data {
|
||||
data[i] = byte(i)
|
||||
}
|
||||
b.Run("md5", func(b *testing.B) {
|
||||
b.ReportAllocs()
|
||||
for i := 0; i < b.N; i++ {
|
||||
h := md5.New()
|
||||
_, _ = h.Write(data)
|
||||
hex.EncodeToString(h.Sum(nil))
|
||||
}
|
||||
})
|
||||
b.Run("xxhash", func(b *testing.B) {
|
||||
b.ReportAllocs()
|
||||
for i := 0; i < b.N; i++ {
|
||||
h := xxhash.New()
|
||||
_, _ = h.Write(data)
|
||||
hex.EncodeToString(h.Sum(nil))
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,111 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"cmp"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"github.com/gtsteffaniak/filebrowser/backend/pkg/indexing"
|
||||
)
|
||||
|
||||
type mergeGroupKey struct {
|
||||
size int64
|
||||
digest string
|
||||
}
|
||||
|
||||
// mergeGroupsByChecksum merges groups that share the same size and sample digest.
|
||||
func mergeGroupsByChecksum(groups []duplicateGroupWithChecksums) []duplicateGroupWithChecksums {
|
||||
if len(groups) <= 1 {
|
||||
return groups
|
||||
}
|
||||
|
||||
keyToGroups := make(map[mergeGroupKey][]int)
|
||||
for i, group := range groups {
|
||||
if group.Checksum == "" {
|
||||
continue
|
||||
}
|
||||
key := mergeGroupKey{size: group.Size, digest: group.Checksum}
|
||||
keyToGroups[key] = append(keyToGroups[key], i)
|
||||
}
|
||||
|
||||
parent := make([]int, len(groups))
|
||||
for i := range parent {
|
||||
parent[i] = i
|
||||
}
|
||||
|
||||
var find func(int) int
|
||||
find = func(x int) int {
|
||||
if parent[x] != x {
|
||||
parent[x] = find(parent[x])
|
||||
}
|
||||
return parent[x]
|
||||
}
|
||||
|
||||
union := func(x, y int) {
|
||||
px, py := find(x), find(y)
|
||||
if px != py {
|
||||
parent[px] = py
|
||||
}
|
||||
}
|
||||
|
||||
for _, indices := range keyToGroups {
|
||||
for i := 1; i < len(indices); i++ {
|
||||
union(indices[0], indices[i])
|
||||
}
|
||||
}
|
||||
|
||||
groupsByRoot := make(map[int][]int)
|
||||
for i := range groups {
|
||||
root := find(i)
|
||||
groupsByRoot[root] = append(groupsByRoot[root], i)
|
||||
}
|
||||
|
||||
merged := make([]duplicateGroupWithChecksums, 0, len(groupsByRoot))
|
||||
for _, indices := range groupsByRoot {
|
||||
if len(indices) == 1 {
|
||||
merged = append(merged, groups[indices[0]])
|
||||
continue
|
||||
}
|
||||
|
||||
combinedFiles := make([]*indexing.SearchResult, 0)
|
||||
var combinedSize int64
|
||||
var combinedChecksum string
|
||||
|
||||
for _, idx := range indices {
|
||||
group := groups[idx]
|
||||
combinedFiles = append(combinedFiles, group.Files...)
|
||||
if combinedChecksum == "" {
|
||||
combinedChecksum = group.Checksum
|
||||
}
|
||||
if combinedSize == 0 {
|
||||
combinedSize = group.Size
|
||||
}
|
||||
}
|
||||
|
||||
if len(combinedFiles) >= 2 {
|
||||
slices.SortFunc(combinedFiles, func(a, b *indexing.SearchResult) int {
|
||||
return strings.Compare(a.Path, b.Path)
|
||||
})
|
||||
merged = append(merged, duplicateGroupWithChecksums{
|
||||
Size: combinedSize,
|
||||
Files: combinedFiles,
|
||||
Checksum: combinedChecksum,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
slices.SortFunc(merged, func(a, b duplicateGroupWithChecksums) int {
|
||||
if c := cmp.Compare(b.Size, a.Size); c != 0 {
|
||||
return c
|
||||
}
|
||||
if c := strings.Compare(a.Checksum, b.Checksum); c != 0 {
|
||||
return c
|
||||
}
|
||||
if len(a.Files) == 0 || len(b.Files) == 0 {
|
||||
return 0
|
||||
}
|
||||
return strings.Compare(a.Files[0].Path, b.Files[0].Path)
|
||||
})
|
||||
|
||||
return merged
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/gtsteffaniak/filebrowser/backend/pkg/indexing"
|
||||
)
|
||||
|
||||
func TestMergeGroupsByChecksumUsesSizeAndDigest(t *testing.T) {
|
||||
groups := []duplicateGroupWithChecksums{
|
||||
{Size: 100, Checksum: "aaaa", Files: []*indexing.SearchResult{{Path: "/a"}, {Path: "/b"}}},
|
||||
{Size: 200, Checksum: "aaaa", Files: []*indexing.SearchResult{{Path: "/c"}, {Path: "/d"}}},
|
||||
{Size: 100, Checksum: "aaaa", Files: []*indexing.SearchResult{{Path: "/e"}, {Path: "/f"}}},
|
||||
}
|
||||
merged := mergeGroupsByChecksum(groups)
|
||||
if len(merged) != 2 {
|
||||
t.Fatalf("expected 2 merged groups, got %d", len(merged))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,204 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"unicode"
|
||||
|
||||
"github.com/gtsteffaniak/filebrowser/backend/pkg/indexing/iteminfo"
|
||||
)
|
||||
|
||||
const filenameSimilarityThreshold = 0.5
|
||||
|
||||
type levenshteinRowBuf struct {
|
||||
prev, cur []int
|
||||
}
|
||||
|
||||
func (b *levenshteinRowBuf) ensure(width int) {
|
||||
if cap(b.prev) < width+1 {
|
||||
b.prev = make([]int, width+1)
|
||||
b.cur = make([]int, width+1)
|
||||
} else {
|
||||
b.prev = b.prev[:width+1]
|
||||
b.cur = b.cur[:width+1]
|
||||
}
|
||||
}
|
||||
|
||||
func maxEditDistanceForSimilarity(maxLen int) int {
|
||||
return maxLen / 2
|
||||
}
|
||||
|
||||
func filenameSimilarWithBudget(name1, name2 string, buf *levenshteinRowBuf) bool {
|
||||
if name1 == name2 {
|
||||
return true
|
||||
}
|
||||
if len(name1) == 0 || len(name2) == 0 {
|
||||
return false
|
||||
}
|
||||
|
||||
maxLen := max(len(name1), len(name2))
|
||||
maxDist := maxEditDistanceForSimilarity(maxLen)
|
||||
if abs(len(name1)-len(name2)) > maxDist {
|
||||
return false
|
||||
}
|
||||
|
||||
distance := boundedLevenshtein(name1, name2, maxDist, buf)
|
||||
if distance < 0 {
|
||||
return false
|
||||
}
|
||||
similarity := 1.0 - float64(distance)/float64(maxLen)
|
||||
return similarity >= filenameSimilarityThreshold
|
||||
}
|
||||
|
||||
func boundedLevenshtein(s1, s2 string, maxDist int, buf *levenshteinRowBuf) int {
|
||||
n, m := len(s1), len(s2)
|
||||
if n == 0 {
|
||||
if m <= maxDist {
|
||||
return m
|
||||
}
|
||||
return -1
|
||||
}
|
||||
if m == 0 {
|
||||
if n <= maxDist {
|
||||
return n
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
buf.ensure(m)
|
||||
prev, cur := buf.prev, buf.cur
|
||||
|
||||
for j := 0; j <= m; j++ {
|
||||
prev[j] = j
|
||||
}
|
||||
|
||||
for i := 1; i <= n; i++ {
|
||||
cur[0] = i
|
||||
rowMin := cur[0]
|
||||
for j := 1; j <= m; j++ {
|
||||
cost := 0
|
||||
if s1[i-1] != s2[j-1] {
|
||||
cost = 1
|
||||
}
|
||||
del := prev[j] + 1
|
||||
ins := cur[j-1] + 1
|
||||
sub := prev[j-1] + cost
|
||||
v := del
|
||||
if ins < v {
|
||||
v = ins
|
||||
}
|
||||
if sub < v {
|
||||
v = sub
|
||||
}
|
||||
cur[j] = v
|
||||
if v < rowMin {
|
||||
rowMin = v
|
||||
}
|
||||
}
|
||||
if rowMin > maxDist {
|
||||
return -1
|
||||
}
|
||||
prev, cur = cur, prev
|
||||
}
|
||||
|
||||
if prev[m] > maxDist {
|
||||
return -1
|
||||
}
|
||||
return prev[m]
|
||||
}
|
||||
|
||||
func groupFilesByFilename(ctx context.Context, files []*iteminfo.FileInfo, expectedSize int64, stats *duplicateProcessingStats) [][]*iteminfo.FileInfo {
|
||||
if len(files) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
normalized := make([]string, len(files))
|
||||
for i, f := range files {
|
||||
if f.Name != "" {
|
||||
normalized[i] = normalizeFilename(f.Name)
|
||||
}
|
||||
}
|
||||
|
||||
var rowBuf levenshteinRowBuf
|
||||
groups := [][]*iteminfo.FileInfo{}
|
||||
used := make(map[int]bool)
|
||||
|
||||
for i := 0; i < len(files); i++ {
|
||||
if ctx.Err() != nil {
|
||||
return groups
|
||||
}
|
||||
if shouldStop, _ := stats.shouldStop(); shouldStop {
|
||||
return groups
|
||||
}
|
||||
|
||||
if used[i] || files[i].Name == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
group := []*iteminfo.FileInfo{files[i]}
|
||||
used[i] = true
|
||||
filename1 := normalized[i]
|
||||
oversized := false
|
||||
|
||||
for j := i + 1; j < len(files); j++ {
|
||||
if ctx.Err() != nil {
|
||||
return groups
|
||||
}
|
||||
if shouldStop, _ := stats.shouldStop(); shouldStop {
|
||||
return groups
|
||||
}
|
||||
|
||||
if used[j] || files[j].Name == "" {
|
||||
continue
|
||||
}
|
||||
if files[j].Size != expectedSize {
|
||||
continue
|
||||
}
|
||||
|
||||
if filenameSimilarWithBudget(filename1, normalized[j], &rowBuf) {
|
||||
group = append(group, files[j])
|
||||
used[j] = true
|
||||
if len(group) > maxFuzzyGroupSize {
|
||||
oversized = true
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if len(group) >= 2 && !oversized {
|
||||
groups = append(groups, group)
|
||||
}
|
||||
}
|
||||
|
||||
return groups
|
||||
}
|
||||
|
||||
func normalizeFilename(filename string) string {
|
||||
filename = strings.ToLower(filename)
|
||||
if idx := strings.LastIndex(filename, "."); idx > 0 {
|
||||
filename = filename[:idx]
|
||||
}
|
||||
|
||||
var result strings.Builder
|
||||
for _, r := range filename {
|
||||
if unicode.IsLetter(r) || unicode.IsDigit(r) || r == ' ' || r == '-' {
|
||||
result.WriteRune(r)
|
||||
}
|
||||
}
|
||||
|
||||
return strings.TrimSpace(result.String())
|
||||
}
|
||||
|
||||
func abs(x int) int {
|
||||
if x < 0 {
|
||||
return -x
|
||||
}
|
||||
return x
|
||||
}
|
||||
|
||||
func max(a, b int) int {
|
||||
if a > b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
@@ -0,0 +1,140 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"context"
|
||||
"math/rand"
|
||||
"strings"
|
||||
"testing"
|
||||
"testing/quick"
|
||||
"time"
|
||||
"unicode"
|
||||
|
||||
"github.com/gtsteffaniak/filebrowser/backend/pkg/indexing/iteminfo"
|
||||
)
|
||||
|
||||
func legacyFilenamesSimilar(name1, name2 string) bool {
|
||||
if name1 == name2 {
|
||||
return true
|
||||
}
|
||||
if len(name1) == 0 || len(name2) == 0 {
|
||||
return false
|
||||
}
|
||||
distance := legacyLevenshtein(name1, name2)
|
||||
maxLen := max(len(name1), len(name2))
|
||||
similarity := 1.0 - float64(distance)/float64(maxLen)
|
||||
return similarity >= 0.5
|
||||
}
|
||||
|
||||
func legacyLevenshtein(s1, s2 string) int {
|
||||
if len(s1) == 0 {
|
||||
return len(s2)
|
||||
}
|
||||
if len(s2) == 0 {
|
||||
return len(s1)
|
||||
}
|
||||
matrix := make([][]int, len(s1)+1)
|
||||
for i := range matrix {
|
||||
matrix[i] = make([]int, len(s2)+1)
|
||||
}
|
||||
for i := 0; i <= len(s1); i++ {
|
||||
matrix[i][0] = i
|
||||
}
|
||||
for j := 0; j <= len(s2); j++ {
|
||||
matrix[0][j] = j
|
||||
}
|
||||
for i := 1; i <= len(s1); i++ {
|
||||
for j := 1; j <= len(s2); j++ {
|
||||
cost := 0
|
||||
if s1[i-1] != s2[j-1] {
|
||||
cost = 1
|
||||
}
|
||||
del := matrix[i-1][j] + 1
|
||||
ins := matrix[i][j-1] + 1
|
||||
sub := matrix[i-1][j-1] + cost
|
||||
v := del
|
||||
if ins < v {
|
||||
v = ins
|
||||
}
|
||||
if sub < v {
|
||||
v = sub
|
||||
}
|
||||
matrix[i][j] = v
|
||||
}
|
||||
}
|
||||
return matrix[len(s1)][len(s2)]
|
||||
}
|
||||
|
||||
var filenameCorpus = []string{
|
||||
"photo", "photo_1", "photo-copy", "IMG_0042", "img 0042",
|
||||
"vacation2023beach", "vacation2023_beach", "user_video", "video",
|
||||
"document-final", "document_final_v2", "a", "ab", "abc", "abcd",
|
||||
"report Q1 2024", "report-q1-2024", "my@file#name", "myfilename",
|
||||
"summer-holiday", "summer holiday", "winter-holiday",
|
||||
}
|
||||
|
||||
func TestFilenameSimilarMatchesLegacyOnCorpus(t *testing.T) {
|
||||
names := append([]string{}, filenameCorpus...)
|
||||
for i := 0; i < 50; i++ {
|
||||
names = append(names, randomNormalizedName(5+corpusRand.Intn(40)))
|
||||
}
|
||||
|
||||
var buf levenshteinRowBuf
|
||||
for _, a := range names {
|
||||
a = normalizeFilename(a + ".jpg")
|
||||
for _, b := range names {
|
||||
b = normalizeFilename(b + ".png")
|
||||
if legacyFilenamesSimilar(a, b) != filenameSimilarWithBudget(a, b, &buf) {
|
||||
t.Fatalf("mismatch %q vs %q", a, b)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestFilenameSimilarMatchesLegacyQuick(t *testing.T) {
|
||||
var buf levenshteinRowBuf
|
||||
if err := quick.Check(func(a, b string) bool {
|
||||
a = trimQuickString(a, 48)
|
||||
b = trimQuickString(b, 48)
|
||||
return legacyFilenamesSimilar(a, b) == filenameSimilarWithBudget(a, b, &buf)
|
||||
}, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func trimQuickString(s string, max int) string {
|
||||
if len(s) > max {
|
||||
s = s[:max]
|
||||
}
|
||||
var b strings.Builder
|
||||
for _, r := range s {
|
||||
if unicode.IsLetter(r) || unicode.IsDigit(r) || r == ' ' || r == '-' {
|
||||
b.WriteRune(r)
|
||||
}
|
||||
}
|
||||
return strings.ToLower(strings.TrimSpace(b.String()))
|
||||
}
|
||||
|
||||
func randomNormalizedName(n int) string {
|
||||
const alphabet = "abcdefghijklmnopqrstuvwxyz0123456789 -"
|
||||
b := make([]byte, n)
|
||||
for i := range b {
|
||||
b[i] = alphabet[corpusRand.Intn(len(alphabet))]
|
||||
}
|
||||
return string(b)
|
||||
}
|
||||
|
||||
func TestGroupFilesByFilenameOversizedExcluded(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
stats := &duplicateProcessingStats{startTime: time.Now()}
|
||||
files := make([]*iteminfo.FileInfo, 12)
|
||||
for i := range files {
|
||||
files[i] = &iteminfo.FileInfo{
|
||||
ItemInfo: iteminfo.ItemInfo{Name: "photo.jpg", Size: 100},
|
||||
}
|
||||
}
|
||||
if len(groupFilesByFilename(ctx, files, 100, stats)) != 0 {
|
||||
t.Fatal("expected no groups for oversized cluster")
|
||||
}
|
||||
}
|
||||
|
||||
var corpusRand = rand.New(rand.NewSource(42))
|
||||
@@ -187,10 +187,10 @@ func fileWatchHandler(w http.ResponseWriter, r *http.Request, d *Context) (int,
|
||||
return http.StatusForbidden, fmt.Errorf("access denied to file")
|
||||
}
|
||||
|
||||
// Get real file path
|
||||
realPath, _, err := idx.GetRealPath(scopePath)
|
||||
// Get real file path (scoped) and enforce ACL on the resolved target
|
||||
realPath, _, err := idx.GetRealPathScoped(userScope, scopePath)
|
||||
if err != nil {
|
||||
return http.StatusNotFound, fmt.Errorf("file not found: %v", err)
|
||||
return realPathErrStatus(err), err
|
||||
}
|
||||
|
||||
// Get file/directory info
|
||||
@@ -198,6 +198,9 @@ func fileWatchHandler(w http.ResponseWriter, r *http.Request, d *Context) (int,
|
||||
if err != nil {
|
||||
return http.StatusNotFound, fmt.Errorf("path not found: %v", err)
|
||||
}
|
||||
if !state.AccessPermittedAtRealPath(idx, d.User.Username, realPath, info.IsDir()) {
|
||||
return http.StatusForbidden, fmt.Errorf("access denied to file")
|
||||
}
|
||||
|
||||
// Get MIME type from the index if available
|
||||
mimeType := "application/octet-stream"
|
||||
@@ -335,10 +338,10 @@ func fileWatchSSEHandler(w http.ResponseWriter, r *http.Request, d *Context) (in
|
||||
return http.StatusForbidden, fmt.Errorf("access denied to file")
|
||||
}
|
||||
|
||||
// Get real file path
|
||||
realPath, _, err := idx.GetRealPath(scopePath)
|
||||
// Get real file path (scoped) and enforce ACL on the resolved target
|
||||
realPath, _, err := idx.GetRealPathScoped(userScope, scopePath)
|
||||
if err != nil {
|
||||
return http.StatusNotFound, fmt.Errorf("path not found: %v", err)
|
||||
return realPathErrStatus(err), err
|
||||
}
|
||||
|
||||
// Get file/directory info
|
||||
@@ -346,6 +349,9 @@ func fileWatchSSEHandler(w http.ResponseWriter, r *http.Request, d *Context) (in
|
||||
if err != nil {
|
||||
return http.StatusNotFound, fmt.Errorf("path not found: %v", err)
|
||||
}
|
||||
if !state.AccessPermittedAtRealPath(idx, d.User.Username, realPath, info.IsDir()) {
|
||||
return http.StatusForbidden, fmt.Errorf("access denied to file")
|
||||
}
|
||||
|
||||
// Set up SSE headers
|
||||
w.Header().Set("Content-Type", "text/event-stream")
|
||||
|
||||
@@ -22,3 +22,37 @@ func TestErrToStatusWrapsUnsupportedPreviewFormat(t *testing.T) {
|
||||
t.Fatalf("ErrToStatus() = %d, want %d", status, http.StatusUnsupportedMediaType)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsViewerOriginalPreview(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
size string
|
||||
file string
|
||||
fileType string
|
||||
docs bool // the server can render documents to an image
|
||||
want bool
|
||||
}{
|
||||
{"Fuji RAW viewer", "original", "DSCF0308.RAF", "image/x-fuji-raf", false, true},
|
||||
{"Canon RAW viewer", "original", "IMG_0001.CR3", "image/x-canon-cr3", false, true},
|
||||
{"Adobe DNG viewer", "original", "a.dng", "image/x-adobe-dng", false, true},
|
||||
{"HEIC viewer", "original", "a.heic", "image/heic", false, true},
|
||||
{"HEIF viewer", "original", "a.heif", "image/heif", false, true},
|
||||
{"rendered document viewer", "original", "a.docx", "application/vnd.openxmlformats-officedocument.wordprocessingml.document", true, true},
|
||||
{"documents need the server to be able to render them", "original", "a.docx", "application/vnd.openxmlformats-officedocument.wordprocessingml.document", false, false},
|
||||
{"RAW thumbnail stays disabled", "small", "DSCF0308.RAF", "image/x-fuji-raf", true, false},
|
||||
{"RAW large stays disabled", "large", "DSCF0308.RAF", "image/x-fuji-raf", true, false},
|
||||
{"no size stays disabled", "", "DSCF0308.RAF", "image/x-fuji-raf", true, false},
|
||||
{"video poster stays disabled", "original", "a.mov", "video/quicktime", true, false},
|
||||
{"audio cover stays disabled", "original", "a.mp3", "audio/mpeg", true, false},
|
||||
{"JPEG is opened directly, not through previews", "original", "a.jpg", "image/jpeg", true, false},
|
||||
{"PDF is opened directly, not through previews", "original", "a.pdf", "application/pdf", true, false},
|
||||
{"text stays disabled", "original", "a.txt", "text/plain", true, false},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if got := isViewerOriginalPreview(tt.size, tt.file, tt.fileType, tt.docs); got != tt.want {
|
||||
t.Fatalf("isViewerOriginalPreview(%q, %q, %q, %v) = %v, want %v", tt.size, tt.file, tt.fileType, tt.docs, got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -3,6 +3,7 @@ package web
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"path/filepath"
|
||||
@@ -12,7 +13,10 @@ import (
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/adapters/fs/files"
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/ffmpeg"
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/gomedia"
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/mediacache"
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/state"
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/utils"
|
||||
"github.com/gtsteffaniak/filebrowser/backend/pkg/indexing"
|
||||
"github.com/gtsteffaniak/filebrowser/backend/pkg/settings"
|
||||
)
|
||||
|
||||
@@ -106,9 +110,34 @@ func subtitlesHandler(w http.ResponseWriter, r *http.Request, d *Context) (int,
|
||||
|
||||
var content string
|
||||
if !embedded {
|
||||
subtitlePath := filepath.Join(filepath.Dir(fileInfo.RealPath), filepath.Base(track.Name))
|
||||
var subtitlePath string
|
||||
if d.Share.Hash != "" {
|
||||
if d.Share.Path == "" {
|
||||
return http.StatusForbidden, fmt.Errorf("share has no path scope")
|
||||
}
|
||||
idx := indexing.GetIndex(source)
|
||||
if idx == nil {
|
||||
return http.StatusInternalServerError, fmt.Errorf("source %s is not available", source)
|
||||
}
|
||||
bound, sidecarIndexPath, scopeErr := shareSubtitleSidecarIndexPaths(d, source, path, track.Name)
|
||||
if scopeErr != nil {
|
||||
return http.StatusForbidden, scopeErr
|
||||
}
|
||||
subtitlePath, _, err = idx.GetRealPathScoped(bound, sidecarIndexPath)
|
||||
if err != nil {
|
||||
return realPathErrStatus(err), err
|
||||
}
|
||||
if !state.AccessPermittedAtRealPath(idx, d.ShareUser.Username, subtitlePath, false) {
|
||||
return http.StatusForbidden, fmt.Errorf("access denied")
|
||||
}
|
||||
} else {
|
||||
subtitlePath = filepath.Join(filepath.Dir(fileInfo.RealPath), filepath.Base(track.Name))
|
||||
}
|
||||
content, err = utils.GetSubtitleSidecarContent(subtitlePath)
|
||||
if err != nil {
|
||||
if errors.Is(err, utils.ErrSubtitleSidecarNotRegular) {
|
||||
return http.StatusForbidden, err
|
||||
}
|
||||
return http.StatusInternalServerError, fmt.Errorf("failed to get subtitle sidecar content: %v", err)
|
||||
}
|
||||
} else {
|
||||
@@ -123,7 +152,8 @@ func subtitlesHandler(w http.ResponseWriter, r *http.Request, d *Context) (int,
|
||||
}
|
||||
// Shadow ffmpeg runs after the fast path; bound it so a detached request cannot
|
||||
// leave extraction running indefinitely (sync fallback still uses r.Context()).
|
||||
legacyCtx, cancel := context.WithTimeout(context.WithoutCancel(r.Context()), time.Minute)
|
||||
// WithoutCache: compare against a fresh extraction, not the value go-media stored.
|
||||
legacyCtx, cancel := context.WithTimeout(mediacache.WithoutCache(context.WithoutCancel(r.Context())), time.Minute)
|
||||
defer cancel()
|
||||
vtt, extractErr := svc.ExtractSubtitle(legacyCtx, fileInfo.RealPath, *track.Index)
|
||||
return vtt, extractErr == nil
|
||||
@@ -160,6 +190,34 @@ func findSubtitleTrack(subtitles []utils.SubtitleTrack, name string, embedded bo
|
||||
return nil
|
||||
}
|
||||
|
||||
// shareSubtitleSidecarIndexPaths maps middleware IndexPath (user-scope-relative) and Share.Path
|
||||
// to scoped index paths for sidecar containment (see publicDeleteHandler).
|
||||
func shareSubtitleSidecarIndexPaths(d *Context, sourceName, videoIndexPath, sidecarFileName string) (bound, sidecarIndexPath string, err error) {
|
||||
userScope, scopeErr := d.ShareUser.GetScopeForSourceName(sourceName)
|
||||
if scopeErr != nil || userScope == "" {
|
||||
return "", "", fmt.Errorf("user has no access to source: %s", sourceName)
|
||||
}
|
||||
if d.Share.Path == "" {
|
||||
return "", "", fmt.Errorf("share has no path scope")
|
||||
}
|
||||
shareRel := strings.TrimPrefix(d.Share.Path, userScope)
|
||||
bound = utils.JoinScopedIndexPath(userScope, shareRel)
|
||||
videoRel := strings.Trim(strings.TrimSpace(videoIndexPath), "/")
|
||||
fullVideo := utils.JoinScopedIndexPath(userScope, videoRel)
|
||||
if strings.TrimSuffix(bound, "/") == strings.TrimSuffix(fullVideo, "/") {
|
||||
parent := utils.GetParentDirectoryPath(strings.TrimSuffix(bound, "/"))
|
||||
if parent == "" {
|
||||
parent = "/"
|
||||
}
|
||||
bound = parent
|
||||
}
|
||||
sidecarIndexPath = utils.JoinPathAsUnix(
|
||||
utils.GetParentDirectoryPath(strings.TrimSuffix(fullVideo, "/")),
|
||||
filepath.Base(sidecarFileName),
|
||||
)
|
||||
return bound, sidecarIndexPath, nil
|
||||
}
|
||||
|
||||
// metadataHandler returns the same directory resource shape as GET /api/resources with metadata enabled,
|
||||
// for client-side patching after a fast listing load.
|
||||
// @Summary Directory with media metadata
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"sort"
|
||||
"testing"
|
||||
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/database/users"
|
||||
@@ -12,19 +13,23 @@ import (
|
||||
|
||||
func withMediaHandlerSettings(t *testing.T) {
|
||||
t.Helper()
|
||||
originalTranscode := settings.Config.Integrations.Media.Transcode
|
||||
originalMedia := settings.CurrentMediaSettings()
|
||||
originalFFmpeg := settings.Env.FFmpegAvailable
|
||||
t.Cleanup(func() {
|
||||
settings.Config.Integrations.Media.Transcode = originalTranscode
|
||||
settings.Env.FFmpegAvailable = originalFFmpeg
|
||||
settings.ApplyMediaSettings(originalMedia)
|
||||
})
|
||||
settings.Env.FFmpegAvailable = true
|
||||
settings.Config.Integrations.Media.Transcode = settings.MediaTranscode{
|
||||
Enabled: true,
|
||||
MaxConcurrent: 3,
|
||||
MaxConcurrentPerUser: 2,
|
||||
Profiles: settings.DefaultTranscodeProfiles(),
|
||||
media := originalMedia
|
||||
media.Transcode = settings.FullAdminTranscode{
|
||||
UserTranscode: settings.UserTranscode{
|
||||
Enabled: true,
|
||||
MaxConcurrentPerUser: 2,
|
||||
},
|
||||
MaxConcurrent: 3,
|
||||
Profiles: settings.DefaultTranscodeProfiles(),
|
||||
}
|
||||
settings.ApplyMediaSettings(media)
|
||||
}
|
||||
|
||||
func mediaHandlerContext(admin bool) *Context {
|
||||
@@ -52,62 +57,111 @@ func callMediaGet(t *testing.T, admin bool) map[string]any {
|
||||
return body
|
||||
}
|
||||
|
||||
// Non-admin users must be able to read the effective media values for the
|
||||
// player, but only the player-relevant subset.
|
||||
// objectField returns obj[key] as a JSON object, failing the test if it is missing or not an object.
|
||||
func objectField(t *testing.T, obj map[string]any, key string) map[string]any {
|
||||
t.Helper()
|
||||
child, ok := obj[key].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("%s is %T, want object", key, obj[key])
|
||||
}
|
||||
return child
|
||||
}
|
||||
|
||||
// keysOf returns the sorted keys of a JSON object.
|
||||
func keysOf(obj map[string]any) []string {
|
||||
keys := make([]string, 0, len(obj))
|
||||
for k := range obj {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
return keys
|
||||
}
|
||||
|
||||
func assertKeys(t *testing.T, path string, got []string, want []string) {
|
||||
t.Helper()
|
||||
sort.Strings(want)
|
||||
if len(got) != len(want) {
|
||||
t.Fatalf("%s keys = %v, want %v", path, got, want)
|
||||
}
|
||||
for i := range got {
|
||||
if got[i] != want[i] {
|
||||
t.Fatalf("%s keys = %v, want %v", path, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Non-admin users receive exactly the PUBLIC payload. Any new key fails this test,
|
||||
// which forces a conscious decision about whether it belongs in UserMedia.
|
||||
func TestSettingsMediaHandlerPlayerPayloadForNonAdmin(t *testing.T) {
|
||||
withMediaHandlerSettings(t)
|
||||
|
||||
body := callMediaGet(t, false)
|
||||
if _, ok := body["transcode"]; ok {
|
||||
t.Fatal("non-admin must not receive the full transcode settings")
|
||||
assertKeys(t, "top-level", keysOf(body), []string{"transcode", "imagePreview"})
|
||||
transcode := objectField(t, body, "transcode")
|
||||
assertKeys(t, "transcode", keysOf(transcode), []string{"enabled", "maxConcurrentPerUser", "enabledProfiles"})
|
||||
|
||||
if enabled, ok := transcode["enabled"].(bool); !ok || !enabled {
|
||||
t.Fatalf("transcode.enabled = %v, want true", transcode["enabled"])
|
||||
}
|
||||
if _, ok := body["lockedFromConfigPaths"]; ok {
|
||||
t.Fatal("non-admin must not receive config lock metadata")
|
||||
if limit, ok := transcode["maxConcurrentPerUser"].(float64); !ok || limit != 2 {
|
||||
t.Fatalf("transcode.maxConcurrentPerUser = %v, want 2", transcode["maxConcurrentPerUser"])
|
||||
}
|
||||
if _, ok := body["lockMessage"]; ok {
|
||||
t.Fatal("non-admin must not receive the config lock message")
|
||||
}
|
||||
if enabled, ok := body["enabled"].(bool); !ok || !enabled {
|
||||
t.Fatalf("enabled = %v, want true", body["enabled"])
|
||||
}
|
||||
if limit, ok := body["maxConcurrentPerUser"].(float64); !ok || limit != 2 {
|
||||
t.Fatalf("maxConcurrentPerUser = %v, want 2", body["maxConcurrentPerUser"])
|
||||
}
|
||||
profiles, ok := body["profiles"].([]any)
|
||||
profiles, ok := transcode["enabledProfiles"].([]any)
|
||||
if !ok || len(profiles) != 3 {
|
||||
t.Fatalf("profiles = %v, want 3 enabled ids", body["profiles"])
|
||||
t.Fatalf("transcode.enabledProfiles = %v, want 3 enabled ids", transcode["enabledProfiles"])
|
||||
}
|
||||
}
|
||||
|
||||
// Admins receive the full stored config. The keys shared with users must have the
|
||||
// same names, so the admin payload is a strict superset of the user payload.
|
||||
func TestSettingsMediaHandlerFullPayloadForAdmin(t *testing.T) {
|
||||
withMediaHandlerSettings(t)
|
||||
|
||||
body := callMediaGet(t, true)
|
||||
transcode, ok := body["transcode"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("admin must receive the full transcode settings, got %T", body["transcode"])
|
||||
}
|
||||
if _, ok := transcode["profiles"].(map[string]any); !ok {
|
||||
t.Fatalf("admin transcode payload must include profiles, got %v", transcode["profiles"])
|
||||
}
|
||||
// Admin payload exposes the full settings surface, not the player subset.
|
||||
for _, key := range []string{"convert", "debug", "hardwareAcceleration"} {
|
||||
if _, ok := body[key]; !ok {
|
||||
admin := callMediaGet(t, true)
|
||||
user := callMediaGet(t, false)
|
||||
|
||||
// lockedFromConfigPaths is omitted when config pins nothing, so it is not required here.
|
||||
for _, key := range []string{"transcode", "imagePreview", "convert", "debug", "hardwareAcceleration", "accelDevice"} {
|
||||
if _, ok := admin[key]; !ok {
|
||||
t.Fatalf("admin payload missing %q", key)
|
||||
}
|
||||
}
|
||||
if _, ok := body["enabled"]; ok {
|
||||
t.Fatal("the admin payload uses the full settings shape, not the player subset")
|
||||
if _, ok := admin["enabled"]; ok {
|
||||
t.Fatal("admin payload must not repeat transcode.enabled at the top level")
|
||||
}
|
||||
|
||||
adminTranscode := objectField(t, admin, "transcode")
|
||||
for _, key := range []string{"maxConcurrent", "profiles", "cacheMaxSizeMB", "encoderBackend", "readRate"} {
|
||||
if _, ok := adminTranscode[key]; !ok {
|
||||
t.Fatalf("admin transcode missing %q", key)
|
||||
}
|
||||
}
|
||||
if _, ok := adminTranscode["profiles"].(map[string]any); !ok {
|
||||
t.Fatalf("admin transcode.profiles must be an object, got %T", adminTranscode["profiles"])
|
||||
}
|
||||
|
||||
// Superset rule: every key a user receives must also exist for admins. Values may
|
||||
// differ by design: admins see stored config (so the UI can save it back unchanged),
|
||||
// users see effective values (for example enabled is false when ffmpeg is missing).
|
||||
for key := range objectField(t, user, "transcode") {
|
||||
if _, ok := adminTranscode[key]; !ok {
|
||||
t.Fatalf("admin transcode missing shared key %q", key)
|
||||
}
|
||||
}
|
||||
if admin["imagePreview"] == nil {
|
||||
t.Fatal("imagePreview must be present for admins")
|
||||
}
|
||||
}
|
||||
|
||||
// Disabling transcoding at runtime must be visible to non-admin players.
|
||||
func TestSettingsMediaHandlerReflectsDisabledTranscode(t *testing.T) {
|
||||
withMediaHandlerSettings(t)
|
||||
settings.Config.Integrations.Media.Transcode.Enabled = false
|
||||
disabled := settings.CurrentMediaSettings()
|
||||
disabled.Transcode.Enabled = false
|
||||
settings.ApplyMediaSettings(disabled)
|
||||
|
||||
body := callMediaGet(t, false)
|
||||
if enabled, ok := body["enabled"].(bool); !ok || enabled {
|
||||
t.Fatalf("enabled = %v, want false", body["enabled"])
|
||||
transcode := objectField(t, callMediaGet(t, false), "transcode")
|
||||
if enabled, ok := transcode["enabled"].(bool); !ok || enabled {
|
||||
t.Fatalf("transcode.enabled = %v, want false", transcode["enabled"])
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,333 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/adapters/fs/files"
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/app"
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/database/share"
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/database/users"
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/state"
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/utils"
|
||||
"github.com/gtsteffaniak/filebrowser/backend/pkg/indexing"
|
||||
"github.com/gtsteffaniak/filebrowser/backend/pkg/indexing/iteminfo"
|
||||
"github.com/gtsteffaniak/filebrowser/backend/pkg/settings"
|
||||
)
|
||||
|
||||
func setupSubtitlesHTTPTest(t *testing.T) (downloadsPath string, admin *users.User) {
|
||||
t.Helper()
|
||||
lockWebGlobals(t)
|
||||
downloadsPath = filepath.Join(t.TempDir(), "downloads")
|
||||
|
||||
dbPath := filepath.Join(t.TempDir(), "subtitles-http-test.sqlite")
|
||||
if _, err := state.Initialize(dbPath); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
app.MustWireServices(state.Default())
|
||||
t.Cleanup(func() {
|
||||
state.Close()
|
||||
})
|
||||
|
||||
settings.Config.Server.SourceMap = map[string]*settings.Source{
|
||||
downloadsPath: {
|
||||
Path: downloadsPath,
|
||||
Name: "Downloads",
|
||||
},
|
||||
}
|
||||
settings.Config.Server.NameToSource = map[string]*settings.Source{
|
||||
"Downloads": settings.Config.Server.SourceMap[downloadsPath],
|
||||
}
|
||||
settings.InitializeUserResolvers()
|
||||
|
||||
adminUser := &users.User{
|
||||
ID: 1,
|
||||
FrontendUser: users.FrontendUser{
|
||||
Username: "admin",
|
||||
Permissions: users.Permissions{Admin: true},
|
||||
},
|
||||
BackendScopes: []users.BackendScope{
|
||||
{Path: downloadsPath, Scope: "/"},
|
||||
},
|
||||
BackendSourcePermissions: map[string]users.SourceFilePermissions{
|
||||
downloadsPath: {
|
||||
View: true,
|
||||
Download: true,
|
||||
Modify: true,
|
||||
Create: true,
|
||||
Delete: true,
|
||||
},
|
||||
},
|
||||
Version: users.SourcePermissionsMigrationVersion,
|
||||
}
|
||||
if err := state.CreateUser(adminUser, ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
adminUser.Permissions = users.Permissions{Admin: true}
|
||||
if err := state.UpdateUser(adminUser, "", "permissions"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := state.UpdateUser(adminUser, "", "backendScopes"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := state.UpdateUser(adminUser, "", "backendSourcePermissions"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if err := os.MkdirAll(downloadsPath, 0755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
indexing.SetTestIndex("Downloads", downloadsPath)
|
||||
t.Cleanup(func() {
|
||||
indexing.ClearTestIndices()
|
||||
})
|
||||
|
||||
loaded, err := state.GetUserByUsername("admin")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return downloadsPath, &loaded
|
||||
}
|
||||
|
||||
func mockVideoWithExternalSubtitle(t *testing.T, trackName, videoRealPath string) {
|
||||
t.Helper()
|
||||
prev := files.FileInfoFasterFunc
|
||||
t.Cleanup(func() { files.FileInfoFasterFunc = prev })
|
||||
files.FileInfoFasterFunc = func(opts utils.FileOptions, user *users.User) (*iteminfo.ExtendedFileInfo, error) {
|
||||
return &iteminfo.ExtendedFileInfo{
|
||||
FileInfo: iteminfo.FileInfo{
|
||||
Path: opts.Path,
|
||||
ItemInfo: iteminfo.ItemInfo{
|
||||
Type: "video/mp4",
|
||||
},
|
||||
},
|
||||
Subtitles: []utils.SubtitleTrack{
|
||||
{Name: trackName, Embedded: false},
|
||||
},
|
||||
RealPath: videoRealPath,
|
||||
}, nil
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubtitlesHandler_RejectsEscapingSidecarSymlink(t *testing.T) {
|
||||
downloadsPath, adminUser := setupSubtitlesHTTPTest(t)
|
||||
videoPath := filepath.Join(downloadsPath, "clip.mov")
|
||||
mockVideoWithExternalSubtitle(t, "clip.srt", videoPath)
|
||||
|
||||
outside := filepath.Join(filepath.Dir(downloadsPath), "outside-secret")
|
||||
if err := os.MkdirAll(outside, 0755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
secret := filepath.Join(outside, "secret.srt")
|
||||
if err := os.WriteFile(secret, []byte("SECRET_OUTSIDE_SUBTITLE"), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(videoPath, []byte("video"), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Symlink(secret, filepath.Join(downloadsPath, "clip.srt")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/media/subtitles?source=Downloads&path=/clip.mov&name=clip.srt&embedded=false", nil)
|
||||
rec := httptest.NewRecorder()
|
||||
status, handlerErr := subtitlesHandler(rec, req, &Context{User: adminUser})
|
||||
if status != http.StatusForbidden {
|
||||
t.Fatalf("status=%d err=%v body=%q", status, handlerErr, rec.Body.String())
|
||||
}
|
||||
if strings.Contains(rec.Body.String(), "SECRET_OUTSIDE_SUBTITLE") {
|
||||
t.Fatalf("response leaked outside subtitle content")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubtitlesHandler_ReturnsInScopeSidecar(t *testing.T) {
|
||||
downloadsPath, adminUser := setupSubtitlesHTTPTest(t)
|
||||
videoPath := filepath.Join(downloadsPath, "clip.mov")
|
||||
mockVideoWithExternalSubtitle(t, "clip.srt", videoPath)
|
||||
|
||||
srtBody := "1\n00:00:01,000 --> 00:00:02,000\nHello in scope"
|
||||
if err := os.WriteFile(videoPath, []byte("video"), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(downloadsPath, "clip.srt"), []byte(srtBody), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/media/subtitles?source=Downloads&path=/clip.mov&name=clip.srt&embedded=false", nil)
|
||||
rec := httptest.NewRecorder()
|
||||
status, handlerErr := subtitlesHandler(rec, req, &Context{User: adminUser})
|
||||
if status != http.StatusOK || handlerErr != nil {
|
||||
t.Fatalf("status=%d err=%v body=%q", status, handlerErr, rec.Body.String())
|
||||
}
|
||||
if !strings.Contains(rec.Body.String(), "Hello in scope") {
|
||||
t.Fatalf("body=%q want in-scope subtitle text", rec.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubtitlesHandler_ShareScopedOwnerLoadsSidecar(t *testing.T) {
|
||||
sourceRoot, adminUser := setupSubtitlesHTTPTest(t)
|
||||
aliceRoot := filepath.Join(sourceRoot, "users", "alice", "shared")
|
||||
if err := os.MkdirAll(aliceRoot, 0755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
srtBody := "1\n00:00:01,000 --> 00:00:02,000\nScoped share OK"
|
||||
videoPath := filepath.Join(aliceRoot, "clip.mov")
|
||||
if err := os.WriteFile(videoPath, []byte("video"), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(aliceRoot, "clip.srt"), []byte(srtBody), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
mockVideoWithExternalSubtitle(t, "clip.srt", videoPath)
|
||||
|
||||
adminUser.BackendScopes = []users.BackendScope{{Path: sourceRoot, Scope: "/users/alice"}}
|
||||
req := httptest.NewRequest(http.MethodGet, "/public/api/media/subtitles?hash=h&path=clip.mov&name=clip.srt&embedded=false", nil)
|
||||
rec := httptest.NewRecorder()
|
||||
status, handlerErr := subtitlesHandler(rec, req, &Context{
|
||||
User: adminUser,
|
||||
ShareUser: adminUser,
|
||||
IndexPath: "/shared/clip.mov/",
|
||||
Share: share.Share{
|
||||
ShareColumns: share.ShareColumns{
|
||||
Hash: "h",
|
||||
Path: "/users/alice/shared",
|
||||
},
|
||||
SourcePath: sourceRoot,
|
||||
},
|
||||
})
|
||||
if status != http.StatusOK || handlerErr != nil {
|
||||
t.Fatalf("status=%d err=%v body=%q", status, handlerErr, rec.Body.String())
|
||||
}
|
||||
if !strings.Contains(rec.Body.String(), "Scoped share OK") {
|
||||
t.Fatalf("body=%q", rec.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubtitlesHandler_ShareSingleFileLoadsSidecar(t *testing.T) {
|
||||
sourceRoot, adminUser := setupSubtitlesHTTPTest(t)
|
||||
srtBody := "single-file share subtitle"
|
||||
videoPath := filepath.Join(sourceRoot, "clip.mov")
|
||||
if err := os.WriteFile(videoPath, []byte("video"), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(sourceRoot, "clip.srt"), []byte(srtBody), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
mockVideoWithExternalSubtitle(t, "clip.srt", videoPath)
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/public/api/media/subtitles?hash=h&path=/&name=clip.srt&embedded=false", nil)
|
||||
rec := httptest.NewRecorder()
|
||||
status, handlerErr := subtitlesHandler(rec, req, &Context{
|
||||
User: adminUser,
|
||||
ShareUser: adminUser,
|
||||
IndexPath: "/clip.mov/",
|
||||
Share: share.Share{
|
||||
ShareColumns: share.ShareColumns{
|
||||
Hash: "h",
|
||||
Path: "/clip.mov",
|
||||
},
|
||||
SourcePath: sourceRoot,
|
||||
},
|
||||
})
|
||||
if status != http.StatusOK || handlerErr != nil {
|
||||
t.Fatalf("status=%d err=%v body=%q", status, handlerErr, rec.Body.String())
|
||||
}
|
||||
if !strings.Contains(rec.Body.String(), "single-file share subtitle") {
|
||||
t.Fatalf("body=%q", rec.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubtitlesHandler_ShareDeniesSidecarSymlinkToDeniedTarget(t *testing.T) {
|
||||
sourceRoot, adminUser := setupSubtitlesHTTPTest(t)
|
||||
sharedDir := filepath.Join(sourceRoot, "shared")
|
||||
privateDir := filepath.Join(sourceRoot, "private")
|
||||
for _, dir := range []string{sharedDir, privateDir} {
|
||||
if err := os.MkdirAll(dir, 0755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
secret := filepath.Join(privateDir, "clip.srt")
|
||||
if err := os.WriteFile(secret, []byte("SECRET_DENIED_SUBTITLE"), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
videoPath := filepath.Join(sharedDir, "clip.mov")
|
||||
if err := os.WriteFile(videoPath, []byte("video"), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Symlink(filepath.Join("..", "private", "clip.srt"), filepath.Join(sharedDir, "clip.srt")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := state.DenyUser(sourceRoot, utils.IndexPathFromNormalized("/private", true), "admin"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
mockVideoWithExternalSubtitle(t, "clip.srt", videoPath)
|
||||
|
||||
adminUser.BackendScopes = []users.BackendScope{{Path: sourceRoot, Scope: "/"}}
|
||||
req := httptest.NewRequest(http.MethodGet, "/public/api/media/subtitles?hash=h&path=clip.mov&name=clip.srt&embedded=false", nil)
|
||||
rec := httptest.NewRecorder()
|
||||
status, handlerErr := subtitlesHandler(rec, req, &Context{
|
||||
User: adminUser,
|
||||
ShareUser: adminUser,
|
||||
IndexPath: "/shared/clip.mov/",
|
||||
Share: share.Share{
|
||||
ShareColumns: share.ShareColumns{
|
||||
Hash: "h",
|
||||
Path: "/shared",
|
||||
},
|
||||
SourcePath: sourceRoot,
|
||||
},
|
||||
})
|
||||
if status != http.StatusForbidden {
|
||||
t.Fatalf("status=%d err=%v body=%q", status, handlerErr, rec.Body.String())
|
||||
}
|
||||
if strings.Contains(rec.Body.String(), "SECRET_DENIED_SUBTITLE") {
|
||||
t.Fatalf("response leaked denied subtitle")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubtitlesHandler_ShareRejectsSidecarPastEscapingDirSymlink(t *testing.T) {
|
||||
sourceRoot, adminUser := setupSubtitlesHTTPTest(t)
|
||||
videoIndexPath := "/shared/link/clip.mov/"
|
||||
mockVideoWithExternalSubtitle(t, "clip.srt", filepath.Join(sourceRoot, "shared", "link", "clip.mov"))
|
||||
|
||||
sharedDir := filepath.Join(sourceRoot, "shared")
|
||||
if err := os.MkdirAll(sharedDir, 0755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
outside := filepath.Join(filepath.Dir(sourceRoot), "outside-share")
|
||||
if err := os.MkdirAll(outside, 0755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(outside, "clip.srt"), []byte("SECRET_PAST_DIR_SYMLINK"), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Symlink(outside, filepath.Join(sharedDir, "link")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/public/api/media/subtitles?hash=h&path=link/clip.mov&name=clip.srt&embedded=false", nil)
|
||||
rec := httptest.NewRecorder()
|
||||
status, handlerErr := subtitlesHandler(rec, req, &Context{
|
||||
User: adminUser,
|
||||
ShareUser: adminUser,
|
||||
IndexPath: videoIndexPath,
|
||||
Share: share.Share{
|
||||
ShareColumns: share.ShareColumns{
|
||||
Hash: "h",
|
||||
Path: "/shared",
|
||||
},
|
||||
SourcePath: sourceRoot,
|
||||
},
|
||||
})
|
||||
if status != http.StatusForbidden {
|
||||
t.Fatalf("status=%d err=%v body=%q", status, handlerErr, rec.Body.String())
|
||||
}
|
||||
if strings.Contains(rec.Body.String(), "SECRET_PAST_DIR_SYMLINK") {
|
||||
t.Fatalf("response leaked subtitle past directory symlink")
|
||||
}
|
||||
}
|
||||
@@ -122,7 +122,10 @@ func withHashFileHelper(fn handleFunc) handleFunc {
|
||||
pathWithoutUserScope = pathWithoutUserScope + "/"
|
||||
}
|
||||
data.IndexPath = pathWithoutUserScope
|
||||
// skip file fetch for certain apis
|
||||
// Skip FileInfoFaster prefetch for performance; handlers must apply equivalent
|
||||
// checks (access rules + GetRealPathScoped with d.Share.Path where bytes are read).
|
||||
// TODO: align all skipped public-share routes on one bound (Share.Path) or stop
|
||||
// skipping FileInfoFaster here for media/metadata/items.
|
||||
if (r.Method == "POST" && strings.Contains(r.URL.Path, "/resources")) ||
|
||||
(r.Method == "POST" && strings.Contains(r.URL.Path, "/resources/view-token")) ||
|
||||
(r.Method == "GET" && strings.Contains(r.URL.Path, "/resources/items")) ||
|
||||
|
||||
@@ -5,7 +5,9 @@ import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -22,6 +24,7 @@ import (
|
||||
)
|
||||
|
||||
func setupTestEnv(t *testing.T) {
|
||||
lockWebGlobals(t)
|
||||
utils.BcryptCost = bcrypt.MinCost
|
||||
dbPath := filepath.Join(t.TempDir(), "test.sqlite")
|
||||
|
||||
@@ -535,6 +538,88 @@ func newHTTPRequest(t *testing.T, hash string, requestModifiers ...func(*http.Re
|
||||
return r
|
||||
}
|
||||
|
||||
func TestPublicGetResourceChecksum(t *testing.T) {
|
||||
setupTestEnv(t)
|
||||
|
||||
filePath := filepath.Join(t.TempDir(), "a.txt")
|
||||
if err := os.WriteFile(filePath, []byte("hello\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
originalFileInfoFaster := FileInfoFasterFunc
|
||||
t.Cleanup(func() { FileInfoFasterFunc = originalFileInfoFaster })
|
||||
FileInfoFasterFunc = func(opts utils.FileOptions, user *users.User) (*iteminfo.ExtendedFileInfo, error) {
|
||||
return &iteminfo.ExtendedFileInfo{
|
||||
FileInfo: iteminfo.FileInfo{
|
||||
Path: opts.Path,
|
||||
ItemInfo: iteminfo.ItemInfo{
|
||||
Name: "a.txt",
|
||||
Type: "text/plain",
|
||||
},
|
||||
},
|
||||
RealPath: filePath,
|
||||
}, nil
|
||||
}
|
||||
|
||||
newOwner := &users.User{
|
||||
FrontendUser: users.FrontendUser{
|
||||
Username: "checksum_owner",
|
||||
Permissions: users.Permissions{Admin: false},
|
||||
},
|
||||
BackendScopes: []users.BackendScope{{Path: "/srv", Scope: "/"}},
|
||||
}
|
||||
if err := state.CreateUser(newOwner, ""); err != nil {
|
||||
t.Fatal("failed to save user:", err)
|
||||
}
|
||||
ownerValue, err := state.GetUserByUsername("checksum_owner")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
owner := &ownerValue
|
||||
|
||||
if err := state.CreateShare(&share.Share{
|
||||
ShareSettings: share.ShareSettings{ShareLimits: share.ShareLimits{SourceName: "srv"}},
|
||||
ShareColumns: share.ShareColumns{Hash: "checksum_hash", Path: "/"},
|
||||
SourcePath: "/srv",
|
||||
UserID: owner.ID,
|
||||
}); err != nil {
|
||||
t.Fatal("failed to create share:", err)
|
||||
}
|
||||
|
||||
originalAuthKey := settings.Config.Auth.Key
|
||||
settings.Config.Auth.Key = "key"
|
||||
t.Cleanup(func() { settings.Config.Auth.Key = originalAuthKey })
|
||||
|
||||
t.Run("anonymous", func(t *testing.T) {
|
||||
recorder := httptest.NewRecorder()
|
||||
req := newTestRequest(t, "checksum_hash", "", "", nil)
|
||||
q := req.URL.Query()
|
||||
q.Set("checksum", "md5")
|
||||
req.URL.RawQuery = q.Encode()
|
||||
withHashFile(publicGetResourceHandler)(recorder, req)
|
||||
if recorder.Code != http.StatusUnauthorized {
|
||||
t.Fatalf("expected status %d, got %d body=%s", http.StatusUnauthorized, recorder.Code, recorder.Body.String())
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("authenticated", func(t *testing.T) {
|
||||
token := testSessionToken(t, owner, time.Hour)
|
||||
recorder := httptest.NewRecorder()
|
||||
req := newTestRequest(t, "checksum_hash", "", "", map[string]string{
|
||||
"Authorization": "Bearer " + token,
|
||||
})
|
||||
q := req.URL.Query()
|
||||
q.Set("checksum", "md5")
|
||||
req.URL.RawQuery = q.Encode()
|
||||
withHashFile(publicGetResourceHandler)(recorder, req)
|
||||
if recorder.Code != http.StatusOK {
|
||||
t.Fatalf("expected status %d, got %d body=%s", http.StatusOK, recorder.Code, recorder.Body.String())
|
||||
}
|
||||
if !strings.Contains(recorder.Body.String(), "b1946ac92492d2347c6235b4d2611184") {
|
||||
t.Fatalf("checksum missing from response: %s", recorder.Body.String())
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestWithTimeoutReturnsJSONTimeoutReason(t *testing.T) {
|
||||
t.Parallel()
|
||||
timeout := 50 * time.Millisecond
|
||||
|
||||
@@ -166,7 +166,11 @@ func onlyofficeClientConfigGetHandler(w http.ResponseWriter, r *http.Request, d
|
||||
}
|
||||
providedPath = cleanPath
|
||||
|
||||
themeMode := utils.Ternary(d.User.DarkMode, "dark", "light")
|
||||
themeMode := "light"
|
||||
if d.User.DarkMode == users.DarkModeDark ||
|
||||
(d.User.DarkMode == users.DarkModeAuto && settings.DefaultDarkMode()) {
|
||||
themeMode = "dark"
|
||||
}
|
||||
var sourceInfo *settings.Source
|
||||
var ok bool
|
||||
if hash != "" {
|
||||
|
||||
@@ -16,6 +16,7 @@ import (
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/adapters/fs/files"
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/preview"
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/utils"
|
||||
"github.com/gtsteffaniak/filebrowser/backend/pkg/indexing"
|
||||
"github.com/gtsteffaniak/filebrowser/backend/pkg/indexing/iteminfo"
|
||||
"github.com/gtsteffaniak/filebrowser/backend/pkg/settings"
|
||||
"github.com/gtsteffaniak/go-logger/logger"
|
||||
@@ -38,6 +39,31 @@ func isClientCancellation(ctx context.Context, err error) bool {
|
||||
return errors.Is(err, context.Canceled)
|
||||
}
|
||||
|
||||
// viewerConvertedDocExtensions are the documents the viewer opens as a rendered image through the
|
||||
// "original" preview (keep in sync with pdfConvertable in frontend Preview.vue).
|
||||
var viewerConvertedDocExtensions = map[string]bool{
|
||||
".xps": true, ".mobi": true, ".fb2": true, ".cbz": true, ".svg": true,
|
||||
".docx": true, ".pptx": true, ".xlsx": true, ".hwp": true, ".hwpx": true,
|
||||
}
|
||||
|
||||
// isViewerOriginalPreview reports whether a preview request is the viewer opening a file the browser can't
|
||||
// display by itself (camera RAW, HEIC/HEIF, rendered documents). Unlike a thumbnail, without it the file can't
|
||||
// be opened at all, so it is served even when thumbnails are disabled. documentRenderable tells whether the
|
||||
// server can render documents of this kind to an image (see iteminfo.HasDocConvertableExtension).
|
||||
func isViewerOriginalPreview(previewSize, fileName, fileType string, documentRenderable bool) bool {
|
||||
if previewSize != "original" {
|
||||
return false
|
||||
}
|
||||
ext := strings.ToLower(filepath.Ext(fileName))
|
||||
if iteminfo.IsRawImage(ext) {
|
||||
return true
|
||||
}
|
||||
if fileType == "image/heic" || fileType == "image/heif" {
|
||||
return true
|
||||
}
|
||||
return documentRenderable && viewerConvertedDocExtensions[ext]
|
||||
}
|
||||
|
||||
// previewHandler handles the preview request for images.
|
||||
// @Summary Get image preview
|
||||
// @Description Returns a preview image based on the requested path and size.
|
||||
@@ -55,7 +81,9 @@ func isClientCancellation(ctx context.Context, err error) bool {
|
||||
// @Failure 501 {object} map[string]string "Preview generation not implemented"
|
||||
// @Router /api/resources/preview [get]
|
||||
func previewHandler(w http.ResponseWriter, r *http.Request, d *Context) (int, error) {
|
||||
if settings.Config.Server.DisablePreviews {
|
||||
previewsDisabled := settings.Config.Server.DisablePreviews
|
||||
// Only the viewer's "original" requests can pass when previews are disabled (checked below once the file is known).
|
||||
if previewsDisabled && r.URL.Query().Get("size") != "original" {
|
||||
return http.StatusNotImplemented, fmt.Errorf("preview is disabled")
|
||||
}
|
||||
path := r.URL.Query().Get("path")
|
||||
@@ -85,6 +113,9 @@ func previewHandler(w http.ResponseWriter, r *http.Request, d *Context) (int, er
|
||||
return ErrToStatus(err), err
|
||||
}
|
||||
d.FileInfo = *fileInfo
|
||||
if previewsDisabled && !isViewerOriginalPreview(r.URL.Query().Get("size"), fileInfo.Name, fileInfo.Type, iteminfo.HasDocConvertableExtension(fileInfo.Name, fileInfo.Type)) {
|
||||
return http.StatusNotImplemented, fmt.Errorf("preview is disabled")
|
||||
}
|
||||
status, err := PreviewHelperFunc(w, r, d)
|
||||
if err != nil {
|
||||
// Error already logged in PreviewHelperFunc or its callees
|
||||
@@ -109,7 +140,9 @@ func previewHandler(w http.ResponseWriter, r *http.Request, d *Context) (int, er
|
||||
// @Failure 501 {object} map[string]string "Previews disabled globally, for this share, or for upload shares"
|
||||
// @Router /public/api/resources/preview [get]
|
||||
func publicPreviewHandler(w http.ResponseWriter, r *http.Request, d *Context) (int, error) {
|
||||
if settings.Config.Server.DisablePreviews || d.Share.DisableThumbnails {
|
||||
previewSize := r.URL.Query().Get("size")
|
||||
if (settings.Config.Server.DisablePreviews || d.Share.DisableThumbnails) &&
|
||||
!isViewerOriginalPreview(previewSize, d.FileInfo.Name, d.FileInfo.Type, iteminfo.HasDocConvertableExtension(d.FileInfo.Name, d.FileInfo.Type)) {
|
||||
return http.StatusNotImplemented, fmt.Errorf("preview is disabled")
|
||||
}
|
||||
if d.Share.ShareType == "upload" {
|
||||
@@ -196,13 +229,35 @@ func getDirectoryPreview(ctx context.Context, r *http.Request, d *Context, frame
|
||||
// force an ffprobe probe and a subtitle scan per frame on the fast path.
|
||||
fileInfo, err := files.FileInfoFaster(
|
||||
utils.FileOptions{
|
||||
Path: path,
|
||||
Source: source,
|
||||
AlbumArt: true,
|
||||
Path: path,
|
||||
Source: source,
|
||||
AlbumArt: true,
|
||||
FollowSymlinks: true,
|
||||
}, user)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if d.Share.Hash != "" {
|
||||
idx := indexing.GetIndex(source)
|
||||
if idx == nil {
|
||||
return nil, fmt.Errorf("source not available")
|
||||
}
|
||||
userScope, scopeErr := d.ShareUser.GetScopeForSourceName(source)
|
||||
if scopeErr != nil {
|
||||
return nil, scopeErr
|
||||
}
|
||||
shareRel := strings.TrimPrefix(d.Share.Path, userScope)
|
||||
bound := utils.JoinScopedIndexPath(userScope, shareRel)
|
||||
childIndexPath := utils.JoinScopedIndexPath(userScope, path)
|
||||
realPath, _, rerr := idx.GetRealPathScoped(bound, childIndexPath)
|
||||
if rerr != nil {
|
||||
if index != 0 {
|
||||
return getDirectoryPreview(ctx, r, d, 0)
|
||||
}
|
||||
return nil, rerr
|
||||
}
|
||||
fileInfo.RealPath = realPath
|
||||
}
|
||||
tempCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||
_, previewErr := preview.GetPreviewForFile(tempCtx, *fileInfo, "small", "", 0)
|
||||
cancel()
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/database/share"
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/database/users"
|
||||
commonerrors "github.com/gtsteffaniak/filebrowser/backend/internal/errors"
|
||||
"github.com/gtsteffaniak/filebrowser/backend/pkg/indexing"
|
||||
"github.com/gtsteffaniak/filebrowser/backend/pkg/indexing/iteminfo"
|
||||
)
|
||||
|
||||
func TestGetDirectoryPreview_ShareRejectsSymlinkChildOutsideShare(t *testing.T) {
|
||||
sourceRoot, adminUser := setupSubtitlesHTTPTest(t)
|
||||
sharedDir := filepath.Join(sourceRoot, "shared")
|
||||
privateDir := filepath.Join(sourceRoot, "private")
|
||||
for _, dir := range []string{sharedDir, privateDir} {
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(privateDir, "secret.jpg"), []byte{0xff, 0xd8, 0xff, 0xd9}, 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Symlink(filepath.Join("..", "private", "secret.jpg"), filepath.Join(sharedDir, "link.jpg")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
req := httptest.NewRequest("GET", "/", nil)
|
||||
adminUser.BackendScopes = []users.BackendScope{{Path: sourceRoot, Scope: "/"}}
|
||||
d := &Context{
|
||||
User: adminUser,
|
||||
ShareUser: adminUser,
|
||||
IndexPath: "/shared/",
|
||||
Share: share.Share{
|
||||
ShareColumns: share.ShareColumns{Hash: "h", Path: "/shared"},
|
||||
SourcePath: sourceRoot,
|
||||
},
|
||||
FileInfo: iteminfo.ExtendedFileInfo{
|
||||
Source: "Downloads",
|
||||
FileInfo: iteminfo.FileInfo{
|
||||
Path: "/shared/",
|
||||
ItemInfo: iteminfo.ItemInfo{
|
||||
Type: "directory",
|
||||
},
|
||||
Files: []iteminfo.ExtendedItemInfo{
|
||||
{ItemInfo: iteminfo.ItemInfo{Name: "link.jpg", Type: "image/jpeg", HasPreview: true}},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
idx := indexing.GetIndex("Downloads")
|
||||
if idx == nil {
|
||||
t.Fatal("missing index")
|
||||
}
|
||||
|
||||
_, err := getDirectoryPreview(ctx, req, d, 0)
|
||||
if err == nil {
|
||||
t.Fatal("expected scope error for share child symlink outside share root")
|
||||
}
|
||||
if !errors.Is(err, commonerrors.ErrPathEscapesScope) {
|
||||
t.Fatalf("error=%v want ErrPathEscapesScope", err)
|
||||
}
|
||||
}
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
)
|
||||
|
||||
func TestGetRemoteIPUsesRightmostForwardedFor(t *testing.T) {
|
||||
lockWebGlobals(t)
|
||||
orig := settings.Config.Http
|
||||
t.Cleanup(func() { settings.Config.Http = orig })
|
||||
settings.Config.Http.TrustProxyHeaders = true
|
||||
@@ -27,6 +28,7 @@ func TestGetRemoteIPUsesRightmostForwardedFor(t *testing.T) {
|
||||
// proxy hops so multi-hop deployments (CDN -> proxy -> filebrowser) resolve the
|
||||
// real client IP instead of collapsing everyone onto the shared edge address.
|
||||
func TestGetRemoteIPTrustedProxies(t *testing.T) {
|
||||
lockWebGlobals(t)
|
||||
orig := settings.Config.Http
|
||||
t.Cleanup(func() { settings.Config.Http = orig })
|
||||
settings.Config.Http.TrustProxyHeaders = true
|
||||
@@ -59,6 +61,7 @@ func TestGetRemoteIPTrustedProxies(t *testing.T) {
|
||||
|
||||
// Bare IPs (not just CIDRs) are accepted in http.trustedProxies.
|
||||
func TestGetRemoteIPTrustedProxiesBareIP(t *testing.T) {
|
||||
lockWebGlobals(t)
|
||||
orig := settings.Config.Http
|
||||
t.Cleanup(func() { settings.Config.Http = orig })
|
||||
settings.Config.Http.TrustProxyHeaders = true
|
||||
@@ -74,6 +77,7 @@ func TestGetRemoteIPTrustedProxiesBareIP(t *testing.T) {
|
||||
|
||||
// Without trustProxyHeaders the XFF chain is ignored entirely.
|
||||
func TestGetRemoteIPIgnoresForwardedForWhenUntrusted(t *testing.T) {
|
||||
lockWebGlobals(t)
|
||||
orig := settings.Config.Http
|
||||
t.Cleanup(func() { settings.Config.Http = orig })
|
||||
settings.Config.Http.TrustProxyHeaders = false
|
||||
@@ -90,6 +94,7 @@ func TestGetRemoteIPIgnoresForwardedForWhenUntrusted(t *testing.T) {
|
||||
// A direct client that is not a trusted proxy cannot spoof its address with
|
||||
// forwarded headers, even when trustProxyHeaders is enabled.
|
||||
func TestGetRemoteIPIgnoresForwardedHeadersFromUntrustedPeer(t *testing.T) {
|
||||
lockWebGlobals(t)
|
||||
orig := settings.Config.Http
|
||||
t.Cleanup(func() { settings.Config.Http = orig })
|
||||
settings.Config.Http.TrustProxyHeaders = true
|
||||
@@ -104,8 +109,26 @@ func TestGetRemoteIPIgnoresForwardedHeadersFromUntrustedPeer(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetRemoteIPUnixSocketHonorsForwardedWithTrustAllLocalPeers(t *testing.T) {
|
||||
lockWebGlobals(t)
|
||||
orig := settings.Config.Http
|
||||
t.Cleanup(func() { settings.Config.Http = orig })
|
||||
settings.Config.Http.TrustProxyHeaders = true
|
||||
settings.Config.Http.Socket = "/var/run/filebrowser.sock"
|
||||
settings.Config.Http.UnixSocketTrustAllLocalPeers = true
|
||||
settings.Config.Http.TrustedProxies = nil
|
||||
|
||||
req := httptest.NewRequest("GET", "http://example.com/", nil)
|
||||
req.RemoteAddr = "@"
|
||||
req.Header.Set("X-Forwarded-For", "203.0.113.50")
|
||||
if got := GetRemoteIP(req); got != "203.0.113.50" {
|
||||
t.Fatalf("GetRemoteIP = %q, want 203.0.113.50", got)
|
||||
}
|
||||
}
|
||||
|
||||
// With no trusted proxies configured, forwarded headers are never honored.
|
||||
func TestGetRemoteIPIgnoresForwardedHeadersWithoutTrustedProxies(t *testing.T) {
|
||||
lockWebGlobals(t)
|
||||
orig := settings.Config.Http
|
||||
t.Cleanup(func() { settings.Config.Http = orig })
|
||||
settings.Config.Http.TrustProxyHeaders = true
|
||||
@@ -121,6 +144,7 @@ func TestGetRemoteIPIgnoresForwardedHeadersWithoutTrustedProxies(t *testing.T) {
|
||||
|
||||
// An IPv4-mapped IPv6 trusted hop in the XFF chain must still be skipped.
|
||||
func TestGetRemoteIPMappedTrustedHop(t *testing.T) {
|
||||
lockWebGlobals(t)
|
||||
orig := settings.Config.Http
|
||||
t.Cleanup(func() { settings.Config.Http = orig })
|
||||
settings.Config.Http.TrustProxyHeaders = true
|
||||
@@ -135,6 +159,7 @@ func TestGetRemoteIPMappedTrustedHop(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestGetScheme(t *testing.T) {
|
||||
lockWebGlobals(t)
|
||||
t.Run("trusted proto", func(t *testing.T) {
|
||||
settings.Config.Http.TrustProxyHeaders = true
|
||||
t.Cleanup(func() { settings.Config.Http.TrustProxyHeaders = false })
|
||||
@@ -158,6 +183,7 @@ func TestGetScheme(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestRequestSchemeTLSFallback(t *testing.T) {
|
||||
lockWebGlobals(t)
|
||||
settings.Config.Http.TrustProxyHeaders = false
|
||||
|
||||
req := httptest.NewRequest("GET", "https://example.com/", nil)
|
||||
@@ -168,6 +194,7 @@ func TestRequestSchemeTLSFallback(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestRequestSchemeIgnoresInvalidProto(t *testing.T) {
|
||||
lockWebGlobals(t)
|
||||
settings.Config.Http.TrustProxyHeaders = true
|
||||
t.Cleanup(func() { settings.Config.Http.TrustProxyHeaders = false })
|
||||
|
||||
@@ -179,6 +206,7 @@ func TestRequestSchemeIgnoresInvalidProto(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestRequestHostTrustProxyHeaders(t *testing.T) {
|
||||
lockWebGlobals(t)
|
||||
settings.Config.Http.TrustProxyHeaders = true
|
||||
t.Cleanup(func() { settings.Config.Http.TrustProxyHeaders = false })
|
||||
|
||||
@@ -192,6 +220,7 @@ func TestRequestHostTrustProxyHeaders(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestRequestSchemeForPublicURL(t *testing.T) {
|
||||
lockWebGlobals(t)
|
||||
orig := settings.Config.Http.TrustProxyHeaders
|
||||
t.Cleanup(func() { settings.Config.Http.TrustProxyHeaders = orig })
|
||||
|
||||
@@ -225,6 +254,7 @@ func TestRequestSchemeForPublicURL(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestRequestPageFullURL(t *testing.T) {
|
||||
lockWebGlobals(t)
|
||||
orig := settings.Config.Http
|
||||
t.Cleanup(func() { settings.Config.Http = orig })
|
||||
settings.Config.Http.ExternalUrl = ""
|
||||
|
||||
@@ -237,6 +237,7 @@ func TestResourcePutHandler_SuccessResponseOmitsRealPath(t *testing.T) {
|
||||
|
||||
func setupPutHTTPTest(t *testing.T) string {
|
||||
t.Helper()
|
||||
lockWebGlobals(t)
|
||||
downloadsPath := filepath.Join(t.TempDir(), "downloads")
|
||||
|
||||
dbPath := filepath.Join(t.TempDir(), "quota-http-test.sqlite")
|
||||
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
"github.com/google/uuid"
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/adapters/fs/files"
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/adapters/fs/fileutils"
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/database/access"
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/database/users"
|
||||
commonerrors "github.com/gtsteffaniak/filebrowser/backend/internal/errors"
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/quota"
|
||||
@@ -171,6 +172,10 @@ func putResourceWithQuota(w http.ResponseWriter, r *http.Request, d *Context, so
|
||||
if err != nil {
|
||||
return sanitizePutPathError(err)
|
||||
}
|
||||
permUser := accessCheckUsername(d)
|
||||
if !state.AccessPermittedAtRealPath(idx, permUser, realPath, access.IsDirAtRealPath(realPath)) {
|
||||
return http.StatusForbidden, fmt.Errorf("access denied to path %s", fullIndexPath)
|
||||
}
|
||||
if stat, statErr := os.Stat(realPath); statErr == nil && stat.IsDir() {
|
||||
return http.StatusMethodNotAllowed, errPutPathIsDirectory
|
||||
}
|
||||
|
||||
@@ -19,6 +19,7 @@ import (
|
||||
|
||||
func setupQuotaHTTPTest(t *testing.T) {
|
||||
t.Helper()
|
||||
lockWebGlobals(t)
|
||||
dbPath := filepath.Join(t.TempDir(), "quota-http-test.sqlite")
|
||||
if _, err := state.Initialize(dbPath); err != nil {
|
||||
t.Fatal(err)
|
||||
|
||||
@@ -16,6 +16,7 @@ import (
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/activity"
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/adapters/fs/files"
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/adapters/fs/fileutils"
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/database/access"
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/database/users"
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/errors"
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/preview"
|
||||
@@ -208,7 +209,9 @@ func resourceGetHandler(w http.ResponseWriter, r *http.Request, d *Context) (int
|
||||
// @Param path query string false "Path within the share to retrieve information for. Defaults to share root."
|
||||
// @Param content query string false "Include file content if true"
|
||||
// @Param metadata query string false "Extract audio/video metadata if true"
|
||||
// @Param checksum query string false "Optional checksum algorithm (md5, sha1, sha256, sha512); requires a logged-in session"
|
||||
// @Success 200 {object} iteminfo.FileInfo "File or directory metadata"
|
||||
// @Failure 401 {object} map[string]string "Login required for checksum generation"
|
||||
// @Failure 403 {object} map[string]string "Share unavailable or access denied"
|
||||
// @Failure 404 {object} map[string]string "Share not found or file not found"
|
||||
// @Failure 500 {object} map[string]string "Internal server error"
|
||||
@@ -218,6 +221,24 @@ func publicGetResourceHandler(w http.ResponseWriter, r *http.Request, d *Context
|
||||
if d.Share.ShareType == "upload" {
|
||||
return http.StatusNotImplemented, fmt.Errorf("browsing is disabled for upload shares")
|
||||
}
|
||||
if algo := r.URL.Query().Get("checksum"); algo != "" {
|
||||
if d.User == nil || d.User.Username == "" || d.User.Username == "anonymous" {
|
||||
return http.StatusUnauthorized, fmt.Errorf("login required to generate checksums")
|
||||
}
|
||||
if d.FileInfo.Type == "directory" {
|
||||
return http.StatusBadRequest, fmt.Errorf("checksum is only supported for files")
|
||||
}
|
||||
if d.Share.DisableFileViewer {
|
||||
return http.StatusForbidden, fmt.Errorf("file viewing is disabled for this share")
|
||||
}
|
||||
checksum, err := utils.GetChecksum(d.FileInfo.RealPath, algo)
|
||||
if err == errors.ErrInvalidOption {
|
||||
return http.StatusBadRequest, nil
|
||||
} else if err != nil {
|
||||
return http.StatusInternalServerError, err
|
||||
}
|
||||
d.FileInfo.Checksums = map[string]string{algo: checksum}
|
||||
}
|
||||
return RenderJSON(w, r, d.FileInfo)
|
||||
}
|
||||
|
||||
@@ -792,6 +813,9 @@ func ResourcePostHandler(w http.ResponseWriter, r *http.Request, d *Context) (in
|
||||
if !state.AccessPermitted(idx.Path, utils.IndexPathFromNormalized(fullIndexPath, true), filePermUser.Username) {
|
||||
return http.StatusForbidden, fmt.Errorf("access denied to path %s", path)
|
||||
}
|
||||
if !state.AccessPermittedAtRealPath(idx, filePermUser.Username, realPath, access.IsDirAtRealPath(realPath)) {
|
||||
return http.StatusForbidden, fmt.Errorf("access denied to path %s", path)
|
||||
}
|
||||
|
||||
// Check for file/folder conflicts before creation
|
||||
if stat, statErr := os.Stat(realPath); statErr == nil {
|
||||
@@ -1528,6 +1552,15 @@ func ResourcePatchHandler(w http.ResponseWriter, r *http.Request, d *Context) (i
|
||||
response.Failed = append(response.Failed, moveCopyWithClientPaths(item, clientFromPath, clientToPath))
|
||||
continue
|
||||
}
|
||||
if !state.AccessPermittedAtRealPath(srcIdx, d.User.Username, realSrc, isSrcDir) {
|
||||
item.Message = "access denied to source path"
|
||||
if d.Share.Hash != "" {
|
||||
response.Failed = append(response.Failed, MoveCopyItem{Message: item.Message})
|
||||
continue
|
||||
}
|
||||
response.Failed = append(response.Failed, moveCopyWithClientPaths(item, clientFromPath, clientToPath))
|
||||
continue
|
||||
}
|
||||
|
||||
// Resolve the destination: copy must reject escaping symlink leaves; move/rename
|
||||
// must replace an existing symlink entry without following it.
|
||||
@@ -1548,10 +1581,28 @@ func ResourcePatchHandler(w http.ResponseWriter, r *http.Request, d *Context) (i
|
||||
response.Failed = append(response.Failed, moveCopyWithClientPaths(item, clientFromPath, clientToPath))
|
||||
continue
|
||||
}
|
||||
if !state.AccessPermittedAtRealPath(dstIdx, d.User.Username, realDest, access.IsDirAtRealPath(realDest)) {
|
||||
item.Message = "access denied to destination path"
|
||||
if d.Share.Hash != "" {
|
||||
response.Failed = append(response.Failed, MoveCopyItem{Message: item.Message})
|
||||
continue
|
||||
}
|
||||
response.Failed = append(response.Failed, moveCopyWithClientPaths(item, clientFromPath, clientToPath))
|
||||
continue
|
||||
}
|
||||
|
||||
// Auto-rename if requested
|
||||
if req.Rename {
|
||||
realDest = addVersionSuffix(realDest)
|
||||
if !state.AccessPermittedAtRealPath(dstIdx, d.User.Username, realDest, isSrcDir) {
|
||||
item.Message = "access denied to destination path"
|
||||
if d.Share.Hash != "" {
|
||||
response.Failed = append(response.Failed, MoveCopyItem{Message: item.Message})
|
||||
continue
|
||||
}
|
||||
response.Failed = append(response.Failed, moveCopyWithClientPaths(item, clientFromPath, clientToPath))
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
// Validate move/rename operation to prevent circular references
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/state"
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/utils"
|
||||
)
|
||||
|
||||
func TestResourcePatchHandler_CopyRename_DeniesDeniedVersionSuffix(t *testing.T) {
|
||||
_, sourceRoot, sourceName, username := setupExtractAccessPolicyTest(t)
|
||||
destDir := filepath.Join(sourceRoot, "dest")
|
||||
if err := os.MkdirAll(destDir, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
src := filepath.Join(sourceRoot, "src.txt")
|
||||
if err := os.WriteFile(src, []byte("src"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
existing := filepath.Join(destDir, "report.txt")
|
||||
if err := os.WriteFile(existing, []byte("existing"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
denied := filepath.Join(destDir, "report(1).txt")
|
||||
if err := state.DenyUser(sourceRoot, utils.IndexPathFromNormalized("/dest/report(1).txt", false), username); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
user, err := state.GetUserByUsername(username)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body := `{"action":"copy","rename":true,"items":[{"fromSource":"` + sourceName +
|
||||
`","fromPath":"/src.txt","toSource":"` + sourceName + `","toPath":"/dest/report.txt"}]}`
|
||||
req := httptest.NewRequest(http.MethodPatch, "/api/resources", strings.NewReader(body))
|
||||
rec := httptest.NewRecorder()
|
||||
status, handlerErr := ResourcePatchHandler(rec, req, &Context{User: &user})
|
||||
if handlerErr != nil {
|
||||
t.Fatalf("err=%v", handlerErr)
|
||||
}
|
||||
if status != http.StatusInternalServerError && status != http.StatusOK {
|
||||
t.Fatalf("status=%d body=%s", status, rec.Body.String())
|
||||
}
|
||||
var resp MoveCopyResponse
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(resp.Failed) != 1 || !strings.Contains(resp.Failed[0].Message, "access denied") {
|
||||
t.Fatalf("failed=%+v", resp.Failed)
|
||||
}
|
||||
if _, err := os.Stat(denied); err == nil {
|
||||
t.Fatal("denied destination file should not exist")
|
||||
}
|
||||
}
|
||||
@@ -17,6 +17,7 @@ import (
|
||||
|
||||
func setupPutQuotaTest(t *testing.T) string {
|
||||
t.Helper()
|
||||
lockWebGlobals(t)
|
||||
downloadsPath := filepath.Join(t.TempDir(), "downloads")
|
||||
|
||||
dbPath := filepath.Join(t.TempDir(), "quota-http-test.sqlite")
|
||||
|
||||
@@ -0,0 +1,82 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/preview"
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/state"
|
||||
)
|
||||
|
||||
func TestResourcePostHandler_RejectsEscapingParentSymlinkMkdir(t *testing.T) {
|
||||
downloadsPath := setupPutHTTPTest(t)
|
||||
outside := filepath.Join(filepath.Dir(downloadsPath), "outside-mkdir")
|
||||
if err := os.MkdirAll(outside, 0755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Symlink(outside, filepath.Join(downloadsPath, "evil")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
adminUser, err := state.GetUserByUsername("admin")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/api/resources?source=Downloads&path=/evil/newdir&isDir=true", nil)
|
||||
rec := httptest.NewRecorder()
|
||||
|
||||
status, handlerErr := ResourcePostHandler(rec, req, &requestContext{User: &adminUser})
|
||||
if status != http.StatusForbidden {
|
||||
t.Fatalf("status=%d err=%v", status, handlerErr)
|
||||
}
|
||||
if handlerErr == nil {
|
||||
t.Fatal("expected mkdir to be forbidden")
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(outside, "newdir")); !os.IsNotExist(err) {
|
||||
t.Fatal("mkdir escaped outside the source via symlink parent")
|
||||
}
|
||||
}
|
||||
|
||||
func TestResourceDeleteHandler_RejectsEscapingIntermediate(t *testing.T) {
|
||||
downloadsPath := setupPutHTTPTest(t)
|
||||
if preview.GetService() == nil {
|
||||
if err := preview.StartPreviewGenerator(1, t.TempDir()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
outsideDir := filepath.Join(filepath.Dir(downloadsPath), "outside-delete")
|
||||
if err := os.MkdirAll(outsideDir, 0755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
victim := filepath.Join(outsideDir, "victim.txt")
|
||||
if err := os.WriteFile(victim, []byte("keep"), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Symlink(outsideDir, filepath.Join(downloadsPath, "evil")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
adminUser, err := state.GetUserByUsername("admin")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
req := httptest.NewRequest(http.MethodDelete, "/api/resources?source=Downloads&path=/evil/victim.txt", nil)
|
||||
rec := httptest.NewRecorder()
|
||||
|
||||
status, handlerErr := resourceDeleteHandler(rec, req, &requestContext{User: &adminUser})
|
||||
if status != http.StatusForbidden {
|
||||
t.Fatalf("status=%d err=%v", status, handlerErr)
|
||||
}
|
||||
if handlerErr == nil {
|
||||
t.Fatal("expected delete to be forbidden")
|
||||
}
|
||||
got, err := os.ReadFile(victim)
|
||||
if err != nil || string(got) != "keep" {
|
||||
t.Fatalf("delete escaped: err=%v content=%q", err, got)
|
||||
}
|
||||
}
|
||||
@@ -87,7 +87,11 @@ func settingsGetHandler(w http.ResponseWriter, r *http.Request, d *Context) (int
|
||||
// filesystem. It is the comment/secret-marker source for both the YAML export and the settings
|
||||
// JSON redactor.
|
||||
func readEmbeddedConfigYaml() (string, error) {
|
||||
content, err := fs.ReadFile(assetFs, "config.generated.yaml")
|
||||
path := "config.generated.yaml"
|
||||
if settings.Env.EmbeddedFs {
|
||||
path = "embed/config.generated.yaml"
|
||||
}
|
||||
content, err := fs.ReadFile(assetFs, path)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
@@ -453,33 +457,17 @@ func settingsShareDefaultsPatchHandler(w http.ResponseWriter, r *http.Request, d
|
||||
return settingsShareDefaultsGetHandler(w, r, d)
|
||||
}
|
||||
|
||||
// settingsMediaGetHandler returns the runtime media settings.
|
||||
// @Summary Get runtime media settings
|
||||
// @Description Admins receive the DB-backed media integration settings; every authenticated user
|
||||
// @Description receives the player-relevant subset (transcode enabled, profiles, per-user limit).
|
||||
// @Tags Settings
|
||||
// @Produce json
|
||||
// @Success 200 {object} mediaSettingsResponse "Media settings"
|
||||
// @Router /api/settings/media [get]
|
||||
// mediaSettingsResponse is the runtime media settings plus config-lock metadata.
|
||||
// mediaSettingsResponse is the ADMIN ONLY media payload: the full stored settings
|
||||
// (FullAdminMedia, which embeds the public UserTranscode fields) plus config-lock
|
||||
// metadata and ffmpeg option lists. Non-admins never receive this type.
|
||||
type mediaSettingsResponse struct {
|
||||
settings.MediaSettings
|
||||
settings.FullAdminMedia
|
||||
LockedFromConfigPaths []string `json:"lockedFromConfigPaths,omitempty"`
|
||||
LockMessage string `json:"lockMessage,omitempty"`
|
||||
AccelDeviceOptions []ffmpeg.AccelDeviceOption `json:"accelDeviceOptions,omitempty"`
|
||||
EncoderBackendOptions []ffmpeg.EncoderBackendOption `json:"encoderBackendOptions,omitempty"`
|
||||
}
|
||||
|
||||
// mediaPlayerSettings is the player-relevant subset of the runtime media
|
||||
// settings. It is safe to expose to any authenticated user so the player can
|
||||
// reflect admin changes without a page reload.
|
||||
type mediaPlayerSettings struct {
|
||||
Enabled bool `json:"enabled"`
|
||||
Profiles []string `json:"profiles"`
|
||||
MaxConcurrentPerUser int `json:"maxConcurrentPerUser"`
|
||||
HardwareAcceleration bool `json:"hardwareAcceleration"`
|
||||
}
|
||||
|
||||
func mediaSettingsLockMeta() (paths []string, message string) {
|
||||
paths = settings.ConfigSpecifiedMediaPaths()
|
||||
if len(paths) == 0 {
|
||||
@@ -491,12 +479,25 @@ func mediaSettingsLockMeta() (paths []string, message string) {
|
||||
// The media settings APIs no longer require ffmpeg. go-media serves probing,
|
||||
// metadata, subtitles, and video thumbnails without it. ffmpeg-only controls
|
||||
// (transcoding, hardware acceleration) report their own availability.
|
||||
// @Summary Get runtime media settings
|
||||
// @Description Returns the media settings the caller is allowed to see.
|
||||
// @Description Admins get the full settings, including ffmpeg and hardware options, so they can edit them.
|
||||
// @Description Everyone else gets a small read-only view: whether transcoding is on, how many
|
||||
// @Description transcodes they can run at once, which quality presets are enabled, and which image
|
||||
// @Description previews the server can generate. Keys in the view are the same for admins and users.
|
||||
// @Tags Settings
|
||||
// @Produce json
|
||||
// @Success 200 {object} mediaSettingsResponse "Media settings"
|
||||
// @Router /api/settings/media [get]
|
||||
func settingsMediaGetHandler(w http.ResponseWriter, r *http.Request, d *Context) (int, error) {
|
||||
// PUBLIC payload: non-admins get only what the player and general UI read.
|
||||
if !d.User.Permissions.Admin {
|
||||
return RenderJSON(w, r, currentMediaPlayerSettings())
|
||||
return RenderJSON(w, r, settings.CurrentUserMedia())
|
||||
}
|
||||
// ADMIN payload: full stored settings with effective and derived values filled in
|
||||
// by PrepMediaForFrontend (a read-only copy; never saved).
|
||||
media := settings.PrepMediaForFrontend(state.GetMediaSettings())
|
||||
lockedPaths, lockMessage := mediaSettingsLockMeta()
|
||||
media := state.GetMediaSettings()
|
||||
svc := ffmpeg.Get()
|
||||
var accelDeviceOptions []ffmpeg.AccelDeviceOption
|
||||
var encoderBackendOptions []ffmpeg.EncoderBackendOption
|
||||
@@ -506,7 +507,7 @@ func settingsMediaGetHandler(w http.ResponseWriter, r *http.Request, d *Context)
|
||||
encoderBackendOptions = svc.EncoderBackendOptions()
|
||||
}
|
||||
return RenderJSON(w, r, mediaSettingsResponse{
|
||||
MediaSettings: media,
|
||||
FullAdminMedia: media,
|
||||
LockedFromConfigPaths: lockedPaths,
|
||||
LockMessage: lockMessage,
|
||||
AccelDeviceOptions: accelDeviceOptions,
|
||||
@@ -514,17 +515,6 @@ func settingsMediaGetHandler(w http.ResponseWriter, r *http.Request, d *Context)
|
||||
})
|
||||
}
|
||||
|
||||
// currentMediaPlayerSettings reads the live media settings for the player UI.
|
||||
func currentMediaPlayerSettings() mediaPlayerSettings {
|
||||
media := state.GetMediaSettings()
|
||||
return mediaPlayerSettings{
|
||||
Enabled: settings.TranscodeEnabled(),
|
||||
Profiles: settings.EnabledTranscodeProfiles(),
|
||||
MaxConcurrentPerUser: settings.TranscodeMaxConcurrentPerUser(),
|
||||
HardwareAcceleration: media.HardwareAcceleration,
|
||||
}
|
||||
}
|
||||
|
||||
// settingsMediaPatchHandler merges a partial update into the runtime media settings.
|
||||
// @Summary Update runtime media settings
|
||||
// @Description Merges a partial JSON patch into the DB-backed media settings and applies them at runtime.
|
||||
|
||||
@@ -13,21 +13,31 @@ import (
|
||||
|
||||
const settingsRedactSentinel = "SENTINEL-SETTINGS-SECRET"
|
||||
|
||||
func swapSettingsConfigForTest(t *testing.T, mutate func(*settings.Settings)) {
|
||||
t.Helper()
|
||||
cfg := settings.Config
|
||||
if mutate != nil {
|
||||
mutate(&cfg)
|
||||
}
|
||||
prev := settings.SwapConfigForTest(cfg)
|
||||
t.Cleanup(func() { settings.SwapConfigForTest(prev) })
|
||||
}
|
||||
|
||||
// TestSettingsGetHandler_redactsSecrets is the end-to-end regression guard for
|
||||
// GHSA-892f-9h5p-mfc8: GET /api/settings (and ?property=auth) must not return cleartext secrets.
|
||||
func TestSettingsGetHandler_redactsSecrets(t *testing.T) {
|
||||
setupTestEnv(t)
|
||||
|
||||
prev := settings.Config
|
||||
t.Cleanup(func() { settings.Config = prev })
|
||||
settings.Config.Auth.Key = settingsRedactSentinel
|
||||
settings.Config.Auth.TotpSecret = settingsRedactSentinel
|
||||
settings.Config.Auth.Methods.PasswordAuth.AdminPassword = settingsRedactSentinel
|
||||
settings.Config.Auth.Methods.PasswordAuth.Recaptcha.Secret = settingsRedactSentinel
|
||||
settings.Config.Auth.Methods.OidcAuth.ClientSecret = settingsRedactSentinel
|
||||
settings.Config.Auth.Methods.LdapAuth.UserPassword = settingsRedactSentinel
|
||||
settings.Config.Auth.Methods.JwtAuth.Secret = settingsRedactSentinel
|
||||
settings.Config.Integrations.OnlyOffice.Secret = settingsRedactSentinel
|
||||
swapSettingsConfigForTest(t, func(cfg *settings.Settings) {
|
||||
cfg.Auth.Key = settingsRedactSentinel
|
||||
cfg.Auth.TotpSecret = settingsRedactSentinel
|
||||
cfg.Auth.Methods.PasswordAuth.AdminPassword = settingsRedactSentinel
|
||||
cfg.Auth.Methods.PasswordAuth.Recaptcha.Secret = settingsRedactSentinel
|
||||
cfg.Auth.Methods.OidcAuth.ClientSecret = settingsRedactSentinel
|
||||
cfg.Auth.Methods.LdapAuth.UserPassword = settingsRedactSentinel
|
||||
cfg.Auth.Methods.JwtAuth.Secret = settingsRedactSentinel
|
||||
cfg.Integrations.OnlyOffice.Secret = settingsRedactSentinel
|
||||
})
|
||||
|
||||
admin := &users.User{
|
||||
FrontendUser: users.FrontendUser{
|
||||
@@ -62,9 +72,9 @@ func TestSettingsGetHandler_redactsSecrets(t *testing.T) {
|
||||
func TestSettingsGetHandler_doesNotMutateLiveConfig(t *testing.T) {
|
||||
setupTestEnv(t)
|
||||
|
||||
prev := settings.Config
|
||||
t.Cleanup(func() { settings.Config = prev })
|
||||
settings.Config.Auth.Key = settingsRedactSentinel
|
||||
swapSettingsConfigForTest(t, func(cfg *settings.Settings) {
|
||||
cfg.Auth.Key = settingsRedactSentinel
|
||||
})
|
||||
|
||||
admin := &users.User{
|
||||
FrontendUser: users.FrontendUser{
|
||||
@@ -92,10 +102,10 @@ func TestSettingsGetHandler_doesNotMutateLiveConfig(t *testing.T) {
|
||||
func TestSettingsGetHandler_keepsNonSecretAuthFields(t *testing.T) {
|
||||
setupTestEnv(t)
|
||||
|
||||
prev := settings.Config
|
||||
t.Cleanup(func() { settings.Config = prev })
|
||||
settings.Config.Auth.Key = settingsRedactSentinel
|
||||
settings.Config.Auth.TokenExpirationHours = 7
|
||||
swapSettingsConfigForTest(t, func(cfg *settings.Settings) {
|
||||
cfg.Auth.Key = settingsRedactSentinel
|
||||
cfg.Auth.TokenExpirationHours = 7
|
||||
})
|
||||
|
||||
admin := &users.User{
|
||||
FrontendUser: users.FrontendUser{
|
||||
|
||||
@@ -151,7 +151,13 @@ func handleWithStaticData(w http.ResponseWriter, r *http.Request, d *requestCont
|
||||
externalLinks = newExternalLinks
|
||||
}
|
||||
|
||||
defaultThemeColor := settings.DefaultBackgroundColor()
|
||||
// Instance default dark mode: htmlVars/defaultDark use a boolean for first paint;
|
||||
// globalVars.darkMode carries the full preference (including "auto") for the SPA.
|
||||
defaultDark := settings.DefaultDarkMode()
|
||||
defaultThemeColor := settings.Config.Frontend.Styling.LightBackground
|
||||
if defaultDark {
|
||||
defaultThemeColor = settings.Config.Frontend.Styling.DarkBackground
|
||||
}
|
||||
staticURL := settings.Config.Http.BaseURL + "public/static"
|
||||
description := settings.Config.Frontend.Description
|
||||
title := settings.Config.Frontend.Name
|
||||
@@ -251,6 +257,7 @@ func handleWithStaticData(w http.ResponseWriter, r *http.Request, d *requestCont
|
||||
"customCSS": template.CSS(settings.Config.Frontend.Styling.CustomCSSRaw),
|
||||
"userSelectedTheme": template.CSS(userSelectedTheme),
|
||||
"defaultDarkMode": settings.DefaultDarkMode(),
|
||||
"autoDarkMode": settings.DefaultDarkModeIsAuto(),
|
||||
"lightBackground": settings.Config.Frontend.Styling.LightBackground,
|
||||
"darkBackground": settings.Config.Frontend.Styling.DarkBackground,
|
||||
"staticURL": staticURL,
|
||||
@@ -275,57 +282,51 @@ func handleWithStaticData(w http.ResponseWriter, r *http.Request, d *requestCont
|
||||
|
||||
// only include constant values that don't change during a users session
|
||||
data["globalVars"] = map[string]interface{}{
|
||||
"name": settings.Config.Frontend.Name,
|
||||
"minSearchLength": settings.Config.Server.MinSearchLength,
|
||||
"searchResultsLimit": settings.Config.Server.SearchResultsLimit,
|
||||
"disableExternal": settings.Config.Frontend.DisableDefaultLinks,
|
||||
"darkMode": settings.Config.UserDefaults.UI.DarkMode,
|
||||
"singleClick": settings.Config.UserDefaults.Listing.SingleClick,
|
||||
"lightBackground": settings.Config.Frontend.Styling.LightBackground,
|
||||
"darkBackground": settings.Config.Frontend.Styling.DarkBackground,
|
||||
"baseURL": settings.Config.Http.BaseURL,
|
||||
"version": versionString,
|
||||
"commitSHA": commitSHAString,
|
||||
"signup": settings.Config.Auth.Methods.PasswordAuth.Signup,
|
||||
"noAuth": settings.Config.Auth.Methods.NoAuth,
|
||||
"enableThumbs": !settings.Config.Server.DisablePreviews,
|
||||
"externalLinks": externalLinks,
|
||||
"externalUrl": strings.TrimSuffix(settings.Config.Http.ExternalUrl, "/"),
|
||||
"onlyOfficeUrl": onlyOfficeUrlForSPA(),
|
||||
"oidcAvailable": settings.Config.Auth.Methods.OidcAuth.Enabled,
|
||||
"jwtAvailable": settings.Config.Auth.Methods.JwtAuth.Enabled,
|
||||
"proxyAvailable": settings.Config.Auth.Methods.ProxyAuth.Enabled,
|
||||
"passwordAvailable": settings.Config.Auth.Methods.PasswordAuth.Enabled,
|
||||
"passwordMinLength": settings.PasswordMinLength(),
|
||||
"ldapAvailable": settings.Config.Auth.Methods.LdapAuth.Enabled,
|
||||
"ffmpegAvailable": settings.FFmpegEnabled(),
|
||||
"hardwareAccelerationAvailable": settings.HardwareAccelerationSupported(),
|
||||
"transcodeEnabled": settings.TranscodeEnabled(),
|
||||
"transcodeProfiles": settings.EnabledTranscodeProfiles(),
|
||||
"transcodeMaxPerUser": settings.TranscodeMaxConcurrentPerUser(),
|
||||
"docPreviewExtensions": iteminfo.DocPreviewExtensionsForSPA(),
|
||||
"updateAvailable": utils.GetUpdateAvailableUrl(),
|
||||
"disableNavButtons": disableNavButtons,
|
||||
"userSelectableThemes": settings.Config.Frontend.Styling.CustomThemeOptions,
|
||||
"enableHeicConversion": settings.CanConvertImage("heic"),
|
||||
"enableJxlConversion": settings.CanConvertImage("jxl"),
|
||||
"eventBasedThemes": !settings.Config.Frontend.Styling.DisableEventBasedThemes,
|
||||
"loginIcon": loginIcon,
|
||||
"disableSidebar": disableSidebar,
|
||||
"shareHash": shareHash,
|
||||
"oidcLoginButtonText": settings.Config.Frontend.OIDCLoginButtonText,
|
||||
"loginButtonText": settings.Config.Frontend.LoginButtonText,
|
||||
"passkeyAvailable": settings.Config.Auth.Methods.PasskeyAuth.Enabled,
|
||||
"passkeyLoginButtonText": settings.Config.Auth.Methods.PasskeyAuth.LoginButtonText,
|
||||
"recaptcha": recaptchaEnabled,
|
||||
"recaptchaHost": recaptchaHost,
|
||||
"recaptchaKey": recaptchaKey,
|
||||
"disablePWAInstall": settings.Config.Frontend.DisablePWAInstall,
|
||||
"devMode": settings.Env.IsDevMode,
|
||||
"playwrightTest": settings.Env.IsPlaywright,
|
||||
"officeProduct": officeProductForSPA(),
|
||||
"officeExtensions": wopiExtensionsForSPA(),
|
||||
"textPreviewMimeTypes": iteminfo.TextPreviewMimeTypes(),
|
||||
"name": settings.Config.Frontend.Name,
|
||||
"minSearchLength": settings.Config.Server.MinSearchLength,
|
||||
"searchResultsLimit": settings.Config.Server.SearchResultsLimit,
|
||||
"disableExternal": settings.Config.Frontend.DisableDefaultLinks,
|
||||
"darkMode": settings.DefaultDarkModePreference(),
|
||||
"singleClick": settings.Config.UserDefaults.Listing.SingleClick,
|
||||
"lightBackground": settings.Config.Frontend.Styling.LightBackground,
|
||||
"darkBackground": settings.Config.Frontend.Styling.DarkBackground,
|
||||
"baseURL": settings.Config.Http.BaseURL,
|
||||
"version": versionString,
|
||||
"commitSHA": commitSHAString,
|
||||
"signup": settings.Config.Auth.Methods.PasswordAuth.Signup,
|
||||
"noAuth": settings.Config.Auth.Methods.NoAuth,
|
||||
"enableThumbs": !settings.Config.Server.DisablePreviews,
|
||||
"externalLinks": externalLinks,
|
||||
"externalUrl": strings.TrimSuffix(settings.Config.Http.ExternalUrl, "/"),
|
||||
"onlyOfficeUrl": onlyOfficeUrlForSPA(),
|
||||
"oidcAvailable": settings.Config.Auth.Methods.OidcAuth.Enabled,
|
||||
"jwtAvailable": settings.Config.Auth.Methods.JwtAuth.Enabled,
|
||||
"proxyAvailable": settings.Config.Auth.Methods.ProxyAuth.Enabled,
|
||||
"passwordAvailable": settings.Config.Auth.Methods.PasswordAuth.Enabled,
|
||||
"passwordMinLength": settings.PasswordMinLength(),
|
||||
"ldapAvailable": settings.Config.Auth.Methods.LdapAuth.Enabled,
|
||||
"ffmpegAvailable": settings.FFmpegEnabled(),
|
||||
"docPreviewExtensions": iteminfo.DocPreviewExtensionsForSPA(),
|
||||
"updateAvailable": utils.GetUpdateAvailableUrl(),
|
||||
"disableNavButtons": disableNavButtons,
|
||||
"userSelectableThemes": settings.Config.Frontend.Styling.CustomThemeOptions,
|
||||
"eventBasedThemes": !settings.Config.Frontend.Styling.DisableEventBasedThemes,
|
||||
"loginIcon": loginIcon,
|
||||
"disableSidebar": disableSidebar,
|
||||
"shareHash": shareHash,
|
||||
"oidcLoginButtonText": settings.Config.Frontend.OIDCLoginButtonText,
|
||||
"loginButtonText": settings.Config.Frontend.LoginButtonText,
|
||||
"passkeyAvailable": settings.Config.Auth.Methods.PasskeyAuth.Enabled,
|
||||
"passkeyLoginButtonText": settings.Config.Auth.Methods.PasskeyAuth.LoginButtonText,
|
||||
"recaptcha": recaptchaEnabled,
|
||||
"recaptchaHost": recaptchaHost,
|
||||
"recaptchaKey": recaptchaKey,
|
||||
"disablePWAInstall": settings.Config.Frontend.DisablePWAInstall,
|
||||
"devMode": settings.Env.IsDevMode,
|
||||
"playwrightTest": settings.Env.IsPlaywright,
|
||||
"officeProduct": officeProductForSPA(),
|
||||
"officeExtensions": wopiExtensionsForSPA(),
|
||||
"textPreviewMimeTypes": iteminfo.TextPreviewMimeTypes(),
|
||||
}
|
||||
|
||||
cspNonce, err := utils.CSPNonce()
|
||||
|
||||
@@ -487,10 +487,16 @@ func ServeSingleFile(w http.ResponseWriter, r *http.Request, d *Context, source
|
||||
bound = userScope
|
||||
}
|
||||
|
||||
realPath, _, err := idx.GetRealPathScoped(bound, scopedFilePath)
|
||||
realPath, isDir, err := idx.GetRealPathScoped(bound, scopedFilePath)
|
||||
if err != nil {
|
||||
return realPathErrStatus(err), err
|
||||
}
|
||||
if st, statErr := os.Stat(realPath); statErr == nil {
|
||||
isDir = st.IsDir()
|
||||
}
|
||||
if !state.AccessPermittedAtRealPath(idx, permUser, realPath, isDir) {
|
||||
return http.StatusForbidden, fmt.Errorf("access denied to path %s", scopedFilePath)
|
||||
}
|
||||
|
||||
isOnlyOffice := IsOnlyOfficeCompatibleFile(displayFileName) && settings.Config.Integrations.OnlyOffice.Url != "" && !settings.Config.Integrations.OnlyOffice.IsCollabora()
|
||||
var documentId string
|
||||
|
||||
@@ -0,0 +1,15 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// webGlobalTestMu serializes tests that mutate process-wide settings.Config or state.
|
||||
var webGlobalTestMu sync.Mutex
|
||||
|
||||
func lockWebGlobals(t testing.TB) {
|
||||
t.Helper()
|
||||
webGlobalTestMu.Lock()
|
||||
t.Cleanup(webGlobalTestMu.Unlock)
|
||||
}
|
||||
@@ -33,7 +33,7 @@ func TestTranscodeManagerIsWiredWithoutFFmpeg(t *testing.T) {
|
||||
// Startup with ffmpeg missing and transcoding disabled.
|
||||
settings.Env.FFmpegAvailable = false
|
||||
disabled := originalMedia
|
||||
disabled.Transcode = settings.MediaTranscode{Enabled: false}
|
||||
disabled.Transcode = settings.FullAdminTranscode{UserTranscode: settings.UserTranscode{Enabled: false}}
|
||||
settings.ApplyMediaSettings(disabled)
|
||||
|
||||
setupTestEnv(t)
|
||||
|
||||
@@ -176,6 +176,8 @@ func usersPostHandler(w http.ResponseWriter, r *http.Request, d *Context) (int,
|
||||
}
|
||||
// Parse the JSON into the UserRequest struct
|
||||
var req UserRequest
|
||||
// Seed with defaults so the decode below only overrides fields the request sent.
|
||||
state.PrefillUserProfileDefaults(&req.User)
|
||||
if err = json.Unmarshal(body, &req); err != nil {
|
||||
return http.StatusBadRequest, err
|
||||
}
|
||||
@@ -199,7 +201,7 @@ func usersPostHandler(w http.ResponseWriter, r *http.Request, d *Context) (int,
|
||||
return status, err
|
||||
}
|
||||
plaintextPassword := req.User.Password
|
||||
err = state.CreateUser(&req.User, plaintextPassword)
|
||||
err = state.CreateUserKeepingProfile(&req.User, plaintextPassword)
|
||||
if err != nil {
|
||||
return http.StatusInternalServerError, err
|
||||
}
|
||||
|
||||
@@ -0,0 +1,73 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/state"
|
||||
"github.com/gtsteffaniak/filebrowser/backend/pkg/settings"
|
||||
)
|
||||
|
||||
// Omitted permissions and omitted profile keys come from userDefaults; a partially
|
||||
// specified fileLoading object only overrides the keys it contains.
|
||||
func TestUsersPostOmittedFieldsFallBackToDefaults(t *testing.T) {
|
||||
setupTestEnv(t)
|
||||
|
||||
defaults := settings.Config.UserDefaults
|
||||
defaults.UI.Locale = "de"
|
||||
defaults.Account.Permissions.Share = true
|
||||
defaults.FileLoading.MaxConcurrent = 7
|
||||
defaults.FileLoading.UploadChunkSize = 9
|
||||
state.SetEffectiveUserDefaultsForTest(t, defaults)
|
||||
|
||||
admin := createPostTestAdmin(t)
|
||||
|
||||
body := []byte(`{"which":[],"data":{"username":"partial","password":"secret","loginMethod":"password",
|
||||
"fileLoading":{"maxConcurrentUpload":3}}}`)
|
||||
req := httptest.NewRequest(http.MethodPost, "/api/users", bytes.NewReader(body))
|
||||
status, err := usersPostHandler(httptest.NewRecorder(), req, &Context{User: admin})
|
||||
if err != nil || status != http.StatusCreated {
|
||||
t.Fatalf("usersPostHandler() status=%d err=%v", status, err)
|
||||
}
|
||||
got, err := state.GetUserByUsername("partial")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got.Locale != "de" {
|
||||
t.Errorf("locale = %q, want default de", got.Locale)
|
||||
}
|
||||
if !got.Permissions.Share {
|
||||
t.Errorf("share = false, want true from default permissions")
|
||||
}
|
||||
if got.FileLoading.MaxConcurrent != 3 {
|
||||
t.Errorf("maxConcurrentUpload = %d, want requested 3", got.FileLoading.MaxConcurrent)
|
||||
}
|
||||
if got.FileLoading.UploadChunkSize != 9 {
|
||||
t.Errorf("uploadChunkSizeMb = %d, want default 9", got.FileLoading.UploadChunkSize)
|
||||
}
|
||||
}
|
||||
|
||||
// A create request must not mutate the shared in-memory userDefaults (json.Unmarshal
|
||||
// reuses slice backing arrays).
|
||||
func TestUsersPostDoesNotMutateUserDefaults(t *testing.T) {
|
||||
setupTestEnv(t)
|
||||
|
||||
defaults := settings.Config.UserDefaults
|
||||
defaults.Listing.NewFileTemplate = []string{"default.txt"}
|
||||
state.SetEffectiveUserDefaultsForTest(t, defaults)
|
||||
|
||||
admin := createPostTestAdmin(t)
|
||||
|
||||
body := []byte(`{"which":[],"data":{"username":"nomutate","password":"secret","loginMethod":"password",
|
||||
"newFileTemplate":["custom.txt"]}}`)
|
||||
req := httptest.NewRequest(http.MethodPost, "/api/users", bytes.NewReader(body))
|
||||
status, err := usersPostHandler(httptest.NewRecorder(), req, &Context{User: admin})
|
||||
if err != nil || status != http.StatusCreated {
|
||||
t.Fatalf("usersPostHandler() status=%d err=%v", status, err)
|
||||
}
|
||||
if got := state.EffectiveUserDefaults().Listing.NewFileTemplate; len(got) != 1 || got[0] != "default.txt" {
|
||||
t.Errorf("userDefaults newFileTemplate = %v, want [default.txt]", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,139 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/database/users"
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/state"
|
||||
"github.com/gtsteffaniak/filebrowser/backend/pkg/settings"
|
||||
)
|
||||
|
||||
func createPostTestAdmin(t *testing.T) *users.User {
|
||||
t.Helper()
|
||||
admin := &users.User{
|
||||
ID: 1,
|
||||
FrontendUser: users.FrontendUser{
|
||||
Username: "admin",
|
||||
LoginMethod: users.LoginMethodOidc,
|
||||
Permissions: users.Permissions{Admin: true},
|
||||
},
|
||||
}
|
||||
if err := state.CreateUser(admin, ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return admin
|
||||
}
|
||||
|
||||
// Issue #3322: profile/account values in POST /api/users must not be replaced by userDefaults.
|
||||
func TestUsersPostKeepsRequestedProfileValues(t *testing.T) {
|
||||
setupTestEnv(t)
|
||||
|
||||
defaults := settings.Config.UserDefaults
|
||||
defaults.UI.Locale = "en"
|
||||
defaults.UI.ThemeColor = "var(--blue)"
|
||||
defaults.UI.DarkMode = users.DarkModeDark
|
||||
defaults.Account.LockPassword = false
|
||||
defaults.Account.DisableSettings = false
|
||||
defaults.Account.ShowAdvancedProfile = false
|
||||
defaults.Listing.PromptRightCloseButton = false
|
||||
defaults.Listing.NewFileTemplate = []string{""}
|
||||
defaults.Listing.ShowHidden = true
|
||||
state.SetEffectiveUserDefaultsForTest(t, defaults)
|
||||
|
||||
admin := createPostTestAdmin(t)
|
||||
|
||||
body := []byte(`{"which":[],"data":{
|
||||
"username":"profiletest","password":"secret","loginMethod":"password",
|
||||
"locale":"zhCN","themeColor":"var(--green)","darkMode":false,
|
||||
"lockPassword":true,"disableSettings":true,"showAdvancedProfile":true,
|
||||
"promptRightCloseButton":true,"newFileTemplate":["custom.txt"],
|
||||
"preview":{"image":false},
|
||||
"permissions":{"admin":false,"api":true,"share":true,"realtime":false},
|
||||
"scopes":[{"name":"/","scope":"/","permissions":{"view":true,"download":true,"modify":true,"create":true,"delete":true}}]
|
||||
}}`)
|
||||
req := httptest.NewRequest(http.MethodPost, "/api/users", bytes.NewReader(body))
|
||||
rec := httptest.NewRecorder()
|
||||
status, err := usersPostHandler(rec, req, &Context{User: admin})
|
||||
if err != nil || status != http.StatusCreated {
|
||||
t.Fatalf("usersPostHandler() status=%d err=%v", status, err)
|
||||
}
|
||||
|
||||
got, err := state.GetUserByUsername("profiletest")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got.Locale != "zhCN" {
|
||||
t.Errorf("locale = %q, want zhCN", got.Locale)
|
||||
}
|
||||
if got.ThemeColor != "var(--green)" {
|
||||
t.Errorf("themeColor = %q, want var(--green)", got.ThemeColor)
|
||||
}
|
||||
if got.DarkMode != users.DarkModeLight {
|
||||
t.Errorf("darkMode = %q, want light", got.DarkMode)
|
||||
}
|
||||
if !got.LockPassword {
|
||||
t.Errorf("lockPassword = false, want true")
|
||||
}
|
||||
if !got.DisableSettings {
|
||||
t.Errorf("disableSettings = false, want true")
|
||||
}
|
||||
if !got.ShowAdvancedProfile {
|
||||
t.Errorf("showAdvancedProfile = false, want true")
|
||||
}
|
||||
if !got.PromptRightCloseButton {
|
||||
t.Errorf("promptRightCloseButton = false, want true")
|
||||
}
|
||||
if len(got.NewFileTemplate) != 1 || got.NewFileTemplate[0] != "custom.txt" {
|
||||
t.Errorf("newFileTemplate = %v, want [custom.txt]", got.NewFileTemplate)
|
||||
}
|
||||
if got.Preview.Image {
|
||||
t.Errorf("preview.image = true, want false")
|
||||
}
|
||||
// Omitted fields still come from userDefaults.
|
||||
if !got.ShowHidden {
|
||||
t.Errorf("showHidden = false, want true from userDefaults")
|
||||
}
|
||||
if !got.Preview.Video {
|
||||
t.Errorf("preview.video = false, want true from userDefaults")
|
||||
}
|
||||
wantPerms := users.Permissions{Api: true, Share: true}
|
||||
if got.Permissions.Admin != wantPerms.Admin || got.Permissions.Api != wantPerms.Api ||
|
||||
got.Permissions.Share != wantPerms.Share || got.Permissions.Realtime != wantPerms.Realtime {
|
||||
t.Errorf("permissions = %+v, want %+v", got.Permissions, wantPerms)
|
||||
}
|
||||
}
|
||||
|
||||
// Enforced userDefaults still win over request values on create.
|
||||
func TestUsersPostEnforcedDefaultsWinOverRequest(t *testing.T) {
|
||||
setupTestEnv(t)
|
||||
|
||||
defaults := settings.Config.UserDefaults
|
||||
defaults.UI.Locale = "en"
|
||||
state.SetEffectiveUserDefaultsForTest(t, defaults)
|
||||
if err := state.PatchUserDefaultsEnforced([]byte(`{"ui":{"locale":true}}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
admin := createPostTestAdmin(t)
|
||||
|
||||
body := []byte(`{"which":[],"data":{"username":"enforced","password":"secret","loginMethod":"password","locale":"zhCN","lockPassword":true}}`)
|
||||
req := httptest.NewRequest(http.MethodPost, "/api/users", bytes.NewReader(body))
|
||||
rec := httptest.NewRecorder()
|
||||
status, err := usersPostHandler(rec, req, &Context{User: admin})
|
||||
if err != nil || status != http.StatusCreated {
|
||||
t.Fatalf("usersPostHandler() status=%d err=%v", status, err)
|
||||
}
|
||||
got, err := state.GetUserByUsername("enforced")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got.Locale != "en" {
|
||||
t.Errorf("locale = %q, want enforced en", got.Locale)
|
||||
}
|
||||
if !got.LockPassword {
|
||||
t.Errorf("lockPassword = false, want true (not enforced)")
|
||||
}
|
||||
}
|
||||
@@ -19,6 +19,7 @@ import (
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/activity"
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/adapters/fs/files"
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/adapters/fs/fileutils"
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/database/access"
|
||||
activitydb "github.com/gtsteffaniak/filebrowser/backend/internal/database/activity"
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/database/users"
|
||||
commonerrors "github.com/gtsteffaniak/filebrowser/backend/internal/errors"
|
||||
@@ -237,6 +238,35 @@ func (ffs *filteredFileSystem) checkScopePath(requestPath string, forRemoval boo
|
||||
return err
|
||||
}
|
||||
|
||||
// checkResolvedAccess enforces per-path DENY rules on the symlink-resolved target.
|
||||
func (ffs *filteredFileSystem) checkResolvedAccess(requestPath string, forRemoval bool) error {
|
||||
idx := indexing.GetIndex(ffs.source)
|
||||
if idx == nil {
|
||||
return fmt.Errorf("source not found")
|
||||
}
|
||||
userScope, err := ffs.user.GetScopeForSourceName(ffs.source)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fullIndexPath := utils.JoinScopedIndexPath(userScope, strings.TrimPrefix(requestPath, "/"))
|
||||
var realPath string
|
||||
if forRemoval {
|
||||
realPath, err = idx.GetRealPathForRemoval(userScope, fullIndexPath)
|
||||
} else {
|
||||
realPath, err = idx.GetRealPathForMutation(userScope, fullIndexPath)
|
||||
}
|
||||
if err != nil {
|
||||
if errors.Is(err, commonerrors.ErrPathEscapesScope) {
|
||||
return os.ErrPermission
|
||||
}
|
||||
return err
|
||||
}
|
||||
if !state.AccessPermittedAtRealPath(idx, ffs.user.Username, realPath, access.IsDirAtRealPath(realPath)) {
|
||||
return os.ErrPermission
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (ffs *filteredFileSystem) requireCreatePermissionForNewFile(requestPath string) error {
|
||||
idx := indexing.GetIndex(ffs.source)
|
||||
if idx == nil {
|
||||
@@ -412,6 +442,9 @@ func (ffs *filteredFileSystem) Mkdir(ctx context.Context, name string, perm os.F
|
||||
logger.Debugf("Mkdir: scope check denied for %s: %v", name, err)
|
||||
return err
|
||||
}
|
||||
if err := ffs.checkResolvedAccess(name, false); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := ffs.fs.Mkdir(ctx, name, perm); err != nil {
|
||||
return err
|
||||
@@ -450,6 +483,9 @@ func (ffs *filteredFileSystem) OpenFile(ctx context.Context, requestPath string,
|
||||
logger.Debugf("OpenFile: scope check denied for %s: %v", requestPath, err)
|
||||
return nil, err
|
||||
}
|
||||
if err := ffs.checkResolvedAccess(requestPath, false); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if (flag&os.O_CREATE) != 0 && !ffs.sourceFilePerms().Create {
|
||||
if err := ffs.requireCreatePermissionForNewFile(requestPath); err != nil {
|
||||
@@ -462,6 +498,9 @@ func (ffs *filteredFileSystem) OpenFile(ctx context.Context, requestPath string,
|
||||
logger.Debugf("OpenFile: read denied by scope for %s", requestPath)
|
||||
return nil, os.ErrPermission
|
||||
}
|
||||
if err := ffs.checkResolvedAccess(requestPath, false); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
var desiredMtime time.Time
|
||||
|
||||
@@ -30,6 +30,7 @@ import (
|
||||
|
||||
// setupWebDAVTestEnv sets up the test environment with multiple sources and users
|
||||
func setupWebDAVTestEnv(t *testing.T) (string, string) {
|
||||
lockWebGlobals(t)
|
||||
// Create temp directories for test sources
|
||||
tempDir := t.TempDir()
|
||||
source1Path := filepath.Join(tempDir, "source1")
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"runtime"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/adapters/fs/fileutils"
|
||||
@@ -24,8 +25,9 @@ import (
|
||||
)
|
||||
|
||||
var (
|
||||
RealPathCache = cache.NewCache[string](48*time.Hour, 72*time.Hour)
|
||||
IsDirCache = cache.NewCache[bool](48*time.Hour, 72*time.Hour)
|
||||
RealPathCache = cache.NewCache[string](48*time.Hour, 72*time.Hour)
|
||||
IsDirCache = cache.NewCache[bool](48*time.Hour, 72*time.Hour)
|
||||
realPathGeneration atomic.Uint64
|
||||
)
|
||||
|
||||
// getDiskUsage returns the actual disk space used by a file in bytes
|
||||
@@ -1222,6 +1224,7 @@ func (idx *Index) getRealPathInternal(boundIndexPath string, enforceScope bool,
|
||||
if ok && cached != "" {
|
||||
return cached, isDir, nil
|
||||
}
|
||||
generation := realPathGeneration.Load()
|
||||
absolutePath, err := filepath.Abs(joinedPath)
|
||||
if err != nil {
|
||||
return absolutePath, false, fmt.Errorf("could not get real path: %v, %s", joinedPath, err)
|
||||
@@ -1239,9 +1242,77 @@ func (idx *Index) getRealPathInternal(boundIndexPath string, enforceScope bool,
|
||||
}
|
||||
RealPathCache.Set(cacheKey, realPath)
|
||||
IsDirCache.Set(cacheKey+":isdir", isDir)
|
||||
if realPathGeneration.Load() != generation {
|
||||
RealPathCache.Delete(cacheKey)
|
||||
IsDirCache.Delete(cacheKey + ":isdir")
|
||||
}
|
||||
return realPath, isDir, nil
|
||||
}
|
||||
|
||||
// realPathCacheJoinedPath builds the filesystem join used in RealPathCache keys.
|
||||
func realPathCacheJoinedPath(sourceRoot, indexPath string) string {
|
||||
seg := utils.CleanIndexPathSegment(indexPath)
|
||||
if seg == "" {
|
||||
return sourceRoot
|
||||
}
|
||||
return filepath.Join(sourceRoot, seg)
|
||||
}
|
||||
|
||||
func realPathCacheIndexAncestors(indexPath string) []string {
|
||||
rel := utils.CleanIndexPathSegment(indexPath)
|
||||
if rel == "" {
|
||||
return []string{"/"}
|
||||
}
|
||||
parts := strings.Split(rel, "/")
|
||||
ancestors := []string{"/"}
|
||||
for i := range parts {
|
||||
ancestors = append(ancestors, "/"+strings.Join(parts[:i+1], "/"))
|
||||
}
|
||||
return ancestors
|
||||
}
|
||||
|
||||
// ClearRealPathCaches drops every cached symlink resolution. The cache library has no
|
||||
// prefix delete, so subtree invalidation uses a full clear; entries repopulate lazily.
|
||||
func ClearRealPathCaches() {
|
||||
realPathGeneration.Add(1)
|
||||
RealPathCache.ClearAll()
|
||||
IsDirCache.ClearAll()
|
||||
}
|
||||
|
||||
// InvalidateRealPathCache drops cached symlink resolutions for indexPath and its
|
||||
// ancestors. scopeBounds should include any caller scope prefixes used with
|
||||
// GetRealPathScoped (e.g. user or share path).
|
||||
func (idx *Index) InvalidateRealPathCache(indexPath string, scopeBounds ...string) {
|
||||
realPathGeneration.Add(1)
|
||||
boundSet := map[string]struct{}{"/": {}, "": {}}
|
||||
for _, b := range scopeBounds {
|
||||
b = strings.TrimRight(strings.TrimSpace(b), "/")
|
||||
if b == "" || b == "/" {
|
||||
boundSet["/"] = struct{}{}
|
||||
boundSet[""] = struct{}{}
|
||||
continue
|
||||
}
|
||||
if !strings.HasPrefix(b, "/") {
|
||||
b = "/" + b
|
||||
}
|
||||
boundSet[b] = struct{}{}
|
||||
}
|
||||
bounds := make([]string, 0, len(boundSet))
|
||||
for b := range boundSet {
|
||||
bounds = append(bounds, b)
|
||||
}
|
||||
for _, ip := range realPathCacheIndexAncestors(indexPath) {
|
||||
joined := realPathCacheJoinedPath(idx.Path, ip)
|
||||
for _, enforceScope := range []bool{false, true} {
|
||||
for _, bound := range bounds {
|
||||
key := fmt.Sprintf("realpath|scoped=%t|bound=%q|path=%q", enforceScope, bound, joined)
|
||||
RealPathCache.Delete(key)
|
||||
IsDirCache.Delete(key + ":isdir")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// GetRealPathForMutation resolves an index path for a mutating filesystem
|
||||
// operation (create/overwrite/mkdir/move-into). When the leaf exists it is fully
|
||||
// resolved; otherwise the deepest existing ancestor is resolved and the missing
|
||||
@@ -1603,7 +1674,10 @@ func setFilePreviewFlags(fileInfo *iteminfo.ItemInfo, realPath string) {
|
||||
}
|
||||
switch simpleType {
|
||||
case "image":
|
||||
if fileInfo.Size > iteminfo.LargeFileSizeThreshold {
|
||||
// Camera RAW previews are the embedded JPEG, which is read without decoding the RAW data,
|
||||
// so large RAW files (medium format, uncompressed) don't need the size limit.
|
||||
isRaw := iteminfo.IsRawImage(strings.ToLower(filepath.Ext(fileInfo.Name)))
|
||||
if fileInfo.Size > iteminfo.LargeFileSizeThreshold && !isRaw {
|
||||
fileInfo.HasPreview = false
|
||||
return
|
||||
}
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
package indexing
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/gtsteffaniak/filebrowser/backend/pkg/indexing/iteminfo"
|
||||
)
|
||||
|
||||
func TestSetFilePreviewFlagsLargeImages(t *testing.T) {
|
||||
large := iteminfo.LargeFileSizeThreshold + 1
|
||||
tests := []struct {
|
||||
name string
|
||||
file string
|
||||
typ string
|
||||
size int64
|
||||
want bool
|
||||
}{
|
||||
{"normal image under the limit", "a.jpg", "image/jpeg", 5 << 20, true},
|
||||
{"normal image over the limit", "a.jpg", "image/jpeg", int64(large), false},
|
||||
{"RAW under the limit", "a.raf", "image/x-fuji-raf", 26 << 20, true},
|
||||
{"large RAW keeps its preview, the embedded JPEG is read without decoding", "a.RAF", "image/x-fuji-raf", int64(large), true},
|
||||
{"large DNG keeps its preview", "a.dng", "image/x-adobe-dng", int64(large), true},
|
||||
{"empty file has no preview", "a.raf", "image/x-fuji-raf", 0, false},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
info := &iteminfo.ItemInfo{Name: tt.file, Type: tt.typ, Size: tt.size}
|
||||
setFilePreviewFlags(info, "/nonexistent/"+tt.file)
|
||||
if info.HasPreview != tt.want {
|
||||
t.Fatalf("HasPreview = %v, want %v", info.HasPreview, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -2,6 +2,7 @@ package indexing
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
@@ -163,6 +164,66 @@ func TestGetRealPathScoped_CacheKeyDoesNotCollideWithUnscoped(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRealPathCacheGeneration_BumpsOnInvalidateAndClear(t *testing.T) {
|
||||
idx := symlinkTestIndex(t.TempDir())
|
||||
gen := realPathGeneration.Load()
|
||||
idx.InvalidateRealPathCache("/any")
|
||||
if realPathGeneration.Load() != gen+1 {
|
||||
t.Fatalf("InvalidateRealPathCache: gen=%d want %d", realPathGeneration.Load(), gen+1)
|
||||
}
|
||||
gen = realPathGeneration.Load()
|
||||
ClearRealPathCaches()
|
||||
if realPathGeneration.Load() != gen+1 {
|
||||
t.Fatalf("ClearRealPathCaches: gen=%d want %d", realPathGeneration.Load(), gen+1)
|
||||
}
|
||||
}
|
||||
|
||||
func TestClearRealPathCaches_ClearsWarmEntries(t *testing.T) {
|
||||
sourceRoot, userScope := setupSymlinkEscapeFixture(t)
|
||||
idx := symlinkTestIndex(sourceRoot)
|
||||
|
||||
userDir := filepath.Join(sourceRoot, "user1")
|
||||
if err := os.WriteFile(filepath.Join(userDir, "foo"), []byte("x"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, _, err := idx.GetRealPathScoped(userScope, "/user1/foo"); err != nil {
|
||||
t.Fatalf("warm cache: %v", err)
|
||||
}
|
||||
joined := filepath.Join(sourceRoot, "user1", "foo")
|
||||
key := fmt.Sprintf("realpath|scoped=%t|bound=%q|path=%q", true, userScope, joined)
|
||||
if _, ok := RealPathCache.Get(key); !ok {
|
||||
t.Fatalf("expected cache key %q", key)
|
||||
}
|
||||
ClearRealPathCaches()
|
||||
if _, ok := RealPathCache.Get(key); ok {
|
||||
t.Fatal("ClearRealPathCaches should drop warm entries")
|
||||
}
|
||||
}
|
||||
|
||||
func TestInvalidateRealPathCache_ClearsGetRealPathEntry(t *testing.T) {
|
||||
sourceRoot, userScope := setupSymlinkEscapeFixture(t)
|
||||
idx := symlinkTestIndex(sourceRoot)
|
||||
|
||||
userDir := filepath.Join(sourceRoot, "user1")
|
||||
if err := os.WriteFile(filepath.Join(userDir, "foo"), []byte("x"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if _, _, err := idx.GetRealPath("/user1/foo"); err != nil {
|
||||
t.Fatalf("warm cache: %v", err)
|
||||
}
|
||||
joined := realPathCacheJoinedPath(sourceRoot, "/user1/foo")
|
||||
key := fmt.Sprintf("realpath|scoped=%t|bound=%q|path=%q", false, "/", joined)
|
||||
if _, ok := RealPathCache.Get(key); !ok {
|
||||
t.Fatalf("expected cache key %q", key)
|
||||
}
|
||||
|
||||
idx.InvalidateRealPathCache("/user1/foo", userScope)
|
||||
if _, ok := RealPathCache.Get(key); ok {
|
||||
t.Fatal("InvalidateRealPathCache should drop unscoped cache entry")
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetRealPathForMutation_ExistingInScope(t *testing.T) {
|
||||
sourceRoot, userScope := setupSymlinkEscapeFixture(t)
|
||||
idx := symlinkTestIndex(sourceRoot)
|
||||
|
||||
Loaded 100 of 237 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user