Compare commits

...
237 changed files with 27158 additions and 21311 deletions

No files matched your search

+1 -1
View File
@@ -15,7 +15,7 @@ jobs:
with:
go-version: 'stable'
- working-directory: backend
run: go test -race -v ./... -timeout 30s
run: go test -race -v ./... -timeout 45s
lint-backend:
runs-on: ubuntu-24.04-arm
timeout-minutes: 5
+5
View File
@@ -12,6 +12,11 @@ All notable changes to this project will be documented in this file. For commit
- Transcoding for media playback with quality presets -- requires ffmpeg.
- Hardware Acceleration device selector (Software / detected GPU devices)
- More operations use native api operations rather than ffmpeg, improving performance in many cases. Added temporary config option to force ffmpeg to be used
- HLS keyframe indexes for MP4/MOV/M4V/MKV/WebM are read from the container index with go-media instead of demuxing every packet with ffprobe; ffprobe still runs when go-media cannot read the file or reports a keyframe gap that would rule out stream copy.
- HEIC/AVIF rotation (irot) for the ffmpeg decode fallback is read with go-media instead of a full `ffmpeg -f null` decode.
- AVIF images get previews through the HEIC preview path (go-media decode, ffmpeg fallback) when HEIC previews are enabled.
- WMV/ASF video thumbnails use go-media for WMV3; WMV1/WMV2 still use ffmpeg.
- ffmpeg and go-media share one media metadata cache keyed by path, size and modification time, so a probe, duration or subtitle result from either backend is reused by the other.
- Document enhancements
- Official Collabora support using WOPI config.
- Document previews use a pure-go in-process package (all versions get it now).
+11 -11
View File
@@ -5,6 +5,7 @@ go 1.27.1
require (
github.com/alecthomas/kong v1.16.1
github.com/asdine/storm/v3 v3.2.1
github.com/cespare/xxhash/v2 v2.3.0
github.com/coreos/go-oidc/v3 v3.21.0
github.com/coreos/go-systemd/v22 v22.7.0
github.com/dhowden/tag v0.0.0-20240417053706-3d75831295e8
@@ -21,21 +22,21 @@ require (
github.com/gtsteffaniak/go-cache v1.1.0
github.com/gtsteffaniak/go-ffmpeg v1.6.2
github.com/gtsteffaniak/go-logger v1.1.0
github.com/gtsteffaniak/go-media v1.2.1
github.com/gtsteffaniak/go-media v1.3.0
github.com/kovidgoyal/imaging v1.9.0
github.com/pquerna/otp v1.5.0
github.com/spf13/afero v1.15.0
github.com/stretchr/testify v1.12.1
github.com/swaggo/swag v1.16.6
golang.org/x/crypto v0.58.0
golang.org/x/image v0.47.0
golang.org/x/image v0.48.0
golang.org/x/mod v0.42.0
golang.org/x/net v0.61.0
golang.org/x/oauth2 v0.37.0
golang.org/x/sync v0.24.0
golang.org/x/sys v0.49.0
golang.org/x/sync v0.25.0
golang.org/x/sys v0.50.0
golang.org/x/term v0.47.0
golang.org/x/text v0.43.0
golang.org/x/text v0.44.0
golang.org/x/time v0.16.0
golang.org/x/tools v0.52.0
gopkg.in/yaml.v3 v3.0.1
@@ -44,7 +45,7 @@ require (
require (
github.com/Azure/go-ntlmssp v0.1.1 // indirect
github.com/Eyevinn/mp4ff v0.61.0 // indirect
github.com/Eyevinn/mp4ff v0.62.1 // indirect
github.com/KyleBanks/depth v1.2.1 // indirect
github.com/abema/go-mp4 v1.7.4 // indirect
github.com/at-wat/ebml-go v0.19.4 // indirect
@@ -54,10 +55,10 @@ require (
github.com/dustin/go-humanize v1.1.0 // indirect
github.com/fxamacker/cbor/v2 v2.9.6 // indirect
github.com/gabriel-vasile/mimetype v1.4.16 // indirect
github.com/gen2brain/gav1d v0.2.6 // indirect
github.com/gen2brain/h265 v0.2.4 // indirect
github.com/gen2brain/jxl v0.2.0 // indirect
github.com/gen2brain/vpx v0.2.1 // indirect
github.com/gen2brain/gav1d v0.2.7 // indirect
github.com/gen2brain/h265 v0.2.5 // indirect
github.com/gen2brain/jxl v0.2.1 // indirect
github.com/gen2brain/vpx v0.2.2 // indirect
github.com/go-asn1-ber/asn1-ber v1.5.8 // indirect
github.com/go-errors/errors v1.5.1 // indirect
github.com/go-jose/go-jose/v4 v4.1.6 // indirect
@@ -80,7 +81,6 @@ require (
github.com/golang/protobuf v1.5.4 // indirect
github.com/golang/snappy v0.0.4 // indirect
github.com/google/go-tpm v0.9.8 // indirect
github.com/gtsteffaniak/go-av1 v1.0.0 // indirect
github.com/gtsteffaniak/go-vpx v1.0.1 // indirect
github.com/kovidgoyal/go-parallel v1.1.1 // indirect
github.com/kovidgoyal/go-shm v1.0.0 // indirect
+22 -24
View File
@@ -2,8 +2,8 @@ github.com/Azure/go-ntlmssp v0.1.1 h1:l+FM/EEMb0U9QZE7mKNEDw5Mu3mFiaa2GKOoTSsNDP
github.com/Azure/go-ntlmssp v0.1.1/go.mod h1:NYqdhxd/8aAct/s4qSYZEerdPuH1liG2/X9DiVTbhpk=
github.com/DataDog/zstd v1.4.1 h1:3oxKN3wbHibqx897utPC2LTQU4J+IHWWJO+glkAkpFM=
github.com/DataDog/zstd v1.4.1/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo=
github.com/Eyevinn/mp4ff v0.61.0 h1:uX3GBauEouxuGepRbv15JB7X2RyVyE2EZEfLpgimL6o=
github.com/Eyevinn/mp4ff v0.61.0/go.mod h1:97zsDIJUiDTYqKpqq/b1K9foamxXmvsgE4GdPrPK3ks=
github.com/Eyevinn/mp4ff v0.62.1 h1:EMuNIrmTCm5acOCVENca5Fzk+Mdek/l4bQZdk2EfReo=
github.com/Eyevinn/mp4ff v0.62.1/go.mod h1:97zsDIJUiDTYqKpqq/b1K9foamxXmvsgE4GdPrPK3ks=
github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc=
github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE=
github.com/Sereal/Sereal v0.0.0-20190618215532-0b8ac451a863 h1:BRrxwOZBolJN4gIwvZMJY1tzqBvQgpaZiQRuIDD40jM=
@@ -25,6 +25,8 @@ github.com/at-wat/ebml-go v0.19.4/go.mod h1:wu3fQj+DWo6SOOl9c4yDi+gFs9Oa96Z2B5qx
github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8=
github.com/boombuler/barcode v1.1.0 h1:ChaYjBR63fr4LFyGn8E8nt7dBSt3MiU3zMOZqFvVkHo=
github.com/boombuler/barcode v1.1.0/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8=
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/coreos/go-oidc/v3 v3.21.0 h1:wZo4Q9Pum8dYEj0eMUPrqR+kvuGkeUplbLpNCkBqoWM=
github.com/coreos/go-oidc/v3 v3.21.0/go.mod h1:DYCf24+ncYi+XkIH97GY1+dqoRlbaSI26KVTCI9SrY4=
github.com/coreos/go-systemd/v22 v22.7.0 h1:LAEzFkke61DFROc7zNLX/WA2i5J8gYqe0rSj9KI28KA=
@@ -62,14 +64,14 @@ github.com/gabriel-vasile/mimetype v1.4.16 h1:5izNFuvA37st1dDRAcA7TtmngBmRgiIPDm
github.com/gabriel-vasile/mimetype v1.4.16/go.mod h1:3oFFwxX8kTklZwzIl2aQEDZT0lKY8xH92ST88bAwba4=
github.com/gen2brain/folio v0.2.8 h1:9iDI7h+zt4mjwEnzm1x8Pl9BrWqLwUwB74X/fnc1PZs=
github.com/gen2brain/folio v0.2.8/go.mod h1:4/PcXfaXIYmW1nwQ4bJeE+aG5ybG58Aw9xrwODXaXII=
github.com/gen2brain/gav1d v0.2.6 h1:7TSkrWTdE22k5Ul7vYZML7Nu80gNW6OovV7FSCITSE0=
github.com/gen2brain/gav1d v0.2.6/go.mod h1:ReFKjHq7pvRhEk+pgRPAo9gOlcrf51sdzVcAbbX7e0E=
github.com/gen2brain/h265 v0.2.4 h1:JUSbGkRAYZpd7ucCL192yx4ViEo75ZO6OEdm1TQt9qM=
github.com/gen2brain/h265 v0.2.4/go.mod h1:kefpLxLhGpNCUG/q/aabJcD6GqB9Nhet3fNi9xX9Wzw=
github.com/gen2brain/jxl v0.2.0 h1:cXSQnfbcC0JGFzaI344VzPMbe7mh6rYc0tjPcBUsE4U=
github.com/gen2brain/jxl v0.2.0/go.mod h1:VjJHRai/8I8Es01mULqyiOihRBOjJr1Zo+NoN/MyIwo=
github.com/gen2brain/vpx v0.2.1 h1:UD6tibcs/zlaIOKNFXeVyZuHBEaW9GYfxKSxvAN9cc8=
github.com/gen2brain/vpx v0.2.1/go.mod h1:EVIdqguMSiapbtjDm4AWgFgYYJ4d+k8trXqjxYZ/4XE=
github.com/gen2brain/gav1d v0.2.7 h1:ehu3jWt+0/46Is5llW/ZyX2OA3hTBJNukKnebzuk8KU=
github.com/gen2brain/gav1d v0.2.7/go.mod h1:ReFKjHq7pvRhEk+pgRPAo9gOlcrf51sdzVcAbbX7e0E=
github.com/gen2brain/h265 v0.2.5 h1:D3qCPx5GU+gHpinvLJtYMLSmQy64RGOm4fzMNGQcA1g=
github.com/gen2brain/h265 v0.2.5/go.mod h1:kefpLxLhGpNCUG/q/aabJcD6GqB9Nhet3fNi9xX9Wzw=
github.com/gen2brain/jxl v0.2.1 h1:74+8uw2RxUHGDVrDp+727PZ2hvAd5/psyV/mpsOQea0=
github.com/gen2brain/jxl v0.2.1/go.mod h1:d7VMurg9ek62xfuyZyVAumFcuaQf9U8NlW57j/k5MQI=
github.com/gen2brain/vpx v0.2.2 h1:kRCIx6Y2LsTe8hercAfhER2QbeHYnBOetovy4pkC4V0=
github.com/gen2brain/vpx v0.2.2/go.mod h1:EVIdqguMSiapbtjDm4AWgFgYYJ4d+k8trXqjxYZ/4XE=
github.com/go-asn1-ber/asn1-ber v1.5.8 h1:H9AZkK22UOmfX8J84ubyaZxKJZ3FMHVwn8swoMML7iQ=
github.com/go-asn1-ber/asn1-ber v1.5.8/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q=
@@ -157,18 +159,14 @@ github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3/go.mod h1:jl5iWTm0/hd
github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/gtsteffaniak/go-av1 v1.0.0 h1:CILp67bcOtiDXvnrQubnH9ctub0OSKnUNA2JJuoIP+c=
github.com/gtsteffaniak/go-av1 v1.0.0/go.mod h1:C/dIrjfdUcHgdyChr2YXwLY3pIVjENI9npSd98PU4lE=
github.com/gtsteffaniak/go-cache v1.1.0 h1:ni/2cJpCxpGmaMZd92iTDIGOy+yUcMcxQuW7bSKhkCE=
github.com/gtsteffaniak/go-cache v1.1.0/go.mod h1:rHUZvf4huhMX/bnNK4/oQ3sNAE2fYC2jFpAWZiBPWac=
github.com/gtsteffaniak/go-ffmpeg v1.6.2 h1:gtvBg3G4PHO1Z/ZIPEK/XP4XT/g+l9S0Rpmmq6v5tnA=
github.com/gtsteffaniak/go-ffmpeg v1.6.2/go.mod h1:FefhoD+KZzHWqBaUpntcKp6lXtUzdFW00F74axnENUQ=
github.com/gtsteffaniak/go-logger v1.1.0 h1:baFds+iibuLLNpNLwygz32bDqdsunI6dGWcdxwMIVKI=
github.com/gtsteffaniak/go-logger v1.1.0/go.mod h1:rapapTju4D7bwt6h4JOdZNUXDjnp0WkTBd6TN5yWglI=
github.com/gtsteffaniak/go-media v1.2.0 h1:lbNL/G86uzzH8NaT45DBHKm1r0P7yndIo9pNC7gdqQs=
github.com/gtsteffaniak/go-media v1.2.0/go.mod h1:EH+5KAVLWfFl0g4W0Xni0qJJH8MUDIhAw4wPcC0mhxY=
github.com/gtsteffaniak/go-media v1.2.1 h1:SjrPqgQTHnenAFuVh8fTRxk8+qXFEaCxiADx3zkM1OI=
github.com/gtsteffaniak/go-media v1.2.1/go.mod h1:EH+5KAVLWfFl0g4W0Xni0qJJH8MUDIhAw4wPcC0mhxY=
github.com/gtsteffaniak/go-media v1.3.0 h1:EOxoaOQq05V3OUPo5NndnPZ0/VYt4gFEwa6l7MdR6rk=
github.com/gtsteffaniak/go-media v1.3.0/go.mod h1:x8k0z759GiawTtKBTXDNXF/pT1UQDhR4e70/eypXpOk=
github.com/gtsteffaniak/go-vpx v1.0.1 h1:9lSI8yoz6omDChaeV0AJrhdfe2YTrWvJrDENI/bnkAM=
github.com/gtsteffaniak/go-vpx v1.0.1/go.mod h1:tRCdDZ15oZZTsiwmQAwyH56PTt1jn1C1O0P03oOkoJ8=
github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8=
@@ -258,8 +256,8 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.58.0 h1:COkYLr4k7nDI4r6QtMML2AfzeIswWHAJthLS6K/M/54=
golang.org/x/crypto v0.58.0/go.mod h1:Xh+kl5A+M0gMiWDytH2Hhr5Z7WbNCpv/eB/EFgfSk+w=
golang.org/x/image v0.47.0 h1:za/8gVz++PPZqmZJBNSISICN5Rrj5zixP/8K++fgmqo=
golang.org/x/image v0.47.0/go.mod h1:IkN5o4A7vq55vXN8FWjjnhg9Rs2GOtffqsSi9mWAKYE=
golang.org/x/image v0.48.0 h1:1AvuPcuoOTJpA7s4VZ5umrb90T3So0rkB/bSGmxkxik=
golang.org/x/image v0.48.0/go.mod h1:mmtg64iqJKGqEgy+r13dfybl85tmxUd2lMnmaiSdWYY=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/mod v0.42.0 h1:ICeL3t2L30ljSPHHxDnR4fqVcBt/zR2zJAb9juavmQg=
golang.org/x/mod v0.42.0/go.mod h1:u5fzIrRInkL8MaO1cbRkUHT64v303YIT5KEqpNr7jRs=
@@ -279,8 +277,8 @@ golang.org/x/oauth2 v0.37.0 h1:JUlcxA8oAtauLfiH8FX2/FkAWHAdi0QtGCGc+hofE98=
golang.org/x/oauth2 v0.37.0/go.mod h1:IxwZNxUULJmpBFf9K/9NTMSIfZZuvuTy1gGxhigP/58=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.24.0 h1:iqiDrGoDgyuvAKrhLQthUEzU/R378N78gXbBjxed7oA=
golang.org/x/sync v0.24.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0=
golang.org/x/sync v0.25.0 h1:dJnifr0nMV8/SBnBvdo4eSGGnrc3O4FFx6IqvtLcmRs=
golang.org/x/sync v0.25.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -292,8 +290,8 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.49.0 h1:XbzkgYJHdqh/8m2Uu0W/dQv8nktxx4BFHp1M0gROTXA=
golang.org/x/sys v0.49.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
golang.org/x/sys v0.50.0 h1:dkKZEFTEKaLr1sjhs0WiKqoi8Jj5nHbS3Njmx1UoMJs=
golang.org/x/sys v0.50.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.47.0 h1:cpyOwQkr2Nzxe2p6KnrTG7Ei2wtzjsuSjhpXXepZ/BM=
@@ -303,8 +301,8 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
golang.org/x/text v0.43.0 h1:1QivrlhwAsnMOcqgOdl9my7s9OyZl8O5/UGUTJVXoWc=
golang.org/x/text v0.43.0/go.mod h1:hbSIYA/amXcRXTFSZqh8tWHEoqRvvbKPJ2uz/KU+a+s=
golang.org/x/text v0.44.0 h1:YItdp/mQNGnsmrgpcdA5yQ3bnRxCfo/Tj/iByGW9GGo=
golang.org/x/text v0.44.0/go.mod h1:S4FR1waQ9LbjQTls97SyRjDaUUVkaM+cXwwuG/G+MwY=
golang.org/x/time v0.16.0 h1:vMb6ptszcQMkcwiRTAuNNU50gom6++Q/6gY2hDM6VDE=
golang.org/x/time v0.16.0/go.mod h1:rVKOqvZeKvrDKTQiAHJ7wmwP0RzleSphoEA9RcdLA0s=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+5 -5
View File
@@ -14,13 +14,13 @@ func TestUserUpdateChangesSkipsSensitiveFields(t *testing.T) {
FrontendUser: users.FrontendUser{
Username: "alice",
NonAdminEditable: users.NonAdminEditable{
DarkMode: true,
DarkMode: users.DarkModeDark,
},
},
TOTPSecret: "secret-value",
}
after := *before
after.DarkMode = false
after.DarkMode = users.DarkModeLight
after.TOTPSecret = "new-secret"
changes := UserUpdateChanges(before, &after, []string{"totpSecret", "darkMode"}, false)
@@ -36,7 +36,7 @@ func TestUserUpdateChangesFiltersUnchangedFields(t *testing.T) {
FrontendUser: users.FrontendUser{
Username: "akadmin",
NonAdminEditable: users.NonAdminEditable{
DarkMode: true,
DarkMode: users.DarkModeDark,
StickySidebar: true,
Locale: "en",
SingleClick: false,
@@ -47,7 +47,7 @@ func TestUserUpdateChangesFiltersUnchangedFields(t *testing.T) {
},
}
after := *before
after.DarkMode = false
after.DarkMode = users.DarkModeLight
after.Sorting = users.Sorting{By: "name", Asc: true}
which := []string{
@@ -60,7 +60,7 @@ func TestUserUpdateChangesFiltersUnchangedFields(t *testing.T) {
if len(changes) != 3 {
t.Fatalf("expected 3 changes, got %d: %#v", len(changes), changes)
}
if changes[0].Field != "darkMode" || changes[0].From != "true" || changes[0].To != "false" {
if changes[0].Field != "darkMode" || changes[0].From != "dark" || changes[0].To != "light" {
t.Fatalf("unexpected darkMode change: %#v", changes[0])
}
if changes[1].Field != "sorting.asc" || changes[1].From != "false" || changes[1].To != "true" {
@@ -9,7 +9,10 @@ import (
"io"
"os"
"path/filepath"
"sort"
"sync"
"sync/atomic"
"time"
"github.com/gtsteffaniak/filebrowser/backend/internal/adapters/fs/fileutils"
"github.com/gtsteffaniak/go-logger/logger"
@@ -21,9 +24,28 @@ type Cache interface {
Set(key string, value []byte) error
}
const (
// evictTargetPercent is the size, as a percentage of the limit, that eviction shrinks the cache to.
// Stopping below the limit avoids running an eviction on every write once the cache is full.
evictTargetPercent = 90
// touchInterval is how old a file's modification time has to be before a read refreshes it. Reads
// refresh it so the files in use are the last ones evicted, but not on every read.
touchInterval = time.Hour
)
// FileCache struct for file-based caching
type FileCache struct {
dir string
// maxBytes is the size limit of the cache in bytes. 0 means no limit.
maxBytes int64
// size is the total size of the cached files in bytes. Only tracked when maxBytes > 0.
size atomic.Int64
// sizeMu keeps size consistent with the files on disk: stores and deletes hold it for reading while
// they change a file and its size, and eviction holds it for writing while it measures the directory,
// so a store can never fall between a measurement and the size being set from it.
sizeMu sync.RWMutex
// evicting is set while an eviction is running so only one runs at a time.
evicting atomic.Bool
// granular locks
scopedLocks struct {
sync.Mutex
@@ -33,6 +55,12 @@ type FileCache struct {
}
func NewFileCache(dir string) (*FileCache, error) {
return NewFileCacheWithLimit(dir, 0)
}
// NewFileCacheWithLimit is like NewFileCache but keeps the cache under maxBytes, deleting the least
// recently used files when it grows past it. 0 means no limit.
func NewFileCacheWithLimit(dir string, maxBytes int64) (*FileCache, error) {
cacheDir := filepath.Join(dir, "diskcache")
// Migrate existing cache files from old structure (if any)
@@ -44,7 +72,13 @@ func NewFileCache(dir string) (*FileCache, error) {
if err := os.MkdirAll(cacheDir, fileutils.PermDir); err != nil {
return nil, fmt.Errorf("failed to create cache directory: %w", err)
}
return &FileCache{dir: cacheDir}, nil
cache := &FileCache{dir: cacheDir}
if maxBytes > 0 {
cache.maxBytes = maxBytes
// measures the cache and trims it right away if it is already over the limit
cache.evictPass()
}
return cache, nil
}
// migrateOldCacheStructure moves cache files from old structure (dir/a/...) to new structure (dir/diskcache/a/...)
@@ -99,19 +133,50 @@ func (f *FileCache) Store(ctx context.Context, key string, value []byte) error {
return err
}
if f.maxBytes > 0 {
f.sizeMu.RLock()
defer f.sizeMu.RUnlock()
}
previousSize := f.fileSize(fileName)
if err := os.WriteFile(fileName, value, fileutils.PermFile); err != nil {
// a failed write can leave a truncated or partial file, so count what is actually there
f.sizeChanged(f.fileSize(fileName) - previousSize)
return err
}
f.sizeChanged(int64(len(value)) - previousSize)
return nil
}
// fileSize returns the size of a cached file, or 0 if it doesn't exist or the size isn't tracked.
func (f *FileCache) fileSize(fileName string) int64 {
if f.maxBytes <= 0 {
return 0
}
info, err := os.Stat(fileName)
if err != nil {
return 0
}
return info.Size()
}
// sizeChanged adds delta to the tracked size and starts an eviction if that put the cache over its limit.
func (f *FileCache) sizeChanged(delta int64) {
if f.maxBytes <= 0 {
return
}
if f.size.Add(delta) > f.maxBytes {
f.evictAsync()
}
}
func (f *FileCache) Load(ctx context.Context, key string) (value []byte, exist bool, err error) {
r, ok, err := f.open(key)
if err != nil || !ok {
return nil, ok, err
}
defer r.Close()
f.touch(r)
value, err = io.ReadAll(r)
if err != nil {
@@ -125,13 +190,119 @@ func (f *FileCache) Delete(ctx context.Context, key string) error {
mu.Lock()
defer mu.Unlock()
if f.maxBytes > 0 {
f.sizeMu.RLock()
defer f.sizeMu.RUnlock()
}
fileName := f.getFileName(key)
removedSize := f.fileSize(fileName)
if err := os.Remove(fileName); err != nil && !errors.Is(err, os.ErrNotExist) {
return err
}
if removedSize > 0 {
f.size.Add(-removedSize)
}
return nil
}
// touch refreshes the modification time of a file that was just read, which is what eviction orders by.
func (f *FileCache) touch(file *os.File) {
if f.maxBytes <= 0 {
return
}
info, err := file.Stat()
if err != nil || time.Since(info.ModTime()) < touchInterval {
return
}
now := time.Now()
_ = os.Chtimes(file.Name(), now, now)
}
type cachedFile struct {
path string
size int64
modTime time.Time
}
// scan lists the cached files and their total size.
func (f *FileCache) scan() ([]cachedFile, int64) {
var files []cachedFile
var total int64
_ = filepath.WalkDir(f.dir, func(path string, entry os.DirEntry, err error) error {
if err != nil || entry.IsDir() {
return nil
}
info, err := entry.Info()
if err != nil || !info.Mode().IsRegular() {
return nil
}
files = append(files, cachedFile{path: path, size: info.Size(), modTime: info.ModTime()})
total += info.Size()
return nil
})
return files, total
}
// evictAsync starts an eviction in the background unless one is already running, so the request that
// stored the file doesn't wait for it.
func (f *FileCache) evictAsync() {
if !f.evicting.CompareAndSwap(false, true) {
return
}
go func() {
for {
removed := f.evictPass()
f.evicting.Store(false)
if removed == 0 {
return // under the limit, or nothing could be removed: don't spin
}
// Files stored while the pass was running were counted but not necessarily seen by it, and a
// store that came in just before the flag was cleared saw it set and didn't start another
// pass, so check again before stopping.
if f.size.Load() <= f.maxBytes || !f.evicting.CompareAndSwap(false, true) {
return
}
}
}()
}
// evictPass measures the cache and, if it is over the limit, deletes the least recently used files until
// it is under evictTargetPercent of the limit. Deleting a file that is being read or written is harmless:
// readers keep their open file and a removed entry is simply rebuilt the next time it is requested.
func (f *FileCache) evictPass() (removed int) {
f.sizeMu.Lock()
files, total := f.scan()
f.size.Store(total)
f.sizeMu.Unlock()
if total <= f.maxBytes {
return 0
}
target := f.maxBytes / 100 * evictTargetPercent
sort.Slice(files, func(i, j int) bool { return files[i].modTime.Before(files[j].modTime) })
var removedBytes int64
for _, file := range files {
if f.size.Load() <= target {
break
}
// Hold the lock so no store or delete changes this file between measuring it and removing it, and
// subtract the size it has now: the listing is from the scan and the file may have been rewritten since.
f.sizeMu.Lock()
info, err := os.Stat(file.path)
if err == nil {
err = os.Remove(file.path)
}
if err == nil {
f.size.Add(-info.Size())
removed++
removedBytes += info.Size()
}
f.sizeMu.Unlock()
}
logger.Debugf("preview cache limit: removed %d files (%d MB), now %d MB of %d MB", removed, removedBytes>>20, f.size.Load()>>20, f.maxBytes>>20)
return removed
}
func (f *FileCache) open(key string) (*os.File, bool, error) {
fileName := f.getFileName(key)
file, err := os.Open(fileName)
@@ -2,9 +2,12 @@ package diskcache
import (
"context"
"fmt"
"os"
"path/filepath"
"sync"
"testing"
"time"
"github.com/gtsteffaniak/filebrowser/backend/internal/adapters/fs/fileutils"
"github.com/stretchr/testify/require"
@@ -70,3 +73,168 @@ func fileExists(filename string) bool {
}
return !info.IsDir()
}
// storeAged stores a value and sets its modification time, so the eviction order is deterministic.
func storeAged(t *testing.T, cache *FileCache, key string, size int, age time.Duration) {
t.Helper()
require.NoError(t, cache.Store(context.Background(), key, make([]byte, size)))
when := time.Now().Add(-age)
require.NoError(t, os.Chtimes(cache.getFileName(key), when, when))
}
func waitFor(t *testing.T, condition func() bool) {
t.Helper()
deadline := time.Now().Add(5 * time.Second)
for !condition() {
if time.Now().After(deadline) {
t.Fatal("condition not met in time")
}
time.Sleep(10 * time.Millisecond)
}
}
func newLimitedCache(t *testing.T, maxBytes int64) *FileCache {
t.Helper()
fileutils.SetFsPermissions(0644, 0755)
cache, err := NewFileCacheWithLimit(t.TempDir(), maxBytes)
require.NoError(t, err)
return cache
}
func TestFileCacheEvictsLeastRecentlyUsedOverLimit(t *testing.T) {
const kb = 1024
cache := newLimitedCache(t, 10*kb)
// 3 files of 3KB are under the 10KB limit
storeAged(t, cache, "oldest", 3*kb, 3*time.Hour)
storeAged(t, cache, "old", 3*kb, 2*time.Hour)
storeAged(t, cache, "recent", 3*kb, time.Hour)
require.True(t, fileExists(cache.getFileName("oldest")), "nothing is removed under the limit")
// the 4th file goes over the limit: the oldest ones are removed until it is under 90% of it
require.NoError(t, cache.Store(context.Background(), "newest", make([]byte, 3*kb)))
waitFor(t, func() bool { return !fileExists(cache.getFileName("oldest")) })
waitFor(t, func() bool { return !cache.evicting.Load() })
require.False(t, fileExists(cache.getFileName("oldest")))
require.False(t, fileExists(cache.getFileName("old")))
require.True(t, fileExists(cache.getFileName("recent")))
require.True(t, fileExists(cache.getFileName("newest")))
require.LessOrEqual(t, cache.size.Load(), int64(9*kb))
}
func TestFileCacheTrimsAnOversizedCacheAtStartup(t *testing.T) {
const kb = 1024
fileutils.SetFsPermissions(0644, 0755)
dir := t.TempDir()
unlimited, err := NewFileCache(dir)
require.NoError(t, err)
storeAged(t, &FileCache{dir: unlimited.dir}, "a", 4*kb, 4*time.Hour)
storeAged(t, &FileCache{dir: unlimited.dir}, "b", 4*kb, 3*time.Hour)
storeAged(t, &FileCache{dir: unlimited.dir}, "c", 4*kb, 2*time.Hour)
cache, err := NewFileCacheWithLimit(dir, 10*kb)
require.NoError(t, err)
// 12KB over a 10KB limit: only the oldest file is removed, which already brings it under 90% (9KB)
require.False(t, fileExists(cache.getFileName("a")))
require.True(t, fileExists(cache.getFileName("b")))
require.True(t, fileExists(cache.getFileName("c")))
require.Equal(t, int64(8*kb), cache.size.Load())
}
func TestFileCacheWithoutLimitKeepsEverything(t *testing.T) {
fileutils.SetFsPermissions(0644, 0755)
cache, err := NewFileCache(t.TempDir())
require.NoError(t, err)
for i := 0; i < 5; i++ {
storeAged(t, cache, fmt.Sprintf("key-%d", i), 1<<20, time.Duration(i)*time.Hour)
}
for i := 0; i < 5; i++ {
require.True(t, fileExists(cache.getFileName(fmt.Sprintf("key-%d", i))))
}
require.Zero(t, cache.size.Load(), "size is only tracked when there is a limit")
}
func TestFileCacheReadRefreshesModTime(t *testing.T) {
cache := newLimitedCache(t, 1<<20)
storeAged(t, cache, "used", 100, 3*time.Hour)
_, ok, err := cache.Load(context.Background(), "used")
require.NoError(t, err)
require.True(t, ok)
info, err := os.Stat(cache.getFileName("used"))
require.NoError(t, err)
require.WithinDuration(t, time.Now(), info.ModTime(), time.Minute, "a file that is read is the last to be evicted")
}
func TestFileCacheDeleteAndOverwriteKeepTheSizeAccurate(t *testing.T) {
const kb = 1024
cache := newLimitedCache(t, 1<<20)
ctx := context.Background()
require.NoError(t, cache.Store(ctx, "k", make([]byte, 5*kb)))
require.Equal(t, int64(5*kb), cache.size.Load())
require.NoError(t, cache.Store(ctx, "k", make([]byte, 2*kb)))
require.Equal(t, int64(2*kb), cache.size.Load(), "overwriting replaces the old size")
require.NoError(t, cache.Delete(ctx, "k"))
require.Zero(t, cache.size.Load())
}
// Stores that land while an eviction is measuring or deleting must not be lost from the tracked size,
// and the cache must end up under its limit even though more files kept arriving during the eviction.
func TestFileCacheStaysUnderTheLimitWithConcurrentStores(t *testing.T) {
const (
kb = 1024
limit = 100 * kb
workers = 16
stores = 40
)
cache := newLimitedCache(t, limit)
var wg sync.WaitGroup
for w := 0; w < workers; w++ {
wg.Add(1)
go func(w int) {
defer wg.Done()
for i := 0; i < stores; i++ {
_ = cache.Store(context.Background(), fmt.Sprintf("worker-%d-key-%d", w, i), make([]byte, 4*kb))
}
}(w)
}
wg.Wait()
waitFor(t, func() bool { return !cache.evicting.Load() })
_, onDisk := cache.scan()
require.LessOrEqual(t, onDisk, int64(limit), "the files left on disk are within the limit")
require.Equal(t, onDisk, cache.size.Load(), "the tracked size matches what is on disk")
}
// Files that are rewritten while an eviction is running must be subtracted at the size they have when
// they are removed, not at the size the scan saw, or the tracked size drifts from the disk.
func TestFileCacheKeepsTheSizeAccurateWhenFilesAreRewrittenDuringEviction(t *testing.T) {
const (
kb = 1024
limit = 60 * kb
workers = 8
rounds = 300
keys = 40
)
cache := newLimitedCache(t, limit)
var wg sync.WaitGroup
for w := 0; w < workers; w++ {
wg.Add(1)
go func(w int) {
defer wg.Done()
for i := 0; i < rounds; i++ {
// the same keys again and again, with a different size each time
key := fmt.Sprintf("key-%d", (w+i)%keys)
_ = cache.Store(context.Background(), key, make([]byte, (1+(w*7+i)%8)*kb))
}
}(w)
}
wg.Wait()
waitFor(t, func() bool { return !cache.evicting.Load() })
_, onDisk := cache.scan()
require.Equal(t, onDisk, cache.size.Load(), "the tracked size matches what is on disk")
require.LessOrEqual(t, onDisk, int64(limit))
}
+57 -27
View File
@@ -17,6 +17,7 @@ import (
"github.com/gtsteffaniak/filebrowser/backend/internal/errors"
"github.com/gtsteffaniak/filebrowser/backend/internal/ffmpeg"
"github.com/gtsteffaniak/filebrowser/backend/internal/gomedia"
"github.com/gtsteffaniak/filebrowser/backend/internal/mediacache"
"github.com/gtsteffaniak/filebrowser/backend/internal/utils"
"github.com/gtsteffaniak/filebrowser/backend/pkg/indexing"
"github.com/gtsteffaniak/filebrowser/backend/pkg/indexing/iteminfo"
@@ -52,7 +53,7 @@ func addMetadataToChildren(response *iteminfo.ExtendedFileInfo, opts utils.FileO
logger.Debugf("failed to extract metadata for file: %s, error: %v", fileItem.Name, err)
}
} else {
if err := extractVideoMetadata(context.Background(), fileItem, fullPath, sharedFFmpegService); err != nil {
if err := extractVideoMetadata(context.Background(), fileItem, fullPath, opts.AlbumArt, sharedFFmpegService); err != nil {
logger.Debugf("failed to extract video metadata for file: %s, error: %v", fileItem.Name, err)
}
}
@@ -380,7 +381,7 @@ func processContent(info *iteminfo.ExtendedFileInfo, idx *indexing.Index, opts u
extItem := &iteminfo.ExtendedItemInfo{
ItemInfo: info.ItemInfo,
}
err := extractVideoMetadata(context.Background(), extItem, info.RealPath, nil)
err := extractVideoMetadata(context.Background(), extItem, info.RealPath, opts.AlbumArt, nil)
if err != nil {
logger.Debugf("failed to extract video metadata for file: %s (%s): %v", info.RealPath, info.Name, err)
} else {
@@ -393,7 +394,8 @@ func processContent(info *iteminfo.ExtendedFileInfo, idx *indexing.Index, opts u
info.GetSubtitles(parentInfo)
}
subtitles, ok := gomedia.EmbeddedSubtitles(info.RealPath, info.ModTime, func() ([]utils.SubtitleTrack, bool) {
return ffmpeg.DetectEmbeddedSubtitles(info.RealPath, info.ModTime), true
// Shadow run: bypass the shared cache so ffprobe is compared fresh.
return ffmpeg.DetectEmbeddedSubtitlesContext(mediacache.WithoutCache(context.Background()), info.RealPath), true
})
if !ok {
subtitles = ffmpeg.DetectEmbeddedSubtitles(info.RealPath, info.ModTime)
@@ -471,23 +473,11 @@ func extractAudioMetadata(ctx context.Context, item *iteminfo.ExtendedItemInfo,
// go-media fast path: tags, lyrics availability, and artwork. Unsupported or
// failed reads fall through to the existing dhowden/tag code below.
if info, ok := gomedia.Audio(realPath, getArt, func() (gomedia.AudioInfo, bool) { return legacyAudioInfo(realPath, getArt) }); ok {
item.Metadata = &iteminfo.MediaMetadata{
Title: info.Tags.Title,
Artist: info.Tags.Artist,
Album: info.Tags.Album,
Year: info.Tags.Year,
Genre: info.Tags.Genre,
Track: info.Tags.Track,
HasLyrics: info.HasLyrics,
}
item.Metadata = &iteminfo.MediaMetadata{HasLyrics: info.HasLyrics}
applyMediaTags(item.Metadata, info, realPath)
if getDuration {
audioDuration(ctx, item, realPath, ffmpegService)
}
if n := len(info.Artwork); n > 0 && n <= 5*1024*1024 {
item.Metadata.AlbumArt = info.Artwork
} else if n > 5*1024*1024 {
logger.Debugf("Skipping album art for %s: too large (%d bytes)", realPath, n)
}
return nil
}
@@ -541,8 +531,9 @@ func audioDuration(ctx context.Context, item *iteminfo.ExtendedItemInfo, realPat
if service == nil {
service = ffmpeg.Get()
}
// The shadow run can outlive the request, so it uses a context that is not cancelled with it.
shadowCtx := context.WithoutCancel(ctx)
// The shadow run can outlive the request, so it uses a context that is not
// cancelled with it. It bypasses the shared cache so ffmpeg is compared fresh.
shadowCtx := mediacache.WithoutCache(context.WithoutCancel(ctx))
legacy := func() (float64, bool) {
d, err := service.GetMediaDuration(shadowCtx, realPath)
return d, err == nil
@@ -707,14 +698,15 @@ func legacyLyrics(realPath string) (gomedia.LyricsResult, bool) {
// extractVideoMetadata extracts duration and codec info from video files using ffprobe.
// If ffmpegService is nil, a new service will be created (for backward compatibility)
func extractVideoMetadata(ctx context.Context, item *iteminfo.ExtendedItemInfo, realPath string, ffmpegService *ffmpeg.FFmpegService) error {
func extractVideoMetadata(ctx context.Context, item *iteminfo.ExtendedItemInfo, realPath string, getArt bool, ffmpegService *ffmpeg.FFmpegService) error {
// go-media fast path. Works without ffmpeg installed.
probeService := ffmpegService
if probeService == nil {
probeService = ffmpeg.Get()
}
// The shadow run can outlive the request, so it uses a context that is not cancelled with it.
shadowCtx := context.WithoutCancel(ctx)
// The shadow run can outlive the request, so it uses a context that is not
// cancelled with it. It bypasses the shared cache so ffprobe is compared fresh.
shadowCtx := mediacache.WithoutCache(context.WithoutCancel(ctx))
legacyProbe := func() (gomedia.ProbeResult, bool) {
info, err := probeService.ProbeFile(shadowCtx, realPath)
if err != nil || info == nil {
@@ -759,12 +751,33 @@ func extractVideoMetadata(ctx context.Context, item *iteminfo.ExtendedItemInfo,
if info.FormatName != "" {
meta.Container = info.FormatName
}
if ok {
if tags, tagsOK := gomedia.VideoTags(realPath, getArt); tagsOK {
applyMediaTags(meta, tags, realPath)
}
}
if meta.Duration > 0 || meta.VideoCodec != "" || meta.AudioCodec != "" || meta.Container != "" {
item.Metadata = meta
}
return nil
}
// applyMediaTags copies go-media tags and embedded cover art into meta. Covers
// over 8MB are skipped.
func applyMediaTags(meta *iteminfo.MediaMetadata, info gomedia.AudioInfo, realPath string) {
meta.Title = info.Tags.Title
meta.Artist = info.Tags.Artist
meta.Album = info.Tags.Album
meta.Year = info.Tags.Year
meta.Genre = info.Tags.Genre
meta.Track = info.Tags.Track
if n := len(info.Artwork); n > 0 && n <= 8*1024*1024 {
meta.AlbumArt = info.Artwork
} else if n > 8*1024*1024 {
logger.Debugf("Skipping album art for %s: too large (%d bytes)", realPath, n)
}
}
// DeleteFiles deletes the file or directory at indexPath. boundIndexPath limits
// symlink resolution to the caller's scope ("" or "/" = source root only). The
// entry itself is removed: a symlink leaf is deleted, never followed, and an
@@ -777,6 +790,7 @@ func DeleteFiles(source, boundIndexPath, indexPath string, isDir bool) error {
if utils.CleanIndexPathSegment(indexPath) == "" {
return fmt.Errorf("refusing to delete source root directory: %s", indexPath)
}
deleteIndexPath := indexPath
realPath, err := index.GetRealPathForRemoval(boundIndexPath, indexPath)
if err != nil {
return err
@@ -795,9 +809,13 @@ func DeleteFiles(source, boundIndexPath, indexPath string, isDir bool) error {
return err
}
// Clear cache entries
indexing.RealPathCache.Delete(realPath)
indexing.IsDirCache.Delete(realPath + ":isdir")
if isDir {
indexing.ClearRealPathCaches()
}
index.InvalidateRealPathCache(deleteIndexPath, boundIndexPath)
if parent := utils.GetParentDirectoryPath(deleteIndexPath); parent != "" && parent != "/" {
index.InvalidateRealPathCache(parent, boundIndexPath)
}
// Remove metadata from index
deleteSuccess := index.DeleteMetadata(indexPath, isDir, isDir)
@@ -849,8 +867,7 @@ func RefreshIndex(source string, path string, isDir bool, recursive bool) error
if !Exists(realPath) {
// Directory no longer exists, remove it from the index
// This clears both Directories and DirectoriesLedger maps
indexing.RealPathCache.Delete(realPath)
indexing.IsDirCache.Delete(realPath + ":isdir")
idx.InvalidateRealPathCache(path)
idx.DeleteMetadata(path, true, false)
return nil
}
@@ -928,6 +945,9 @@ func moveResourceImpl(isSrcDir bool, sourceIndex, destIndex, realsrc, realdst st
if err != nil {
return err
}
if isSrcDir {
indexing.ClearRealPathCaches()
}
// Handle SOURCE cleanup (treat as deletion)
// Run async to avoid blocking the HTTP response
@@ -1088,6 +1108,11 @@ func WriteDirectory(boundIndexPath string, opts utils.FileOptions) error {
}
}
idx.InvalidateRealPathCache(opts.Path, boundIndexPath)
if parent := utils.GetParentDirectoryPath(opts.Path); parent != "" && parent != "/" {
idx.InvalidateRealPathCache(parent, boundIndexPath)
}
return nil
}
@@ -1163,6 +1188,11 @@ func writeFile(source, boundIndexPath, path string, in io.Reader) error {
}
}
idx.InvalidateRealPathCache(path, boundIndexPath)
if parent := utils.GetParentDirectoryPath(path); parent != "" && parent != "/" {
idx.InvalidateRealPathCache(parent, boundIndexPath)
}
return nil
}
@@ -0,0 +1,44 @@
package access
import (
"os"
"path/filepath"
"strings"
"github.com/gtsteffaniak/filebrowser/backend/internal/utils"
"github.com/gtsteffaniak/filebrowser/backend/pkg/indexing"
)
// PermittedAtRealPath reports whether username may access realPath after symlink
// resolution. DENY rules apply to the target index path, not only the link path.
func (s *Storage) PermittedAtRealPath(index *indexing.Index, username, realPath string, isDir bool) bool {
if s == nil || index == nil || username == "" || realPath == "" {
return false
}
lexical := realPath
if !strings.HasPrefix(realPath, index.Path) {
root, err := filepath.EvalSymlinks(index.Path)
if err != nil {
return false
}
rel, err := filepath.Rel(root, realPath)
if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) {
return false
}
lexical = index.Path
if rel != "." {
lexical = filepath.Join(index.Path, rel)
}
}
indexPath := index.MakeIndexPath(lexical, isDir).String()
return s.Permitted(index.Path, utils.IndexPathFromNormalized(indexPath, true), username)
}
// IsDirAtRealPath reports whether an existing path is a directory; missing paths are treated as files.
func IsDirAtRealPath(realPath string) bool {
st, err := os.Stat(realPath)
if err != nil {
return false
}
return st.IsDir()
}
@@ -0,0 +1,140 @@
package access
import (
"os"
"path/filepath"
"testing"
"github.com/gtsteffaniak/filebrowser/backend/internal/utils"
"github.com/gtsteffaniak/filebrowser/backend/pkg/indexing"
"github.com/gtsteffaniak/filebrowser/backend/pkg/settings"
)
func setupSymlinkDenyFixture(t *testing.T) (sourceRoot string, idx *indexing.Index, store *Storage, username string) {
t.Helper()
sourceRoot = filepath.Join(t.TempDir(), "data")
aliceDir := filepath.Join(sourceRoot, "alice")
bobDir := filepath.Join(sourceRoot, "bob")
for _, dir := range []string{aliceDir, bobDir} {
if err := os.MkdirAll(dir, 0o755); err != nil {
t.Fatal(err)
}
}
if err := os.WriteFile(filepath.Join(bobDir, "secret.txt"), []byte("secret"), 0o644); err != nil {
t.Fatal(err)
}
if err := os.Symlink("../bob/secret.txt", filepath.Join(aliceDir, "link.txt")); err != nil {
t.Skipf("symlinks required: %v", err)
}
if err := os.Symlink("../bob", filepath.Join(aliceDir, "bobdir")); err != nil {
t.Skipf("symlinks required: %v", err)
}
settings.Config.Server.SourceMap = map[string]*settings.Source{
sourceRoot: {Path: sourceRoot, Config: settings.SourceConfig{DenyByDefault: false}},
}
store = NewStorage(nil)
username = "scoped-user"
if err := store.DenyUser(sourceRoot, utils.IndexPathFromNormalized("/bob", true), username); err != nil {
t.Fatal(err)
}
idx = &indexing.Index{Source: settings.Source{Path: sourceRoot}}
return sourceRoot, idx, store, username
}
func TestPermittedAtRealPath_AllowsLinkPathDeniesResolvedTarget(t *testing.T) {
_, idx, store, user := setupSymlinkDenyFixture(t)
if !store.Permitted(idx.Path, utils.IndexPathFromNormalized("/alice/link.txt", false), user) {
t.Fatal("index path at the symlink should be allowed")
}
if store.Permitted(idx.Path, utils.IndexPathFromNormalized("/bob/secret.txt", false), user) {
t.Fatal("direct path under /bob should be denied")
}
realPath, _, err := idx.GetRealPathScoped("/", "/alice/link.txt")
if err != nil {
t.Fatal(err)
}
if store.PermittedAtRealPath(idx, user, realPath, false) {
t.Fatal("resolved target under denied /bob must not be permitted")
}
}
func TestPermittedAtRealPath_DeniesWriteThroughDirectorySymlink(t *testing.T) {
_, idx, store, user := setupSymlinkDenyFixture(t)
realPath, err := idx.GetRealPathForMutation("/", "/alice/bobdir/planted.txt")
if err != nil {
t.Fatal(err)
}
if store.PermittedAtRealPath(idx, user, realPath, false) {
t.Fatal("resolved write target under denied /bob must not be permitted")
}
}
func TestPermittedAtRealPath_DeniesCopySourceResolved(t *testing.T) {
_, idx, store, user := setupSymlinkDenyFixture(t)
realSrc, isDir, err := idx.GetRealPathScoped("/", "/alice/link.txt")
if err != nil {
t.Fatal(err)
}
if isDir {
t.Fatal("expected file")
}
if store.PermittedAtRealPath(idx, user, realSrc, isDir) {
t.Fatal("copy source resolved to denied target must not be permitted")
}
}
func TestPermittedAtRealPath_SymlinkedSourceRoot(t *testing.T) {
sourceRoot, _, _, user := setupSymlinkDenyFixture(t)
linkRoot := filepath.Join(filepath.Dir(sourceRoot), "root-link")
if err := os.Symlink(sourceRoot, linkRoot); err != nil {
t.Skipf("symlinks required: %v", err)
}
if _, err := filepath.EvalSymlinks(linkRoot); err != nil {
t.Fatal(err)
}
settings.Config.Server.SourceMap = map[string]*settings.Source{
linkRoot: {Path: linkRoot, Config: settings.SourceConfig{DenyByDefault: false}},
}
store := NewStorage(nil)
if err := store.DenyUser(linkRoot, utils.IndexPathFromNormalized("/bob", true), user); err != nil {
t.Fatal(err)
}
idx := &indexing.Index{Source: settings.Source{Path: linkRoot}}
realPath, _, err := idx.GetRealPathScoped("/", "/alice/link.txt")
if err != nil {
t.Fatal(err)
}
realPath, err = filepath.EvalSymlinks(realPath)
if err != nil {
t.Fatal(err)
}
if store.PermittedAtRealPath(idx, user, realPath, false) {
t.Fatal("resolved target under denied /bob must not be permitted with symlinked source root")
}
}
func TestIsDirAtRealPath(t *testing.T) {
dir := t.TempDir()
if err := os.MkdirAll(filepath.Join(dir, "nested"), 0o755); err != nil {
t.Fatal(err)
}
file := filepath.Join(dir, "file.txt")
if err := os.WriteFile(file, []byte("x"), 0o644); err != nil {
t.Fatal(err)
}
if !IsDirAtRealPath(dir) {
t.Fatal("expected directory")
}
if IsDirAtRealPath(file) {
t.Fatal("expected file")
}
if IsDirAtRealPath(filepath.Join(dir, "missing")) {
t.Fatal("missing path should not be treated as directory")
}
}
@@ -25,8 +25,8 @@ func TestFinishUserLoad_legacyProfileRowPreservesAbsentSettingsFields(t *testing
if !loaded.ShowFirstLogin {
t.Fatal("expected top-level showFirstLogin to be preserved when settings omits it")
}
if !loaded.DarkMode {
t.Fatal("expected profile darkMode=true")
if !loaded.DarkMode.IsDark() {
t.Fatalf("expected profile darkMode=true, got %q", loaded.DarkMode)
}
if len(loaded.PasskeyCredentials) != 0 {
t.Fatalf("expected no passkeys when settings omits passkeyCredentials, got %#v", loaded.PasskeyCredentials)
@@ -95,7 +95,7 @@ func TestUserDataRoundTrip_profileSettingsOverlay(t *testing.T) {
LoginMethod: users.LoginMethodPassword,
ShowFirstLogin: true,
NonAdminEditable: users.NonAdminEditable{
DarkMode: true,
DarkMode: users.DarkModeDark,
SidebarLinks: []users.SidebarLink{
{Name: "Home", Target: "/files"},
},
@@ -0,0 +1,73 @@
package users
import (
"encoding/json"
"fmt"
"strings"
)
// DarkMode is the user's theme preference: "auto" (follow the client's
// system preference), "dark", or "light". It unmarshals from legacy JSON
// booleans for backward compatibility with existing stored users.
type DarkMode string
const (
DarkModeAuto DarkMode = "auto"
DarkModeDark DarkMode = "dark"
DarkModeLight DarkMode = "light"
)
// UnmarshalJSON accepts legacy boolean values (true -> dark, false -> light),
// the tri-state strings ("auto", "dark", "light"), and null (unset).
func (d *DarkMode) UnmarshalJSON(data []byte) error {
var v interface{}
if err := json.Unmarshal(data, &v); err != nil {
return err
}
switch val := v.(type) {
case nil:
*d = ""
case bool:
if val {
*d = DarkModeDark
} else {
*d = DarkModeLight
}
case string:
switch DarkMode(strings.ToLower(val)) {
case "":
*d = ""
case "true":
// Some YAML decoders cast scalar booleans to strings.
*d = DarkModeDark
case "false":
*d = DarkModeLight
case DarkModeAuto:
*d = DarkModeAuto
case DarkModeDark:
*d = DarkModeDark
case DarkModeLight:
*d = DarkModeLight
default:
return fmt.Errorf("invalid darkMode value %q: must be \"auto\", \"dark\" or \"light\"", val)
}
default:
return fmt.Errorf("invalid darkMode value: must be a boolean or one of \"auto\", \"dark\", \"light\"")
}
return nil
}
// IsDark reports the effective boolean value for server-side rendering
// contexts. "auto" cannot be resolved server-side; callers that can know the
// client preference should handle DarkModeAuto explicitly.
func (d DarkMode) IsDark() bool {
return d == DarkModeDark
}
// Or returns d unless it is unset (""), in which case it returns def.
func (d DarkMode) Or(def DarkMode) DarkMode {
if d == "" {
return def
}
return d
}
+6 -5
View File
@@ -308,7 +308,7 @@ type NonAdminEditable struct {
DeleteWithoutConfirming bool `json:"deleteWithoutConfirming"` // delete files without confirmation
Preview Preview `json:"preview"`
StickySidebar bool `json:"stickySidebar"` // keep sidebar open when navigating
DarkMode bool `json:"darkMode"` // should dark mode be enabled
DarkMode DarkMode `json:"darkMode"` // theme preference: "auto" (follow system), "dark", or "light"
Password string `json:"password,omitempty"`
Locale string `json:"locale"` // language to use: eg. de, en, or fr
ViewMode string `json:"viewMode"` // view mode to use: eg. normal, list, grid, or compact
@@ -346,10 +346,11 @@ type NonAdminEditable struct {
}
type FileLoading struct {
MaxConcurrent int `json:"maxConcurrentUpload"`
UploadChunkSize int `json:"uploadChunkSizeMb"`
ClearAll bool `json:"clearAll"`
DownloadChunkSize int `json:"downloadChunkSizeMb"`
MaxConcurrent int `json:"maxConcurrentUpload"`
UploadChunkSize int `json:"uploadChunkSizeMb"`
ClearAll bool `json:"clearAll"`
DownloadChunkSize int `json:"downloadChunkSizeMb"`
UploadStallTimeoutSeconds int `json:"uploadStallTimeoutSeconds"` // seconds without progress before an upload is paused, 0 uses the default (10)
}
// SidebarLink represents a customizable link in the sidebar.
-14
View File
@@ -1,14 +0,0 @@
package ffmpeg
import (
"time"
"github.com/gtsteffaniak/filebrowser/backend/internal/utils"
"github.com/gtsteffaniak/go-cache/cache"
)
var (
MediaCache = cache.NewCache[[]utils.SubtitleTrack](24 * time.Hour) // subtitle track lists from ffprobe
SubtitleContentCache = cache.NewCache[string](24 * time.Hour) // extracted embedded subtitle content
MetadataCache = cache.NewCache[float64](24 * time.Hour) // media duration from ffprobe
)
@@ -9,13 +9,51 @@ import (
"os/exec"
"strconv"
"strings"
"sync/atomic"
"github.com/kovidgoyal/imaging"
)
// HEICDisplayRotationReader reads the display rotation that a HEIC/HEIF/AVIF
// file's item properties (irot) ask for, without decoding it. The rotation is
// in degrees counter-clockwise, normalized to (-180, 180], which is the value
// ffmpeg prints as "Display Matrix: rotation of ..." for the same file. ok is
// false when the reader could not tell; the ffmpeg probe is used instead.
type HEICDisplayRotationReader func(path string) (rotation float64, ok bool)
// heicDisplayRotationReader is the header reader set by
// SetHEICDisplayRotationReader, or nil.
var heicDisplayRotationReader atomic.Pointer[HEICDisplayRotationReader]
// SetHEICDisplayRotationReader installs the reader HEICDisplayRotation tries
// before running ffmpeg. The gomedia package installs go-media's reader when it
// is loaded; it cannot be called from here directly because gomedia imports
// this package. Passing nil removes the reader.
func SetHEICDisplayRotationReader(fn HEICDisplayRotationReader) {
if fn == nil {
heicDisplayRotationReader.Store(nil)
return
}
heicDisplayRotationReader.Store(&fn)
}
// HEICDisplayRotation returns the display rotation of heicPath and whether it is
// known. The installed header reader (go-media) answers first; the
// "ffmpeg -f null" stderr probe runs only when no reader is installed or the
// reader could not read the file.
func (s *Service) HEICDisplayRotation(ctx context.Context, heicPath string) (float64, bool) {
if p := heicDisplayRotationReader.Load(); p != nil {
if rotation, ok := (*p)(heicPath); ok {
return rotation, true
}
}
return s.GetHEICDisplayMatrixRotation(ctx, heicPath)
}
// GetHEICDisplayMatrixRotation returns the largest rotation (degrees) ffmpeg reports for a
// HEIC display matrix. The second return value is false when the probe could not run or
// parse stderr — callers must not treat that as “no rotation was applied”.
// HEICDisplayRotation is preferred; it only runs this probe as a fallback.
func (s *Service) GetHEICDisplayMatrixRotation(ctx context.Context, heicPath string) (float64, bool) {
if s == nil {
return 0, false
@@ -36,6 +74,8 @@ func (s *Service) GetHEICDisplayMatrixRotation(ctx context.Context, heicPath str
return rotation, ok
}
// parseDisplayMatrixRotation returns the largest-magnitude "Display Matrix"
// rotation in ffmpeg stderr, and whether any was found.
func parseDisplayMatrixRotation(stderr string) (float64, bool) {
var best float64
var found bool
@@ -65,6 +105,7 @@ func parseDisplayMatrixRotation(stderr string) (float64, bool) {
return best, found
}
// isNormalOrientation reports whether an EXIF orientation needs no change.
func isNormalOrientation(orientation string) bool {
switch orientation {
case "", "Horizontal (normal)", "Top-left", "1":
@@ -74,6 +115,8 @@ func isNormalOrientation(orientation string) bool {
}
}
// isPureRotationOrientation reports whether an EXIF orientation is a rotation
// without a mirror.
func isPureRotationOrientation(orientation string) bool {
switch orientation {
case "Rotate 90 CW", "Right-top", "6",
@@ -100,6 +143,8 @@ func orientationNeedsPostCorrection(orientation string, displayRotation float64,
return true
}
// applyOrientationToJPEG rotates or flips JPEG bytes for an EXIF orientation.
// The input is returned unchanged when it cannot be decoded or re-encoded.
func applyOrientationToJPEG(jpegBytes []byte, orientation string) []byte {
if len(jpegBytes) < 100 || isNormalOrientation(orientation) {
return jpegBytes
@@ -50,3 +50,35 @@ func TestOrientationNeedsPostCorrection(t *testing.T) {
}
}
}
func TestHEICDisplayRotationPrefersReader(t *testing.T) {
prev := heicDisplayRotationReader.Load()
t.Cleanup(func() { heicDisplayRotationReader.Store(prev) })
var calls int
SetHEICDisplayRotationReader(func(path string) (float64, bool) {
calls++
if path == "known.heic" {
return -90, true
}
return 0, false
})
var svc *Service // nil: the ffmpeg probe cannot run
got, ok := svc.HEICDisplayRotation(t.Context(), "known.heic")
if !ok || got != -90 {
t.Fatalf("HEICDisplayRotation(known) = %v, %v; want -90, true", got, ok)
}
// The reader cannot tell: the ffmpeg probe runs, and with no service it
// reports unknown rather than "no rotation".
if _, ok := svc.HEICDisplayRotation(t.Context(), "other.heic"); ok {
t.Fatal("HEICDisplayRotation(other) ok = true, want the ffmpeg probe's false")
}
if calls != 2 {
t.Fatalf("reader calls = %d, want 2", calls)
}
SetHEICDisplayRotationReader(nil)
if _, ok := svc.HEICDisplayRotation(t.Context(), "known.heic"); ok {
t.Fatal("with no reader, HEICDisplayRotation must use the ffmpeg probe")
}
}
+5 -3
View File
@@ -81,14 +81,16 @@ func (s *Service) ConvertHEICToJPEG(ctx context.Context, heicPath string, target
return s.CorrectHEICJPEGOrientation(ctx, heicPath, jpegBytes), nil
}
// CorrectHEICJPEGOrientation applies irot / display-matrix correction after a full-resolution
// HEIC decode when pixels are still in coded orientation (go-media and ffmpeg filter-free paths).
// CorrectHEICJPEGOrientation applies EXIF orientation after a filter-free ffmpeg HEIC decode
// when the pixels are still in coded orientation. The display rotation (irot) comes from
// HEICDisplayRotation: go-media reads it from the item properties, and the ffmpeg stderr probe
// is only a fallback.
func (s *Service) CorrectHEICJPEGOrientation(ctx context.Context, heicPath string, jpegBytes []byte) []byte {
orientation, err := s.GetHEICOrientation(heicPath)
if err != nil {
orientation = "Horizontal (normal)"
}
displayRotation, displayKnown := s.GetHEICDisplayMatrixRotation(ctx, heicPath)
displayRotation, displayKnown := s.HEICDisplayRotation(ctx, heicPath)
if orientationNeedsPostCorrection(orientation, displayRotation, displayKnown) {
return applyOrientationToJPEG(jpegBytes, orientation)
}
@@ -0,0 +1,115 @@
package ffmpeg
import (
"context"
"os"
"os/exec"
"path/filepath"
"sync"
"testing"
"github.com/gtsteffaniak/filebrowser/backend/internal/mediacache"
)
const cacheFixtureMP4 = "../../../frontend/tests/playwright-files/myfolder/testdata/sample-640x480.mp4"
var (
testServiceOnce sync.Once
testServiceErr error
)
// initTestService starts the ffmpeg service once, without hardware probes, or
// skips.
func initTestService(t *testing.T) *Service {
t.Helper()
if testing.Short() {
t.Skip("starts ffmpeg")
}
if _, err := exec.LookPath("ffmpeg"); err != nil {
t.Skip("ffmpeg not installed")
}
if _, err := exec.LookPath("ffprobe"); err != nil {
t.Skip("ffprobe not installed")
}
testServiceOnce.Do(func() {
testServiceErr = Initialize(context.Background(), InitOptions{SkipHWTests: true, CacheDir: os.TempDir()})
})
if testServiceErr != nil {
t.Skipf("ffmpeg init failed: %v", testServiceErr)
}
svc := Get()
if svc == nil {
t.Skip("ffmpeg service unavailable")
}
return svc
}
// copyFixture copies the MP4 fixture so the test owns its cache key.
func copyFixture(t *testing.T) (string, mediacache.Key) {
t.Helper()
data, err := os.ReadFile(cacheFixtureMP4)
if err != nil {
t.Fatal(err)
}
path := filepath.Join(t.TempDir(), "sample.mp4")
if err = os.WriteFile(path, data, 0o644); err != nil {
t.Fatal(err)
}
key, err := mediacache.Stat(path)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() {
mediacache.Probes.Delete(key.String())
mediacache.Durations.Delete(key.String())
})
return path, key
}
// TestProbeFileUsesSharedCache checks that ffprobe results go into the shared
// cache, cached values are reused, and WithoutCache bypasses both.
func TestProbeFileUsesSharedCache(t *testing.T) {
svc := initTestService(t)
path, key := copyFixture(t)
got, err := svc.ProbeFile(context.Background(), path)
if err != nil {
t.Fatal(err)
}
cached, ok := mediacache.Probes.Get(key.String())
if !ok || cached != *got {
t.Fatalf("ProbeFile must store its result in the shared cache: cached=%+v ok=%v got=%+v", cached, ok, *got)
}
sentinel := FileProbeInfo{Duration: 1, VideoCodec: "sentinel"}
mediacache.Probes.Set(key.String(), sentinel)
got, err = svc.ProbeFile(context.Background(), path)
if err != nil || *got != sentinel {
t.Fatalf("ProbeFile must reuse a cached value (e.g. from go-media): got=%+v err=%v", got, err)
}
fresh, err := svc.ProbeFile(mediacache.WithoutCache(context.Background()), path)
if err != nil || *fresh == sentinel {
t.Fatalf("WithoutCache must probe fresh: got=%+v err=%v", fresh, err)
}
if cached, _ := mediacache.Probes.Get(key.String()); cached != sentinel {
t.Fatal("WithoutCache must not overwrite the shared cache")
}
}
// TestGetMediaDurationUsesSharedCache checks that cached durations are reused
// and WithoutCache probes fresh.
func TestGetMediaDurationUsesSharedCache(t *testing.T) {
svc := initTestService(t)
path, key := copyFixture(t)
mediacache.Durations.Set(key.String(), 4242)
d, err := svc.GetMediaDuration(context.Background(), path)
if err != nil || d != 4242 {
t.Fatalf("GetMediaDuration must reuse a cached value: d=%v err=%v", d, err)
}
d, err = svc.GetMediaDuration(mediacache.WithoutCache(context.Background()), path)
if err != nil || d == 4242 || d <= 0 {
t.Fatalf("WithoutCache must probe fresh: d=%v err=%v", d, err)
}
}
+32 -16
View File
@@ -8,6 +8,7 @@ import (
"strings"
"time"
"github.com/gtsteffaniak/filebrowser/backend/internal/mediacache"
goffmpeg "github.com/gtsteffaniak/go-ffmpeg"
)
@@ -49,24 +50,29 @@ func resolveLocalMediaPath(mediaPath string) (string, os.FileInfo, error) {
return clean, info, nil
}
// FileProbeInfo holds ffprobe results for a local media file.
type FileProbeInfo struct {
Duration float64
VideoCodec string
AudioCodec string
FormatName string
}
// FileProbeInfo holds probe results for a local media file. It is the shared
// cache type, so go-media and ffprobe results are interchangeable.
type FileProbeInfo = mediacache.ProbeInfo
// ProbeFile extracts duration and codec info from a media file.
// ProbeFile extracts duration and codec info from a media file. Results are
// stored in the shared media cache, which the go-media probe also reads, unless
// ctx is mediacache.WithoutCache.
func (s *Service) ProbeFile(ctx context.Context, mediaPath string) (*FileProbeInfo, error) {
if s == nil || s.inner == nil {
return nil, fmt.Errorf("ffmpeg service not available")
}
localPath, _, err := resolveLocalMediaPath(mediaPath)
localPath, fileInfo, err := resolveLocalMediaPath(mediaPath)
if err != nil {
return nil, err
}
useCache := !mediacache.Bypassed(ctx)
cacheKey := mediacache.FromFileInfo(localPath, fileInfo).String()
if useCache {
if cached, ok := mediacache.Probes.Get(cacheKey); ok {
return &cached, nil
}
}
info, err := s.inner.ProbeStream(ctx, goffmpeg.ProbeStreamOptions{
URL: localPath,
@@ -77,15 +83,20 @@ func (s *Service) ProbeFile(ctx context.Context, mediaPath string) (*FileProbeIn
return nil, err
}
return &FileProbeInfo{
res := FileProbeInfo{
Duration: info.Duration,
VideoCodec: info.VideoCodec,
AudioCodec: info.AudioCodec,
FormatName: info.FormatName,
}, nil
}
if useCache {
mediacache.Probes.Set(cacheKey, res)
}
return &res, nil
}
// GetMediaDuration extracts the duration of a media file in seconds.
// GetMediaDuration extracts the duration of a media file in seconds. Results
// are stored in the shared media cache unless ctx is mediacache.WithoutCache.
func (s *Service) GetMediaDuration(ctx context.Context, mediaPath string) (float64, error) {
if s == nil || s.inner == nil {
return 0, fmt.Errorf("ffmpeg service not available")
@@ -95,9 +106,12 @@ func (s *Service) GetMediaDuration(ctx context.Context, mediaPath string) (float
if err != nil {
return 0, err
}
cacheKey := fmt.Sprintf("%s:%d", localPath, fileInfo.ModTime().Unix())
if duration, found := MetadataCache.Get(cacheKey); found {
return duration, nil
useCache := !mediacache.Bypassed(ctx)
cacheKey := mediacache.FromFileInfo(localPath, fileInfo).String()
if useCache {
if duration, found := mediacache.Durations.Get(cacheKey); found {
return duration, nil
}
}
duration, err := s.inner.GetMediaDuration(ctx, localPath)
@@ -105,6 +119,8 @@ func (s *Service) GetMediaDuration(ctx context.Context, mediaPath string) (float
return 0, err
}
MetadataCache.Set(cacheKey, duration)
if useCache {
mediacache.Durations.Set(cacheKey, duration)
}
return duration, nil
}
+12 -5
View File
@@ -9,6 +9,7 @@ import (
"sync"
"sync/atomic"
"github.com/gtsteffaniak/filebrowser/backend/internal/mediacache"
goffmpeg "github.com/gtsteffaniak/go-ffmpeg"
"github.com/gtsteffaniak/go-ffmpeg/capabilities"
"github.com/gtsteffaniak/go-ffmpeg/ops"
@@ -460,7 +461,8 @@ func (s *Service) VideoPreview(ctx context.Context, w io.Writer, opts ops.Previe
return s.inner.CaptureFrame(ctx, w, frame)
}
// ExtractSubtitle returns embedded subtitle content as WebVTT.
// ExtractSubtitle returns embedded subtitle content as WebVTT. Results are
// stored in the shared media cache unless ctx is mediacache.WithoutCache.
func (s *Service) ExtractSubtitle(ctx context.Context, videoPath string, streamIndex int) (string, error) {
if s == nil || s.inner == nil {
return "", fmt.Errorf("ffmpeg service not available")
@@ -470,9 +472,12 @@ func (s *Service) ExtractSubtitle(ctx context.Context, videoPath string, streamI
if err != nil {
return "", fmt.Errorf("failed to stat file: %w", err)
}
cacheKey := fmt.Sprintf("subtitle_content:%s:%d:%d", videoPath, streamIndex, fileInfo.ModTime().Unix())
if content, ok := SubtitleContentCache.Get(cacheKey); ok {
return content, nil
useCache := !mediacache.Bypassed(ctx)
cacheKey := mediacache.FromFileInfo(videoPath, fileInfo).Stream(streamIndex)
if useCache {
if content, ok := mediacache.SubtitleContent.Get(cacheKey); ok {
return content, nil
}
}
content, err := s.inner.ExtractSubtitle(ctx, videoPath, streamIndex)
@@ -480,7 +485,9 @@ func (s *Service) ExtractSubtitle(ctx context.Context, videoPath string, streamI
return "", err
}
SubtitleContentCache.Set(cacheKey, content)
if useCache {
mediacache.SubtitleContent.Set(cacheKey, content)
}
return content, nil
}
+22 -6
View File
@@ -7,16 +7,30 @@ import (
"strings"
"time"
"github.com/gtsteffaniak/filebrowser/backend/internal/mediacache"
"github.com/gtsteffaniak/filebrowser/backend/internal/utils"
goffmpeg "github.com/gtsteffaniak/go-ffmpeg"
"github.com/gtsteffaniak/go-logger/logger"
)
// DetectEmbeddedSubtitles detects embedded subtitle streams.
// DetectEmbeddedSubtitles detects embedded subtitle streams. Results are stored
// in the shared media cache, keyed by the file's current size and modification
// time; modtime is kept for API compatibility and is not used for the key.
func DetectEmbeddedSubtitles(videoPath string, modtime time.Time) []utils.SubtitleTrack {
key := "embedded_subtitles:" + videoPath + ":" + modtime.Format(time.RFC3339)
if cached, ok := MediaCache.Get(key); ok {
return cached
_ = modtime
return DetectEmbeddedSubtitlesContext(context.Background(), videoPath)
}
// DetectEmbeddedSubtitlesContext is DetectEmbeddedSubtitles with a context. The
// shared media cache is skipped when ctx is mediacache.WithoutCache.
func DetectEmbeddedSubtitlesContext(ctx context.Context, videoPath string) []utils.SubtitleTrack {
useCache := !mediacache.Bypassed(ctx)
key, statErr := mediacache.Stat(videoPath)
useCache = useCache && statErr == nil
if useCache {
if cached, ok := mediacache.SubtitleTracks.Get(key.String()); ok {
return cached
}
}
svc := Get()
@@ -24,14 +38,16 @@ func DetectEmbeddedSubtitles(videoPath string, modtime time.Time) []utils.Subtit
return nil
}
tracks, err := svc.inner.DetectSubtitles(context.Background(), videoPath)
tracks, err := svc.inner.DetectSubtitles(ctx, videoPath)
if err != nil {
logger.Debug("ffprobe failed for file: " + videoPath + ", error: " + err.Error())
return nil
}
subtitles := mapSubtitleTracks(tracks)
MediaCache.Set(key, subtitles)
if useCache {
mediacache.SubtitleTracks.Set(key.String(), subtitles)
}
return subtitles
}
@@ -0,0 +1,73 @@
package gomedia
import (
"os"
"os/exec"
"path/filepath"
"testing"
"github.com/stretchr/testify/require"
)
const fixtureWMV2 = fixtureDir + "/file_example_WMV_480_1_2MB.wmv"
// TestContainerGates pins which extensions go-media handles for probes and
// for thumbnails.
func TestContainerGates(t *testing.T) {
cases := []struct {
name string
probe, thumbnail bool
}{
{"a.mp4", true, true},
{"a.M4V", true, true},
{"a.mov", true, true},
{"a.mkv", true, true},
{"a.webm", true, true},
{"a.wmv", false, true},
{"a.ASF", false, true},
{"a.avi", false, false},
{"a.ts", false, false},
{"a.m2ts", false, false},
{"a.mts", false, false},
{"a.mpg", false, false},
{"a.mpeg", false, false},
{"a.vob", false, false},
{"a.flv", false, false},
}
for _, c := range cases {
require.Equal(t, c.probe, supportsVideoContainer(c.name), "probe gate for %s", c.name)
require.Equal(t, c.thumbnail, supportsVideoThumbnail(c.name), "thumbnail gate for %s", c.name)
}
for _, name := range []string{"a.heic", "a.HEIF", "a.heics", "a.avif"} {
require.True(t, supportsHEIFImage(name), name)
}
require.False(t, supportsHEIFImage("a.jpg"))
}
// TestWMV2FallsBackToFFmpeg checks that a WMV2 file is not probed by go-media
// and that its thumbnail falls back to ffmpeg (go-media has no WMV1/WMV2
// decoder).
func TestWMV2FallsBackToFFmpeg(t *testing.T) {
withGate(t, false)
_, ok := Probe(fixtureWMV2, nil)
require.False(t, ok, "WMV probes stay on ffprobe")
_, ok = VideoThumbnail(fixtureWMV2, 320, 240, 80, 10, nil)
require.False(t, ok, "WMV2 thumbnails must fall back to ffmpeg")
}
// TestWMV1FallsBackToFFmpeg is TestWMV2FallsBackToFFmpeg for a WMV1 stream
// encoded on the fly.
func TestWMV1FallsBackToFFmpeg(t *testing.T) {
withGate(t, false)
requireTool(t, "ffmpeg")
path := filepath.Join(t.TempDir(), "wmv1.wmv")
out, err := exec.Command("ffmpeg", "-v", "error", "-y", "-f", "lavfi", "-i", "testsrc2=size=160x120:rate=25:duration=2",
"-c:v", "wmv1", path).CombinedOutput()
if err != nil {
t.Skipf("ffmpeg cannot encode wmv1: %v: %s", err, out)
}
_, err = os.Stat(path)
require.NoError(t, err)
_, ok := VideoThumbnail(path, 160, 120, 80, 10, nil)
require.False(t, ok, "WMV1 thumbnails must fall back to ffmpeg")
}
+146 -75
View File
@@ -20,6 +20,7 @@ import (
"time"
"github.com/gtsteffaniak/filebrowser/backend/internal/ffmpeg"
"github.com/gtsteffaniak/filebrowser/backend/internal/mediacache"
"github.com/gtsteffaniak/filebrowser/backend/internal/utils"
"github.com/gtsteffaniak/filebrowser/backend/pkg/settings"
"github.com/gtsteffaniak/go-cache/cache"
@@ -28,20 +29,27 @@ import (
"golang.org/x/sync/singleflight"
)
var (
subtitleListCache = cache.NewCache[[]utils.SubtitleTrack](24 * time.Hour)
subtitleContentCache = cache.NewCache[string](24 * time.Hour)
// Probe, duration, and subtitle results are cached in the mediacache package,
// which the ffmpeg code path shares. A directory listing probes every video
// child, and the preview flow probes the focused file again; the cache keeps
// each request from re-reading the same containers. Keys include size and
// modification time, so an edited file is read again.
// probeCache caches successful probe results by file identity (path, modtime,
// size). A directory listing probes every video child, and the preview flow
// probes the focused file again; without this each request re-read the same
// containers. Keying on modtime+size means an edited file is re-probed.
probeCache = cache.NewCache[ProbeResult](24 * time.Hour)
// probeGroup coalesces concurrent probes of the same file so a burst of
// requests for one container reads it once.
var probeGroup singleflight.Group
// probeGroup coalesces concurrent probes of the same file so a burst of
// requests for one container reads it once.
probeGroup singleflight.Group
)
// tagsCache caches the embedded part of a tag read by file identity. A listing
// reads tags for every media child on each uncached request. Sidecar lyrics
// are not cached: adding an .lrc next to a file does not change its modtime.
var tagsCache = cache.NewCache[AudioInfo](24 * time.Hour)
func init() {
// The ffmpeg HEIC decode path corrects orientation after decoding and needs
// the file's irot. internal/ffmpeg cannot import this package, so go-media's
// header reader is installed here.
ffmpeg.SetHEICDisplayRotationReader(HEICDisplayRotation)
}
// Enabled reports whether the go-media fast path may run. The admin
// "Always use ffmpeg" setting turns it off.
@@ -60,9 +68,15 @@ func videoDimensions(info *media.ProbeInfo) (w, h int, ok bool) {
}
// videoContainerExts are the containers go-media parses for video probes,
// thumbnails, and embedded subtitles. Other video containers (AVI, WMV, TS, ...)
// are not go-media targets, so they go straight to ffmpeg with no file open and
// no log line. go-media reads MOV as MP4.
// thumbnails, embedded subtitles, and keyframe indexes. go-media reads MOV as
// MP4.
//
// go-media also demuxes AVI, ASF/WMV, MPEG-TS/M2TS, and MPEG-PS (VOB), but their
// probes are left to ffmpeg on purpose: go-media's results for those containers
// do not yet match ffprobe (TS/M2TS and PS list no audio streams, PS durations
// are missing or wrong, AVI misnames MPEG-4 Part 2 and AAC, ASF reports WMA Pro
// as wmav2). Those files go straight to ffmpeg with no file open and no log
// line. See videoThumbnailExts for the thumbnail exception.
var videoContainerExts = map[string]bool{
".mp4": true,
".m4v": true,
@@ -71,11 +85,31 @@ var videoContainerExts = map[string]bool{
".webm": true,
}
// videoThumbnailExts are the containers go-media decodes video thumbnails from:
// videoContainerExts plus ASF/WMV, whose WMV3 keyframes go-media decodes
// bit-exact with ffmpeg. WMV1, WMV2, MS-MPEG4, and WVC1 Advanced Profile return
// an unsupported error and fall back to ffmpeg. AVI, TS/M2TS, and PS stay on
// ffmpeg: go-media's TS and PS thumbnails scan the whole file for the duration
// (several times slower than ffmpeg on large files), and AVI seeks by a
// duration that can be wrong.
var videoThumbnailExts = map[string]bool{
".mp4": true,
".m4v": true,
".mov": true,
".mkv": true,
".webm": true,
".wmv": true,
".asf": true,
}
// heifImageExts are still-image containers go-media decodes for HEIC preview.
// AVIF is a HEIF container with an AV1 image item; go-media decodes it in pure
// Go and applies its irot/imir, like HEIC.
var heifImageExts = map[string]bool{
".heic": true,
".heif": true,
".heics": true,
".avif": true,
}
const heicDecodeMaxEdge = 16384
@@ -93,15 +127,27 @@ var cameraRawExts = map[string]bool{
".dc2": true, ".x3f": true, ".iiq": true, ".nkc": true, ".r3d": true,
}
// supportsVideoContainer reports whether go-media targets path's container.
// supportsVideoContainer reports whether go-media targets path's container for
// probes, embedded subtitles, and keyframe indexes.
func supportsVideoContainer(path string) bool {
return videoContainerExts[strings.ToLower(filepath.Ext(path))]
}
// supportsVideoThumbnail reports whether go-media targets path's container for
// video thumbnails.
func supportsVideoThumbnail(path string) bool {
return videoThumbnailExts[strings.ToLower(filepath.Ext(path))]
}
// supportsHEIFImage reports whether path is a HEIF-family still image
// (HEIC/HEIF/AVIF) go-media decodes.
func supportsHEIFImage(path string) bool {
return heifImageExts[strings.ToLower(filepath.Ext(path))]
}
// supportsCameraRAW reports whether path is a camera RAW format go-media
// extracts a preview from.
func supportsCameraRAW(path string) bool {
return cameraRawExts[strings.ToLower(filepath.Ext(path))]
}
@@ -133,16 +179,26 @@ func logUnsupported(op, path, reason string) {
// openFile opens path for go-media, which reads through io.ReaderAt.
func openFile(path string) (*os.File, int64, error) {
f, err := os.Open(path)
f, info, err := openFileInfo(path)
if err != nil {
return nil, 0, err
}
return f, info.Size(), nil
}
// openFileInfo opens path and returns its FileInfo, which also gives the
// shared cache key.
func openFileInfo(path string) (*os.File, os.FileInfo, error) {
f, err := os.Open(path)
if err != nil {
return nil, nil, err
}
info, err := f.Stat()
if err != nil {
_ = f.Close()
return nil, 0, err
return nil, nil, err
}
return f, info.Size(), nil
return f, info, nil
}
// ffprobeFormatNames maps go-media containers to the format_name strings ffprobe
@@ -179,30 +235,25 @@ func probeFast(path string) (ProbeResult, bool) {
return ProbeResult{}, false
}
f, size, err := openFile(path)
f, info, err := openFileInfo(path)
if err != nil {
logFallback("probe", path, err)
return ProbeResult{}, false
}
defer f.Close()
info, err := f.Stat()
if err != nil {
logFallback("probe", path, err)
return ProbeResult{}, false
}
cacheKey := probeCacheKey(path, info.ModTime().UnixNano(), size)
if res, ok := probeCache.Get(cacheKey); ok {
cacheKey := mediacache.FromFileInfo(path, info).String()
if res, ok := mediacache.Probes.Get(cacheKey); ok {
return res, true
}
// Coalesce concurrent probes of the same file into one container read.
v, err, _ := probeGroup.Do(cacheKey, func() (any, error) {
res, ok := parseProbe(path, f, size)
res, ok := parseProbe(path, f, info.Size())
if !ok {
return nil, errProbeUnsupported
}
probeCache.Set(cacheKey, res)
mediacache.Probes.Set(cacheKey, res)
return res, nil
})
if err != nil {
@@ -219,12 +270,6 @@ func probeFast(path string) (ProbeResult, bool) {
// no detail because parseProbe already logged the specific reason.
var errProbeUnsupported = errors.New("probe unsupported")
// probeCacheKey identifies a file version for caching: path plus modtime and size.
// An edited file gets a new key, so stale metadata is never served.
func probeCacheKey(path string, modTimeNanos, size int64) string {
return fmt.Sprintf("probe:%s:%d:%d", path, modTimeNanos, size)
}
// parseProbe reads a container and builds a ProbeResult. It reports false when
// go-media should defer to ffmpeg, having logged why.
func parseProbe(path string, f *os.File, size int64) (ProbeResult, bool) {
@@ -280,14 +325,18 @@ func durationFast(path string) (float64, bool) {
if !Enabled() {
return 0, false
}
f, size, err := openFile(path)
f, info, err := openFileInfo(path)
if err != nil {
logFallback("duration", path, err)
return 0, false
}
defer f.Close()
res, err := media.Duration(f, size)
cacheKey := mediacache.FromFileInfo(path, info).String()
if d, ok := mediacache.Durations.Get(cacheKey); ok {
return d, true
}
res, err := media.Duration(f, info.Size())
if err != nil {
logFallback("duration", path, err)
return 0, false
@@ -296,7 +345,9 @@ func durationFast(path string) (float64, bool) {
// Unknown duration is not an error; ffmpeg may still find one.
return 0, false
}
return res.Duration.Seconds(), true
d := res.Duration.Seconds()
mediacache.Durations.Set(cacheKey, d)
return d, true
}
// AudioInfo is the tag, lyrics availability, and cover art FileBrowser reads
@@ -311,38 +362,60 @@ type AudioInfo struct {
// is true. A missing cover is not an error.
// legacy is the dhowden tag read, run only in shadow mode.
func Audio(path string, withArt bool, legacy Legacy[AudioInfo]) (AudioInfo, bool) {
return timed("tags", path, legacy, audioDiffer(withArt), func() (AudioInfo, bool) { return audioFast(path, withArt) })
return timed("tags", path, legacy, audioDiffer(withArt), func() (AudioInfo, bool) { return audioFast(path, withArt, true) })
}
func audioFast(path string, withArt bool) (AudioInfo, bool) {
func audioFast(path string, withArt, withSidecar bool) (AudioInfo, bool) {
if !Enabled() {
return AudioInfo{}, false
}
f, size, err := openFile(path)
f, st, err := openFileInfo(path)
if err != nil {
logFallback("tags", path, err)
return AudioInfo{}, false
}
defer f.Close()
// One container parse gives tags, the embedded lyrics flag, and cover bytes.
// Separate ReadTags/HasLyrics/EmbeddedArtwork calls each re-parsed the file.
audio, err := media.InspectAudio(f, size, media.AudioOptions{WithArtwork: withArt})
if err != nil {
logFallback("tags", path, err)
return AudioInfo{}, false
size := st.Size()
cacheKey := tagsCacheKey(path, st.ModTime().UnixNano(), size)
var info AudioInfo
cached := false
if !withArt {
info, cached = tagsCache.Get(cacheKey)
}
info := AudioInfo{Tags: audio.Tags, HasLyrics: audio.HasEmbeddedLyrics}
if !info.HasLyrics {
if !cached {
// One container parse gives tags, the embedded lyrics flag, and cover bytes.
// Separate ReadTags/HasLyrics/EmbeddedArtwork calls each re-parsed the file.
audio, err := media.InspectAudio(f, size, media.AudioOptions{WithArtwork: withArt})
if err != nil {
logFallback("tags", path, err)
return AudioInfo{}, false
}
info = AudioInfo{Tags: audio.Tags, HasLyrics: audio.HasEmbeddedLyrics}
stored := info
stored.Tags.Raw = nil
tagsCache.Set(cacheKey, stored)
if withArt {
info.Artwork = audio.Artwork
}
}
if withSidecar && !info.HasLyrics {
// Sidecar lyrics are only checked when the file has no embedded lyrics.
_, _, info.HasLyrics = media.LyricsSidecar(path)
}
if withArt {
info.Artwork = audio.Artwork
}
return info, true
}
// tagsCacheKey identifies a file version for the tag cache: path plus modtime and size.
func tagsCacheKey(path string, modTimeNanos, size int64) string {
return fmt.Sprintf("tags:%s:%d:%d", path, modTimeNanos, size)
}
// VideoTags reads embedded tags and cover from a video container
func VideoTags(path string, withArt bool) (AudioInfo, bool) {
return timed("tags", path, nil, nil, func() (AudioInfo, bool) { return audioFast(path, withArt, false) })
}
// Lyrics returns lyrics from a sidecar file, or else from embedded tags. The
// format is "embedded" for tags or the sidecar extension. An empty result with
// ok=true means go-media checked and found no lyrics.
@@ -384,7 +457,9 @@ func lyricsFast(path string) (text, format string, ok bool) {
// EmbeddedSubtitles lists embedded subtitle tracks. go-media supports MKV, WebM,
// and MP4 only. Other containers are reported as unsupported, not as "no
// subtitles", because go-media returns an empty list for them.
// subtitles", because go-media returns an empty list for them. modtime is kept
// for API compatibility; the cache key comes from the file's current size and
// modification time.
// legacy is the ffmpeg subtitle detection, run only in shadow mode.
func EmbeddedSubtitles(path string, modtime time.Time, legacy Legacy[[]utils.SubtitleTrack]) ([]utils.SubtitleTrack, bool) {
return timed("subtitle list", path, legacy, subtitleListDiff, func() ([]utils.SubtitleTrack, bool) {
@@ -399,19 +474,20 @@ func embeddedSubtitlesFast(path string, modtime time.Time) ([]utils.SubtitleTrac
if !supportsVideoContainer(path) {
return nil, false
}
key := fmt.Sprintf("embedded_subtitles:%s:%d", path, modtime.Unix())
if cached, ok := subtitleListCache.Get(key); ok {
return cached, true
}
f, size, err := openFile(path)
_ = modtime
f, st, err := openFileInfo(path)
if err != nil {
logFallback("subtitle list", path, err)
return nil, false
}
defer f.Close()
info, err := media.Probe(f, size)
key := mediacache.FromFileInfo(path, st).String()
if cached, ok := mediacache.SubtitleTracks.Get(key); ok {
return cached, true
}
info, err := media.Probe(f, st.Size())
if err != nil {
logFallback("subtitle list", path, err)
return nil, false
@@ -433,7 +509,7 @@ func embeddedSubtitlesFast(path string, modtime time.Time) ([]utils.SubtitleTrac
})
}
tracks := ffmpeg.BuildSubtitleTracks(streams)
subtitleListCache.Set(key, tracks)
mediacache.SubtitleTracks.Set(key, tracks)
return tracks, true
}
@@ -453,24 +529,19 @@ func extractSubtitleFast(path string, streamIndex int) (string, bool) {
if !supportsVideoContainer(path) {
return "", false
}
st, err := os.Stat(path)
if err != nil {
logFallback("subtitle extract", path, err)
return "", false
}
key := fmt.Sprintf("subtitle_content:%s:%d:%d", path, streamIndex, st.ModTime().Unix())
if content, ok := subtitleContentCache.Get(key); ok {
return content, true
}
f, size, err := openFile(path)
f, st, err := openFileInfo(path)
if err != nil {
logFallback("subtitle extract", path, err)
return "", false
}
defer f.Close()
vtt, err := media.ExtractEmbeddedSubtitle(f, size, streamIndex)
key := mediacache.FromFileInfo(path, st).Stream(streamIndex)
if content, ok := mediacache.SubtitleContent.Get(key); ok {
return content, true
}
vtt, err := media.ExtractEmbeddedSubtitle(f, st.Size(), streamIndex)
if err != nil {
logFallback("subtitle extract", path, err)
return "", false
@@ -479,7 +550,7 @@ func extractSubtitleFast(path string, streamIndex int) (string, bool) {
logFallback("subtitle extract", path, errors.New("empty subtitle output"))
return "", false
}
subtitleContentCache.Set(key, vtt)
mediacache.SubtitleContent.Set(key, vtt)
return vtt, true
}
@@ -501,7 +572,7 @@ func videoThumbnailFast(path string, width, height, quality int, seekPercent flo
if !Enabled() {
return nil, false
}
if !supportsVideoContainer(path) {
if !supportsVideoThumbnail(path) {
return nil, false
}
f, size, err := openFile(path)
+66 -13
View File
@@ -11,6 +11,8 @@ import (
"testing"
"time"
"github.com/gtsteffaniak/filebrowser/backend/internal/mediacache"
"github.com/gtsteffaniak/filebrowser/backend/internal/utils"
"github.com/gtsteffaniak/filebrowser/backend/pkg/settings"
"github.com/stretchr/testify/require"
)
@@ -131,27 +133,78 @@ func TestProbeUsesCache(t *testing.T) {
require.NoError(t, err)
require.NoError(t, os.WriteFile(path, data, 0o644))
st, err := os.Stat(path)
key, err := mediacache.Stat(path)
require.NoError(t, err)
key := probeCacheKey(path, st.ModTime().UnixNano(), st.Size())
sentinel := ProbeResult{Duration: 123, VideoCodec: "sentinel", AudioCodec: "sentinel", FormatName: "sentinel"}
probeCache.Set(key, sentinel)
t.Cleanup(func() { probeCache.Delete(key) })
// The ffmpeg probe stores into the same shared cache, so this also stands in
// for a value ffprobe produced earlier.
mediacache.Probes.Set(key.String(), sentinel)
t.Cleanup(func() { mediacache.Probes.Delete(key.String()) })
got, ok := Probe(path, nil)
require.True(t, ok)
require.Equal(t, sentinel, got, "probe must return the cached value for an unchanged file")
// An edit (new size and mtime) must miss the cached value.
require.NoError(t, os.WriteFile(path, append(data, 0), 0o644))
future := time.Now().Add(time.Hour)
require.NoError(t, os.Chtimes(path, future, future))
got, ok = Probe(path, nil)
require.True(t, ok)
require.NotEqual(t, sentinel, got, "an edited file must be probed again")
edited, err := mediacache.Stat(path)
require.NoError(t, err)
t.Cleanup(func() { mediacache.Probes.Delete(edited.String()) })
}
// TestProbeCacheKeyTracksFileIdentity checks that path, modtime, and size all
// participate in the cache key, so an edited file is never served stale metadata.
func TestProbeCacheKeyTracksFileIdentity(t *testing.T) {
base := probeCacheKey("f.mp4", 100, 200)
require.Equal(t, base, probeCacheKey("f.mp4", 100, 200), "identical identity must hash the same")
require.NotEqual(t, base, probeCacheKey("f.mp4", 101, 200), "modtime change must change the key")
require.NotEqual(t, base, probeCacheKey("f.mp4", 100, 201), "size change must change the key")
require.NotEqual(t, base, probeCacheKey("g.mp4", 100, 200), "path change must change the key")
require.NotEqual(t, base, probeCacheKey("f.mp4", 100, 201), "key must not collide across fields")
// TestDurationUsesSharedCache checks that Duration reads and fills the shared
// cache that ffmpeg's GetMediaDuration also uses.
func TestDurationUsesSharedCache(t *testing.T) {
withGate(t, false)
path := filepath.Join(t.TempDir(), "sample.mp4")
data, err := os.ReadFile(fixtureMP4)
require.NoError(t, err)
require.NoError(t, os.WriteFile(path, data, 0o644))
key, err := mediacache.Stat(path)
require.NoError(t, err)
t.Cleanup(func() { mediacache.Durations.Delete(key.String()) })
d, ok := Duration(path, nil)
require.True(t, ok)
cached, found := mediacache.Durations.Get(key.String())
require.True(t, found, "go-media duration must be stored in the shared cache")
require.Equal(t, d, cached)
mediacache.Durations.Set(key.String(), 4242)
d, ok = Duration(path, nil)
require.True(t, ok)
require.Equal(t, 4242.0, d, "a cached duration (from either backend) must be reused")
}
// TestSubtitleCachesUseFileIdentity checks the subtitle list cache is keyed by
// the file's real size and mtime, not by the caller's modtime argument.
func TestSubtitleCachesUseFileIdentity(t *testing.T) {
withGate(t, false)
path := filepath.Join(t.TempDir(), "sample.mp4")
data, err := os.ReadFile(fixtureMP4)
require.NoError(t, err)
require.NoError(t, os.WriteFile(path, data, 0o644))
key, err := mediacache.Stat(path)
require.NoError(t, err)
sentinel := []utils.SubtitleTrack{{Name: "sentinel"}}
mediacache.SubtitleTracks.Set(key.String(), sentinel)
t.Cleanup(func() { mediacache.SubtitleTracks.Delete(key.String()) })
// The modtime argument is ignored for the key.
got, ok := EmbeddedSubtitles(path, time.Unix(1, 0), nil)
require.True(t, ok)
require.Equal(t, sentinel, got)
mediacache.SubtitleContent.Set(key.Stream(7), "WEBVTT\n\nsentinel")
t.Cleanup(func() { mediacache.SubtitleContent.Delete(key.Stream(7)) })
vtt, ok := ExtractSubtitle(path, 7, nil)
require.True(t, ok)
require.Equal(t, "WEBVTT\n\nsentinel", vtt)
}
func TestProbeWebMMatchesFFprobe(t *testing.T) {
+101
View File
@@ -0,0 +1,101 @@
package gomedia
import (
"context"
"fmt"
"math"
"time"
"github.com/gtsteffaniak/filebrowser/backend/internal/ffmpeg"
"github.com/gtsteffaniak/go-media/pkg/media"
)
// heicRotationShadowTimeout bounds the dev-mode ffmpeg comparison probe.
const heicRotationShadowTimeout = time.Minute
// HEICDisplayRotation returns the display rotation that a HEIC/HEIF/AVIF
// file's irot property asks for, read from the item properties without
// decoding. The value is degrees counter-clockwise in (-180, 180], the same
// number ffmpeg prints as "Display Matrix: rotation of ..." for the file. A
// file with no irot returns 0 and ok=true. ok is false when go-media is
// disabled or cannot read the item properties; the caller then runs the ffmpeg
// probe.
//
// It is installed as ffmpeg's HEIC display rotation reader. In shadow mode the
// ffmpeg stderr probe runs for comparison.
func HEICDisplayRotation(path string) (float64, bool) {
return timed("heic rotation", path, heicRotationLegacy(path), heicRotationDiff, func() (float64, bool) {
return heicDisplayRotationFast(path)
})
}
// heicRotationLegacy returns the ffmpeg stderr probe for shadow mode, or nil
// when ffmpeg is not available.
func heicRotationLegacy(path string) Legacy[float64] {
if !ShadowEnabled() {
return nil
}
svc := ffmpeg.Get()
if svc == nil {
return nil
}
return func() (float64, bool) {
ctx, cancel := context.WithTimeout(context.Background(), heicRotationShadowTimeout)
defer cancel()
return svc.GetHEICDisplayMatrixRotation(ctx, path)
}
}
// heicDisplayRotationFast reads the primary item's irot with go-media and
// converts it to ffmpeg's display rotation.
func heicDisplayRotationFast(path string) (float64, bool) {
if !Enabled() {
return 0, false
}
// No extension check: the resolved path can have another name (symlink
// target). go-media sniffs the content and reports other files unsupported.
f, size, err := openFile(path)
if err != nil {
logFallback("heic rotation", path, err)
return 0, false
}
defer f.Close()
t, err := media.HEIFTransform(f, size)
if err != nil {
logFallback("heic rotation", path, err)
return 0, false
}
return displayRotation(t), true
}
// displayRotation converts go-media's counter-clockwise irot angle to ffmpeg's
// display matrix convention: counter-clockwise degrees normalized to
// (-180, 180]. ffmpeg builds the matrix with av_display_rotation_set(-irot)
// and prints av_display_rotation_get, so irot 270 is reported as -90. A mirror
// does not change the rotation value.
func displayRotation(t media.ImageTransform) float64 {
r := ((t.Rotation % 360) + 360) % 360
if r > 180 {
r -= 360
}
return float64(r)
}
// heicRotationDiff compares go-media's display rotation with ffmpeg's. 180 and
// -180 are the same rotation.
func heicRotationDiff(fast, legacy float64) []string {
norm := func(v float64) float64 {
v = math.Mod(v, 360)
if v <= -180 {
v += 360
} else if v > 180 {
v -= 360
}
return v
}
if math.Abs(norm(fast)-norm(legacy)) > 0.5 {
return []string{fmt.Sprintf("display rotation go-media=%.2f ffmpeg=%.2f", fast, legacy)}
}
return nil
}
@@ -0,0 +1,82 @@
package gomedia
import (
"math"
"os"
"path/filepath"
"testing"
"github.com/gtsteffaniak/filebrowser/backend/internal/ffmpeg"
"github.com/gtsteffaniak/go-media/pkg/media"
"github.com/stretchr/testify/require"
)
// fixtureAVIFRotated has irot=1 (90° counter-clockwise) and imir axis 0.
const fixtureAVIFRotated = "testdata/irot1_imir0.avif"
// TestDisplayRotation checks the irot to ffmpeg display rotation mapping.
func TestDisplayRotation(t *testing.T) {
cases := []struct {
rotation int
want float64
}{
{0, 0}, {90, 90}, {180, 180}, {270, -90}, {-90, -90}, {360, 0}, {450, 90},
}
for _, c := range cases {
require.Equal(t, c.want, displayRotation(media.ImageTransform{Rotation: c.rotation}), "rotation %d", c.rotation)
}
require.Equal(t, 90.0, displayRotation(media.ImageTransform{Rotation: 90, Mirror: true}), "a mirror does not change the rotation")
}
// TestHEICDisplayRotationReadsIrot checks the rotation read from a fixture
// with irot=1.
func TestHEICDisplayRotationReadsIrot(t *testing.T) {
withGate(t, false)
got, ok := HEICDisplayRotation(fixtureAVIFRotated)
require.True(t, ok)
require.Equal(t, 90.0, got)
}
// TestHEICDisplayRotationFallsBack checks that non-HEIF input, broken input,
// and the Always use ffmpeg setting leave the answer to ffmpeg.
func TestHEICDisplayRotationFallsBack(t *testing.T) {
withGate(t, false)
_, ok := HEICDisplayRotation(fixtureMP4)
require.False(t, ok, "a non-HEIF file must fall back to the ffmpeg probe")
junk := filepath.Join(t.TempDir(), "junk.heic")
require.NoError(t, os.WriteFile(junk, []byte("not heif"), 0o644))
_, ok = HEICDisplayRotation(junk)
require.False(t, ok)
withGate(t, true)
_, ok = HEICDisplayRotation(fixtureAVIFRotated)
require.False(t, ok, "the Always use ffmpeg setting must skip go-media")
}
// TestHEICDisplayRotationInstalled checks that loading this package installs
// go-media as the ffmpeg package's rotation reader.
func TestHEICDisplayRotationInstalled(t *testing.T) {
withGate(t, false)
var svc *ffmpeg.Service // nil: the ffmpeg probe cannot run, so a result proves the reader answered
got, ok := svc.HEICDisplayRotation(t.Context(), fixtureAVIFRotated)
require.True(t, ok)
require.Equal(t, 90.0, got)
}
// TestHEICRotationDiff checks the shadow comparison of display rotations.
func TestHEICRotationDiff(t *testing.T) {
require.Empty(t, heicRotationDiff(-90, -90))
require.Empty(t, heicRotationDiff(180, -180))
require.Empty(t, heicRotationDiff(0, math.Copysign(0, -1)))
require.Len(t, heicRotationDiff(90, -90), 1)
}
// TestAVIFDecodesThroughHEICPath checks that an AVIF file is accepted by the
// HEIC decode path.
func TestAVIFDecodesThroughHEICPath(t *testing.T) {
withGate(t, false)
data, ok := HEICJPEG(fixtureAVIFRotated, 80, nil)
require.True(t, ok)
require.NotEmpty(t, data)
}
+145
View File
@@ -0,0 +1,145 @@
package gomedia
import (
"context"
"fmt"
"io"
"math"
"sort"
"github.com/gtsteffaniak/go-media/pkg/media"
)
// keyframeTimeDedupeSec matches go-ffmpeg's ffprobe keyframe probe, which
// drops keyframes within 1 ms of the previous one.
const keyframeTimeDedupeSec = 0.001
// keyframeShadowTolerance is how far apart two backends' keyframe times can be
// before the shadow comparison reports them. ffprobe prints six decimals.
const keyframeShadowTolerance = 0.002
// KeyframeTimes returns the presentation times, in seconds, of the first video
// stream's keyframes, for HLS segment planning. The result is sorted, has no
// negative times, and drops keyframes within 1 ms of the previous one, the same
// as go-ffmpeg's ffprobe probe. go-media reads only the container index (MP4
// stss/stts/ctts, Matroska Cues or block headers), so it is much faster than
// demuxing every packet with ffprobe.
//
// ok is false when go-media is disabled, the container is not one go-media
// indexes, or the read fails; the caller then runs ffprobe. ctx cancels the
// file reads. legacy is the ffprobe probe, run only in shadow mode.
//
// go-media can report fewer keyframes than ffprobe (ffprobe's parsers also flag
// IDR frames the container index omits, and some Matroska muxers index only
// part of the keyframes in Cues). It never reports a keyframe ffprobe would not.
func KeyframeTimes(ctx context.Context, path string, legacy Legacy[[]float64]) ([]float64, bool) {
return timed("keyframe index", path, legacy, keyframeDiff, func() ([]float64, bool) {
return keyframeTimesFast(ctx, path, 0)
})
}
// KeyframeTimesWithin is KeyframeTimes limited to keyframes within windowSec
// seconds of the first keyframe. It is the pre-flight check that can rule out
// stream copy without reading the whole index (Matroska files without Cues).
// It has no shadow comparison.
func KeyframeTimesWithin(ctx context.Context, path string, windowSec float64) ([]float64, bool) {
return timed("keyframe preflight", path, nil, nil, func() ([]float64, bool) {
return keyframeTimesFast(ctx, path, windowSec)
})
}
// keyframeTimesFast reads the keyframe index with go-media. windowSec > 0
// limits it to keyframes within that many seconds of the first one.
func keyframeTimesFast(ctx context.Context, path string, windowSec float64) ([]float64, bool) {
if !Enabled() {
return nil, false
}
if !supportsVideoContainer(path) {
return nil, false
}
if err := ctx.Err(); err != nil {
return nil, false
}
f, size, err := openFile(path)
if err != nil {
logFallback("keyframe index", path, err)
return nil, false
}
defer f.Close()
r := ctxReaderAt{ctx: ctx, r: f}
var times []float64
if windowSec > 0 {
times, err = media.KeyframeTimesWithin(r, size, windowSec)
} else {
times, err = media.KeyframeTimes(r, size)
}
if err != nil {
if ctx.Err() == nil {
logFallback("keyframe index", path, err)
}
return nil, false
}
times = dedupeKeyframeTimes(times)
if len(times) == 0 {
logUnsupported("keyframe index", path, "no keyframes in container index")
return nil, false
}
return times, true
}
// dedupeKeyframeTimes sorts times, drops negative values, and drops times
// within keyframeTimeDedupeSec of the previous kept time, as go-ffmpeg does for
// ffprobe output.
func dedupeKeyframeTimes(times []float64) []float64 {
sorted := append([]float64(nil), times...)
sort.Float64s(sorted)
out := make([]float64, 0, len(sorted))
for _, t := range sorted {
if t < 0 || math.IsNaN(t) || math.IsInf(t, 0) {
continue
}
if len(out) == 0 || t > out[len(out)-1]+keyframeTimeDedupeSec {
out = append(out, t)
}
}
return out
}
// ctxReaderAt fails reads once ctx is done, so a cancelled request or the plan
// timeout stops a long index walk on a slow share.
type ctxReaderAt struct {
ctx context.Context
r io.ReaderAt
}
// ReadAt reads from the wrapped reader unless the context is done.
func (c ctxReaderAt) ReadAt(p []byte, off int64) (int, error) {
if err := c.ctx.Err(); err != nil {
return 0, err
}
return c.r.ReadAt(p, off)
}
// keyframeDiff compares go-media's keyframe index with ffprobe's. go-media
// reporting a subset of ffprobe's keyframes is expected (parser-flagged IDR
// frames, partial Cues) and is logged at info level.
func keyframeDiff(fast, legacy []float64) []string {
missing := 0
j := 0
for _, t := range fast {
for j < len(legacy) && legacy[j] < t-keyframeShadowTolerance {
j++
missing++
}
if j >= len(legacy) || math.Abs(legacy[j]-t) > keyframeShadowTolerance {
return []string{fmt.Sprintf("keyframe at %.6fs from go-media not in ffmpeg index (go-media=%d ffmpeg=%d keyframes)", t, len(fast), len(legacy))}
}
j++
}
missing += len(legacy) - j
if missing > 0 {
return []string{fmt.Sprintf("%sgo-media found %d of %d ffmpeg keyframes (keyframes outside the container index)", expectedDiffPrefix, len(fast), len(legacy))}
}
return nil
}
+176
View File
@@ -0,0 +1,176 @@
package gomedia
import (
"context"
"math"
"os"
"os/exec"
"path/filepath"
"sort"
"strconv"
"strings"
"testing"
"github.com/stretchr/testify/require"
)
// ffprobeKeyframes runs the same packet probe as go-ffmpeg's
// ProbeVideoKeyframeTimes: key packets of v:0, pts_time falling back to
// dts_time, sorted and de-duplicated within 1 ms.
func ffprobeKeyframes(t *testing.T, path string) []float64 {
t.Helper()
requireTool(t, "ffprobe")
out, err := exec.Command("ffprobe", "-v", "error", "-select_streams", "v:0",
"-show_entries", "packet=pts_time,dts_time,flags", "-of", "csv=p=0", path).Output()
require.NoError(t, err)
var times []float64
for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") {
f := strings.Split(strings.TrimSpace(line), ",")
if len(f) < 3 || !strings.Contains(f[2], "K") {
continue
}
v, perr := strconv.ParseFloat(f[0], 64)
if perr != nil {
v, perr = strconv.ParseFloat(f[1], 64)
}
if perr == nil {
times = append(times, v)
}
}
sort.Float64s(times)
return dedupeKeyframeTimes(times)
}
// encodeKeyframeClip makes a short clip with a fixed GOP. B-frames give MP4
// files a ctts table and an edit list, the hard case for the timestamp model.
func encodeKeyframeClip(t *testing.T, name string, args ...string) string {
t.Helper()
requireTool(t, "ffmpeg")
path := filepath.Join(t.TempDir(), name)
full := []string{"-v", "error", "-y", "-f", "lavfi", "-i", "testsrc2=size=160x120:rate=25:duration=6"}
full = append(full, args...)
full = append(full, path)
if out, err := exec.Command("ffmpeg", full...).CombinedOutput(); err != nil {
t.Skipf("ffmpeg cannot encode %s: %v: %s", name, err, out)
}
return path
}
// requireSameKeyframes fails unless got matches want within the shadow
// tolerance.
func requireSameKeyframes(t *testing.T, want, got []float64) {
t.Helper()
require.Equal(t, len(want), len(got), "keyframe count: ffprobe=%v go-media=%v", want, got)
for i := range want {
require.InDelta(t, want[i], got[i], keyframeShadowTolerance, "keyframe %d: ffprobe=%v go-media=%v", i, want, got)
}
}
// TestKeyframeTimesMatchFFprobe compares go-media keyframe indexes with
// ffprobe for fixtures and encoded clips, including B-frame MP4/MOV.
func TestKeyframeTimesMatchFFprobe(t *testing.T) {
if testing.Short() {
t.Skip("encodes clips with ffmpeg")
}
withGate(t, false)
x264 := []string{"-c:v", "libx264", "-g", "50", "-keyint_min", "50", "-sc_threshold", "0", "-bf", "2", "-pix_fmt", "yuv420p"}
cases := map[string]string{
"fixture.mp4": fixtureMP4,
"fixture.webm": fixtureWebM,
"bframes.mp4": encodeKeyframeClip(t, "bframes.mp4", x264...),
"bframes.mov": encodeKeyframeClip(t, "bframes.mov", x264...),
"bframes.mkv": encodeKeyframeClip(t, "bframes.mkv", x264...),
"mpeg4.m4v": encodeKeyframeClip(t, "mpeg4.m4v", "-c:v", "mpeg4", "-g", "30", "-f", "mp4"),
}
for name, path := range cases {
t.Run(name, func(t *testing.T) {
want := ffprobeKeyframes(t, path)
got, ok := KeyframeTimes(context.Background(), path, nil)
require.True(t, ok)
requireSameKeyframes(t, want, got)
within, ok := KeyframeTimesWithin(context.Background(), path, 2.5)
require.True(t, ok)
require.NotEmpty(t, within)
require.LessOrEqual(t, len(within), len(got))
for i := range within {
require.Equal(t, got[i], within[i])
require.LessOrEqual(t, within[i]-got[0], 2.5+1e-9)
}
})
}
}
// TestKeyframeTimesFallsBack checks the cases that must leave the index to
// ffprobe.
func TestKeyframeTimesFallsBack(t *testing.T) {
withGate(t, false)
dir := t.TempDir()
// Not an indexed container: no file open, ffprobe answers.
ts := filepath.Join(dir, "clip.ts")
require.NoError(t, os.WriteFile(ts, []byte{0x47}, 0o644))
_, ok := KeyframeTimes(context.Background(), ts, nil)
require.False(t, ok)
// Indexed extension but not a real container.
junk := filepath.Join(dir, "junk.mp4")
require.NoError(t, os.WriteFile(junk, []byte("not a video"), 0o644))
_, ok = KeyframeTimes(context.Background(), junk, nil)
require.False(t, ok)
// Missing file.
_, ok = KeyframeTimes(context.Background(), filepath.Join(dir, "missing.mp4"), nil)
require.False(t, ok)
// Cancelled context.
ctx, cancel := context.WithCancel(context.Background())
cancel()
_, ok = KeyframeTimes(ctx, fixtureMP4, nil)
require.False(t, ok)
}
// TestKeyframeTimesGate checks that the Always use ffmpeg setting disables
// the go-media keyframe index.
func TestKeyframeTimesGate(t *testing.T) {
withGate(t, true)
_, ok := KeyframeTimes(context.Background(), fixtureMP4, nil)
require.False(t, ok, "KeyframeTimes must skip go-media when the gate is on")
_, ok = KeyframeTimesWithin(context.Background(), fixtureMP4, 10)
require.False(t, ok)
}
// TestDedupeKeyframeTimes checks sorting, filtering, and the 1 ms dedupe.
func TestDedupeKeyframeTimes(t *testing.T) {
got := dedupeKeyframeTimes([]float64{2, -1, 0, 0.0005, 1, 1.001, 1.0021, math.NaN(), math.Inf(1)})
require.Equal(t, []float64{0, 1, 1.0021, 2}, got)
require.Empty(t, dedupeKeyframeTimes(nil))
}
// TestCtxReaderAt checks that reads stop once the context is cancelled.
func TestCtxReaderAt(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
r := ctxReaderAt{ctx: ctx, r: strings.NewReader("abcdef")}
buf := make([]byte, 3)
n, err := r.ReadAt(buf, 1)
require.NoError(t, err)
require.Equal(t, "bcd", string(buf[:n]))
cancel()
_, err = r.ReadAt(buf, 0)
require.ErrorIs(t, err, context.Canceled)
}
// TestKeyframeDiff checks the shadow comparison: a subset is expected, an
// extra keyframe is a real difference.
func TestKeyframeDiff(t *testing.T) {
require.Empty(t, keyframeDiff([]float64{0, 2, 4}, []float64{0, 2.001, 4}))
subset := keyframeDiff([]float64{0, 4}, []float64{0, 2, 4, 6})
require.Len(t, subset, 1)
require.True(t, strings.HasPrefix(subset[0], expectedDiffPrefix), subset[0])
require.Contains(t, subset[0], "2 of 4")
extra := keyframeDiff([]float64{0, 1, 2}, []float64{0, 2})
require.Len(t, extra, 1)
require.False(t, strings.HasPrefix(extra[0], expectedDiffPrefix), extra[0])
}
+2 -2
View File
@@ -59,8 +59,8 @@ func runShadow[T any](op, path string, goLatency time.Duration, fast T, legacy L
ffStart := time.Now()
legacyVal, ffOK := legacy()
ffLatency := time.Since(ffStart)
shadowLogf("go-media shadow %s for %s: go-media=%.2fms legacy(ffmpeg/dhowden)=%.2fms diff(legacy-go)=%+.2fms legacy_ok=%t",
op, path, ms(goLatency), ms(ffLatency), ms(ffLatency)-ms(goLatency), ffOK)
shadowLogf("go-media shadow %s for %s: go-media=%.2fms legacy(ffmpeg/dhowden)=%.2fms diff(go-legacy)=%+.2fms",
op, path, ms(goLatency), ms(ffLatency), ms(goLatency)-ms(ffLatency))
if !ffOK {
// go-media read the file but the legacy path failed. That is an outcome
Binary file not shown.

After

Width:  |  Height:  |  Size: 4.7 KiB

+4 -5
View File
@@ -9,6 +9,7 @@ import (
"path/filepath"
"testing"
"github.com/gtsteffaniak/filebrowser/backend/internal/database/users"
"github.com/gtsteffaniak/filebrowser/backend/pkg/settings"
"github.com/kovidgoyal/imaging"
)
@@ -122,23 +123,21 @@ func TestDefaultBackgroundColor(t *testing.T) {
saved := settings.Config
defer func() { settings.Config = saved }()
dark := true
light := false
settings.Config.Frontend.Styling.DarkBackground = "#111111"
settings.Config.Frontend.Styling.LightBackground = "#ffffff"
settings.Config.UserDefaults.UI.DarkMode = &dark
settings.Config.UserDefaults.UI.DarkMode = users.DarkModeDark
if got := settings.DefaultBackgroundColor(); got != "#111111" {
t.Fatalf("dark branch: got %q, want %q", got, "#111111")
}
settings.Config.UserDefaults.UI.DarkMode = &light
settings.Config.UserDefaults.UI.DarkMode = users.DarkModeLight
if got := settings.DefaultBackgroundColor(); got != "#ffffff" {
t.Fatalf("light branch (dark disabled): got %q, want %q", got, "#ffffff")
}
settings.Config.UserDefaults.UI.DarkMode = nil
settings.Config.UserDefaults.UI.DarkMode = ""
if got := settings.DefaultBackgroundColor(); got != "#ffffff" {
t.Fatalf("light branch (unset): got %q, want %q", got, "#ffffff")
}
+109
View File
@@ -0,0 +1,109 @@
// Package mediacache holds the media metadata caches shared by the ffmpeg and
// go-media code paths. Both backends read and write the same entries, so a
// value found by one is reused by the other instead of being probed again.
//
// Every entry is keyed by file identity: path, size, and modification time in
// nanoseconds. An edited file gets a new key, so a stale value is never served
// for it; the old entry ages out after TTL.
package mediacache
import (
"context"
"fmt"
"io/fs"
"os"
"time"
"github.com/gtsteffaniak/filebrowser/backend/internal/utils"
"github.com/gtsteffaniak/go-cache/cache"
)
// TTL is how long a cached media value is kept after it is stored.
const TTL = 24 * time.Hour
// ProbeInfo is the container summary both backends produce: ffprobe through
// go-ffmpeg, and go-media. FormatName uses ffprobe's format_name strings.
type ProbeInfo struct {
Duration float64
VideoCodec string
AudioCodec string
FormatName string
}
var (
// Probes caches container probes (duration, first video and audio codec,
// format name).
Probes = cache.NewCache[ProbeInfo](TTL)
// Durations caches media length in seconds.
Durations = cache.NewCache[float64](TTL)
// SubtitleTracks caches the embedded subtitle track list of a video.
SubtitleTracks = cache.NewCache[[]utils.SubtitleTrack](TTL)
// SubtitleContent caches one embedded subtitle stream converted to WebVTT.
// Use Key.Stream for its key.
SubtitleContent = cache.NewCache[string](TTL)
)
// Key identifies one version of a file. The zero Key is never produced for a
// real file, because Path is always set.
type Key struct {
Path string
Size int64
ModTime int64 // modification time in Unix nanoseconds
}
// FromFileInfo returns the Key for path using an already-read FileInfo, so
// callers that stat or open the file anyway do not stat it twice.
func FromFileInfo(path string, info fs.FileInfo) Key {
return Key{Path: path, Size: info.Size(), ModTime: info.ModTime().UnixNano()}
}
// Stat returns the Key for path. It follows symlinks, like os.Stat.
func Stat(path string) (Key, error) {
info, err := os.Stat(path)
if err != nil {
return Key{}, err
}
return FromFileInfo(path, info), nil
}
// String returns the cache key for k. The path is quoted so a path containing
// the separator cannot collide with another file's key.
func (k Key) String() string {
return fmt.Sprintf("%q:%d:%d", k.Path, k.Size, k.ModTime)
}
// Stream returns the cache key for one stream of k, used by SubtitleContent.
func (k Key) Stream(index int) string {
return fmt.Sprintf("%s:%d", k.String(), index)
}
// ClearAll drops every cached media value. It is used when the backend that
// produced the values is switched (the admin "Always use ffmpeg" setting), so
// the newly selected backend answers fresh.
func ClearAll() {
Probes.ClearAll()
Durations.ClearAll()
SubtitleTracks.ClearAll()
SubtitleContent.ClearAll()
}
// bypassKey is the context key set by WithoutCache.
type bypassKey struct{}
// WithoutCache returns a context under which the ffmpeg code paths neither read
// nor write the shared caches. Dev-mode shadow runs use it, so the ffmpeg result
// they compare against go-media is computed fresh rather than read back from the
// value go-media just stored.
func WithoutCache(ctx context.Context) context.Context {
return context.WithValue(ctx, bypassKey{}, true)
}
// Bypassed reports whether ctx was returned by WithoutCache. A nil context is
// not bypassed.
func Bypassed(ctx context.Context) bool {
if ctx == nil {
return false
}
v, _ := ctx.Value(bypassKey{}).(bool)
return v
}
@@ -0,0 +1,157 @@
package mediacache
import (
"context"
"io/fs"
"os"
"path/filepath"
"testing"
"time"
)
// TestStatKeyChangesWithSizeAndModTime checks on disk that a size change
// or an mtime change gives a new key. The mtimes differ by whole seconds so
// the check holds on filesystems with coarse timestamps; the 1 ns case is
// covered by TestFromFileInfoNanosecondModTime.
func TestStatKeyChangesWithSizeAndModTime(t *testing.T) {
path := filepath.Join(t.TempDir(), "a.mp4")
if err := os.WriteFile(path, []byte("one"), 0o600); err != nil {
t.Fatal(err)
}
k1, err := Stat(path)
if err != nil {
t.Fatal(err)
}
if k1.Path != path || k1.Size != 3 || k1.ModTime == 0 {
t.Fatalf("unexpected key %+v", k1)
}
// Same size, different mtime.
mt := time.Unix(1_700_000_000, 0)
if err = os.Chtimes(path, mt, mt); err != nil {
t.Fatal(err)
}
k2, err := Stat(path)
if err != nil {
t.Fatal(err)
}
if k2.String() == k1.String() {
t.Fatalf("mtime change kept key %s", k2)
}
mt2 := mt.Add(time.Second)
if err = os.Chtimes(path, mt2, mt2); err != nil {
t.Fatal(err)
}
k3, _ := Stat(path)
if k3.String() == k2.String() {
t.Fatalf("1s mtime change kept key %s", k3)
}
// Same mtime, different size.
if err = os.WriteFile(path, []byte("three"), 0o600); err != nil {
t.Fatal(err)
}
if err = os.Chtimes(path, mt2, mt2); err != nil {
t.Fatal(err)
}
k4, _ := Stat(path)
if k4.String() == k3.String() {
t.Fatalf("size change kept key %s", k4)
}
}
// stubFileInfo is an fs.FileInfo with a fixed size and mtime.
type stubFileInfo struct {
size int64
modTime time.Time
}
// Name implements fs.FileInfo.
func (s stubFileInfo) Name() string { return "a.mp4" }
// Size implements fs.FileInfo.
func (s stubFileInfo) Size() int64 { return s.size }
// Mode implements fs.FileInfo.
func (s stubFileInfo) Mode() fs.FileMode { return 0o600 }
// ModTime implements fs.FileInfo.
func (s stubFileInfo) ModTime() time.Time { return s.modTime }
// IsDir implements fs.FileInfo.
func (s stubFileInfo) IsDir() bool { return false }
// Sys implements fs.FileInfo.
func (s stubFileInfo) Sys() any { return nil }
// TestFromFileInfoNanosecondModTime checks that mtimes 1 ns apart give
// different keys, independent of what a filesystem can store.
func TestFromFileInfoNanosecondModTime(t *testing.T) {
a := FromFileInfo("/m/a.mp4", stubFileInfo{size: 3, modTime: time.Unix(1_700_000_000, 123)})
b := FromFileInfo("/m/a.mp4", stubFileInfo{size: 3, modTime: time.Unix(1_700_000_000, 124)})
if a.String() == b.String() {
t.Fatalf("1ns mtime change kept key %s", a)
}
if a.ModTime != time.Unix(1_700_000_000, 123).UnixNano() {
t.Fatalf("ModTime = %d, want nanoseconds", a.ModTime)
}
}
// TestStatMissingFile checks that Stat reports a missing file.
func TestStatMissingFile(t *testing.T) {
if _, err := Stat(filepath.Join(t.TempDir(), "missing")); err == nil {
t.Fatal("expected an error for a missing file")
}
}
// TestKeyStringDoesNotCollide checks that keys of different files, and of
// different streams of one file, never collide.
func TestKeyStringDoesNotCollide(t *testing.T) {
a := Key{Path: "a:1", Size: 2, ModTime: 3}
b := Key{Path: "a", Size: 1, ModTime: 2}
if a.String() == b.String() {
t.Fatalf("keys collide: %s", a)
}
if a.Stream(1) == a.Stream(2) || a.Stream(1) == a.String() {
t.Fatalf("stream keys collide: %s %s", a.Stream(1), a.Stream(2))
}
}
// TestClearAll checks that ClearAll empties every shared cache.
func TestClearAll(t *testing.T) {
k := Key{Path: "/clear", Size: 1, ModTime: 1}
Probes.Set(k.String(), ProbeInfo{Duration: 1})
Durations.Set(k.String(), 1)
SubtitleContent.Set(k.Stream(0), "x")
ClearAll()
if _, ok := Probes.Get(k.String()); ok {
t.Fatal("probe survived ClearAll")
}
if _, ok := Durations.Get(k.String()); ok {
t.Fatal("duration survived ClearAll")
}
if _, ok := SubtitleContent.Get(k.Stream(0)); ok {
t.Fatal("subtitle content survived ClearAll")
}
}
// TestWithoutCache checks the cache bypass marker and that derived contexts
// keep it.
func TestWithoutCache(t *testing.T) {
if Bypassed(context.Background()) {
t.Fatal("background context must not bypass")
}
//nolint:staticcheck // nil is part of the documented contract.
if Bypassed(nil) {
t.Fatal("nil context must not bypass")
}
ctx := WithoutCache(context.Background())
if !Bypassed(ctx) {
t.Fatal("WithoutCache context must bypass")
}
child, cancel := context.WithCancel(ctx)
defer cancel()
if !Bypassed(child) {
t.Fatal("bypass must survive derived contexts")
}
}
+122
View File
@@ -0,0 +1,122 @@
package preview
import (
"fmt"
"image"
"image/color"
"runtime"
"testing"
"github.com/kovidgoyal/imaging"
"github.com/stretchr/testify/require"
)
// patternYCbCr is a 4:2:0 image (what a camera JPEG decodes to) with a pattern that differs in
// every row and column, so a crop off by one pixel shows up in the resized output.
func patternYCbCr(w, h int) *image.YCbCr {
return patternYCbCrRatio(w, h, image.YCbCrSubsampleRatio420)
}
// patternYCbCrRatio is patternYCbCr with the given chroma subsampling.
func patternYCbCrRatio(w, h int, r image.YCbCrSubsampleRatio) *image.YCbCr {
img := image.NewYCbCr(image.Rect(0, 0, w, h), r)
for y := 0; y < h; y++ {
for x := 0; x < w; x++ {
img.Y[img.YOffset(x, y)] = uint8((x*7 + y*13) % 251)
}
}
for i := range img.Cb {
img.Cb[i] = uint8((i*3 + i/img.CStride) % 256)
img.Cr[i] = uint8((i + 5*(i/img.CStride)) % 256)
}
return img
}
// patternNRGBA is an NRGBA image whose pixels differ in every row and column.
func patternNRGBA(w, h int) *image.NRGBA {
img := image.NewNRGBA(image.Rect(0, 0, w, h))
for y := 0; y < h; y++ {
for x := 0; x < w; x++ {
img.SetNRGBA(x, y, color.NRGBA{uint8(x % 256), uint8(y % 256), uint8((x + y) % 256), 255})
}
}
return img
}
// requireSamePixels fails unless want and got are the same size and every pixel is equal,
// comparing from each image's own origin.
func requireSamePixels(t *testing.T, want, got image.Image) {
t.Helper()
require.Equal(t, want.Bounds().Size(), got.Bounds().Size())
wb, gb := want.Bounds().Min, got.Bounds().Min
for y := 0; y < want.Bounds().Dy(); y++ {
for x := 0; x < want.Bounds().Dx(); x++ {
wr, wg, wbl, wa := want.At(wb.X+x, wb.Y+y).RGBA()
gr, gg, gbl, ga := got.At(gb.X+x, gb.Y+y).RGBA()
if wr != gr || wg != gg || wbl != gbl || wa != ga {
t.Fatalf("pixel (%d,%d) differs: imaging.Fill %v, fillCenter %v", x, y,
[4]uint32{wr, wg, wbl, wa}, [4]uint32{gr, gg, gbl, ga})
}
}
}
}
// fillCenter must produce exactly what imaging.Fill(…, imaging.Center, …) produces: the same
// pixels and the same image type, so the encoded preview is the same bytes.
func TestFillCenter_MatchesImagingFill(t *testing.T) {
sources := map[string]image.Image{
"ycbcr landscape": patternYCbCr(1200, 801),
"ycbcr portrait": patternYCbCr(601, 1000),
"nrgba square": patternNRGBA(700, 700),
"nrgba wide": patternNRGBA(1500, 300),
"small (falls back to imaging.Fill)": patternNRGBA(90, 60),
"exact target size": patternNRGBA(256, 256),
"offset origin": patternNRGBA(900, 700).SubImage(image.Rect(37, 11, 837, 611)),
}
// Every subsampling ratio, in shapes whose centered crop has an x (or y) offset and reaches
// the source's last row (or column) -- where a SubImage's buffer ends short of a full stride.
for _, r := range []image.YCbCrSubsampleRatio{
image.YCbCrSubsampleRatio444, image.YCbCrSubsampleRatio422, image.YCbCrSubsampleRatio420,
image.YCbCrSubsampleRatio440, image.YCbCrSubsampleRatio411, image.YCbCrSubsampleRatio410,
} {
for _, sz := range [][2]int{{600, 264}, {264, 600}, {601, 263}} {
sources[fmt.Sprintf("ycbcr %v %dx%d", r, sz[0], sz[1])] = patternYCbCrRatio(sz[0], sz[1], r)
}
}
sources["nycbcra 444 600x264"] = &image.NYCbCrA{YCbCr: *patternYCbCrRatio(600, 264, image.YCbCrSubsampleRatio444), A: make([]uint8, 600*264), AStride: 600}
// 264x264 is the crop of the 600x264 and 264x600 sources itself: nothing to scale.
targets := [][2]int{{256, 256}, {320, 180}, {180, 320}, {264, 264}}
filters := map[string]imaging.ResampleFilter{
"nearest": imaging.NearestNeighbor, "box": imaging.Box, "catmullrom": imaging.CatmullRom,
}
for name, src := range sources {
for _, tg := range targets {
for fname, f := range filters {
t.Run(fmt.Sprintf("%s/%dx%d/%s", name, tg[0], tg[1], fname), func(t *testing.T) {
t.Parallel()
want := imaging.Fill(src, tg[0], tg[1], imaging.Center, f)
got := fillCenter(src, tg[0], tg[1], f)
require.Equal(t, fmt.Sprintf("%T", want), fmt.Sprintf("%T", got), "image type")
requireSamePixels(t, want, got)
})
}
}
}
}
// A square thumbnail of a large photo must not allocate a full-resolution copy of it.
func TestFillCenter_NoFullSizeCopy(t *testing.T) {
const w, h = 4000, 3000
src := patternYCbCr(w, h)
fullCopy := uint64(h * h * 4) // the square crop imaging.Fill materializes as NRGBA
var before, after runtime.MemStats
runtime.ReadMemStats(&before)
out := fillCenter(src, 256, 256, imaging.Box)
runtime.ReadMemStats(&after)
require.Equal(t, image.Pt(256, 256), out.Bounds().Size())
allocated := after.TotalAlloc - before.TotalAlloc
require.Less(t, allocated, fullCopy/4,
"fillCenter allocated %d bytes; a copy of the crop would be %d", allocated, fullCopy)
}
+72 -2
View File
@@ -7,9 +7,11 @@ import (
"image"
"image/jpeg"
"io"
"math"
exif "github.com/dsoprea/go-exif/v3"
exifcommon "github.com/dsoprea/go-exif/v3/common"
"github.com/gtsteffaniak/filebrowser/backend/internal/embeddedpreview"
"github.com/kovidgoyal/imaging"
)
@@ -203,6 +205,50 @@ func (s *Service) ResizeWithSize(ctx context.Context, in io.Reader, out io.Write
return err
}
// fillCenter is imaging.Fill(img, width, height, imaging.Center, filter) without a full-size copy
// of the source. For sources of 100px or more, imaging.Fill first crops into a new NRGBA image
// (4 bytes per pixel of the cropped area, about 144 MB for a 48 MP photo) and only then resizes.
// Here the crop is a SubImage view into the decoded pixels, so only the resize allocates. The
// crop rectangle is computed as imaging's cropAndResize computes it, so the output is the same.
//
// Two cases still go through imaging.Fill: a crop that is already the target size (nothing to
// scale, and Fill's small copy is what keeps the output type the same), and 4:4:4 YCbCr sources,
// whose scanner in imaging v1.9.0 advances one full parent stride past the end of a SubImage's
// last row and panics when the crop has an x offset and reaches the bottom row.
func fillCenter(img image.Image, width, height int, filter imaging.ResampleFilter) image.Image {
b := img.Bounds()
sub, ok := img.(interface {
SubImage(image.Rectangle) image.Image
})
if !ok || is444(img) || width <= 0 || height <= 0 || b.Dx() < 100 || b.Dy() < 100 || (b.Dx() == width && b.Dy() == height) {
return imaging.Fill(img, width, height, imaging.Center, filter)
}
cropW, cropH := b.Dx(), b.Dy()
if float64(b.Dx())/float64(b.Dy()) < float64(width)/float64(height) {
cropH = int(math.Max(1, float64(b.Dx())*float64(height)/float64(width)) + 0.5)
} else {
cropW = int(math.Max(1, float64(b.Dy())*float64(width)/float64(height)) + 0.5)
}
if cropW == width && cropH == height {
return imaging.Fill(img, width, height, imaging.Center, filter)
}
x0 := b.Min.X + (b.Dx()-cropW)/2
y0 := b.Min.Y + (b.Dy()-cropH)/2
crop := image.Rect(x0, y0, x0+cropW, y0+cropH).Intersect(b)
return imaging.Resize(sub.SubImage(crop), width, height, filter)
}
// is444 reports whether img is a YCbCr image without chroma subsampling.
func is444(img image.Image) bool {
switch p := img.(type) {
case *image.YCbCr:
return p.SubsampleRatio == image.YCbCrSubsampleRatio444
case *image.NYCbCrA:
return p.SubsampleRatio == image.YCbCrSubsampleRatio444
}
return false
}
func (s *Service) resizeWithSize(in io.Reader, out io.Writer, fileSize int64, opts ResizeOptions) error {
// Set defaults
if opts.ResizeMode == 0 {
@@ -280,7 +326,7 @@ func (s *Service) resizeWithSize(in io.Reader, out io.Writer, fileSize int64, op
resampleFilter := opts.Quality.resampleFilter()
switch opts.ResizeMode {
case ResizeModeFill:
img = imaging.Fill(img, opts.Width, opts.Height, imaging.Center, resampleFilter)
img = fillCenter(img, opts.Width, opts.Height, resampleFilter)
case ResizeModeFit:
fallthrough
default:
@@ -406,8 +452,32 @@ func getEmbeddedThumbnail(in io.Reader) ([]byte, io.Reader, error) {
return nil, wrappedReader, exif.ErrNoThumbnail
}
// Read IFD1's Orientation first; fall back to IFD0's Orientation if not specified (per MWG guidelines)
orientStr := ""
tags1, err1 := ifd.FindTagWithName("Orientation")
if err1 == nil && len(tags1) > 0 {
orientStr, _ = tags1[0].FormatFirst()
}
if orientStr == "" {
tags0, err0 := index.RootIfd.FindTagWithName("Orientation")
if err0 == nil && len(tags0) > 0 {
orientStr, _ = tags0[0].FormatFirst()
}
}
thm, err := ifd.Thumbnail()
return thm, wrappedReader, err
if err != nil {
return nil, wrappedReader, err
}
if orientStr != "" && orientStr != "1" {
if !embeddedpreview.Usable(thm) {
return nil, wrappedReader, fmt.Errorf("embedded thumbnail exceeds decode limits")
}
thm = applyOrientationToPreviewBytes(thm, orientStr)
}
return thm, wrappedReader, nil
}
// CreateThumbnail decodes an image and creates a fixed-size thumbnail.
+39
View File
@@ -26,6 +26,7 @@ func TestMain(m *testing.M) {
if fileutils.PermDir == 0 {
fileutils.SetFsPermissions(0644, 0755)
}
configureImagingParallelism()
// Run the tests
code := m.Run()
@@ -549,3 +550,41 @@ func TestReadImageFileDimensions(t *testing.T) {
require.Greater(t, width, 0)
require.Greater(t, height, 0)
}
func TestApplyOrientationToPreviewBytes(t *testing.T) {
t.Parallel()
// Generate a 100x60 test JPEG image in memory (landscape)
img := image.NewRGBA(image.Rect(0, 0, 100, 60))
var buf bytes.Buffer
err := jpeg.Encode(&buf, img, nil)
require.NoError(t, err)
rawBytes := buf.Bytes()
tests := []struct {
orientation string
wantWidth int
wantHeight int
}{
{orientation: "1", wantWidth: 100, wantHeight: 60},
{orientation: "Horizontal (normal)", wantWidth: 100, wantHeight: 60},
{orientation: "8", wantWidth: 60, wantHeight: 100},
{orientation: "Rotate 270 CW", wantWidth: 60, wantHeight: 100},
{orientation: "6", wantWidth: 60, wantHeight: 100},
{orientation: "Rotate 90 CW", wantWidth: 60, wantHeight: 100},
{orientation: "3", wantWidth: 100, wantHeight: 60},
{orientation: "Rotate 180", wantWidth: 100, wantHeight: 60},
}
for _, tt := range tests {
t.Run(tt.orientation, func(t *testing.T) {
rotated := applyOrientationToPreviewBytes(rawBytes, tt.orientation)
require.NotEmpty(t, rotated)
cfg, _, decodeErr := image.DecodeConfig(bytes.NewReader(rotated))
require.NoError(t, decodeErr)
require.Equal(t, tt.wantWidth, cfg.Width)
require.Equal(t, tt.wantHeight, cfg.Height)
})
}
}
+7 -4
View File
@@ -64,7 +64,7 @@ func NewPreviewGenerator(concurrencyLimit int, cacheDir string) *Service {
var fileCache diskcache.Interface
// Use file cache if cacheDir is specified
var err error
fileCache, err = diskcache.NewFileCache(actualCacheDir)
fileCache, err = diskcache.NewFileCacheWithLimit(actualCacheDir, max(settings.Config.Server.CacheDirMaxSizeMB, 0)*1024*1024)
if err != nil {
logger.Error("The cache dir could not be created. Make sure the user that you executed the program with has access to create directories in the local path. See ")
logger.Fatalf("failed to create file cache path, which is now require to run the server: %v", err)
@@ -272,8 +272,9 @@ func determinePreviewType(file iteminfo.ExtendedFileInfo) filePreviewType {
return previewTypeOffice
}
// Check HEIC with conversion enabled
if strings.HasPrefix(file.Type, "image/heic") &&
// Check HEIC with conversion enabled. AVIF is a HEIF container too and uses
// the same path (go-media decode, ffmpeg fallback) under the HEIC setting.
if (strings.HasPrefix(file.Type, "image/heic") || file.Type == "image/avif") &&
settings.CanConvertImage(string(settings.HEICImagePreview)) {
return previewTypeHEIC
}
@@ -503,7 +504,9 @@ func GeneratePreviewWithMD5(ctx context.Context, file iteminfo.ExtendedFileInfo,
}
// Enforce file size limit for image preview generation to prevent memory exhaustion
if strings.HasPrefix(file.Type, "image") && file.Size > iteminfo.LargeFileSizeThreshold {
// Camera RAW is exempt: its preview is the embedded JPEG and the RAW data is never decoded.
if strings.HasPrefix(file.Type, "image") && file.Size > iteminfo.LargeFileSizeThreshold &&
!iteminfo.IsRawImage(strings.ToLower(filepath.Ext(file.Name))) {
message := fmt.Sprintf("Image file too large for preview: %s (size: %d bytes, limit: %d bytes)",
file.Name, file.Size, iteminfo.LargeFileSizeThreshold)
logger.Warning(message)
@@ -0,0 +1,49 @@
package preview
import (
"testing"
"github.com/gtsteffaniak/filebrowser/backend/pkg/indexing/iteminfo"
"github.com/gtsteffaniak/filebrowser/backend/pkg/settings"
)
// withHEICConversion sets ffmpeg availability and the HEIC preview setting for
// one test.
func withHEICConversion(t *testing.T, enabled bool) {
t.Helper()
prevAvail := settings.Env.FFmpegAvailable
prevPreview := settings.Config.Integrations.Media.Convert.ImagePreview
t.Cleanup(func() {
settings.Env.FFmpegAvailable = prevAvail
settings.Config.Integrations.Media.Convert.ImagePreview = prevPreview
})
settings.Env.FFmpegAvailable = true
settings.Config.Integrations.Media.Convert.ImagePreview = map[settings.ImagePreviewType]*bool{
settings.HEICImagePreview: &enabled,
}
}
// previewFile returns a file info with only a name and MIME type set.
func previewFile(name, mimeType string) iteminfo.ExtendedFileInfo {
var f iteminfo.ExtendedFileInfo
f.Name = name
f.Type = mimeType
return f
}
// TestDeterminePreviewTypeAVIF checks that AVIF uses the HEIF decode path when
// HEIC conversion is enabled, and the plain image path otherwise.
func TestDeterminePreviewTypeAVIF(t *testing.T) {
withHEICConversion(t, true)
if got := determinePreviewType(previewFile("a.avif", "image/avif")); got != previewTypeHEIC {
t.Fatalf("avif with HEIC conversion = %v, want previewTypeHEIC", got)
}
if got := determinePreviewType(previewFile("a.heic", "image/heic")); got != previewTypeHEIC {
t.Fatalf("heic with HEIC conversion = %v, want previewTypeHEIC", got)
}
withHEICConversion(t, false)
if got := determinePreviewType(previewFile("a.avif", "image/avif")); got != previewTypeImage {
t.Fatalf("avif without HEIC conversion = %v, want previewTypeImage", got)
}
}
+5 -2
View File
@@ -28,8 +28,11 @@ func (s *Service) GenerateVideoPreview(ctx context.Context, videoPath, previewSi
return nil, err
}
// go-media fast path: fit within the box, no crop. Falls back to ffmpeg when
// unsupported (e.g. HEVC, VP9, AV1) or failed.
// go-media fast path: fit within the box, no crop. go-media decodes H.264,
// HEVC, VP8, VP9, AV1, MPEG-4 Part 2, and WMV3 keyframes in pure Go. It falls
// back to ffmpeg when the container is not a go-media target (AVI, MPEG-TS,
// MPEG-PS, ...), the codec or profile is unsupported (WMV1/WMV2, VC-1
// Advanced, AV1 above 4K, ...), or the decode fails.
// In shadow mode (dev only) the ffmpeg preview also runs, for comparison.
// The shadow run can outlive the request, so it uses a context that is not cancelled with it.
shadowCtx := context.WithoutCancel(ctx)
+9
View File
@@ -19,6 +19,15 @@ func AccessPermitted(sourcePath string, indexPath utils.IndexPath, username stri
return accessDb.Permitted(sourcePath, indexPath, username)
}
// AccessPermittedAtRealPath reports whether username may access realPath after symlink
// resolution on the given index. When accessDb is not initialized, access is allowed.
func AccessPermittedAtRealPath(idx *indexing.Index, username, realPath string, isDir bool) bool {
if accessDb == nil {
return true
}
return accessDb.PermittedAtRealPath(idx, username, realPath, isDir)
}
func AllowUser(sourcePath string, indexPath utils.IndexPath, username string) error {
return accessDb.AllowUser(sourcePath, indexPath, username)
}
+20 -10
View File
@@ -14,14 +14,21 @@ const (
var (
mediaSettingsMu sync.RWMutex
mediaSettingsDefault settings.MediaSettings
mediaSettingsDefault settings.FullAdminMedia
)
// saveMediaSettings is the only write path for media settings. It clears derived
// fields (EnabledProfiles, ImagePreview) so read-time values are never persisted.
func saveMediaSettings(key string, ms settings.FullAdminMedia) error {
settings.ClearMediaDerived(&ms)
return sqlDb.SaveSetting(key, ms)
}
// InitMediaSettings loads persisted runtime media settings from SQLite and seeds from config when missing.
func InitMediaSettings() error {
if _, err := sqlDb.GetSetting(mediaSettingsDefaultKey); err != nil {
seed := settings.SanitizeMediaSettings(settings.CurrentMediaSettings())
if saveErr := sqlDb.SaveSetting(mediaSettingsDefaultKey, seed); saveErr != nil {
if saveErr := saveMediaSettings(mediaSettingsDefaultKey, seed); saveErr != nil {
return fmt.Errorf("seed media settings: %w", saveErr)
}
}
@@ -43,7 +50,7 @@ func InitMediaSettings() error {
// sanitize, so clamps (e.g. the readRate floor) must run again.
overlaid = settings.SanitizeMediaSettings(overlaid)
if !settings.MediaSettingsEqual(loaded, overlaid) {
if saveErr := sqlDb.SaveSetting(mediaSettingsDefaultKey, overlaid); saveErr != nil {
if saveErr := saveMediaSettings(mediaSettingsDefaultKey, overlaid); saveErr != nil {
return fmt.Errorf("save config-synced media settings: %w", saveErr)
}
loaded = overlaid
@@ -57,22 +64,22 @@ func InitMediaSettings() error {
return nil
}
func loadMediaSettingsSetting(key string) (settings.MediaSettings, error) {
func loadMediaSettingsSetting(key string) (settings.FullAdminMedia, error) {
raw, err := sqlDb.GetSetting(key)
if err != nil {
return settings.MediaSettings{}, err
return settings.FullAdminMedia{}, err
}
var ms settings.MediaSettings
var ms settings.FullAdminMedia
if err := json.Unmarshal(raw, &ms); err != nil {
return settings.MediaSettings{}, fmt.Errorf("parse %s: %w", key, err)
return settings.FullAdminMedia{}, fmt.Errorf("parse %s: %w", key, err)
}
return ms, nil
}
// GetMediaSettings returns the runtime media settings. The value is deep-copied
// GetMediaSettings returns the stored runtime media settings. The value is deep-copied
// so callers cannot mutate the stored settings (or the live config they are
// applied to) through the conversion maps.
func GetMediaSettings() settings.MediaSettings {
func GetMediaSettings() settings.FullAdminMedia {
mediaSettingsMu.RLock()
defer mediaSettingsMu.RUnlock()
return settings.CloneMediaSettings(mediaSettingsDefault)
@@ -90,8 +97,11 @@ func PatchMediaSettings(patchJSON []byte) error {
return mergeErr
}
merged = settings.SanitizeMediaSettings(merged)
// A patch may echo derived fields back (e.g. imagePreview from the admin UI).
// Clear them before the value reaches memory or the database.
settings.ClearMediaDerived(&merged)
if saveErr := sqlDb.SaveSetting(mediaSettingsDefaultKey, merged); saveErr != nil {
if saveErr := saveMediaSettings(mediaSettingsDefaultKey, merged); saveErr != nil {
mediaSettingsMu.Unlock()
return fmt.Errorf("save media settings: %w", saveErr)
}
@@ -1,6 +1,7 @@
package state
import (
"encoding/json"
"testing"
"github.com/gtsteffaniak/filebrowser/backend/pkg/settings"
@@ -34,6 +35,56 @@ func TestGetMediaSettingsReturnsIndependentCopy(t *testing.T) {
}
}
// Derived read-time values must never reach the database, even when a caller
// passes a struct that has them populated.
func TestSaveMediaSettingsNeverPersistsDerivedFields(t *testing.T) {
initSidebarLinkTestDB(t)
ms := GetMediaSettings()
ms.Transcode.EnabledProfiles = []string{"quality"}
ms.ImagePreview = map[settings.ImagePreviewType]bool{settings.JPEGImagePreview: true}
if err := saveMediaSettings(mediaSettingsDefaultKey, ms); err != nil {
t.Fatalf("saveMediaSettings: %v", err)
}
raw, err := sqlDb.GetSetting(mediaSettingsDefaultKey)
if err != nil {
t.Fatalf("GetSetting: %v", err)
}
var stored map[string]any
if err := json.Unmarshal(raw, &stored); err != nil {
t.Fatalf("decode stored settings: %v", err)
}
transcode, ok := stored["transcode"].(map[string]any)
if !ok {
t.Fatalf("stored transcode = %T, want object", stored["transcode"])
}
if _, ok := transcode["enabledProfiles"]; ok {
t.Fatal("enabledProfiles (derived) was persisted")
}
if _, ok := stored["imagePreview"]; ok {
t.Fatal("imagePreview (derived) was persisted")
}
}
// The admin UI echoes its whole payload back on PATCH, derived fields included.
// Those must be dropped before the value reaches memory, not only before the database.
func TestPatchMediaSettingsDropsEchoedDerivedFields(t *testing.T) {
initSidebarLinkTestDB(t)
patch := []byte(`{"imagePreview":{"jpeg":true},"transcode":{"enabledProfiles":["quality"]}}`)
if err := PatchMediaSettings(patch); err != nil {
t.Fatalf("PatchMediaSettings: %v", err)
}
got := GetMediaSettings()
if got.ImagePreview != nil {
t.Fatal("echoed imagePreview stayed in memory")
}
if got.Transcode.EnabledProfiles != nil {
t.Fatal("echoed enabledProfiles stayed in memory")
}
}
// A fresh install seeds the stored media settings from config.
func TestMediaSettingsSeedFromConfigOnFirstInit(t *testing.T) {
initSidebarLinkTestDB(t)
+12
View File
@@ -4,6 +4,7 @@ import (
"encoding/json"
"fmt"
"reflect"
"slices"
"strings"
"sync"
@@ -297,6 +298,17 @@ func ApplyUserDefaults(u *users.User) {
applyToolAccessDefaults(u)
}
// PrefillUserProfileDefaults seeds u with the effective profile defaults so a JSON decode
// overlays only the keys the request sent (encoding/json leaves absent keys untouched).
// Permissions are left zero so omitted permissions stay distinguishable; CreateUser merges them.
func PrefillUserProfileDefaults(u *users.User) {
settings.ApplyFullProfileFromDefaults(u, EffectiveUserDefaults())
// json.Unmarshal reuses an existing slice's backing array, so clone it or a request
// would overwrite the shared in-memory defaults.
u.NewFileTemplate = slices.Clone(u.NewFileTemplate)
u.Permissions = users.Permissions{}
}
func applyUserSettingsDefaults(u *users.User) {
settings.ApplyUserDefaultsFrom(u, EffectiveUserDefaults())
}
@@ -90,7 +90,7 @@ func TestStateInit_jwtYAML_organizedDefaultsOnNewUser(t *testing.T) {
if err != nil {
t.Fatal(err)
}
if !loaded.DarkMode || !loaded.Preview.Image {
if !loaded.DarkMode.IsDark() || !loaded.Preview.Image {
t.Fatalf("profile fields: darkMode=%v preview.image=%v", loaded.DarkMode, loaded.Preview.Image)
}
if loaded.BackendScopes[0].Permissions.Modify != access.Modify {
+19
View File
@@ -202,6 +202,18 @@ func UserForShareOwner(link share.Share) (users.User, error) {
// may apply default-enabled sources. In-memory/cache users always keep FrontendScopes nil; GET handlers use
// PrepForFrontend to repopulate FrontendScopes from BackendScopes for responses.
func CreateUser(user *users.User, plaintextPassword string) error {
return createUser(user, plaintextPassword, false)
}
// CreateUserKeepingProfile is CreateUser for callers whose user was seeded with
// PrefillUserProfileDefaults before decoding the request: the profile fields already hold
// defaults overlaid with the request, so they are not replaced by userDefaults again.
// Enforced defaults still win.
func CreateUserKeepingProfile(user *users.User, plaintextPassword string) error {
return createUser(user, plaintextPassword, true)
}
func createUser(user *users.User, plaintextPassword string, keepProfile bool) error {
if err := users.ValidateUsername(user.Username); err != nil {
return err
}
@@ -256,12 +268,19 @@ func CreateUser(user *users.User, plaintextPassword string) error {
requestedScopePermissions[backendScopeKeyFor(scope)] = scope.Permissions
}
}
var requestedProfile settings.UserProfile
if keepProfile {
requestedProfile = settings.ProfileFromUser(user)
}
// Apply profile/scope defaults using the final account permissions (admin flag drives scope defaults).
if permissionsRequested {
settings.ApplyUserDefaultsFromKeepingAccountPermissions(user, EffectiveUserDefaults())
} else {
applyUserSettingsDefaults(user)
}
if keepProfile {
settings.ExpandProfileIntoUser(user, requestedProfile)
}
if preserveRequirePasswordChange {
user.RequirePasswordChange = true
}
+1 -1
View File
@@ -133,7 +133,7 @@ func (e *MediaEngine) BuildPlan(ctx context.Context, realPath string, profile Pr
copyImpossible := false
plan.segmentSec = defaults.SegmentDurationSec
if plan.remux || plan.videoCopy {
rawKeyframes, kfErr := probeKeyframesCached(ctx, svc, realPath, stat.Size(), stat.ModTime().UnixNano(), plan.segmentSec, true)
rawKeyframes, kfErr := probeKeyframesCached(ctx, svc, realPath, stat.Size(), stat.ModTime().UnixNano(), plan.segmentSec, mediaSpan{durationSec: info.Duration, startTimeSec: info.StartTime}, true)
if kfErr != nil {
copyImpossible = true
logger.Warningf("transcode: keyframe probe failed on %s (%v) — forcing video transcode for HLS", realPath, kfErr)
+79 -1
View File
@@ -12,6 +12,7 @@ import (
"time"
"github.com/gtsteffaniak/filebrowser/backend/internal/ffmpeg"
"github.com/gtsteffaniak/filebrowser/backend/internal/gomedia"
"github.com/gtsteffaniak/filebrowser/backend/pkg/settings"
"github.com/gtsteffaniak/go-logger/logger"
)
@@ -55,6 +56,9 @@ type keyframeCache struct {
var defaultKeyframeCache = &keyframeCache{entries: map[keyframeKey]*keyframeEntry{}}
// get returns the keyframe timeline for key from memory, then disk, then
// probe. Concurrent callers for one key wait on a single probe; failures are
// not cached.
func (c *keyframeCache) get(ctx context.Context, key keyframeKey, probe func(context.Context) ([]float64, error)) ([]float64, error) {
c.mu.Lock()
e := c.entries[key]
@@ -105,12 +109,15 @@ func keyframeDiskCacheEnabled() bool {
return settings.Config.Server.CacheDir != ""
}
// keyframeDiskPath returns the disk cache file for key's path.
func keyframeDiskPath(key keyframeKey) string {
sum := sha256.Sum256([]byte(key.path))
name := hex.EncodeToString(sum[:])[:32] + ".json"
return filepath.Join(settings.TranscodeKeyframeCacheDir(), name)
}
// readKeyframeDiskCache returns the persisted timeline for key when it exists
// and matches the file's size, mtime, and the cache version.
func readKeyframeDiskCache(key keyframeKey) ([]float64, bool) {
if !keyframeDiskCacheEnabled() {
return nil, false
@@ -132,6 +139,8 @@ func readKeyframeDiskCache(key keyframeKey) ([]float64, bool) {
return stamp.Times, true
}
// writeKeyframeDiskCache persists a timeline for key atomically. Errors are
// ignored; the cache is an optimization.
func writeKeyframeDiskCache(key keyframeKey, times []float64) {
if len(times) == 0 || !keyframeDiskCacheEnabled() {
return
@@ -219,19 +228,88 @@ func probeKeyframesWithPreflight(ctx context.Context, svc *ffmpeg.Service, realP
)
}
// mediaSpan is the ffprobe format duration and start time of a file, the
// values the HLS planner turns into its media timeline. The zero value means
// unknown.
type mediaSpan struct {
durationSec float64
startTimeSec float64
}
// trailingGapExceedsCopyCap reports whether the stretch from the last
// keyframe to the end of the media is longer than the copy cap, measured on
// the same normalized timeline the planner builds: the final segment runs
// from the last keyframe to the media end, so such a gap forces a transcode
// just like a gap between two keyframes. An unknown span reports false.
func trailingGapExceedsCopyCap(keyframes []float64, span mediaSpan, segmentSec float64) bool {
if len(keyframes) == 0 || span.durationSec <= 0 {
return false
}
kf, mediaDur := ffmpeg.PrepareHLSMediaTimeline(keyframes, span.durationSec, span.startTimeSec)
if len(kf) == 0 || mediaDur <= 0 {
return false
}
return mediaDur-kf[len(kf)-1] > maxCopySegmentDurationSec(segmentSec)
}
// keyframeProbeTimeout bounds a single full-file keyframe probe. The default
// demux has no deadline of its own, so a slow network share would otherwise
// stall plan building (and the client's session POST) indefinitely.
const keyframeProbeTimeout = 2 * time.Minute
func probeKeyframesCached(ctx context.Context, svc *ffmpeg.Service, realPath string, size, modTime int64, segmentSec float64, preflight bool) ([]float64, error) {
// probeKeyframesCached returns the keyframe timeline of a file version from
// the memory or disk cache, or probes it: go-media reads the container index
// first, and ffprobe runs only when go-media cannot answer or reports a gap
// that would rule out stream copy. span is the file's duration and start
// time, used to check the gap after the last keyframe; the zero value skips
// that check.
func probeKeyframesCached(ctx context.Context, svc *ffmpeg.Service, realPath string, size, modTime int64, segmentSec float64, span mediaSpan, preflight bool) ([]float64, error) {
key := keyframeKey{path: realPath, size: size, modTime: modTime}
return defaultKeyframeCache.get(ctx, key, func(ctx context.Context) ([]float64, error) {
probeCtx, cancel := context.WithTimeout(ctx, keyframeProbeTimeout)
defer cancel()
if times, ok := gomediaKeyframes(probeCtx, svc, realPath, segmentSec, span, preflight); ok {
return times, nil
}
if preflight {
return probeKeyframesWithPreflight(probeCtx, svc, realPath, segmentSec)
}
return svc.ProbeVideoKeyframeTimes(probeCtx, realPath)
})
}
// gomediaKeyframes reads the keyframe timeline from the container index with
// go-media. It reports ok=false when go-media cannot answer, and also when
// preflight is set and the timeline has a gap past the copy cap, including
// the gap from the last keyframe to the end of span: go-media can miss
// keyframes that ffprobe's parsers flag (IDR frames not in the index,
// partial Matroska Cues) but never adds one, so only ffprobe may decide that
// stream copy is impossible. A bounded go-media read runs first, so a file
// whose head already shows such a gap does not pay for a full index walk.
func gomediaKeyframes(ctx context.Context, svc *ffmpeg.Service, realPath string, segmentSec float64, span mediaSpan, preflight bool) ([]float64, bool) {
if preflight {
if head, ok := gomedia.KeyframeTimesWithin(ctx, realPath, keyframeBoundedPreflightWindowSec); ok && keyframesExceedCopyCap(head, segmentSec) {
logger.Debugf("transcode: go-media keyframe pre-flight shows a gap past the copy cap on %s; confirming with ffprobe", realPath)
return nil, false
}
}
legacy := func() ([]float64, bool) {
shadowCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), keyframeProbeTimeout)
defer cancel()
times, err := svc.ProbeVideoKeyframeTimes(shadowCtx, realPath)
return times, err == nil
}
times, ok := gomedia.KeyframeTimes(ctx, realPath, legacy)
if !ok {
return nil, false
}
if preflight && keyframesExceedCopyCap(times, segmentSec) {
logger.Debugf("transcode: go-media keyframe index shows a gap past the copy cap on %s; confirming with ffprobe", realPath)
return nil, false
}
if preflight && trailingGapExceedsCopyCap(times, span, segmentSec) {
logger.Debugf("transcode: go-media keyframe index ends more than the copy cap before the end of %s; confirming with ffprobe", realPath)
return nil, false
}
return times, true
}
@@ -177,3 +177,33 @@ func TestKeyframeCacheDoesNotCacheFailures(t *testing.T) {
t.Fatalf("retry after failure: %v, %v", got, err)
}
}
// TestTrailingGapExceedsCopyCap checks the gap from the last keyframe to the
// end of the media, on the planner's start-time-normalized timeline.
func TestTrailingGapExceedsCopyCap(t *testing.T) {
every2 := func(from, to float64) []float64 {
var out []float64
for v := from; v < to; v += 2 {
out = append(out, v)
}
return out
}
cases := []struct {
name string
kf []float64
span mediaSpan
want bool
}{
{"partial index", []float64{0, 2}, mediaSpan{durationSec: 60}, true},
{"full index", every2(0, 60), mediaSpan{durationSec: 60}, false},
{"unknown span", []float64{0, 2}, mediaSpan{}, false},
{"start time, partial", []float64{10, 12}, mediaSpan{durationSec: 70, startTimeSec: 10}, true},
{"start time, full", every2(10, 70), mediaSpan{durationSec: 70, startTimeSec: 10}, false},
{"no keyframes", nil, mediaSpan{durationSec: 60}, false},
}
for _, c := range cases {
if got := trailingGapExceedsCopyCap(c.kf, c.span, 4); got != c.want {
t.Errorf("%s: got %v, want %v", c.name, got, c.want)
}
}
}
@@ -0,0 +1,111 @@
package transcoding
import (
"context"
"os"
"os/exec"
"path/filepath"
"testing"
"github.com/gtsteffaniak/filebrowser/backend/pkg/settings"
)
const keyframeFixtureMP4 = "../../../frontend/tests/playwright-files/myfolder/testdata/sample-640x480.mp4"
// TestProbeKeyframesCachedUsesGoMedia checks that an MP4 keyframe timeline is
// built without ffprobe: with no ffmpeg service, only go-media can answer.
func TestProbeKeyframesCachedUsesGoMedia(t *testing.T) {
if settings.AlwaysUseFfmpegEnabled() {
t.Skip("go-media disabled by settings")
}
prevCacheDir := settings.Config.Server.CacheDir
settings.Config.Server.CacheDir = "" // keep the disk cache out of the test
t.Cleanup(func() { settings.Config.Server.CacheDir = prevCacheDir })
path := filepath.Join(t.TempDir(), "sample.mp4")
data, err := os.ReadFile(keyframeFixtureMP4)
if err != nil {
t.Fatal(err)
}
if err = os.WriteFile(path, data, 0o644); err != nil {
t.Fatal(err)
}
st, err := os.Stat(path)
if err != nil {
t.Fatal(err)
}
times, err := probeKeyframesCached(context.Background(), nil, path, st.Size(), st.ModTime().UnixNano(), 4, mediaSpan{}, true)
if err != nil {
t.Fatalf("probeKeyframesCached: %v", err)
}
if len(times) == 0 || times[0] < 0 {
t.Fatalf("unexpected keyframes %v", times)
}
for i := 1; i < len(times); i++ {
if times[i] <= times[i-1] {
t.Fatalf("keyframes not strictly increasing: %v", times)
}
}
}
// TestGomediaKeyframesDefersCopyDecision checks that a go-media timeline with a
// gap past the copy cap is not used for the pre-flight decision (ffprobe must
// confirm it), while a plain probe still returns it.
func TestGomediaKeyframesDefersCopyDecision(t *testing.T) {
if testing.Short() {
t.Skip("encodes a clip with ffmpeg")
}
if settings.AlwaysUseFfmpegEnabled() {
t.Skip("go-media disabled by settings")
}
if _, err := exec.LookPath("ffmpeg"); err != nil {
t.Skip("ffmpeg not installed")
}
path := filepath.Join(t.TempDir(), "longgop.mp4")
out, err := exec.Command("ffmpeg", "-v", "error", "-y", "-f", "lavfi", "-i", "testsrc2=size=128x96:rate=25:duration=40",
"-c:v", "libx264", "-g", "750", "-keyint_min", "750", "-sc_threshold", "0", "-pix_fmt", "yuv420p", path).CombinedOutput()
if err != nil {
t.Skipf("ffmpeg cannot encode libx264: %v: %s", err, out)
}
if times, ok := gomediaKeyframes(context.Background(), nil, path, 4, mediaSpan{}, true); ok {
t.Fatalf("pre-flight must defer a 30s GOP to ffprobe, got %v", times)
}
times, ok := gomediaKeyframes(context.Background(), nil, path, 4, mediaSpan{}, false)
if !ok {
t.Fatal("a plain probe must return the go-media timeline")
}
if len(times) != 2 || times[0] != 0 || times[1] < 29.9 || times[1] > 30.1 {
t.Fatalf("unexpected keyframes %v, want [0 30]", times)
}
}
// TestGomediaKeyframesDefersTrailingGap checks that a go-media index whose
// last keyframe is more than the copy cap before the end of the media is not
// used for the pre-flight decision, so ffprobe can confirm it, while the
// same index is used when it reaches the end.
func TestGomediaKeyframesDefersTrailingGap(t *testing.T) {
if testing.Short() {
t.Skip("encodes a clip with ffmpeg")
}
if settings.AlwaysUseFfmpegEnabled() {
t.Skip("go-media disabled by settings")
}
if _, err := exec.LookPath("ffmpeg"); err != nil {
t.Skip("ffmpeg not installed")
}
path := filepath.Join(t.TempDir(), "short.mp4")
out, err := exec.Command("ffmpeg", "-v", "error", "-y", "-f", "lavfi", "-i", "testsrc2=size=128x96:rate=25:duration=4",
"-c:v", "libx264", "-g", "50", "-keyint_min", "50", "-sc_threshold", "0", "-pix_fmt", "yuv420p", path).CombinedOutput()
if err != nil {
t.Skipf("ffmpeg cannot encode libx264: %v: %s", err, out)
}
// The index ends at 2 s; a 60 s span leaves a 58 s final segment.
if times, ok := gomediaKeyframes(context.Background(), nil, path, 4, mediaSpan{durationSec: 60}, true); ok {
t.Fatalf("pre-flight must defer a 58s trailing gap to ffprobe, got %v", times)
}
times, ok := gomediaKeyframes(context.Background(), nil, path, 4, mediaSpan{durationSec: 4}, true)
if !ok || len(times) != 2 {
t.Fatalf("index reaching the end must be used, got %v, %v", times, ok)
}
}
+31 -8
View File
@@ -1098,14 +1098,14 @@ func TestManagerSuspendsProducerWhenAllClientsPaused(t *testing.T) {
in := testInput(t, "pauser", "/v.mp4")
sess := mustStart(t, mgr, in)
waitForStarts(t, engine, 1)
job := engine.LastJob()
if err := mgr.Heartbeat(in.Username, sess.ID, HeartbeatRequest{ClientID: "c1", Paused: true}); err != nil {
t.Fatal(err)
}
waitForProducerVODJobAttached(t, mgr, sess.ID)
suspendPausedProducers(t, mgr, time.Now().Add(time.Hour))
pollUntil(t, 2*time.Second, func() bool { return !job.Running() }, "suspended job to stop")
waitForProducerParked(t, mgr, sess.ID)
if snap := snapshotProducer(t, mgr, sess.ID); !snap.exists || !snap.suspended {
t.Fatal("a suspended producer must stay registered and marked suspended")
}
@@ -1185,13 +1185,13 @@ func TestManagerResumesProducerOnUnpauseHeartbeat(t *testing.T) {
pollUntil(t, 2*time.Second, func() bool {
return snapshotProducer(t, mgr, sess.ID).encodeHead >= 1
}, "encode head to advance past segment 0")
job := engine.LastJob()
if err := mgr.Heartbeat(in.Username, sess.ID, HeartbeatRequest{ClientID: "c1", Paused: true}); err != nil {
t.Fatal(err)
}
waitForProducerVODJobAttached(t, mgr, sess.ID)
suspendPausedProducers(t, mgr, time.Now().Add(time.Hour))
pollUntil(t, 2*time.Second, func() bool { return !job.Running() }, "suspended job to stop")
waitForProducerParked(t, mgr, sess.ID)
if err := mgr.Heartbeat(in.Username, sess.ID, HeartbeatRequest{ClientID: "c1", Paused: false}); err != nil {
t.Fatal(err)
@@ -1216,13 +1216,13 @@ func TestManagerResumesProducerOnSegmentRequest(t *testing.T) {
in := testInput(t, "pauser", "/v.mp4")
sess := mustStart(t, mgr, in)
waitForStarts(t, engine, 1)
job := engine.LastJob()
if err := mgr.Heartbeat(in.Username, sess.ID, HeartbeatRequest{ClientID: "c1", Paused: true}); err != nil {
t.Fatal(err)
}
waitForProducerVODJobAttached(t, mgr, sess.ID)
suspendPausedProducers(t, mgr, time.Now().Add(time.Hour))
pollUntil(t, 2*time.Second, func() bool { return !job.Running() }, "suspended job to stop")
waitForProducerParked(t, mgr, sess.ID)
// The suspended job reads as !Running, so a request beyond its head takes
// the restart path: the relaunched job produces the requested segment.
@@ -1246,13 +1246,13 @@ func TestManagerStopReleasesSuspendedProducer(t *testing.T) {
in := testInput(t, "pauser", "/v.mp4")
sess := mustStart(t, mgr, in)
waitForStarts(t, engine, 1)
job := engine.LastJob()
if err := mgr.Heartbeat(in.Username, sess.ID, HeartbeatRequest{ClientID: "c1", Paused: true}); err != nil {
t.Fatal(err)
}
waitForProducerVODJobAttached(t, mgr, sess.ID)
suspendPausedProducers(t, mgr, time.Now().Add(time.Hour))
pollUntil(t, 2*time.Second, func() bool { return !job.Running() }, "suspended job to stop")
waitForProducerParked(t, mgr, sess.ID)
if err := mgr.Stop(in.Username, sess.ID); err != nil {
t.Fatal(err)
@@ -1308,6 +1308,29 @@ func mustStart(t *testing.T, mgr *Manager, in StartInput) SessionInfo {
return info
}
// waitForProducerVODJobAttached waits until the session's producer has the VOD
// job wired up. StartCount can advance before runProducerJob assigns p.job.
func waitForProducerVODJobAttached(t *testing.T, mgr *Manager, sessionID string) {
t.Helper()
pollUntil(t, 5*time.Second, func() bool {
mgr.mu.Lock()
p := mgr.producers[sessionID]
ok := p != nil && p.job != nil
mgr.mu.Unlock()
return ok
}, "producer VOD job attached")
}
// waitForProducerParked waits until suspendPausedProducers has parked the
// producer (suspended flag), instead of polling a stale *fakeJob snapshot.
func waitForProducerParked(t *testing.T, mgr *Manager, sessionID string) {
t.Helper()
pollUntil(t, 5*time.Second, func() bool {
snap := snapshotProducer(t, mgr, sessionID)
return snap.exists && snap.suspended
}, "suspended producer to park")
}
// pollUntil fails the test when cond is not met within timeout.
func pollUntil(t *testing.T, timeout time.Duration, cond func() bool, msg string) {
t.Helper()
+8 -2
View File
@@ -15,12 +15,18 @@ type SubtitleTrack struct {
Embedded bool `json:"embedded"` // true for embedded subtitles, false for external files
}
// ErrSubtitleSidecarNotRegular is returned when an external subtitle path is not a regular file (e.g. symlink).
var ErrSubtitleSidecarNotRegular = errors.New("subtitle sidecar is not a regular file")
func GetSubtitleSidecarContent(subtitlePath string) (string, error) {
// validate the size is not too large
info, err := os.Stat(subtitlePath)
// Sidecars sit beside the media file; do not follow symlink leaves (see lyricsSidecar).
info, err := os.Lstat(subtitlePath)
if err != nil {
return "", err
}
if !info.Mode().IsRegular() {
return "", ErrSubtitleSidecarNotRegular
}
if info.Size() > 1024*1024*50 { // 50MB
return "", errors.New("subtitle file is too large")
}
+1
View File
@@ -22,6 +22,7 @@ import (
func setupAccessHTTPTest(t *testing.T) {
t.Helper()
lockWebGlobals(t)
dbPath := filepath.Join(t.TempDir(), "access-http-test.sqlite")
if _, err := state.Initialize(dbPath); err != nil {
t.Fatal(err)
+11 -8
View File
@@ -279,19 +279,21 @@ func archiveCreateHandler(w http.ResponseWriter, r *http.Request, d *Context) (i
if fullDestParent != fullDestFile && !state.AccessPermitted(idxTo.Path, utils.IndexPathFromNormalized(fullDestParent, true), d.User.Username) {
return http.StatusForbidden, fmt.Errorf("access denied to destination parent of %s", req.Destination)
}
parentReal, err := idxTo.GetRealPathForMutation(userScopeTo, fullDestParent)
if err != nil {
return http.StatusBadRequest, fmt.Errorf("destination parent path invalid: %v", err)
}
if err = os.MkdirAll(parentReal, fileutils.EffectiveDirPerm()); err != nil {
return http.StatusInternalServerError, fmt.Errorf("cannot create destination parent directory: %v", err)
}
destRel := strings.TrimLeft(filepath.ToSlash(req.Destination), "/")
fileName := path.Base(destRel)
if fileName == "." || fileName == "/" {
return http.StatusBadRequest, fmt.Errorf("invalid destination file name")
}
destFileReal := filepath.Join(parentReal, fileName)
destFileReal, err := idxTo.GetRealPathForMutation(userScopeTo, fullDestFile)
if err != nil {
return http.StatusBadRequest, fmt.Errorf("destination path invalid: %v", err)
}
if !state.AccessPermittedAtRealPath(idxTo, d.User.Username, destFileReal, false) {
return http.StatusForbidden, fmt.Errorf("access denied to destination %s", req.Destination)
}
if err = os.MkdirAll(filepath.Dir(destFileReal), fileutils.EffectiveDirPerm()); err != nil {
return http.StatusInternalServerError, fmt.Errorf("cannot create destination parent directory: %v", err)
}
format := strings.ToLower(strings.TrimSpace(req.Format))
if format == "" {
@@ -1270,6 +1272,7 @@ func extractArchivedSymlink(destRoot, destPath, target string, modTime time.Time
logger.Debugf("unarchive: symlink: %v", err)
return fmt.Errorf("failed to create symlink from archive entry")
}
indexing.ClearRealPathCaches()
return applyArchivedTimesAndPerm(destPath, fs.ModeSymlink, modTime)
}
@@ -0,0 +1,86 @@
package web
import (
"encoding/json"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"
"github.com/gtsteffaniak/filebrowser/backend/internal/state"
"github.com/gtsteffaniak/filebrowser/backend/internal/utils"
"github.com/gtsteffaniak/filebrowser/backend/pkg/settings"
)
func TestArchiveCreate_DeniesResolvedSymlinkDestination(t *testing.T) {
_, sourceRoot, sourceName, username := setupExtractAccessPolicyTest(t)
if err := state.DenyUser(sourceRoot, utils.IndexPathFromNormalized("/denied", true), username); err != nil {
t.Fatal(err)
}
allowedDir := filepath.Join(sourceRoot, "allowed")
deniedDir := filepath.Join(sourceRoot, "denied")
for _, dir := range []string{allowedDir, deniedDir} {
if err := os.MkdirAll(dir, 0o755); err != nil {
t.Fatal(err)
}
}
victim := filepath.Join(deniedDir, "victim")
if err := os.WriteFile(victim, []byte("keep"), 0o644); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(sourceRoot, "a.txt"), []byte("payload"), 0o644); err != nil {
t.Fatal(err)
}
link := filepath.Join(allowedDir, "report.zip")
if err := os.Symlink(filepath.Join("..", "denied", "victim"), link); err != nil {
t.Fatal(err)
}
user, err := state.GetUserByUsername(username)
if err != nil {
t.Fatal(err)
}
settings.Config.Server.SourceMap[sourceRoot].Config.DenyByDefault = false
body := `{"fromSource":"` + sourceName + `","paths":["/a.txt"],"destination":"/allowed/report.zip","format":"zip"}`
req := httptest.NewRequest(http.MethodPost, "/api/resources/archive", strings.NewReader(body))
rec := httptest.NewRecorder()
status, handlerErr := archiveCreateHandler(rec, req, &Context{User: &user})
if status != http.StatusForbidden || handlerErr == nil {
t.Fatalf("status=%d err=%v", status, handlerErr)
}
got, err := os.ReadFile(victim)
if err != nil || string(got) != "keep" {
t.Fatalf("victim content=%q err=%v", got, err)
}
}
func TestArchiveCreate_AllowsNonSymlinkDestination(t *testing.T) {
_, sourceRoot, sourceName, username := setupExtractAccessPolicyTest(t)
dest := filepath.Join(sourceRoot, "out.zip")
if err := os.WriteFile(filepath.Join(sourceRoot, "a.txt"), []byte("payload"), 0o644); err != nil {
t.Fatal(err)
}
user, err := state.GetUserByUsername(username)
if err != nil {
t.Fatal(err)
}
body := `{"fromSource":"` + sourceName + `","paths":["/a.txt"],"destination":"/out.zip","format":"zip"}`
req := httptest.NewRequest(http.MethodPost, "/api/resources/archive", strings.NewReader(body))
rec := httptest.NewRecorder()
status, handlerErr := archiveCreateHandler(rec, req, &Context{User: &user})
if status != http.StatusOK || handlerErr != nil {
t.Fatalf("status=%d err=%v body=%s", status, handlerErr, rec.Body.String())
}
var resp map[string]string
if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil {
t.Fatal(err)
}
if _, err := os.Stat(dest); err != nil {
t.Fatalf("expected archive at %s: %v", dest, err)
}
}
@@ -23,6 +23,7 @@ import (
func setupExtractAccessPolicyTest(t *testing.T) (destDir, sourceRoot, sourceName, username string) {
t.Helper()
lockWebGlobals(t)
tempDir := t.TempDir()
dbPath := filepath.Join(tempDir, "access.sqlite")
if _, err := state.Initialize(dbPath); err != nil {
+21
View File
@@ -3,11 +3,16 @@ package web
import (
"archive/tar"
"archive/zip"
"errors"
"io"
"os"
"path/filepath"
"strings"
"testing"
commonerrors "github.com/gtsteffaniak/filebrowser/backend/internal/errors"
"github.com/gtsteffaniak/filebrowser/backend/pkg/indexing"
"github.com/gtsteffaniak/filebrowser/backend/pkg/settings"
)
func TestNormalizeArchiveEntryName(t *testing.T) {
@@ -557,3 +562,19 @@ func TestExtractZip_symlinkDestination(t *testing.T) {
t.Fatalf("victim.txt should be unchanged: %q, %v", victimData, err)
}
}
func TestArchiveCreateDestination_RejectsEscapingSymlinkLeaf(t *testing.T) {
t.Parallel()
root := t.TempDir()
outside := filepath.Join(filepath.Dir(root), "outside-archive")
if err := os.MkdirAll(outside, 0o755); err != nil {
t.Fatal(err)
}
if err := os.Symlink(outside, filepath.Join(root, "report.zip")); err != nil {
t.Fatal(err)
}
idx := &indexing.Index{Source: settings.Source{Path: root}}
if _, err := idx.GetRealPathForMutation("", "/report.zip"); !errors.Is(err, commonerrors.ErrPathEscapesScope) {
t.Fatalf("expected ErrPathEscapesScope, got %v", err)
}
}
@@ -25,6 +25,7 @@ import (
func setupShareArchiveDownloadTest(t *testing.T) (sourceRoot string, sourceName string, owner *users.User) {
t.Helper()
lockWebGlobals(t)
dbPath := filepath.Join(t.TempDir(), "share-dl-test.sqlite")
if _, err := state.Initialize(dbPath); err != nil {
t.Fatal(err)
+39 -469
View File
@@ -1,16 +1,13 @@
package web
import (
"crypto/md5"
"context"
"fmt"
"io"
"net/http"
"os"
"path/filepath"
"strings"
"sync"
"time"
"unicode"
"github.com/gtsteffaniak/filebrowser/backend/internal/activity"
activitydb "github.com/gtsteffaniak/filebrowser/backend/internal/database/activity"
@@ -39,9 +36,6 @@ const (
// duplicateResultsCache caches duplicate search results for 15 seconds
var duplicateResultsCache = cache.NewCache[duplicateResponse](15 * time.Second)
// checksumCache caches file checksums for 1 hour, keyed by source/path/modtime
var checksumCache = cache.NewCache[string](1 * time.Hour)
type duplicateGroup struct {
Size int64 `json:"size"`
Count int `json:"count"`
@@ -127,10 +121,12 @@ func (s *duplicateProcessingStats) shouldStop() (bool, string) {
// identical content but different filenames)
//
// Performance Optimizations:
// - Checksums are cached for 1 hour (keyed by path/size/modtime) to speed up subsequent requests
// - Files are processed in batches to balance memory usage vs SQL query count
// - Resource limits prevent timeouts: max 2 minutes, max 10K checksum operations
// - Large fuzzy groups are skipped to avoid expensive checksum operations on false positives
// - Checksums are cached for 1 hour (keyed by path/size/modtime) to speed up subsequent requests
// The checksum budget counts only cache misses (real disk I/O); cached digests are free,
// so a re-run proceeds past the previously-examined files instead of stalling at the limit.
// - Files are processed in batches to balance memory usage vs SQL query count
// - Resource limits prevent timeouts: max 2 minutes, max 10K checksum operations
// - Large fuzzy groups are skipped to avoid expensive checksum operations on false positives
//
// Response includes incomplete flag if processing was stopped early due to resource limits.
//
@@ -170,7 +166,10 @@ func duplicatesHandler(w http.ResponseWriter, r *http.Request, d *Context) (int,
// Safety check: Reject duplicate search during active indexing to prevent resource contention
// This protects against CPU/memory spikes and ensures indexing performance is not degraded
if string(index.Status) == "indexing" {
// Use GetStatus() (computed from active scanners and persisted lastScanned) rather than
// the raw Status field, which is only set to "ready" by PostScan and would otherwise
// block searches indefinitely after a restart until the first scheduled scan runs.
if index.GetStatus() == indexing.INDEXING {
return http.StatusServiceUnavailable, fmt.Errorf("duplicate search is not available while indexing is in progress - please try again when indexing completes")
}
@@ -238,7 +237,13 @@ func duplicatesHandler(w http.ResponseWriter, r *http.Request, d *Context) (int,
startTime: time.Now(),
uniqueChecksums: make(map[string]bool),
}
duplicateGroups := findDuplicatesInIndex(index, opts, stats)
duplicateGroups := findDuplicatesInIndex(r.Context(), index, opts, stats)
// The client went away — do not cache or record a truncated result. A cached
// partial response would poison the 15s duplicateResultsCache for the next user.
if err := r.Context().Err(); err != nil {
return http.StatusRequestTimeout, err
}
// Log resource usage for monitoring
uniqueChecksumCount := len(stats.uniqueChecksums)
@@ -254,8 +259,7 @@ func duplicatesHandler(w http.ResponseWriter, r *http.Request, d *Context) (int,
Reason: stats.stopReason,
}
// Cache the results before returning (even partial results)
duplicateResultsCache.Set(cacheKey, response)
cacheDuplicateResponse(r.Context(), cacheKey, response)
if stats.stopped {
w.Header().Set("X-Search-Incomplete", "true")
@@ -267,7 +271,7 @@ func duplicatesHandler(w http.ResponseWriter, r *http.Request, d *Context) (int,
}
// findDuplicatesInIndex finds duplicates using the shared IndexDB with resource limits
func findDuplicatesInIndex(index *indexing.Index, opts *duplicatesOptions, stats *duplicateProcessingStats) []duplicateGroup {
func findDuplicatesInIndex(ctx context.Context, index *indexing.Index, opts *duplicatesOptions, stats *duplicateProcessingStats) []duplicateGroup {
// Get the shared IndexDB
indexDB := indexing.GetIndexDB()
@@ -290,6 +294,11 @@ func findDuplicatesInIndex(index *indexing.Index, opts *duplicatesOptions, stats
// Process sizes in batches
for batchStart := 0; batchStart < len(sizes); batchStart += bulkSizeQueryLimit {
if ctx.Err() != nil {
stats.stopped = true
stats.stopReason = ctx.Err().Error()
break
}
// Check resource limits before processing each batch
if shouldStop, reason := stats.shouldStop(); shouldStop {
stats.stopped = true
@@ -327,6 +336,11 @@ func findDuplicatesInIndex(index *indexing.Index, opts *duplicatesOptions, stats
// Memory optimization: Delete entries from filesBySize immediately after processing
// to allow GC to reclaim memory as we go, rather than holding all data until batch completes
for _, size := range sizeBatch {
if ctx.Err() != nil {
stats.stopped = true
stats.stopReason = ctx.Err().Error()
break
}
files, ok := filesBySize[size]
if !ok || len(files) < 2 {
// Clear entry even if not processing to free memory
@@ -345,8 +359,8 @@ func findDuplicatesInIndex(index *indexing.Index, opts *duplicatesOptions, stats
continue
}
// Group files by MIME type in memory (already ordered by type from SQL)
filesByType := groupFilesByType(files)
// Group files by MIME (+ extension when MIME is ambiguous) within this size bucket.
filesByType := groupFilesByTypeBucket(files)
// Process each MIME type group
for _, typeFiles := range filesByType {
@@ -364,29 +378,13 @@ func findDuplicatesInIndex(index *indexing.Index, opts *duplicatesOptions, stats
// Use fuzzy filename matching to group files within this MIME type
// This reduces checksum operations by grouping similar filenames
filenameGroups := groupFilesByFilename(typeFiles, size)
filenameGroups := groupFilesByFilename(ctx, typeFiles, size, stats)
totalFilesInGroups := 0
for _, g := range filenameGroups {
if len(g) >= 2 {
totalFilesInGroups += len(g)
}
}
// Process each fuzzy filename group
for _, fileGroup := range filenameGroups {
if len(fileGroup) < 2 {
continue
}
// Skip checksumming if fuzzy group is too large - groups with many files
// are unlikely to be true duplicates (fuzzy matching is too permissive)
// This optimization prevents expensive checksum operations on false positives
if len(fileGroup) > maxFuzzyGroupSize {
logger.Debugf("[Duplicates] Skipping checksum for fuzzy group with %d files (exceeds limit of %d)", len(fileGroup), maxFuzzyGroupSize)
continue
}
// Stop if we've hit the group limit
if len(groupsWithChecksums) >= maxGroups {
break
@@ -403,7 +401,7 @@ func findDuplicatesInIndex(index *indexing.Index, opts *duplicatesOptions, stats
// Verify with checksums using 3-pass progressive verification
// At this point, files match on: size + MIME type + fuzzy filename similarity (50%+)
// Large fuzzy groups (>10 files) are skipped above to avoid expensive false positives
verifiedGroups := groupFilesByChecksum(fileGroup, index, size, stats)
verifiedGroups := groupFilesByChecksum(ctx, fileGroup, index, size, stats)
// Create SearchResult objects and track checksums for merging
for _, checksumGroup := range verifiedGroups {
@@ -483,100 +481,15 @@ func findDuplicatesInIndex(index *indexing.Index, opts *duplicatesOptions, stats
})
}
// Groups are already sorted by size (largest to smallest) from SQL query
return duplicateGroups
}
// mergeGroupsByChecksum merges groups that share any checksum values
// This catches cases where files have identical content but were grouped separately
func mergeGroupsByChecksum(groups []duplicateGroupWithChecksums) []duplicateGroupWithChecksums {
if len(groups) <= 1 {
return groups
// cacheDuplicateResponse stores a duplicate search response when the client context is still active.
func cacheDuplicateResponse(ctx context.Context, cacheKey string, response duplicateResponse) {
if ctx.Err() != nil {
return
}
// Build a map: checksum -> list of group indices that contain this checksum
checksumToGroups := make(map[string][]int)
for i, group := range groups {
if group.Checksum != "" {
checksumToGroups[group.Checksum] = append(checksumToGroups[group.Checksum], i)
}
}
// Use union-find to merge groups that share checksums
parent := make([]int, len(groups))
for i := range parent {
parent[i] = i
}
var find func(int) int
find = func(x int) int {
if parent[x] != x {
parent[x] = find(parent[x]) // Path compression
}
return parent[x]
}
union := func(x, y int) {
px, py := find(x), find(y)
if px != py {
parent[px] = py
}
}
// Merge groups that share any checksum
for _, groupIndices := range checksumToGroups {
if len(groupIndices) < 2 {
continue
}
// Merge all groups that share this checksum
for i := 1; i < len(groupIndices); i++ {
union(groupIndices[0], groupIndices[i])
}
}
// Group indices by their root parent
groupsByRoot := make(map[int][]int)
for i := range groups {
root := find(i)
groupsByRoot[root] = append(groupsByRoot[root], i)
}
// Merge groups that belong to the same root
merged := make([]duplicateGroupWithChecksums, 0, len(groupsByRoot))
for _, indices := range groupsByRoot {
if len(indices) == 1 {
// No merging needed, just add the group
merged = append(merged, groups[indices[0]])
continue
}
// Merge multiple groups
combinedFiles := make([]*indexing.SearchResult, 0)
var combinedSize int64
var combinedChecksum string
for _, idx := range indices {
group := groups[idx]
combinedFiles = append(combinedFiles, group.Files...)
if combinedChecksum == "" {
combinedChecksum = group.Checksum
}
if combinedSize == 0 {
combinedSize = group.Size
}
}
// Only include merged groups with 2+ files
if len(combinedFiles) >= 2 {
merged = append(merged, duplicateGroupWithChecksums{
Size: combinedSize,
Files: combinedFiles,
Checksum: combinedChecksum,
})
}
}
return merged
duplicateResultsCache.Set(cacheKey, response)
}
func prepDuplicatesOptions(r *http.Request, d *Context) (*duplicatesOptions, error) {
@@ -624,349 +537,6 @@ func prepDuplicatesOptions(r *http.Request, d *Context) (*duplicatesOptions, err
}, nil
}
// groupFilesByChecksum groups files by partial checksum and returns groups with their checksums
func groupFilesByChecksum(files []*iteminfo.FileInfo, index *indexing.Index, fileSize int64, stats *duplicateProcessingStats) []checksumGroup {
if len(files) < 2 {
return nil
}
// PASS 1: Header checksums only (first 8KB)
// This is the fastest way to eliminate files that are definitely different
// ONLY checksum files that already matched on filename+size
headerGroups := make(map[string][]*iteminfo.FileInfo)
for _, file := range files {
// Check if we've hit the checksum operation limit
if stats.checksumOperations >= maxChecksumOperations {
logger.Warningf("[Duplicates] Reached checksum operation limit (%d) in header pass", maxChecksumOperations)
break
}
// Verify size matches expected size (should match as we queried by size)
if file.Size != fileSize {
continue
}
// Construct filesystem path for checksum computation
// index.Path is the absolute filesystem root, file.Path is index-relative
filePath := filepath.Join(index.Path, file.Path)
headerChecksum, err := computeHeaderChecksum(index.Path, filePath, fileSize, file.ModTime)
if err != nil {
continue
}
stats.checksumOperations++
stats.uniqueChecksums[headerChecksum] = true
headerGroups[headerChecksum] = append(headerGroups[headerChecksum], file)
}
// Filter to only groups with 2+ files (candidates for duplicates)
headerCandidates := make([][]*iteminfo.FileInfo, 0)
filesAfterHeaderPass := 0
for _, group := range headerGroups {
if len(group) >= 2 {
headerCandidates = append(headerCandidates, group)
filesAfterHeaderPass += len(group)
}
}
if len(headerCandidates) == 0 {
return nil
}
// PASS 2: Middle checksums (header + middle) for header-matched groups
// Only process files that matched on header to save disk I/O
middleGroups := make(map[string][]*iteminfo.FileInfo)
for _, headerGroup := range headerCandidates {
if len(headerGroup) < 2 {
continue
}
for _, file := range headerGroup {
// Check if we've hit the checksum operation limit
if stats.checksumOperations >= maxChecksumOperations {
logger.Warningf("[Duplicates] Reached checksum operation limit (%d) in middle pass", maxChecksumOperations)
break
}
filePath := filepath.Join(index.Path, file.Path)
middleChecksum, err := computeMiddleChecksum(index.Path, filePath, fileSize, file.ModTime)
if err != nil {
continue
}
stats.checksumOperations++
stats.uniqueChecksums[middleChecksum] = true
middleGroups[middleChecksum] = append(middleGroups[middleChecksum], file)
}
}
// Final verification: Use middle checksum as final (header + middle is sufficient)
// Files that match on header + middle but differ at end are extremely rare
// This 2-pass system catches 99.9%+ of differences while being faster
groups := make([]checksumGroup, 0, len(middleGroups))
for checksum, group := range middleGroups {
if len(group) >= 2 {
groups = append(groups, checksumGroup{
Files: group,
Checksum: checksum,
})
}
}
return groups
}
// computeHeaderChecksum calculates MD5 hash of only the first 8KB of a file
// This is the fastest initial pass to eliminate non-matching files
func computeHeaderChecksum(sourcePath, filePath string, size int64, modTime time.Time) (string, error) {
// Generate cache key for header-only checksum
cacheKey := fmt.Sprintf("%s:%s:%d:%d:header", sourcePath, filePath, size, modTime.Unix())
// Check cache first
if cachedChecksum, ok := checksumCache.Get(cacheKey); ok {
return cachedChecksum, nil
}
file, err := os.Open(filePath)
if err != nil {
return "", err
}
defer file.Close()
hash := md5.New()
buf := make([]byte, 8192) // 8KB buffer
// Read only first 8KB (or entire file if smaller)
n, err := io.ReadFull(file, buf)
if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF {
return "", err
}
hash.Write(buf[:n])
checksum := fmt.Sprintf("%x", hash.Sum(nil))
// Cache the result
checksumCache.Set(cacheKey, checksum)
return checksum, nil
}
// computeMiddleChecksum calculates MD5 hash of header + middle portion
// Only called when header checksums match (progressive verification)
func computeMiddleChecksum(sourcePath, filePath string, size int64, modTime time.Time) (string, error) {
// Generate cache key for header+middle checksum
cacheKey := fmt.Sprintf("%s:%s:%d:%d:middle", sourcePath, filePath, size, modTime.Unix())
// Check cache first
if cachedChecksum, ok := checksumCache.Get(cacheKey); ok {
return cachedChecksum, nil
}
file, err := os.Open(filePath)
if err != nil {
return "", err
}
defer file.Close()
hash := md5.New()
buf := make([]byte, 8192) // 8KB buffer
// Read first 8KB
n, err := io.ReadFull(file, buf)
if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF {
return "", err
}
hash.Write(buf[:n])
// For larger files, also read middle 8KB
if size > 16384 { // 16KB
middleOffset := size / 2
if _, err := file.Seek(middleOffset, 0); err == nil {
n, err := io.ReadFull(file, buf)
if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF {
return "", err
}
hash.Write(buf[:n])
}
}
checksum := fmt.Sprintf("%x", hash.Sum(nil))
// Cache the result
checksumCache.Set(cacheKey, checksum)
return checksum, nil
}
// groupFilesByFilename groups files by fuzzy filename matching
// Works with iteminfo.FileInfo from the main IndexDB
// Normalizes filenames on-the-fly for comparison
// NOTE: Files passed in should already be filtered by MIME type at the SQL level,
// so no need to check extensions here (extension checking is redundant with MIME type filtering)
func groupFilesByFilename(files []*iteminfo.FileInfo, expectedSize int64) [][]*iteminfo.FileInfo {
if len(files) == 0 {
return nil
}
groups := [][]*iteminfo.FileInfo{}
used := make(map[int]bool)
for i := 0; i < len(files); i++ {
if used[i] || files[i].Name == "" {
continue
}
group := []*iteminfo.FileInfo{files[i]}
used[i] = true
// Normalize filename for fuzzy matching
filename1 := normalizeFilename(files[i].Name)
for j := i + 1; j < len(files); j++ {
if used[j] || files[j].Name == "" {
continue
}
// Check size (should match, but safe to check)
if files[j].Size != expectedSize {
continue
}
filename2 := normalizeFilename(files[j].Name)
// Check if filenames are similar enough using levenshtein distance
if filenamesSimilar(filename1, filename2) {
group = append(group, files[j])
used[j] = true
}
}
if len(group) >= 2 {
groups = append(groups, group)
}
}
return groups
}
// normalizeFilename converts filename to lowercase and removes special characters
// to make fuzzy matching more effective
func normalizeFilename(filename string) string {
// Convert to lowercase
filename = strings.ToLower(filename)
// Remove file extension for comparison
if idx := strings.LastIndex(filename, "."); idx > 0 {
filename = filename[:idx]
}
// Remove non-alphanumeric characters except spaces and hyphens
var result strings.Builder
for _, r := range filename {
if unicode.IsLetter(r) || unicode.IsDigit(r) || r == ' ' || r == '-' {
result.WriteRune(r)
}
}
return strings.TrimSpace(result.String())
}
// filenamesSimilar checks if two normalized filenames are similar enough to be considered duplicates
func filenamesSimilar(name1, name2 string) bool {
// Exact match
if name1 == name2 {
return true
}
// Empty names don't match
if len(name1) == 0 || len(name2) == 0 {
return false
}
// Calculate Levenshtein distance
distance := levenshteinDistance(name1, name2)
maxLen := max(len(name1), len(name2))
// Require at least 50% similarity (lowered to catch more duplicates with username prefixes)
// Examples: "video.mp4" vs "username_video.mp4" should match
similarity := 1.0 - float64(distance)/float64(maxLen)
return similarity >= 0.5
}
// levenshteinDistance calculates the edit distance between two strings
func levenshteinDistance(s1, s2 string) int {
if len(s1) == 0 {
return len(s2)
}
if len(s2) == 0 {
return len(s1)
}
// Create matrix
matrix := make([][]int, len(s1)+1)
for i := range matrix {
matrix[i] = make([]int, len(s2)+1)
}
// Initialize first row and column
for i := 0; i <= len(s1); i++ {
matrix[i][0] = i
}
for j := 0; j <= len(s2); j++ {
matrix[0][j] = j
}
// Fill matrix
for i := 1; i <= len(s1); i++ {
for j := 1; j <= len(s2); j++ {
cost := 0
if s1[i-1] != s2[j-1] {
cost = 1
}
matrix[i][j] = min(
matrix[i-1][j]+1, // deletion
matrix[i][j-1]+1, // insertion
matrix[i-1][j-1]+cost, // substitution
)
}
}
return matrix[len(s1)][len(s2)]
}
// min returns the minimum of three integers
func min(a, b, c int) int {
if a < b {
if a < c {
return a
}
return c
}
if b < c {
return b
}
return c
}
// max returns the maximum of two integers
func max(a, b int) int {
if a > b {
return a
}
return b
}
// groupFilesByType groups files by MIME type
// Assumes files are already sorted by type (from SQL ORDER BY)
func groupFilesByType(files []*iteminfo.FileInfo) map[string][]*iteminfo.FileInfo {
grouped := make(map[string][]*iteminfo.FileInfo)
for _, file := range files {
grouped[file.Type] = append(grouped[file.Type], file)
}
return grouped
}
// filterFilesByPermission filters files to only include those the user is permitted to access
// This is called early in the duplicate search process to avoid processing files the user can't see
func filterFilesByPermission(files []*iteminfo.FileInfo, index *indexing.Index, username string) []*iteminfo.FileInfo {
@@ -0,0 +1,157 @@
package web
import (
"context"
"fmt"
"os"
"path/filepath"
"testing"
"time"
"github.com/gtsteffaniak/filebrowser/backend/pkg/indexing"
"github.com/gtsteffaniak/filebrowser/backend/pkg/indexing/iteminfo"
)
type duplicateCorpus struct {
dir string
files []*iteminfo.FileInfo
}
func buildDuplicateCorpus(b *testing.B) *duplicateCorpus {
b.Helper()
dir := b.TempDir()
now := time.Now()
write := func(name string, payload []byte, mutateTail bool) {
if mutateTail && len(payload) > 32 {
dup := append([]byte(nil), payload...)
dup[len(dup)-1] ^= 0xff
payload = dup
}
if err := os.WriteFile(filepath.Join(dir, name), payload, 0o600); err != nil {
b.Fatal(err)
}
}
jpegHeader := []byte{0xff, 0xd8, 0xff, 0xe0}
pngHeader := []byte{0x89, 0x50, 0x4e, 0x47}
pdfHeader := []byte("%PDF-1.4\n")
mp4Header := []byte{0x00, 0x00, 0x00, 0x18, 'f', 't', 'y', 'p'}
mkPayload := func(header []byte, size int) []byte {
out := make([]byte, size)
copy(out, header)
for i := len(header); i < len(out); i++ {
out[i] = byte(i)
}
return out
}
baseJPEG := mkPayload(jpegHeader, 65536)
basePNG := mkPayload(pngHeader, 65536)
basePDF := mkPayload(pdfHeader, 65536)
baseMP4 := mkPayload(mp4Header, 65536)
entries := []struct {
name string
typ string
data []byte
}{
{"exact-a.jpg", "image/jpeg", baseJPEG},
{"exact-b.jpg", "image/jpeg", baseJPEG},
{"renamed-copy.jpeg", "image/jpeg", baseJPEG},
{"user_exact-a.jpg", "image/jpeg", baseJPEG},
{"png-a.png", "image/png", basePNG},
{"png-b.png", "image/png", basePNG},
{"doc-a.pdf", "application/pdf", basePDF},
{"doc-b.pdf", "application/pdf", basePDF},
{"clip-a.mp4", "video/mp4", baseMP4},
{"clip-b.mp4", "video/mp4", baseMP4},
{"sized-a.bin", "application/octet-stream", baseJPEG},
{"sized-b.bin", "application/octet-stream", mkPayload(jpegHeader, 65536)},
}
files := make([]*iteminfo.FileInfo, 0, len(entries)+50)
for _, e := range entries {
write(e.name, e.data, false)
files = append(files, &iteminfo.FileInfo{
ItemInfo: iteminfo.ItemInfo{Name: e.name, Size: int64(len(e.data)), ModTime: now, Type: e.typ},
Path: "/" + e.name,
})
}
write("tail-b.jpg", baseJPEG, true)
for i := 0; i < 12; i++ {
name := fmt.Sprintf("fuzzy_photo_%d.jpg", i)
write(name, baseJPEG, false)
files = append(files, &iteminfo.FileInfo{
ItemInfo: iteminfo.ItemInfo{Name: name, Size: int64(len(baseJPEG)), ModTime: now, Type: "image/jpeg"},
Path: "/" + name,
})
}
for i := 0; i < 30; i++ {
name := fmt.Sprintf("bucket_%04d.jpg", i)
write(name, mkPayload(jpegHeader, 65536), i%7 == 0)
files = append(files, &iteminfo.FileInfo{
ItemInfo: iteminfo.ItemInfo{Name: name, Size: 65536, ModTime: now, Type: "image/jpeg"},
Path: "/" + name,
})
}
return &duplicateCorpus{dir: dir, files: files}
}
func BenchmarkDuplicateTypeBuckets(b *testing.B) {
corpus := buildDuplicateCorpus(b)
b.ResetTimer()
b.ReportAllocs()
for i := 0; i < b.N; i++ {
_ = groupFilesByTypeBucket(corpus.files)
}
}
func BenchmarkDuplicateFilenameGroups(b *testing.B) {
corpus := buildDuplicateCorpus(b)
subset := corpus.files[len(corpus.files)-30:]
ctx := context.Background()
stats := &duplicateProcessingStats{startTime: time.Now()}
b.ResetTimer()
b.ReportAllocs()
for i := 0; i < b.N; i++ {
_ = groupFilesByFilename(ctx, subset, 65536, stats)
}
}
func BenchmarkDuplicateChecksumGrouping(b *testing.B) {
corpus := buildDuplicateCorpus(b)
pair := []*iteminfo.FileInfo{corpus.files[0], corpus.files[1]}
idx := &indexing.Index{Path: corpus.dir}
ctx := context.Background()
stats := &duplicateProcessingStats{startTime: time.Now(), uniqueChecksums: map[string]bool{}}
b.ResetTimer()
b.ReportAllocs()
for i := 0; i < b.N; i++ {
_ = groupFilesByChecksum(ctx, pair, idx, pair[0].Size, stats)
}
}
func BenchmarkDuplicateMerge(b *testing.B) {
mkFiles := func(n int, prefix string) []*indexing.SearchResult {
out := make([]*indexing.SearchResult, n)
for i := range out {
out[i] = &indexing.SearchResult{Path: fmt.Sprintf("/%s/%d", prefix, i)}
}
return out
}
groups := []duplicateGroupWithChecksums{
{Size: 100, Checksum: "d1", Files: mkFiles(4, "a")},
{Size: 100, Checksum: "d1", Files: mkFiles(3, "b")},
{Size: 200, Checksum: "d2", Files: mkFiles(2, "c")},
}
b.ResetTimer()
b.ReportAllocs()
for i := 0; i < b.N; i++ {
_ = mergeGroupsByChecksum(groups)
}
}
@@ -0,0 +1,55 @@
package web
import (
"path/filepath"
"strings"
"github.com/gtsteffaniak/filebrowser/backend/pkg/indexing/iteminfo"
)
// typeBucketKey returns the MIME/extension bucket used inside a fixed-size group.
// When the MIME type already implies a single on-disk format, extension is omitted
// as a redundant pre-filter. Otherwise a normalized lowercase extension is appended.
//
// JPEG aliases: ".jpg", ".jpeg", and ".jpe" normalize to "jpg" so differently named
// JPEGs stay in one bucket without splitting the MIME-implied image/jpeg group.
func typeBucketKey(file *iteminfo.FileInfo) string {
mime := file.Type
if mimeImpliesExtension(mime) {
return mime
}
return mime + "\x1f" + normalizedFileExtension(file.Name)
}
func mimeImpliesExtension(mime string) bool {
switch mime {
case "image/jpeg", "image/png", "image/gif", "image/webp", "image/bmp", "image/tiff",
"video/mp4", "video/webm", "video/quicktime", "video/x-msvideo",
"audio/mpeg", "audio/wav", "audio/flac", "audio/ogg",
"application/pdf", "application/zip", "application/gzip",
"text/plain", "text/html", "text/css", "application/json":
return true
default:
return false
}
}
// normalizedFileExtension returns a lowercase extension without the dot, folding JPEG aliases to "jpg".
func normalizedFileExtension(name string) string {
ext := strings.ToLower(strings.TrimPrefix(filepath.Ext(name), "."))
switch ext {
case "jpeg", "jpe":
return "jpg"
default:
return ext
}
}
func groupFilesByTypeBucket(files []*iteminfo.FileInfo) map[string][]*iteminfo.FileInfo {
grouped := make(map[string][]*iteminfo.FileInfo)
for _, file := range files {
key := typeBucketKey(file)
grouped[key] = append(grouped[key], file)
}
return grouped
}
@@ -0,0 +1,23 @@
package web
import (
"testing"
"github.com/gtsteffaniak/filebrowser/backend/pkg/indexing/iteminfo"
)
func TestTypeBucketKeyJPEGAliases(t *testing.T) {
jpg := &iteminfo.FileInfo{ItemInfo: iteminfo.ItemInfo{Name: "a.jpg", Type: "image/jpeg"}}
jpeg := &iteminfo.FileInfo{ItemInfo: iteminfo.ItemInfo{Name: "b.jpeg", Type: "image/jpeg"}}
if typeBucketKey(jpg) != typeBucketKey(jpeg) {
t.Fatalf("jpg and jpeg should share bucket: %q vs %q", typeBucketKey(jpg), typeBucketKey(jpeg))
}
}
func TestTypeBucketKeyAmbiguousMIMEUsesExtension(t *testing.T) {
a := &iteminfo.FileInfo{ItemInfo: iteminfo.ItemInfo{Name: "a.bin", Type: "application/octet-stream"}}
b := &iteminfo.FileInfo{ItemInfo: iteminfo.ItemInfo{Name: "b.dat", Type: "application/octet-stream"}}
if typeBucketKey(a) == typeBucketKey(b) {
t.Fatal("different extensions should not share bucket for ambiguous MIME")
}
}
@@ -0,0 +1,60 @@
package web
import (
"context"
"testing"
"time"
)
func TestChecksumCacheRejectsExpiredDigest(t *testing.T) {
key := checksumCacheKey{
sourcePath: "/src",
filePath: "/file-a",
size: 4096,
modTime: 42,
kind: checksumKindHeader,
}
encoded := checksumCacheKeyString(key)
checksumTypedCache.SetWithExp(encoded, "digest", time.Millisecond)
time.Sleep(5 * time.Millisecond)
if _, ok := getChecksumFromCache(key); ok {
t.Fatal("expired digest should not be returned")
}
}
func TestChecksumCacheMissAfterFileIdentityChange(t *testing.T) {
modA := checksumCacheKey{
sourcePath: "/src",
filePath: "/file-a",
size: 4096,
modTime: 100,
kind: checksumKindHeader,
}
modB := modA
modB.modTime = 200
checksumTypedCache.Set(checksumCacheKeyString(modA), "digest-a")
if _, ok := getChecksumFromCache(modB); ok {
t.Fatal("modified file identity must not hit the prior cache key")
}
}
func TestCacheDuplicateResponseSkipsCanceledContext(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
cancel()
cacheKey := "cancel-test:" + time.Now().Format(time.RFC3339Nano)
cacheDuplicateResponse(ctx, cacheKey, duplicateResponse{
Groups: []duplicateGroup{{Size: 1, Count: 2}},
})
if _, ok := duplicateResultsCache.Get(cacheKey); ok {
t.Fatal("canceled context must not cache duplicate results")
}
cacheDuplicateResponse(context.Background(), cacheKey, duplicateResponse{
Groups: []duplicateGroup{{Size: 2, Count: 2}},
})
if _, ok := duplicateResultsCache.Get(cacheKey); !ok {
t.Fatal("active context should cache duplicate results")
}
}
+264
View File
@@ -0,0 +1,264 @@
package web
import (
"context"
"encoding/hex"
"fmt"
"hash"
"io"
"os"
"path/filepath"
"sync"
"time"
"github.com/cespare/xxhash/v2"
"github.com/gtsteffaniak/filebrowser/backend/pkg/indexing"
"github.com/gtsteffaniak/filebrowser/backend/pkg/indexing/iteminfo"
"github.com/gtsteffaniak/go-cache/cache"
)
const (
checksumSampleSize = 8192
checksumSmallFile = 16384
checksumCacheTTL = time.Hour
)
type checksumSampleKind uint8
const (
checksumKindHeader checksumSampleKind = iota
checksumKindMiddle
)
// checksumCacheKey identifies a cached sample digest for a file snapshot.
type checksumCacheKey struct {
sourcePath string
filePath string
size int64
modTime int64
kind checksumSampleKind
}
var (
// checksumTypedCache stores sample digests keyed by string-encoded checksumCacheKey.
// The go-cache backend sweeps expired entries on an hourly ticker, so stale keys from
// renamed/deleted/modified files are evicted without a lookup — unlike a bare sync.Map
// which would grow without bound. Cleanup interval is the second arg (1h, matching TTL).
checksumTypedCache = cache.NewCache[string](checksumCacheTTL, checksumCacheTTL)
checksumBufPool = sync.Pool{
New: func() any {
buf := make([]byte, checksumSampleSize)
return &buf
},
}
)
type fileSampleDigests struct {
header string
middle string
}
func duplicateChecksumHasher() hash.Hash {
return xxhash.New()
}
func digestHex(h hash.Hash) string {
sum := h.Sum(nil)
dst := make([]byte, hex.EncodedLen(len(sum)))
hex.Encode(dst, sum)
return string(dst)
}
// checksumCacheKeyString encodes a typed cache key as a single string so it can
// be stored in the TTL-sweeping go-cache. Length prefixes keep fields unambiguous.
func checksumCacheKeyString(key checksumCacheKey) string {
return fmt.Sprintf("%d|%d|%d|%d|%s|%s",
key.kind, key.size, key.modTime, len(key.sourcePath),
key.sourcePath, key.filePath)
}
func getChecksumFromCache(key checksumCacheKey) (string, bool) {
return checksumTypedCache.Get(checksumCacheKeyString(key))
}
func setChecksumCache(key checksumCacheKey, digest string) {
checksumTypedCache.Set(checksumCacheKeyString(key), digest)
}
func readFileSampleDigests(ctx context.Context, sourcePath, filePath string, size int64, modTime time.Time, stats *duplicateProcessingStats) (fileSampleDigests, error) {
headerKey := checksumCacheKey{sourcePath: sourcePath, filePath: filePath, size: size, modTime: modTime.UnixNano(), kind: checksumKindHeader}
middleKey := checksumCacheKey{sourcePath: sourcePath, filePath: filePath, size: size, modTime: modTime.UnixNano(), kind: checksumKindMiddle}
var out fileSampleDigests
if h, ok := getChecksumFromCache(headerKey); ok {
out.header = h
}
if m, ok := getChecksumFromCache(middleKey); ok {
out.middle = m
}
if out.header != "" && out.middle != "" {
return out, nil
}
if ctx.Err() != nil {
return fileSampleDigests{}, ctx.Err()
}
// Only count operations that actually touch disk. Cache hits are O(1) lookups
// and must not consume the checksum budget — otherwise a second search over
// the same files stalls at the limit without reading a single new byte.
// The budget check lives here (post-cache) so cached digests are always served.
if shouldStop, _ := stats.shouldStop(); shouldStop {
return fileSampleDigests{}, fmt.Errorf("duplicate search budget exhausted")
}
file, err := os.Open(filePath)
if err != nil {
return fileSampleDigests{}, err
}
defer file.Close()
// A disk read is happening — charge it against the checksum budget.
stats.checksumOperations++
raw := checksumBufPool.Get()
bufPtr, ok := raw.(*[]byte)
if !ok {
return fileSampleDigests{}, fmt.Errorf("invalid checksum buffer pool type")
}
buf := *bufPtr
defer checksumBufPool.Put(bufPtr)
if size <= checksumSmallFile {
h := duplicateChecksumHasher()
remain := size
for remain > 0 {
if ctx.Err() != nil {
return fileSampleDigests{}, ctx.Err()
}
toRead := int64(len(buf))
if toRead > remain {
toRead = remain
}
n, readErr := file.Read(buf[:toRead])
if n > 0 {
_, _ = h.Write(buf[:n])
remain -= int64(n)
}
if readErr == io.EOF {
break
}
if readErr != nil {
return fileSampleDigests{}, readErr
}
if n == 0 {
break
}
}
d := digestHex(h)
out.header = d
out.middle = d
} else if out.header == "" || out.middle == "" {
n, readErr := io.ReadFull(file, buf)
if readErr != nil && readErr != io.EOF && readErr != io.ErrUnexpectedEOF {
return fileSampleDigests{}, readErr
}
if out.header == "" {
h := duplicateChecksumHasher()
_, _ = h.Write(buf[:n])
out.header = digestHex(h)
}
if out.middle == "" {
middle := duplicateChecksumHasher()
_, _ = middle.Write(buf[:n])
middleOffset := size / 2
if _, err := file.Seek(middleOffset, io.SeekStart); err != nil {
return fileSampleDigests{}, err
}
n, readErr := io.ReadFull(file, buf)
if readErr != nil && readErr != io.EOF && readErr != io.ErrUnexpectedEOF {
return fileSampleDigests{}, readErr
}
_, _ = middle.Write(buf[:n])
out.middle = digestHex(middle)
}
}
if out.header != "" {
setChecksumCache(headerKey, out.header)
}
if out.middle != "" {
setChecksumCache(middleKey, out.middle)
}
return out, nil
}
func groupFilesByChecksum(ctx context.Context, files []*iteminfo.FileInfo, index *indexing.Index, fileSize int64, stats *duplicateProcessingStats) []checksumGroup {
if len(files) < 2 {
return nil
}
headerGroups := make(map[string][]*iteminfo.FileInfo)
digestByPath := make(map[string]fileSampleDigests, len(files))
for _, file := range files {
if ctx.Err() != nil {
return nil
}
// Budget checks live inside readFileSampleDigests (after the cache lookup)
// so cached digests still count toward grouping without consuming budget.
// Checking here would stall on cached files and look stuck.
if file.Size != fileSize {
continue
}
filePath := filepath.Join(index.Path, file.Path)
digests, err := readFileSampleDigests(ctx, index.Path, filePath, fileSize, file.ModTime, stats)
if err != nil {
continue
}
digestByPath[file.Path] = digests
stats.uniqueChecksums[digests.header] = true
headerGroups[digests.header] = append(headerGroups[digests.header], file)
}
headerCandidates := make([][]*iteminfo.FileInfo, 0)
for _, group := range headerGroups {
if len(group) >= 2 {
headerCandidates = append(headerCandidates, group)
}
}
if len(headerCandidates) == 0 {
return nil
}
middleGroups := make(map[string][]*iteminfo.FileInfo)
for _, headerGroup := range headerCandidates {
for _, file := range headerGroup {
if ctx.Err() != nil {
return nil
}
if shouldStop, _ := stats.shouldStop(); shouldStop {
break
}
digests := digestByPath[file.Path]
if digests.middle == "" {
continue
}
stats.uniqueChecksums[digests.middle] = true
middleGroups[digests.middle] = append(middleGroups[digests.middle], file)
}
}
groups := make([]checksumGroup, 0, len(middleGroups))
for checksum, group := range middleGroups {
if len(group) >= 2 {
groups = append(groups, checksumGroup{
Files: group,
Checksum: checksum,
})
}
}
return groups
}
@@ -0,0 +1,116 @@
package web
import (
"context"
"crypto/md5"
"encoding/hex"
"os"
"path/filepath"
"testing"
"time"
"github.com/cespare/xxhash/v2"
)
func TestReadFileSampleDigestsSmallFileUsesWholeFile(t *testing.T) {
dir := t.TempDir()
data := make([]byte, 12_000)
path := filepath.Join(dir, "small.bin")
if err := os.WriteFile(path, data, 0o600); err != nil {
t.Fatal(err)
}
stats := &duplicateProcessingStats{startTime: time.Now()}
d, err := readFileSampleDigests(context.Background(), dir, path, int64(len(data)), time.Now(), stats)
if err != nil {
t.Fatal(err)
}
if d.header != d.middle {
t.Fatalf("small file header and middle digest should match: %q %q", d.header, d.middle)
}
}
func TestHasherUsesXXHash(t *testing.T) {
if _, ok := any(duplicateChecksumHasher()).(*xxhash.Digest); !ok {
t.Fatal("expected xxhash digest")
}
}
func TestCachedDigestsDoNotConsumeChecksumBudget(t *testing.T) {
dir := t.TempDir()
dataA := make([]byte, 65536)
for i := range dataA {
dataA[i] = byte(i % 251)
}
// Use a distinct payload so its digests differ from other tests' files.
pathA := filepath.Join(dir, "budget-a.bin")
pathB := filepath.Join(dir, "budget-b.bin")
if err := os.WriteFile(pathA, dataA, 0o600); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(pathB, dataA, 0o600); err != nil {
t.Fatal(err)
}
modTime := time.Date(2026, 11, 10, 0, 0, 0, 0, time.UTC)
// Prime the cache with one op-worth of budget.
prime := &duplicateProcessingStats{startTime: time.Now(), uniqueChecksums: map[string]bool{}}
d1, err := readFileSampleDigests(context.Background(), dir, pathA, int64(len(dataA)), modTime, prime)
if err != nil {
t.Fatal(err)
}
d2, err := readFileSampleDigests(context.Background(), dir, pathB, int64(len(dataA)), modTime, prime)
if err != nil {
t.Fatal(err)
}
if d1.header != d2.header || d1.middle != d2.middle {
t.Fatal("expected equal digests for identical files")
}
if prime.checksumOperations != 2 {
t.Fatalf("expected 2 disk operations for two cache-miss reads, got %d", prime.checksumOperations)
}
// Exhaust the budget: any further uncached file should be refused.
prime.checksumOperations = maxChecksumOperations
uncached := filepath.Join(dir, "budget-c.bin")
if err = os.WriteFile(uncached, dataA, 0o600); err != nil {
t.Fatal(err)
}
if _, err = readFileSampleDigests(context.Background(), dir, uncached, int64(len(dataA)), modTime, prime); err == nil {
t.Fatal("expected budget-exhausted error for uncached file")
}
// But the cached files must still be served, for free.
cached, err := readFileSampleDigests(context.Background(), dir, pathA, int64(len(dataA)), modTime, prime)
if err != nil {
t.Fatalf("cached digest should be served despite exhausted budget: %v", err)
}
if cached.header != d1.header || cached.middle != d1.middle {
t.Fatal("cached digest mismatch")
}
if prime.checksumOperations != maxChecksumOperations {
t.Fatalf("cached digest must not consume budget, got %d", prime.checksumOperations)
}
}
func BenchmarkDuplicateHashers(b *testing.B) {
data := make([]byte, checksumSampleSize)
for i := range data {
data[i] = byte(i)
}
b.Run("md5", func(b *testing.B) {
b.ReportAllocs()
for i := 0; i < b.N; i++ {
h := md5.New()
_, _ = h.Write(data)
hex.EncodeToString(h.Sum(nil))
}
})
b.Run("xxhash", func(b *testing.B) {
b.ReportAllocs()
for i := 0; i < b.N; i++ {
h := xxhash.New()
_, _ = h.Write(data)
hex.EncodeToString(h.Sum(nil))
}
})
}
+111
View File
@@ -0,0 +1,111 @@
package web
import (
"cmp"
"slices"
"strings"
"github.com/gtsteffaniak/filebrowser/backend/pkg/indexing"
)
type mergeGroupKey struct {
size int64
digest string
}
// mergeGroupsByChecksum merges groups that share the same size and sample digest.
func mergeGroupsByChecksum(groups []duplicateGroupWithChecksums) []duplicateGroupWithChecksums {
if len(groups) <= 1 {
return groups
}
keyToGroups := make(map[mergeGroupKey][]int)
for i, group := range groups {
if group.Checksum == "" {
continue
}
key := mergeGroupKey{size: group.Size, digest: group.Checksum}
keyToGroups[key] = append(keyToGroups[key], i)
}
parent := make([]int, len(groups))
for i := range parent {
parent[i] = i
}
var find func(int) int
find = func(x int) int {
if parent[x] != x {
parent[x] = find(parent[x])
}
return parent[x]
}
union := func(x, y int) {
px, py := find(x), find(y)
if px != py {
parent[px] = py
}
}
for _, indices := range keyToGroups {
for i := 1; i < len(indices); i++ {
union(indices[0], indices[i])
}
}
groupsByRoot := make(map[int][]int)
for i := range groups {
root := find(i)
groupsByRoot[root] = append(groupsByRoot[root], i)
}
merged := make([]duplicateGroupWithChecksums, 0, len(groupsByRoot))
for _, indices := range groupsByRoot {
if len(indices) == 1 {
merged = append(merged, groups[indices[0]])
continue
}
combinedFiles := make([]*indexing.SearchResult, 0)
var combinedSize int64
var combinedChecksum string
for _, idx := range indices {
group := groups[idx]
combinedFiles = append(combinedFiles, group.Files...)
if combinedChecksum == "" {
combinedChecksum = group.Checksum
}
if combinedSize == 0 {
combinedSize = group.Size
}
}
if len(combinedFiles) >= 2 {
slices.SortFunc(combinedFiles, func(a, b *indexing.SearchResult) int {
return strings.Compare(a.Path, b.Path)
})
merged = append(merged, duplicateGroupWithChecksums{
Size: combinedSize,
Files: combinedFiles,
Checksum: combinedChecksum,
})
}
}
slices.SortFunc(merged, func(a, b duplicateGroupWithChecksums) int {
if c := cmp.Compare(b.Size, a.Size); c != 0 {
return c
}
if c := strings.Compare(a.Checksum, b.Checksum); c != 0 {
return c
}
if len(a.Files) == 0 || len(b.Files) == 0 {
return 0
}
return strings.Compare(a.Files[0].Path, b.Files[0].Path)
})
return merged
}
@@ -0,0 +1,19 @@
package web
import (
"testing"
"github.com/gtsteffaniak/filebrowser/backend/pkg/indexing"
)
func TestMergeGroupsByChecksumUsesSizeAndDigest(t *testing.T) {
groups := []duplicateGroupWithChecksums{
{Size: 100, Checksum: "aaaa", Files: []*indexing.SearchResult{{Path: "/a"}, {Path: "/b"}}},
{Size: 200, Checksum: "aaaa", Files: []*indexing.SearchResult{{Path: "/c"}, {Path: "/d"}}},
{Size: 100, Checksum: "aaaa", Files: []*indexing.SearchResult{{Path: "/e"}, {Path: "/f"}}},
}
merged := mergeGroupsByChecksum(groups)
if len(merged) != 2 {
t.Fatalf("expected 2 merged groups, got %d", len(merged))
}
}
+204
View File
@@ -0,0 +1,204 @@
package web
import (
"context"
"strings"
"unicode"
"github.com/gtsteffaniak/filebrowser/backend/pkg/indexing/iteminfo"
)
const filenameSimilarityThreshold = 0.5
type levenshteinRowBuf struct {
prev, cur []int
}
func (b *levenshteinRowBuf) ensure(width int) {
if cap(b.prev) < width+1 {
b.prev = make([]int, width+1)
b.cur = make([]int, width+1)
} else {
b.prev = b.prev[:width+1]
b.cur = b.cur[:width+1]
}
}
func maxEditDistanceForSimilarity(maxLen int) int {
return maxLen / 2
}
func filenameSimilarWithBudget(name1, name2 string, buf *levenshteinRowBuf) bool {
if name1 == name2 {
return true
}
if len(name1) == 0 || len(name2) == 0 {
return false
}
maxLen := max(len(name1), len(name2))
maxDist := maxEditDistanceForSimilarity(maxLen)
if abs(len(name1)-len(name2)) > maxDist {
return false
}
distance := boundedLevenshtein(name1, name2, maxDist, buf)
if distance < 0 {
return false
}
similarity := 1.0 - float64(distance)/float64(maxLen)
return similarity >= filenameSimilarityThreshold
}
func boundedLevenshtein(s1, s2 string, maxDist int, buf *levenshteinRowBuf) int {
n, m := len(s1), len(s2)
if n == 0 {
if m <= maxDist {
return m
}
return -1
}
if m == 0 {
if n <= maxDist {
return n
}
return -1
}
buf.ensure(m)
prev, cur := buf.prev, buf.cur
for j := 0; j <= m; j++ {
prev[j] = j
}
for i := 1; i <= n; i++ {
cur[0] = i
rowMin := cur[0]
for j := 1; j <= m; j++ {
cost := 0
if s1[i-1] != s2[j-1] {
cost = 1
}
del := prev[j] + 1
ins := cur[j-1] + 1
sub := prev[j-1] + cost
v := del
if ins < v {
v = ins
}
if sub < v {
v = sub
}
cur[j] = v
if v < rowMin {
rowMin = v
}
}
if rowMin > maxDist {
return -1
}
prev, cur = cur, prev
}
if prev[m] > maxDist {
return -1
}
return prev[m]
}
func groupFilesByFilename(ctx context.Context, files []*iteminfo.FileInfo, expectedSize int64, stats *duplicateProcessingStats) [][]*iteminfo.FileInfo {
if len(files) == 0 {
return nil
}
normalized := make([]string, len(files))
for i, f := range files {
if f.Name != "" {
normalized[i] = normalizeFilename(f.Name)
}
}
var rowBuf levenshteinRowBuf
groups := [][]*iteminfo.FileInfo{}
used := make(map[int]bool)
for i := 0; i < len(files); i++ {
if ctx.Err() != nil {
return groups
}
if shouldStop, _ := stats.shouldStop(); shouldStop {
return groups
}
if used[i] || files[i].Name == "" {
continue
}
group := []*iteminfo.FileInfo{files[i]}
used[i] = true
filename1 := normalized[i]
oversized := false
for j := i + 1; j < len(files); j++ {
if ctx.Err() != nil {
return groups
}
if shouldStop, _ := stats.shouldStop(); shouldStop {
return groups
}
if used[j] || files[j].Name == "" {
continue
}
if files[j].Size != expectedSize {
continue
}
if filenameSimilarWithBudget(filename1, normalized[j], &rowBuf) {
group = append(group, files[j])
used[j] = true
if len(group) > maxFuzzyGroupSize {
oversized = true
break
}
}
}
if len(group) >= 2 && !oversized {
groups = append(groups, group)
}
}
return groups
}
func normalizeFilename(filename string) string {
filename = strings.ToLower(filename)
if idx := strings.LastIndex(filename, "."); idx > 0 {
filename = filename[:idx]
}
var result strings.Builder
for _, r := range filename {
if unicode.IsLetter(r) || unicode.IsDigit(r) || r == ' ' || r == '-' {
result.WriteRune(r)
}
}
return strings.TrimSpace(result.String())
}
func abs(x int) int {
if x < 0 {
return -x
}
return x
}
func max(a, b int) int {
if a > b {
return a
}
return b
}
@@ -0,0 +1,140 @@
package web
import (
"context"
"math/rand"
"strings"
"testing"
"testing/quick"
"time"
"unicode"
"github.com/gtsteffaniak/filebrowser/backend/pkg/indexing/iteminfo"
)
func legacyFilenamesSimilar(name1, name2 string) bool {
if name1 == name2 {
return true
}
if len(name1) == 0 || len(name2) == 0 {
return false
}
distance := legacyLevenshtein(name1, name2)
maxLen := max(len(name1), len(name2))
similarity := 1.0 - float64(distance)/float64(maxLen)
return similarity >= 0.5
}
func legacyLevenshtein(s1, s2 string) int {
if len(s1) == 0 {
return len(s2)
}
if len(s2) == 0 {
return len(s1)
}
matrix := make([][]int, len(s1)+1)
for i := range matrix {
matrix[i] = make([]int, len(s2)+1)
}
for i := 0; i <= len(s1); i++ {
matrix[i][0] = i
}
for j := 0; j <= len(s2); j++ {
matrix[0][j] = j
}
for i := 1; i <= len(s1); i++ {
for j := 1; j <= len(s2); j++ {
cost := 0
if s1[i-1] != s2[j-1] {
cost = 1
}
del := matrix[i-1][j] + 1
ins := matrix[i][j-1] + 1
sub := matrix[i-1][j-1] + cost
v := del
if ins < v {
v = ins
}
if sub < v {
v = sub
}
matrix[i][j] = v
}
}
return matrix[len(s1)][len(s2)]
}
var filenameCorpus = []string{
"photo", "photo_1", "photo-copy", "IMG_0042", "img 0042",
"vacation2023beach", "vacation2023_beach", "user_video", "video",
"document-final", "document_final_v2", "a", "ab", "abc", "abcd",
"report Q1 2024", "report-q1-2024", "my@file#name", "myfilename",
"summer-holiday", "summer holiday", "winter-holiday",
}
func TestFilenameSimilarMatchesLegacyOnCorpus(t *testing.T) {
names := append([]string{}, filenameCorpus...)
for i := 0; i < 50; i++ {
names = append(names, randomNormalizedName(5+corpusRand.Intn(40)))
}
var buf levenshteinRowBuf
for _, a := range names {
a = normalizeFilename(a + ".jpg")
for _, b := range names {
b = normalizeFilename(b + ".png")
if legacyFilenamesSimilar(a, b) != filenameSimilarWithBudget(a, b, &buf) {
t.Fatalf("mismatch %q vs %q", a, b)
}
}
}
}
func TestFilenameSimilarMatchesLegacyQuick(t *testing.T) {
var buf levenshteinRowBuf
if err := quick.Check(func(a, b string) bool {
a = trimQuickString(a, 48)
b = trimQuickString(b, 48)
return legacyFilenamesSimilar(a, b) == filenameSimilarWithBudget(a, b, &buf)
}, nil); err != nil {
t.Fatal(err)
}
}
func trimQuickString(s string, max int) string {
if len(s) > max {
s = s[:max]
}
var b strings.Builder
for _, r := range s {
if unicode.IsLetter(r) || unicode.IsDigit(r) || r == ' ' || r == '-' {
b.WriteRune(r)
}
}
return strings.ToLower(strings.TrimSpace(b.String()))
}
func randomNormalizedName(n int) string {
const alphabet = "abcdefghijklmnopqrstuvwxyz0123456789 -"
b := make([]byte, n)
for i := range b {
b[i] = alphabet[corpusRand.Intn(len(alphabet))]
}
return string(b)
}
func TestGroupFilesByFilenameOversizedExcluded(t *testing.T) {
ctx := context.Background()
stats := &duplicateProcessingStats{startTime: time.Now()}
files := make([]*iteminfo.FileInfo, 12)
for i := range files {
files[i] = &iteminfo.FileInfo{
ItemInfo: iteminfo.ItemInfo{Name: "photo.jpg", Size: 100},
}
}
if len(groupFilesByFilename(ctx, files, 100, stats)) != 0 {
t.Fatal("expected no groups for oversized cluster")
}
}
var corpusRand = rand.New(rand.NewSource(42))
+12 -6
View File
@@ -187,10 +187,10 @@ func fileWatchHandler(w http.ResponseWriter, r *http.Request, d *Context) (int,
return http.StatusForbidden, fmt.Errorf("access denied to file")
}
// Get real file path
realPath, _, err := idx.GetRealPath(scopePath)
// Get real file path (scoped) and enforce ACL on the resolved target
realPath, _, err := idx.GetRealPathScoped(userScope, scopePath)
if err != nil {
return http.StatusNotFound, fmt.Errorf("file not found: %v", err)
return realPathErrStatus(err), err
}
// Get file/directory info
@@ -198,6 +198,9 @@ func fileWatchHandler(w http.ResponseWriter, r *http.Request, d *Context) (int,
if err != nil {
return http.StatusNotFound, fmt.Errorf("path not found: %v", err)
}
if !state.AccessPermittedAtRealPath(idx, d.User.Username, realPath, info.IsDir()) {
return http.StatusForbidden, fmt.Errorf("access denied to file")
}
// Get MIME type from the index if available
mimeType := "application/octet-stream"
@@ -335,10 +338,10 @@ func fileWatchSSEHandler(w http.ResponseWriter, r *http.Request, d *Context) (in
return http.StatusForbidden, fmt.Errorf("access denied to file")
}
// Get real file path
realPath, _, err := idx.GetRealPath(scopePath)
// Get real file path (scoped) and enforce ACL on the resolved target
realPath, _, err := idx.GetRealPathScoped(userScope, scopePath)
if err != nil {
return http.StatusNotFound, fmt.Errorf("path not found: %v", err)
return realPathErrStatus(err), err
}
// Get file/directory info
@@ -346,6 +349,9 @@ func fileWatchSSEHandler(w http.ResponseWriter, r *http.Request, d *Context) (in
if err != nil {
return http.StatusNotFound, fmt.Errorf("path not found: %v", err)
}
if !state.AccessPermittedAtRealPath(idx, d.User.Username, realPath, info.IsDir()) {
return http.StatusForbidden, fmt.Errorf("access denied to file")
}
// Set up SSE headers
w.Header().Set("Content-Type", "text/event-stream")
+34
View File
@@ -22,3 +22,37 @@ func TestErrToStatusWrapsUnsupportedPreviewFormat(t *testing.T) {
t.Fatalf("ErrToStatus() = %d, want %d", status, http.StatusUnsupportedMediaType)
}
}
func TestIsViewerOriginalPreview(t *testing.T) {
tests := []struct {
name string
size string
file string
fileType string
docs bool // the server can render documents to an image
want bool
}{
{"Fuji RAW viewer", "original", "DSCF0308.RAF", "image/x-fuji-raf", false, true},
{"Canon RAW viewer", "original", "IMG_0001.CR3", "image/x-canon-cr3", false, true},
{"Adobe DNG viewer", "original", "a.dng", "image/x-adobe-dng", false, true},
{"HEIC viewer", "original", "a.heic", "image/heic", false, true},
{"HEIF viewer", "original", "a.heif", "image/heif", false, true},
{"rendered document viewer", "original", "a.docx", "application/vnd.openxmlformats-officedocument.wordprocessingml.document", true, true},
{"documents need the server to be able to render them", "original", "a.docx", "application/vnd.openxmlformats-officedocument.wordprocessingml.document", false, false},
{"RAW thumbnail stays disabled", "small", "DSCF0308.RAF", "image/x-fuji-raf", true, false},
{"RAW large stays disabled", "large", "DSCF0308.RAF", "image/x-fuji-raf", true, false},
{"no size stays disabled", "", "DSCF0308.RAF", "image/x-fuji-raf", true, false},
{"video poster stays disabled", "original", "a.mov", "video/quicktime", true, false},
{"audio cover stays disabled", "original", "a.mp3", "audio/mpeg", true, false},
{"JPEG is opened directly, not through previews", "original", "a.jpg", "image/jpeg", true, false},
{"PDF is opened directly, not through previews", "original", "a.pdf", "application/pdf", true, false},
{"text stays disabled", "original", "a.txt", "text/plain", true, false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := isViewerOriginalPreview(tt.size, tt.file, tt.fileType, tt.docs); got != tt.want {
t.Fatalf("isViewerOriginalPreview(%q, %q, %q, %v) = %v, want %v", tt.size, tt.file, tt.fileType, tt.docs, got, tt.want)
}
})
}
}
+60 -2
View File
@@ -3,6 +3,7 @@ package web
import (
"bytes"
"context"
"errors"
"fmt"
"net/http"
"path/filepath"
@@ -12,7 +13,10 @@ import (
"github.com/gtsteffaniak/filebrowser/backend/internal/adapters/fs/files"
"github.com/gtsteffaniak/filebrowser/backend/internal/ffmpeg"
"github.com/gtsteffaniak/filebrowser/backend/internal/gomedia"
"github.com/gtsteffaniak/filebrowser/backend/internal/mediacache"
"github.com/gtsteffaniak/filebrowser/backend/internal/state"
"github.com/gtsteffaniak/filebrowser/backend/internal/utils"
"github.com/gtsteffaniak/filebrowser/backend/pkg/indexing"
"github.com/gtsteffaniak/filebrowser/backend/pkg/settings"
)
@@ -106,9 +110,34 @@ func subtitlesHandler(w http.ResponseWriter, r *http.Request, d *Context) (int,
var content string
if !embedded {
subtitlePath := filepath.Join(filepath.Dir(fileInfo.RealPath), filepath.Base(track.Name))
var subtitlePath string
if d.Share.Hash != "" {
if d.Share.Path == "" {
return http.StatusForbidden, fmt.Errorf("share has no path scope")
}
idx := indexing.GetIndex(source)
if idx == nil {
return http.StatusInternalServerError, fmt.Errorf("source %s is not available", source)
}
bound, sidecarIndexPath, scopeErr := shareSubtitleSidecarIndexPaths(d, source, path, track.Name)
if scopeErr != nil {
return http.StatusForbidden, scopeErr
}
subtitlePath, _, err = idx.GetRealPathScoped(bound, sidecarIndexPath)
if err != nil {
return realPathErrStatus(err), err
}
if !state.AccessPermittedAtRealPath(idx, d.ShareUser.Username, subtitlePath, false) {
return http.StatusForbidden, fmt.Errorf("access denied")
}
} else {
subtitlePath = filepath.Join(filepath.Dir(fileInfo.RealPath), filepath.Base(track.Name))
}
content, err = utils.GetSubtitleSidecarContent(subtitlePath)
if err != nil {
if errors.Is(err, utils.ErrSubtitleSidecarNotRegular) {
return http.StatusForbidden, err
}
return http.StatusInternalServerError, fmt.Errorf("failed to get subtitle sidecar content: %v", err)
}
} else {
@@ -123,7 +152,8 @@ func subtitlesHandler(w http.ResponseWriter, r *http.Request, d *Context) (int,
}
// Shadow ffmpeg runs after the fast path; bound it so a detached request cannot
// leave extraction running indefinitely (sync fallback still uses r.Context()).
legacyCtx, cancel := context.WithTimeout(context.WithoutCancel(r.Context()), time.Minute)
// WithoutCache: compare against a fresh extraction, not the value go-media stored.
legacyCtx, cancel := context.WithTimeout(mediacache.WithoutCache(context.WithoutCancel(r.Context())), time.Minute)
defer cancel()
vtt, extractErr := svc.ExtractSubtitle(legacyCtx, fileInfo.RealPath, *track.Index)
return vtt, extractErr == nil
@@ -160,6 +190,34 @@ func findSubtitleTrack(subtitles []utils.SubtitleTrack, name string, embedded bo
return nil
}
// shareSubtitleSidecarIndexPaths maps middleware IndexPath (user-scope-relative) and Share.Path
// to scoped index paths for sidecar containment (see publicDeleteHandler).
func shareSubtitleSidecarIndexPaths(d *Context, sourceName, videoIndexPath, sidecarFileName string) (bound, sidecarIndexPath string, err error) {
userScope, scopeErr := d.ShareUser.GetScopeForSourceName(sourceName)
if scopeErr != nil || userScope == "" {
return "", "", fmt.Errorf("user has no access to source: %s", sourceName)
}
if d.Share.Path == "" {
return "", "", fmt.Errorf("share has no path scope")
}
shareRel := strings.TrimPrefix(d.Share.Path, userScope)
bound = utils.JoinScopedIndexPath(userScope, shareRel)
videoRel := strings.Trim(strings.TrimSpace(videoIndexPath), "/")
fullVideo := utils.JoinScopedIndexPath(userScope, videoRel)
if strings.TrimSuffix(bound, "/") == strings.TrimSuffix(fullVideo, "/") {
parent := utils.GetParentDirectoryPath(strings.TrimSuffix(bound, "/"))
if parent == "" {
parent = "/"
}
bound = parent
}
sidecarIndexPath = utils.JoinPathAsUnix(
utils.GetParentDirectoryPath(strings.TrimSuffix(fullVideo, "/")),
filepath.Base(sidecarFileName),
)
return bound, sidecarIndexPath, nil
}
// metadataHandler returns the same directory resource shape as GET /api/resources with metadata enabled,
// for client-side patching after a fast listing load.
// @Summary Directory with media metadata
@@ -4,6 +4,7 @@ import (
"encoding/json"
"net/http"
"net/http/httptest"
"sort"
"testing"
"github.com/gtsteffaniak/filebrowser/backend/internal/database/users"
@@ -12,19 +13,23 @@ import (
func withMediaHandlerSettings(t *testing.T) {
t.Helper()
originalTranscode := settings.Config.Integrations.Media.Transcode
originalMedia := settings.CurrentMediaSettings()
originalFFmpeg := settings.Env.FFmpegAvailable
t.Cleanup(func() {
settings.Config.Integrations.Media.Transcode = originalTranscode
settings.Env.FFmpegAvailable = originalFFmpeg
settings.ApplyMediaSettings(originalMedia)
})
settings.Env.FFmpegAvailable = true
settings.Config.Integrations.Media.Transcode = settings.MediaTranscode{
Enabled: true,
MaxConcurrent: 3,
MaxConcurrentPerUser: 2,
Profiles: settings.DefaultTranscodeProfiles(),
media := originalMedia
media.Transcode = settings.FullAdminTranscode{
UserTranscode: settings.UserTranscode{
Enabled: true,
MaxConcurrentPerUser: 2,
},
MaxConcurrent: 3,
Profiles: settings.DefaultTranscodeProfiles(),
}
settings.ApplyMediaSettings(media)
}
func mediaHandlerContext(admin bool) *Context {
@@ -52,62 +57,111 @@ func callMediaGet(t *testing.T, admin bool) map[string]any {
return body
}
// Non-admin users must be able to read the effective media values for the
// player, but only the player-relevant subset.
// objectField returns obj[key] as a JSON object, failing the test if it is missing or not an object.
func objectField(t *testing.T, obj map[string]any, key string) map[string]any {
t.Helper()
child, ok := obj[key].(map[string]any)
if !ok {
t.Fatalf("%s is %T, want object", key, obj[key])
}
return child
}
// keysOf returns the sorted keys of a JSON object.
func keysOf(obj map[string]any) []string {
keys := make([]string, 0, len(obj))
for k := range obj {
keys = append(keys, k)
}
sort.Strings(keys)
return keys
}
func assertKeys(t *testing.T, path string, got []string, want []string) {
t.Helper()
sort.Strings(want)
if len(got) != len(want) {
t.Fatalf("%s keys = %v, want %v", path, got, want)
}
for i := range got {
if got[i] != want[i] {
t.Fatalf("%s keys = %v, want %v", path, got, want)
}
}
}
// Non-admin users receive exactly the PUBLIC payload. Any new key fails this test,
// which forces a conscious decision about whether it belongs in UserMedia.
func TestSettingsMediaHandlerPlayerPayloadForNonAdmin(t *testing.T) {
withMediaHandlerSettings(t)
body := callMediaGet(t, false)
if _, ok := body["transcode"]; ok {
t.Fatal("non-admin must not receive the full transcode settings")
assertKeys(t, "top-level", keysOf(body), []string{"transcode", "imagePreview"})
transcode := objectField(t, body, "transcode")
assertKeys(t, "transcode", keysOf(transcode), []string{"enabled", "maxConcurrentPerUser", "enabledProfiles"})
if enabled, ok := transcode["enabled"].(bool); !ok || !enabled {
t.Fatalf("transcode.enabled = %v, want true", transcode["enabled"])
}
if _, ok := body["lockedFromConfigPaths"]; ok {
t.Fatal("non-admin must not receive config lock metadata")
if limit, ok := transcode["maxConcurrentPerUser"].(float64); !ok || limit != 2 {
t.Fatalf("transcode.maxConcurrentPerUser = %v, want 2", transcode["maxConcurrentPerUser"])
}
if _, ok := body["lockMessage"]; ok {
t.Fatal("non-admin must not receive the config lock message")
}
if enabled, ok := body["enabled"].(bool); !ok || !enabled {
t.Fatalf("enabled = %v, want true", body["enabled"])
}
if limit, ok := body["maxConcurrentPerUser"].(float64); !ok || limit != 2 {
t.Fatalf("maxConcurrentPerUser = %v, want 2", body["maxConcurrentPerUser"])
}
profiles, ok := body["profiles"].([]any)
profiles, ok := transcode["enabledProfiles"].([]any)
if !ok || len(profiles) != 3 {
t.Fatalf("profiles = %v, want 3 enabled ids", body["profiles"])
t.Fatalf("transcode.enabledProfiles = %v, want 3 enabled ids", transcode["enabledProfiles"])
}
}
// Admins receive the full stored config. The keys shared with users must have the
// same names, so the admin payload is a strict superset of the user payload.
func TestSettingsMediaHandlerFullPayloadForAdmin(t *testing.T) {
withMediaHandlerSettings(t)
body := callMediaGet(t, true)
transcode, ok := body["transcode"].(map[string]any)
if !ok {
t.Fatalf("admin must receive the full transcode settings, got %T", body["transcode"])
}
if _, ok := transcode["profiles"].(map[string]any); !ok {
t.Fatalf("admin transcode payload must include profiles, got %v", transcode["profiles"])
}
// Admin payload exposes the full settings surface, not the player subset.
for _, key := range []string{"convert", "debug", "hardwareAcceleration"} {
if _, ok := body[key]; !ok {
admin := callMediaGet(t, true)
user := callMediaGet(t, false)
// lockedFromConfigPaths is omitted when config pins nothing, so it is not required here.
for _, key := range []string{"transcode", "imagePreview", "convert", "debug", "hardwareAcceleration", "accelDevice"} {
if _, ok := admin[key]; !ok {
t.Fatalf("admin payload missing %q", key)
}
}
if _, ok := body["enabled"]; ok {
t.Fatal("the admin payload uses the full settings shape, not the player subset")
if _, ok := admin["enabled"]; ok {
t.Fatal("admin payload must not repeat transcode.enabled at the top level")
}
adminTranscode := objectField(t, admin, "transcode")
for _, key := range []string{"maxConcurrent", "profiles", "cacheMaxSizeMB", "encoderBackend", "readRate"} {
if _, ok := adminTranscode[key]; !ok {
t.Fatalf("admin transcode missing %q", key)
}
}
if _, ok := adminTranscode["profiles"].(map[string]any); !ok {
t.Fatalf("admin transcode.profiles must be an object, got %T", adminTranscode["profiles"])
}
// Superset rule: every key a user receives must also exist for admins. Values may
// differ by design: admins see stored config (so the UI can save it back unchanged),
// users see effective values (for example enabled is false when ffmpeg is missing).
for key := range objectField(t, user, "transcode") {
if _, ok := adminTranscode[key]; !ok {
t.Fatalf("admin transcode missing shared key %q", key)
}
}
if admin["imagePreview"] == nil {
t.Fatal("imagePreview must be present for admins")
}
}
// Disabling transcoding at runtime must be visible to non-admin players.
func TestSettingsMediaHandlerReflectsDisabledTranscode(t *testing.T) {
withMediaHandlerSettings(t)
settings.Config.Integrations.Media.Transcode.Enabled = false
disabled := settings.CurrentMediaSettings()
disabled.Transcode.Enabled = false
settings.ApplyMediaSettings(disabled)
body := callMediaGet(t, false)
if enabled, ok := body["enabled"].(bool); !ok || enabled {
t.Fatalf("enabled = %v, want false", body["enabled"])
transcode := objectField(t, callMediaGet(t, false), "transcode")
if enabled, ok := transcode["enabled"].(bool); !ok || enabled {
t.Fatalf("transcode.enabled = %v, want false", transcode["enabled"])
}
}
@@ -0,0 +1,333 @@
package web
import (
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"
"github.com/gtsteffaniak/filebrowser/backend/internal/adapters/fs/files"
"github.com/gtsteffaniak/filebrowser/backend/internal/app"
"github.com/gtsteffaniak/filebrowser/backend/internal/database/share"
"github.com/gtsteffaniak/filebrowser/backend/internal/database/users"
"github.com/gtsteffaniak/filebrowser/backend/internal/state"
"github.com/gtsteffaniak/filebrowser/backend/internal/utils"
"github.com/gtsteffaniak/filebrowser/backend/pkg/indexing"
"github.com/gtsteffaniak/filebrowser/backend/pkg/indexing/iteminfo"
"github.com/gtsteffaniak/filebrowser/backend/pkg/settings"
)
func setupSubtitlesHTTPTest(t *testing.T) (downloadsPath string, admin *users.User) {
t.Helper()
lockWebGlobals(t)
downloadsPath = filepath.Join(t.TempDir(), "downloads")
dbPath := filepath.Join(t.TempDir(), "subtitles-http-test.sqlite")
if _, err := state.Initialize(dbPath); err != nil {
t.Fatal(err)
}
app.MustWireServices(state.Default())
t.Cleanup(func() {
state.Close()
})
settings.Config.Server.SourceMap = map[string]*settings.Source{
downloadsPath: {
Path: downloadsPath,
Name: "Downloads",
},
}
settings.Config.Server.NameToSource = map[string]*settings.Source{
"Downloads": settings.Config.Server.SourceMap[downloadsPath],
}
settings.InitializeUserResolvers()
adminUser := &users.User{
ID: 1,
FrontendUser: users.FrontendUser{
Username: "admin",
Permissions: users.Permissions{Admin: true},
},
BackendScopes: []users.BackendScope{
{Path: downloadsPath, Scope: "/"},
},
BackendSourcePermissions: map[string]users.SourceFilePermissions{
downloadsPath: {
View: true,
Download: true,
Modify: true,
Create: true,
Delete: true,
},
},
Version: users.SourcePermissionsMigrationVersion,
}
if err := state.CreateUser(adminUser, ""); err != nil {
t.Fatal(err)
}
adminUser.Permissions = users.Permissions{Admin: true}
if err := state.UpdateUser(adminUser, "", "permissions"); err != nil {
t.Fatal(err)
}
if err := state.UpdateUser(adminUser, "", "backendScopes"); err != nil {
t.Fatal(err)
}
if err := state.UpdateUser(adminUser, "", "backendSourcePermissions"); err != nil {
t.Fatal(err)
}
if err := os.MkdirAll(downloadsPath, 0755); err != nil {
t.Fatal(err)
}
indexing.SetTestIndex("Downloads", downloadsPath)
t.Cleanup(func() {
indexing.ClearTestIndices()
})
loaded, err := state.GetUserByUsername("admin")
if err != nil {
t.Fatal(err)
}
return downloadsPath, &loaded
}
func mockVideoWithExternalSubtitle(t *testing.T, trackName, videoRealPath string) {
t.Helper()
prev := files.FileInfoFasterFunc
t.Cleanup(func() { files.FileInfoFasterFunc = prev })
files.FileInfoFasterFunc = func(opts utils.FileOptions, user *users.User) (*iteminfo.ExtendedFileInfo, error) {
return &iteminfo.ExtendedFileInfo{
FileInfo: iteminfo.FileInfo{
Path: opts.Path,
ItemInfo: iteminfo.ItemInfo{
Type: "video/mp4",
},
},
Subtitles: []utils.SubtitleTrack{
{Name: trackName, Embedded: false},
},
RealPath: videoRealPath,
}, nil
}
}
func TestSubtitlesHandler_RejectsEscapingSidecarSymlink(t *testing.T) {
downloadsPath, adminUser := setupSubtitlesHTTPTest(t)
videoPath := filepath.Join(downloadsPath, "clip.mov")
mockVideoWithExternalSubtitle(t, "clip.srt", videoPath)
outside := filepath.Join(filepath.Dir(downloadsPath), "outside-secret")
if err := os.MkdirAll(outside, 0755); err != nil {
t.Fatal(err)
}
secret := filepath.Join(outside, "secret.srt")
if err := os.WriteFile(secret, []byte("SECRET_OUTSIDE_SUBTITLE"), 0644); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(videoPath, []byte("video"), 0644); err != nil {
t.Fatal(err)
}
if err := os.Symlink(secret, filepath.Join(downloadsPath, "clip.srt")); err != nil {
t.Fatal(err)
}
req := httptest.NewRequest(http.MethodGet, "/api/media/subtitles?source=Downloads&path=/clip.mov&name=clip.srt&embedded=false", nil)
rec := httptest.NewRecorder()
status, handlerErr := subtitlesHandler(rec, req, &Context{User: adminUser})
if status != http.StatusForbidden {
t.Fatalf("status=%d err=%v body=%q", status, handlerErr, rec.Body.String())
}
if strings.Contains(rec.Body.String(), "SECRET_OUTSIDE_SUBTITLE") {
t.Fatalf("response leaked outside subtitle content")
}
}
func TestSubtitlesHandler_ReturnsInScopeSidecar(t *testing.T) {
downloadsPath, adminUser := setupSubtitlesHTTPTest(t)
videoPath := filepath.Join(downloadsPath, "clip.mov")
mockVideoWithExternalSubtitle(t, "clip.srt", videoPath)
srtBody := "1\n00:00:01,000 --> 00:00:02,000\nHello in scope"
if err := os.WriteFile(videoPath, []byte("video"), 0644); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(downloadsPath, "clip.srt"), []byte(srtBody), 0644); err != nil {
t.Fatal(err)
}
req := httptest.NewRequest(http.MethodGet, "/api/media/subtitles?source=Downloads&path=/clip.mov&name=clip.srt&embedded=false", nil)
rec := httptest.NewRecorder()
status, handlerErr := subtitlesHandler(rec, req, &Context{User: adminUser})
if status != http.StatusOK || handlerErr != nil {
t.Fatalf("status=%d err=%v body=%q", status, handlerErr, rec.Body.String())
}
if !strings.Contains(rec.Body.String(), "Hello in scope") {
t.Fatalf("body=%q want in-scope subtitle text", rec.Body.String())
}
}
func TestSubtitlesHandler_ShareScopedOwnerLoadsSidecar(t *testing.T) {
sourceRoot, adminUser := setupSubtitlesHTTPTest(t)
aliceRoot := filepath.Join(sourceRoot, "users", "alice", "shared")
if err := os.MkdirAll(aliceRoot, 0755); err != nil {
t.Fatal(err)
}
srtBody := "1\n00:00:01,000 --> 00:00:02,000\nScoped share OK"
videoPath := filepath.Join(aliceRoot, "clip.mov")
if err := os.WriteFile(videoPath, []byte("video"), 0644); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(aliceRoot, "clip.srt"), []byte(srtBody), 0644); err != nil {
t.Fatal(err)
}
mockVideoWithExternalSubtitle(t, "clip.srt", videoPath)
adminUser.BackendScopes = []users.BackendScope{{Path: sourceRoot, Scope: "/users/alice"}}
req := httptest.NewRequest(http.MethodGet, "/public/api/media/subtitles?hash=h&path=clip.mov&name=clip.srt&embedded=false", nil)
rec := httptest.NewRecorder()
status, handlerErr := subtitlesHandler(rec, req, &Context{
User: adminUser,
ShareUser: adminUser,
IndexPath: "/shared/clip.mov/",
Share: share.Share{
ShareColumns: share.ShareColumns{
Hash: "h",
Path: "/users/alice/shared",
},
SourcePath: sourceRoot,
},
})
if status != http.StatusOK || handlerErr != nil {
t.Fatalf("status=%d err=%v body=%q", status, handlerErr, rec.Body.String())
}
if !strings.Contains(rec.Body.String(), "Scoped share OK") {
t.Fatalf("body=%q", rec.Body.String())
}
}
func TestSubtitlesHandler_ShareSingleFileLoadsSidecar(t *testing.T) {
sourceRoot, adminUser := setupSubtitlesHTTPTest(t)
srtBody := "single-file share subtitle"
videoPath := filepath.Join(sourceRoot, "clip.mov")
if err := os.WriteFile(videoPath, []byte("video"), 0644); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(sourceRoot, "clip.srt"), []byte(srtBody), 0644); err != nil {
t.Fatal(err)
}
mockVideoWithExternalSubtitle(t, "clip.srt", videoPath)
req := httptest.NewRequest(http.MethodGet, "/public/api/media/subtitles?hash=h&path=/&name=clip.srt&embedded=false", nil)
rec := httptest.NewRecorder()
status, handlerErr := subtitlesHandler(rec, req, &Context{
User: adminUser,
ShareUser: adminUser,
IndexPath: "/clip.mov/",
Share: share.Share{
ShareColumns: share.ShareColumns{
Hash: "h",
Path: "/clip.mov",
},
SourcePath: sourceRoot,
},
})
if status != http.StatusOK || handlerErr != nil {
t.Fatalf("status=%d err=%v body=%q", status, handlerErr, rec.Body.String())
}
if !strings.Contains(rec.Body.String(), "single-file share subtitle") {
t.Fatalf("body=%q", rec.Body.String())
}
}
func TestSubtitlesHandler_ShareDeniesSidecarSymlinkToDeniedTarget(t *testing.T) {
sourceRoot, adminUser := setupSubtitlesHTTPTest(t)
sharedDir := filepath.Join(sourceRoot, "shared")
privateDir := filepath.Join(sourceRoot, "private")
for _, dir := range []string{sharedDir, privateDir} {
if err := os.MkdirAll(dir, 0755); err != nil {
t.Fatal(err)
}
}
secret := filepath.Join(privateDir, "clip.srt")
if err := os.WriteFile(secret, []byte("SECRET_DENIED_SUBTITLE"), 0644); err != nil {
t.Fatal(err)
}
videoPath := filepath.Join(sharedDir, "clip.mov")
if err := os.WriteFile(videoPath, []byte("video"), 0644); err != nil {
t.Fatal(err)
}
if err := os.Symlink(filepath.Join("..", "private", "clip.srt"), filepath.Join(sharedDir, "clip.srt")); err != nil {
t.Fatal(err)
}
if err := state.DenyUser(sourceRoot, utils.IndexPathFromNormalized("/private", true), "admin"); err != nil {
t.Fatal(err)
}
mockVideoWithExternalSubtitle(t, "clip.srt", videoPath)
adminUser.BackendScopes = []users.BackendScope{{Path: sourceRoot, Scope: "/"}}
req := httptest.NewRequest(http.MethodGet, "/public/api/media/subtitles?hash=h&path=clip.mov&name=clip.srt&embedded=false", nil)
rec := httptest.NewRecorder()
status, handlerErr := subtitlesHandler(rec, req, &Context{
User: adminUser,
ShareUser: adminUser,
IndexPath: "/shared/clip.mov/",
Share: share.Share{
ShareColumns: share.ShareColumns{
Hash: "h",
Path: "/shared",
},
SourcePath: sourceRoot,
},
})
if status != http.StatusForbidden {
t.Fatalf("status=%d err=%v body=%q", status, handlerErr, rec.Body.String())
}
if strings.Contains(rec.Body.String(), "SECRET_DENIED_SUBTITLE") {
t.Fatalf("response leaked denied subtitle")
}
}
func TestSubtitlesHandler_ShareRejectsSidecarPastEscapingDirSymlink(t *testing.T) {
sourceRoot, adminUser := setupSubtitlesHTTPTest(t)
videoIndexPath := "/shared/link/clip.mov/"
mockVideoWithExternalSubtitle(t, "clip.srt", filepath.Join(sourceRoot, "shared", "link", "clip.mov"))
sharedDir := filepath.Join(sourceRoot, "shared")
if err := os.MkdirAll(sharedDir, 0755); err != nil {
t.Fatal(err)
}
outside := filepath.Join(filepath.Dir(sourceRoot), "outside-share")
if err := os.MkdirAll(outside, 0755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(outside, "clip.srt"), []byte("SECRET_PAST_DIR_SYMLINK"), 0644); err != nil {
t.Fatal(err)
}
if err := os.Symlink(outside, filepath.Join(sharedDir, "link")); err != nil {
t.Fatal(err)
}
req := httptest.NewRequest(http.MethodGet, "/public/api/media/subtitles?hash=h&path=link/clip.mov&name=clip.srt&embedded=false", nil)
rec := httptest.NewRecorder()
status, handlerErr := subtitlesHandler(rec, req, &Context{
User: adminUser,
ShareUser: adminUser,
IndexPath: videoIndexPath,
Share: share.Share{
ShareColumns: share.ShareColumns{
Hash: "h",
Path: "/shared",
},
SourcePath: sourceRoot,
},
})
if status != http.StatusForbidden {
t.Fatalf("status=%d err=%v body=%q", status, handlerErr, rec.Body.String())
}
if strings.Contains(rec.Body.String(), "SECRET_PAST_DIR_SYMLINK") {
t.Fatalf("response leaked subtitle past directory symlink")
}
}
+4 -1
View File
@@ -122,7 +122,10 @@ func withHashFileHelper(fn handleFunc) handleFunc {
pathWithoutUserScope = pathWithoutUserScope + "/"
}
data.IndexPath = pathWithoutUserScope
// skip file fetch for certain apis
// Skip FileInfoFaster prefetch for performance; handlers must apply equivalent
// checks (access rules + GetRealPathScoped with d.Share.Path where bytes are read).
// TODO: align all skipped public-share routes on one bound (Share.Path) or stop
// skipping FileInfoFaster here for media/metadata/items.
if (r.Method == "POST" && strings.Contains(r.URL.Path, "/resources")) ||
(r.Method == "POST" && strings.Contains(r.URL.Path, "/resources/view-token")) ||
(r.Method == "GET" && strings.Contains(r.URL.Path, "/resources/items")) ||
+85
View File
@@ -5,7 +5,9 @@ import (
"encoding/json"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"
"time"
@@ -22,6 +24,7 @@ import (
)
func setupTestEnv(t *testing.T) {
lockWebGlobals(t)
utils.BcryptCost = bcrypt.MinCost
dbPath := filepath.Join(t.TempDir(), "test.sqlite")
@@ -535,6 +538,88 @@ func newHTTPRequest(t *testing.T, hash string, requestModifiers ...func(*http.Re
return r
}
func TestPublicGetResourceChecksum(t *testing.T) {
setupTestEnv(t)
filePath := filepath.Join(t.TempDir(), "a.txt")
if err := os.WriteFile(filePath, []byte("hello\n"), 0o600); err != nil {
t.Fatal(err)
}
originalFileInfoFaster := FileInfoFasterFunc
t.Cleanup(func() { FileInfoFasterFunc = originalFileInfoFaster })
FileInfoFasterFunc = func(opts utils.FileOptions, user *users.User) (*iteminfo.ExtendedFileInfo, error) {
return &iteminfo.ExtendedFileInfo{
FileInfo: iteminfo.FileInfo{
Path: opts.Path,
ItemInfo: iteminfo.ItemInfo{
Name: "a.txt",
Type: "text/plain",
},
},
RealPath: filePath,
}, nil
}
newOwner := &users.User{
FrontendUser: users.FrontendUser{
Username: "checksum_owner",
Permissions: users.Permissions{Admin: false},
},
BackendScopes: []users.BackendScope{{Path: "/srv", Scope: "/"}},
}
if err := state.CreateUser(newOwner, ""); err != nil {
t.Fatal("failed to save user:", err)
}
ownerValue, err := state.GetUserByUsername("checksum_owner")
if err != nil {
t.Fatal(err)
}
owner := &ownerValue
if err := state.CreateShare(&share.Share{
ShareSettings: share.ShareSettings{ShareLimits: share.ShareLimits{SourceName: "srv"}},
ShareColumns: share.ShareColumns{Hash: "checksum_hash", Path: "/"},
SourcePath: "/srv",
UserID: owner.ID,
}); err != nil {
t.Fatal("failed to create share:", err)
}
originalAuthKey := settings.Config.Auth.Key
settings.Config.Auth.Key = "key"
t.Cleanup(func() { settings.Config.Auth.Key = originalAuthKey })
t.Run("anonymous", func(t *testing.T) {
recorder := httptest.NewRecorder()
req := newTestRequest(t, "checksum_hash", "", "", nil)
q := req.URL.Query()
q.Set("checksum", "md5")
req.URL.RawQuery = q.Encode()
withHashFile(publicGetResourceHandler)(recorder, req)
if recorder.Code != http.StatusUnauthorized {
t.Fatalf("expected status %d, got %d body=%s", http.StatusUnauthorized, recorder.Code, recorder.Body.String())
}
})
t.Run("authenticated", func(t *testing.T) {
token := testSessionToken(t, owner, time.Hour)
recorder := httptest.NewRecorder()
req := newTestRequest(t, "checksum_hash", "", "", map[string]string{
"Authorization": "Bearer " + token,
})
q := req.URL.Query()
q.Set("checksum", "md5")
req.URL.RawQuery = q.Encode()
withHashFile(publicGetResourceHandler)(recorder, req)
if recorder.Code != http.StatusOK {
t.Fatalf("expected status %d, got %d body=%s", http.StatusOK, recorder.Code, recorder.Body.String())
}
if !strings.Contains(recorder.Body.String(), "b1946ac92492d2347c6235b4d2611184") {
t.Fatalf("checksum missing from response: %s", recorder.Body.String())
}
})
}
func TestWithTimeoutReturnsJSONTimeoutReason(t *testing.T) {
t.Parallel()
timeout := 50 * time.Millisecond
+5 -1
View File
@@ -166,7 +166,11 @@ func onlyofficeClientConfigGetHandler(w http.ResponseWriter, r *http.Request, d
}
providedPath = cleanPath
themeMode := utils.Ternary(d.User.DarkMode, "dark", "light")
themeMode := "light"
if d.User.DarkMode == users.DarkModeDark ||
(d.User.DarkMode == users.DarkModeAuto && settings.DefaultDarkMode()) {
themeMode = "dark"
}
var sourceInfo *settings.Source
var ok bool
if hash != "" {
+60 -5
View File
@@ -16,6 +16,7 @@ import (
"github.com/gtsteffaniak/filebrowser/backend/internal/adapters/fs/files"
"github.com/gtsteffaniak/filebrowser/backend/internal/preview"
"github.com/gtsteffaniak/filebrowser/backend/internal/utils"
"github.com/gtsteffaniak/filebrowser/backend/pkg/indexing"
"github.com/gtsteffaniak/filebrowser/backend/pkg/indexing/iteminfo"
"github.com/gtsteffaniak/filebrowser/backend/pkg/settings"
"github.com/gtsteffaniak/go-logger/logger"
@@ -38,6 +39,31 @@ func isClientCancellation(ctx context.Context, err error) bool {
return errors.Is(err, context.Canceled)
}
// viewerConvertedDocExtensions are the documents the viewer opens as a rendered image through the
// "original" preview (keep in sync with pdfConvertable in frontend Preview.vue).
var viewerConvertedDocExtensions = map[string]bool{
".xps": true, ".mobi": true, ".fb2": true, ".cbz": true, ".svg": true,
".docx": true, ".pptx": true, ".xlsx": true, ".hwp": true, ".hwpx": true,
}
// isViewerOriginalPreview reports whether a preview request is the viewer opening a file the browser can't
// display by itself (camera RAW, HEIC/HEIF, rendered documents). Unlike a thumbnail, without it the file can't
// be opened at all, so it is served even when thumbnails are disabled. documentRenderable tells whether the
// server can render documents of this kind to an image (see iteminfo.HasDocConvertableExtension).
func isViewerOriginalPreview(previewSize, fileName, fileType string, documentRenderable bool) bool {
if previewSize != "original" {
return false
}
ext := strings.ToLower(filepath.Ext(fileName))
if iteminfo.IsRawImage(ext) {
return true
}
if fileType == "image/heic" || fileType == "image/heif" {
return true
}
return documentRenderable && viewerConvertedDocExtensions[ext]
}
// previewHandler handles the preview request for images.
// @Summary Get image preview
// @Description Returns a preview image based on the requested path and size.
@@ -55,7 +81,9 @@ func isClientCancellation(ctx context.Context, err error) bool {
// @Failure 501 {object} map[string]string "Preview generation not implemented"
// @Router /api/resources/preview [get]
func previewHandler(w http.ResponseWriter, r *http.Request, d *Context) (int, error) {
if settings.Config.Server.DisablePreviews {
previewsDisabled := settings.Config.Server.DisablePreviews
// Only the viewer's "original" requests can pass when previews are disabled (checked below once the file is known).
if previewsDisabled && r.URL.Query().Get("size") != "original" {
return http.StatusNotImplemented, fmt.Errorf("preview is disabled")
}
path := r.URL.Query().Get("path")
@@ -85,6 +113,9 @@ func previewHandler(w http.ResponseWriter, r *http.Request, d *Context) (int, er
return ErrToStatus(err), err
}
d.FileInfo = *fileInfo
if previewsDisabled && !isViewerOriginalPreview(r.URL.Query().Get("size"), fileInfo.Name, fileInfo.Type, iteminfo.HasDocConvertableExtension(fileInfo.Name, fileInfo.Type)) {
return http.StatusNotImplemented, fmt.Errorf("preview is disabled")
}
status, err := PreviewHelperFunc(w, r, d)
if err != nil {
// Error already logged in PreviewHelperFunc or its callees
@@ -109,7 +140,9 @@ func previewHandler(w http.ResponseWriter, r *http.Request, d *Context) (int, er
// @Failure 501 {object} map[string]string "Previews disabled globally, for this share, or for upload shares"
// @Router /public/api/resources/preview [get]
func publicPreviewHandler(w http.ResponseWriter, r *http.Request, d *Context) (int, error) {
if settings.Config.Server.DisablePreviews || d.Share.DisableThumbnails {
previewSize := r.URL.Query().Get("size")
if (settings.Config.Server.DisablePreviews || d.Share.DisableThumbnails) &&
!isViewerOriginalPreview(previewSize, d.FileInfo.Name, d.FileInfo.Type, iteminfo.HasDocConvertableExtension(d.FileInfo.Name, d.FileInfo.Type)) {
return http.StatusNotImplemented, fmt.Errorf("preview is disabled")
}
if d.Share.ShareType == "upload" {
@@ -196,13 +229,35 @@ func getDirectoryPreview(ctx context.Context, r *http.Request, d *Context, frame
// force an ffprobe probe and a subtitle scan per frame on the fast path.
fileInfo, err := files.FileInfoFaster(
utils.FileOptions{
Path: path,
Source: source,
AlbumArt: true,
Path: path,
Source: source,
AlbumArt: true,
FollowSymlinks: true,
}, user)
if err != nil {
return nil, err
}
if d.Share.Hash != "" {
idx := indexing.GetIndex(source)
if idx == nil {
return nil, fmt.Errorf("source not available")
}
userScope, scopeErr := d.ShareUser.GetScopeForSourceName(source)
if scopeErr != nil {
return nil, scopeErr
}
shareRel := strings.TrimPrefix(d.Share.Path, userScope)
bound := utils.JoinScopedIndexPath(userScope, shareRel)
childIndexPath := utils.JoinScopedIndexPath(userScope, path)
realPath, _, rerr := idx.GetRealPathScoped(bound, childIndexPath)
if rerr != nil {
if index != 0 {
return getDirectoryPreview(ctx, r, d, 0)
}
return nil, rerr
}
fileInfo.RealPath = realPath
}
tempCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
_, previewErr := preview.GetPreviewForFile(tempCtx, *fileInfo, "small", "", 0)
cancel()
@@ -0,0 +1,70 @@
package web
import (
"context"
"errors"
"net/http/httptest"
"os"
"path/filepath"
"testing"
"github.com/gtsteffaniak/filebrowser/backend/internal/database/share"
"github.com/gtsteffaniak/filebrowser/backend/internal/database/users"
commonerrors "github.com/gtsteffaniak/filebrowser/backend/internal/errors"
"github.com/gtsteffaniak/filebrowser/backend/pkg/indexing"
"github.com/gtsteffaniak/filebrowser/backend/pkg/indexing/iteminfo"
)
func TestGetDirectoryPreview_ShareRejectsSymlinkChildOutsideShare(t *testing.T) {
sourceRoot, adminUser := setupSubtitlesHTTPTest(t)
sharedDir := filepath.Join(sourceRoot, "shared")
privateDir := filepath.Join(sourceRoot, "private")
for _, dir := range []string{sharedDir, privateDir} {
if err := os.MkdirAll(dir, 0o755); err != nil {
t.Fatal(err)
}
}
if err := os.WriteFile(filepath.Join(privateDir, "secret.jpg"), []byte{0xff, 0xd8, 0xff, 0xd9}, 0o644); err != nil {
t.Fatal(err)
}
if err := os.Symlink(filepath.Join("..", "private", "secret.jpg"), filepath.Join(sharedDir, "link.jpg")); err != nil {
t.Fatal(err)
}
ctx := context.Background()
req := httptest.NewRequest("GET", "/", nil)
adminUser.BackendScopes = []users.BackendScope{{Path: sourceRoot, Scope: "/"}}
d := &Context{
User: adminUser,
ShareUser: adminUser,
IndexPath: "/shared/",
Share: share.Share{
ShareColumns: share.ShareColumns{Hash: "h", Path: "/shared"},
SourcePath: sourceRoot,
},
FileInfo: iteminfo.ExtendedFileInfo{
Source: "Downloads",
FileInfo: iteminfo.FileInfo{
Path: "/shared/",
ItemInfo: iteminfo.ItemInfo{
Type: "directory",
},
Files: []iteminfo.ExtendedItemInfo{
{ItemInfo: iteminfo.ItemInfo{Name: "link.jpg", Type: "image/jpeg", HasPreview: true}},
},
},
},
}
idx := indexing.GetIndex("Downloads")
if idx == nil {
t.Fatal("missing index")
}
_, err := getDirectoryPreview(ctx, req, d, 0)
if err == nil {
t.Fatal("expected scope error for share child symlink outside share root")
}
if !errors.Is(err, commonerrors.ErrPathEscapesScope) {
t.Fatalf("error=%v want ErrPathEscapesScope", err)
}
}
+30
View File
@@ -9,6 +9,7 @@ import (
)
func TestGetRemoteIPUsesRightmostForwardedFor(t *testing.T) {
lockWebGlobals(t)
orig := settings.Config.Http
t.Cleanup(func() { settings.Config.Http = orig })
settings.Config.Http.TrustProxyHeaders = true
@@ -27,6 +28,7 @@ func TestGetRemoteIPUsesRightmostForwardedFor(t *testing.T) {
// proxy hops so multi-hop deployments (CDN -> proxy -> filebrowser) resolve the
// real client IP instead of collapsing everyone onto the shared edge address.
func TestGetRemoteIPTrustedProxies(t *testing.T) {
lockWebGlobals(t)
orig := settings.Config.Http
t.Cleanup(func() { settings.Config.Http = orig })
settings.Config.Http.TrustProxyHeaders = true
@@ -59,6 +61,7 @@ func TestGetRemoteIPTrustedProxies(t *testing.T) {
// Bare IPs (not just CIDRs) are accepted in http.trustedProxies.
func TestGetRemoteIPTrustedProxiesBareIP(t *testing.T) {
lockWebGlobals(t)
orig := settings.Config.Http
t.Cleanup(func() { settings.Config.Http = orig })
settings.Config.Http.TrustProxyHeaders = true
@@ -74,6 +77,7 @@ func TestGetRemoteIPTrustedProxiesBareIP(t *testing.T) {
// Without trustProxyHeaders the XFF chain is ignored entirely.
func TestGetRemoteIPIgnoresForwardedForWhenUntrusted(t *testing.T) {
lockWebGlobals(t)
orig := settings.Config.Http
t.Cleanup(func() { settings.Config.Http = orig })
settings.Config.Http.TrustProxyHeaders = false
@@ -90,6 +94,7 @@ func TestGetRemoteIPIgnoresForwardedForWhenUntrusted(t *testing.T) {
// A direct client that is not a trusted proxy cannot spoof its address with
// forwarded headers, even when trustProxyHeaders is enabled.
func TestGetRemoteIPIgnoresForwardedHeadersFromUntrustedPeer(t *testing.T) {
lockWebGlobals(t)
orig := settings.Config.Http
t.Cleanup(func() { settings.Config.Http = orig })
settings.Config.Http.TrustProxyHeaders = true
@@ -104,8 +109,26 @@ func TestGetRemoteIPIgnoresForwardedHeadersFromUntrustedPeer(t *testing.T) {
}
}
func TestGetRemoteIPUnixSocketHonorsForwardedWithTrustAllLocalPeers(t *testing.T) {
lockWebGlobals(t)
orig := settings.Config.Http
t.Cleanup(func() { settings.Config.Http = orig })
settings.Config.Http.TrustProxyHeaders = true
settings.Config.Http.Socket = "/var/run/filebrowser.sock"
settings.Config.Http.UnixSocketTrustAllLocalPeers = true
settings.Config.Http.TrustedProxies = nil
req := httptest.NewRequest("GET", "http://example.com/", nil)
req.RemoteAddr = "@"
req.Header.Set("X-Forwarded-For", "203.0.113.50")
if got := GetRemoteIP(req); got != "203.0.113.50" {
t.Fatalf("GetRemoteIP = %q, want 203.0.113.50", got)
}
}
// With no trusted proxies configured, forwarded headers are never honored.
func TestGetRemoteIPIgnoresForwardedHeadersWithoutTrustedProxies(t *testing.T) {
lockWebGlobals(t)
orig := settings.Config.Http
t.Cleanup(func() { settings.Config.Http = orig })
settings.Config.Http.TrustProxyHeaders = true
@@ -121,6 +144,7 @@ func TestGetRemoteIPIgnoresForwardedHeadersWithoutTrustedProxies(t *testing.T) {
// An IPv4-mapped IPv6 trusted hop in the XFF chain must still be skipped.
func TestGetRemoteIPMappedTrustedHop(t *testing.T) {
lockWebGlobals(t)
orig := settings.Config.Http
t.Cleanup(func() { settings.Config.Http = orig })
settings.Config.Http.TrustProxyHeaders = true
@@ -135,6 +159,7 @@ func TestGetRemoteIPMappedTrustedHop(t *testing.T) {
}
func TestGetScheme(t *testing.T) {
lockWebGlobals(t)
t.Run("trusted proto", func(t *testing.T) {
settings.Config.Http.TrustProxyHeaders = true
t.Cleanup(func() { settings.Config.Http.TrustProxyHeaders = false })
@@ -158,6 +183,7 @@ func TestGetScheme(t *testing.T) {
}
func TestRequestSchemeTLSFallback(t *testing.T) {
lockWebGlobals(t)
settings.Config.Http.TrustProxyHeaders = false
req := httptest.NewRequest("GET", "https://example.com/", nil)
@@ -168,6 +194,7 @@ func TestRequestSchemeTLSFallback(t *testing.T) {
}
func TestRequestSchemeIgnoresInvalidProto(t *testing.T) {
lockWebGlobals(t)
settings.Config.Http.TrustProxyHeaders = true
t.Cleanup(func() { settings.Config.Http.TrustProxyHeaders = false })
@@ -179,6 +206,7 @@ func TestRequestSchemeIgnoresInvalidProto(t *testing.T) {
}
func TestRequestHostTrustProxyHeaders(t *testing.T) {
lockWebGlobals(t)
settings.Config.Http.TrustProxyHeaders = true
t.Cleanup(func() { settings.Config.Http.TrustProxyHeaders = false })
@@ -192,6 +220,7 @@ func TestRequestHostTrustProxyHeaders(t *testing.T) {
}
func TestRequestSchemeForPublicURL(t *testing.T) {
lockWebGlobals(t)
orig := settings.Config.Http.TrustProxyHeaders
t.Cleanup(func() { settings.Config.Http.TrustProxyHeaders = orig })
@@ -225,6 +254,7 @@ func TestRequestSchemeForPublicURL(t *testing.T) {
}
func TestRequestPageFullURL(t *testing.T) {
lockWebGlobals(t)
orig := settings.Config.Http
t.Cleanup(func() { settings.Config.Http = orig })
settings.Config.Http.ExternalUrl = ""
+1
View File
@@ -237,6 +237,7 @@ func TestResourcePutHandler_SuccessResponseOmitsRealPath(t *testing.T) {
func setupPutHTTPTest(t *testing.T) string {
t.Helper()
lockWebGlobals(t)
downloadsPath := filepath.Join(t.TempDir(), "downloads")
dbPath := filepath.Join(t.TempDir(), "quota-http-test.sqlite")
+5
View File
@@ -12,6 +12,7 @@ import (
"github.com/google/uuid"
"github.com/gtsteffaniak/filebrowser/backend/internal/adapters/fs/files"
"github.com/gtsteffaniak/filebrowser/backend/internal/adapters/fs/fileutils"
"github.com/gtsteffaniak/filebrowser/backend/internal/database/access"
"github.com/gtsteffaniak/filebrowser/backend/internal/database/users"
commonerrors "github.com/gtsteffaniak/filebrowser/backend/internal/errors"
"github.com/gtsteffaniak/filebrowser/backend/internal/quota"
@@ -171,6 +172,10 @@ func putResourceWithQuota(w http.ResponseWriter, r *http.Request, d *Context, so
if err != nil {
return sanitizePutPathError(err)
}
permUser := accessCheckUsername(d)
if !state.AccessPermittedAtRealPath(idx, permUser, realPath, access.IsDirAtRealPath(realPath)) {
return http.StatusForbidden, fmt.Errorf("access denied to path %s", fullIndexPath)
}
if stat, statErr := os.Stat(realPath); statErr == nil && stat.IsDir() {
return http.StatusMethodNotAllowed, errPutPathIsDirectory
}
+1
View File
@@ -19,6 +19,7 @@ import (
func setupQuotaHTTPTest(t *testing.T) {
t.Helper()
lockWebGlobals(t)
dbPath := filepath.Join(t.TempDir(), "quota-http-test.sqlite")
if _, err := state.Initialize(dbPath); err != nil {
t.Fatal(err)
+51
View File
@@ -16,6 +16,7 @@ import (
"github.com/gtsteffaniak/filebrowser/backend/internal/activity"
"github.com/gtsteffaniak/filebrowser/backend/internal/adapters/fs/files"
"github.com/gtsteffaniak/filebrowser/backend/internal/adapters/fs/fileutils"
"github.com/gtsteffaniak/filebrowser/backend/internal/database/access"
"github.com/gtsteffaniak/filebrowser/backend/internal/database/users"
"github.com/gtsteffaniak/filebrowser/backend/internal/errors"
"github.com/gtsteffaniak/filebrowser/backend/internal/preview"
@@ -208,7 +209,9 @@ func resourceGetHandler(w http.ResponseWriter, r *http.Request, d *Context) (int
// @Param path query string false "Path within the share to retrieve information for. Defaults to share root."
// @Param content query string false "Include file content if true"
// @Param metadata query string false "Extract audio/video metadata if true"
// @Param checksum query string false "Optional checksum algorithm (md5, sha1, sha256, sha512); requires a logged-in session"
// @Success 200 {object} iteminfo.FileInfo "File or directory metadata"
// @Failure 401 {object} map[string]string "Login required for checksum generation"
// @Failure 403 {object} map[string]string "Share unavailable or access denied"
// @Failure 404 {object} map[string]string "Share not found or file not found"
// @Failure 500 {object} map[string]string "Internal server error"
@@ -218,6 +221,24 @@ func publicGetResourceHandler(w http.ResponseWriter, r *http.Request, d *Context
if d.Share.ShareType == "upload" {
return http.StatusNotImplemented, fmt.Errorf("browsing is disabled for upload shares")
}
if algo := r.URL.Query().Get("checksum"); algo != "" {
if d.User == nil || d.User.Username == "" || d.User.Username == "anonymous" {
return http.StatusUnauthorized, fmt.Errorf("login required to generate checksums")
}
if d.FileInfo.Type == "directory" {
return http.StatusBadRequest, fmt.Errorf("checksum is only supported for files")
}
if d.Share.DisableFileViewer {
return http.StatusForbidden, fmt.Errorf("file viewing is disabled for this share")
}
checksum, err := utils.GetChecksum(d.FileInfo.RealPath, algo)
if err == errors.ErrInvalidOption {
return http.StatusBadRequest, nil
} else if err != nil {
return http.StatusInternalServerError, err
}
d.FileInfo.Checksums = map[string]string{algo: checksum}
}
return RenderJSON(w, r, d.FileInfo)
}
@@ -792,6 +813,9 @@ func ResourcePostHandler(w http.ResponseWriter, r *http.Request, d *Context) (in
if !state.AccessPermitted(idx.Path, utils.IndexPathFromNormalized(fullIndexPath, true), filePermUser.Username) {
return http.StatusForbidden, fmt.Errorf("access denied to path %s", path)
}
if !state.AccessPermittedAtRealPath(idx, filePermUser.Username, realPath, access.IsDirAtRealPath(realPath)) {
return http.StatusForbidden, fmt.Errorf("access denied to path %s", path)
}
// Check for file/folder conflicts before creation
if stat, statErr := os.Stat(realPath); statErr == nil {
@@ -1528,6 +1552,15 @@ func ResourcePatchHandler(w http.ResponseWriter, r *http.Request, d *Context) (i
response.Failed = append(response.Failed, moveCopyWithClientPaths(item, clientFromPath, clientToPath))
continue
}
if !state.AccessPermittedAtRealPath(srcIdx, d.User.Username, realSrc, isSrcDir) {
item.Message = "access denied to source path"
if d.Share.Hash != "" {
response.Failed = append(response.Failed, MoveCopyItem{Message: item.Message})
continue
}
response.Failed = append(response.Failed, moveCopyWithClientPaths(item, clientFromPath, clientToPath))
continue
}
// Resolve the destination: copy must reject escaping symlink leaves; move/rename
// must replace an existing symlink entry without following it.
@@ -1548,10 +1581,28 @@ func ResourcePatchHandler(w http.ResponseWriter, r *http.Request, d *Context) (i
response.Failed = append(response.Failed, moveCopyWithClientPaths(item, clientFromPath, clientToPath))
continue
}
if !state.AccessPermittedAtRealPath(dstIdx, d.User.Username, realDest, access.IsDirAtRealPath(realDest)) {
item.Message = "access denied to destination path"
if d.Share.Hash != "" {
response.Failed = append(response.Failed, MoveCopyItem{Message: item.Message})
continue
}
response.Failed = append(response.Failed, moveCopyWithClientPaths(item, clientFromPath, clientToPath))
continue
}
// Auto-rename if requested
if req.Rename {
realDest = addVersionSuffix(realDest)
if !state.AccessPermittedAtRealPath(dstIdx, d.User.Username, realDest, isSrcDir) {
item.Message = "access denied to destination path"
if d.Share.Hash != "" {
response.Failed = append(response.Failed, MoveCopyItem{Message: item.Message})
continue
}
response.Failed = append(response.Failed, moveCopyWithClientPaths(item, clientFromPath, clientToPath))
continue
}
}
// Validate move/rename operation to prevent circular references
@@ -0,0 +1,60 @@
package web
import (
"encoding/json"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"
"github.com/gtsteffaniak/filebrowser/backend/internal/state"
"github.com/gtsteffaniak/filebrowser/backend/internal/utils"
)
func TestResourcePatchHandler_CopyRename_DeniesDeniedVersionSuffix(t *testing.T) {
_, sourceRoot, sourceName, username := setupExtractAccessPolicyTest(t)
destDir := filepath.Join(sourceRoot, "dest")
if err := os.MkdirAll(destDir, 0o755); err != nil {
t.Fatal(err)
}
src := filepath.Join(sourceRoot, "src.txt")
if err := os.WriteFile(src, []byte("src"), 0o644); err != nil {
t.Fatal(err)
}
existing := filepath.Join(destDir, "report.txt")
if err := os.WriteFile(existing, []byte("existing"), 0o644); err != nil {
t.Fatal(err)
}
denied := filepath.Join(destDir, "report(1).txt")
if err := state.DenyUser(sourceRoot, utils.IndexPathFromNormalized("/dest/report(1).txt", false), username); err != nil {
t.Fatal(err)
}
user, err := state.GetUserByUsername(username)
if err != nil {
t.Fatal(err)
}
body := `{"action":"copy","rename":true,"items":[{"fromSource":"` + sourceName +
`","fromPath":"/src.txt","toSource":"` + sourceName + `","toPath":"/dest/report.txt"}]}`
req := httptest.NewRequest(http.MethodPatch, "/api/resources", strings.NewReader(body))
rec := httptest.NewRecorder()
status, handlerErr := ResourcePatchHandler(rec, req, &Context{User: &user})
if handlerErr != nil {
t.Fatalf("err=%v", handlerErr)
}
if status != http.StatusInternalServerError && status != http.StatusOK {
t.Fatalf("status=%d body=%s", status, rec.Body.String())
}
var resp MoveCopyResponse
if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil {
t.Fatal(err)
}
if len(resp.Failed) != 1 || !strings.Contains(resp.Failed[0].Message, "access denied") {
t.Fatalf("failed=%+v", resp.Failed)
}
if _, err := os.Stat(denied); err == nil {
t.Fatal("denied destination file should not exist")
}
}
@@ -17,6 +17,7 @@ import (
func setupPutQuotaTest(t *testing.T) string {
t.Helper()
lockWebGlobals(t)
downloadsPath := filepath.Join(t.TempDir(), "downloads")
dbPath := filepath.Join(t.TempDir(), "quota-http-test.sqlite")
@@ -0,0 +1,82 @@
package web
import (
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"testing"
"github.com/gtsteffaniak/filebrowser/backend/internal/preview"
"github.com/gtsteffaniak/filebrowser/backend/internal/state"
)
func TestResourcePostHandler_RejectsEscapingParentSymlinkMkdir(t *testing.T) {
downloadsPath := setupPutHTTPTest(t)
outside := filepath.Join(filepath.Dir(downloadsPath), "outside-mkdir")
if err := os.MkdirAll(outside, 0755); err != nil {
t.Fatal(err)
}
if err := os.Symlink(outside, filepath.Join(downloadsPath, "evil")); err != nil {
t.Fatal(err)
}
adminUser, err := state.GetUserByUsername("admin")
if err != nil {
t.Fatal(err)
}
req := httptest.NewRequest(http.MethodPost, "/api/resources?source=Downloads&path=/evil/newdir&isDir=true", nil)
rec := httptest.NewRecorder()
status, handlerErr := ResourcePostHandler(rec, req, &requestContext{User: &adminUser})
if status != http.StatusForbidden {
t.Fatalf("status=%d err=%v", status, handlerErr)
}
if handlerErr == nil {
t.Fatal("expected mkdir to be forbidden")
}
if _, err := os.Stat(filepath.Join(outside, "newdir")); !os.IsNotExist(err) {
t.Fatal("mkdir escaped outside the source via symlink parent")
}
}
func TestResourceDeleteHandler_RejectsEscapingIntermediate(t *testing.T) {
downloadsPath := setupPutHTTPTest(t)
if preview.GetService() == nil {
if err := preview.StartPreviewGenerator(1, t.TempDir()); err != nil {
t.Fatal(err)
}
}
outsideDir := filepath.Join(filepath.Dir(downloadsPath), "outside-delete")
if err := os.MkdirAll(outsideDir, 0755); err != nil {
t.Fatal(err)
}
victim := filepath.Join(outsideDir, "victim.txt")
if err := os.WriteFile(victim, []byte("keep"), 0644); err != nil {
t.Fatal(err)
}
if err := os.Symlink(outsideDir, filepath.Join(downloadsPath, "evil")); err != nil {
t.Fatal(err)
}
adminUser, err := state.GetUserByUsername("admin")
if err != nil {
t.Fatal(err)
}
req := httptest.NewRequest(http.MethodDelete, "/api/resources?source=Downloads&path=/evil/victim.txt", nil)
rec := httptest.NewRecorder()
status, handlerErr := resourceDeleteHandler(rec, req, &requestContext{User: &adminUser})
if status != http.StatusForbidden {
t.Fatalf("status=%d err=%v", status, handlerErr)
}
if handlerErr == nil {
t.Fatal("expected delete to be forbidden")
}
got, err := os.ReadFile(victim)
if err != nil || string(got) != "keep" {
t.Fatalf("delete escaped: err=%v content=%q", err, got)
}
}
+25 -35
View File
@@ -87,7 +87,11 @@ func settingsGetHandler(w http.ResponseWriter, r *http.Request, d *Context) (int
// filesystem. It is the comment/secret-marker source for both the YAML export and the settings
// JSON redactor.
func readEmbeddedConfigYaml() (string, error) {
content, err := fs.ReadFile(assetFs, "config.generated.yaml")
path := "config.generated.yaml"
if settings.Env.EmbeddedFs {
path = "embed/config.generated.yaml"
}
content, err := fs.ReadFile(assetFs, path)
if err != nil {
return "", err
}
@@ -453,33 +457,17 @@ func settingsShareDefaultsPatchHandler(w http.ResponseWriter, r *http.Request, d
return settingsShareDefaultsGetHandler(w, r, d)
}
// settingsMediaGetHandler returns the runtime media settings.
// @Summary Get runtime media settings
// @Description Admins receive the DB-backed media integration settings; every authenticated user
// @Description receives the player-relevant subset (transcode enabled, profiles, per-user limit).
// @Tags Settings
// @Produce json
// @Success 200 {object} mediaSettingsResponse "Media settings"
// @Router /api/settings/media [get]
// mediaSettingsResponse is the runtime media settings plus config-lock metadata.
// mediaSettingsResponse is the ADMIN ONLY media payload: the full stored settings
// (FullAdminMedia, which embeds the public UserTranscode fields) plus config-lock
// metadata and ffmpeg option lists. Non-admins never receive this type.
type mediaSettingsResponse struct {
settings.MediaSettings
settings.FullAdminMedia
LockedFromConfigPaths []string `json:"lockedFromConfigPaths,omitempty"`
LockMessage string `json:"lockMessage,omitempty"`
AccelDeviceOptions []ffmpeg.AccelDeviceOption `json:"accelDeviceOptions,omitempty"`
EncoderBackendOptions []ffmpeg.EncoderBackendOption `json:"encoderBackendOptions,omitempty"`
}
// mediaPlayerSettings is the player-relevant subset of the runtime media
// settings. It is safe to expose to any authenticated user so the player can
// reflect admin changes without a page reload.
type mediaPlayerSettings struct {
Enabled bool `json:"enabled"`
Profiles []string `json:"profiles"`
MaxConcurrentPerUser int `json:"maxConcurrentPerUser"`
HardwareAcceleration bool `json:"hardwareAcceleration"`
}
func mediaSettingsLockMeta() (paths []string, message string) {
paths = settings.ConfigSpecifiedMediaPaths()
if len(paths) == 0 {
@@ -491,12 +479,25 @@ func mediaSettingsLockMeta() (paths []string, message string) {
// The media settings APIs no longer require ffmpeg. go-media serves probing,
// metadata, subtitles, and video thumbnails without it. ffmpeg-only controls
// (transcoding, hardware acceleration) report their own availability.
// @Summary Get runtime media settings
// @Description Returns the media settings the caller is allowed to see.
// @Description Admins get the full settings, including ffmpeg and hardware options, so they can edit them.
// @Description Everyone else gets a small read-only view: whether transcoding is on, how many
// @Description transcodes they can run at once, which quality presets are enabled, and which image
// @Description previews the server can generate. Keys in the view are the same for admins and users.
// @Tags Settings
// @Produce json
// @Success 200 {object} mediaSettingsResponse "Media settings"
// @Router /api/settings/media [get]
func settingsMediaGetHandler(w http.ResponseWriter, r *http.Request, d *Context) (int, error) {
// PUBLIC payload: non-admins get only what the player and general UI read.
if !d.User.Permissions.Admin {
return RenderJSON(w, r, currentMediaPlayerSettings())
return RenderJSON(w, r, settings.CurrentUserMedia())
}
// ADMIN payload: full stored settings with effective and derived values filled in
// by PrepMediaForFrontend (a read-only copy; never saved).
media := settings.PrepMediaForFrontend(state.GetMediaSettings())
lockedPaths, lockMessage := mediaSettingsLockMeta()
media := state.GetMediaSettings()
svc := ffmpeg.Get()
var accelDeviceOptions []ffmpeg.AccelDeviceOption
var encoderBackendOptions []ffmpeg.EncoderBackendOption
@@ -506,7 +507,7 @@ func settingsMediaGetHandler(w http.ResponseWriter, r *http.Request, d *Context)
encoderBackendOptions = svc.EncoderBackendOptions()
}
return RenderJSON(w, r, mediaSettingsResponse{
MediaSettings: media,
FullAdminMedia: media,
LockedFromConfigPaths: lockedPaths,
LockMessage: lockMessage,
AccelDeviceOptions: accelDeviceOptions,
@@ -514,17 +515,6 @@ func settingsMediaGetHandler(w http.ResponseWriter, r *http.Request, d *Context)
})
}
// currentMediaPlayerSettings reads the live media settings for the player UI.
func currentMediaPlayerSettings() mediaPlayerSettings {
media := state.GetMediaSettings()
return mediaPlayerSettings{
Enabled: settings.TranscodeEnabled(),
Profiles: settings.EnabledTranscodeProfiles(),
MaxConcurrentPerUser: settings.TranscodeMaxConcurrentPerUser(),
HardwareAcceleration: media.HardwareAcceleration,
}
}
// settingsMediaPatchHandler merges a partial update into the runtime media settings.
// @Summary Update runtime media settings
// @Description Merges a partial JSON patch into the DB-backed media settings and applies them at runtime.
+27 -17
View File
@@ -13,21 +13,31 @@ import (
const settingsRedactSentinel = "SENTINEL-SETTINGS-SECRET"
func swapSettingsConfigForTest(t *testing.T, mutate func(*settings.Settings)) {
t.Helper()
cfg := settings.Config
if mutate != nil {
mutate(&cfg)
}
prev := settings.SwapConfigForTest(cfg)
t.Cleanup(func() { settings.SwapConfigForTest(prev) })
}
// TestSettingsGetHandler_redactsSecrets is the end-to-end regression guard for
// GHSA-892f-9h5p-mfc8: GET /api/settings (and ?property=auth) must not return cleartext secrets.
func TestSettingsGetHandler_redactsSecrets(t *testing.T) {
setupTestEnv(t)
prev := settings.Config
t.Cleanup(func() { settings.Config = prev })
settings.Config.Auth.Key = settingsRedactSentinel
settings.Config.Auth.TotpSecret = settingsRedactSentinel
settings.Config.Auth.Methods.PasswordAuth.AdminPassword = settingsRedactSentinel
settings.Config.Auth.Methods.PasswordAuth.Recaptcha.Secret = settingsRedactSentinel
settings.Config.Auth.Methods.OidcAuth.ClientSecret = settingsRedactSentinel
settings.Config.Auth.Methods.LdapAuth.UserPassword = settingsRedactSentinel
settings.Config.Auth.Methods.JwtAuth.Secret = settingsRedactSentinel
settings.Config.Integrations.OnlyOffice.Secret = settingsRedactSentinel
swapSettingsConfigForTest(t, func(cfg *settings.Settings) {
cfg.Auth.Key = settingsRedactSentinel
cfg.Auth.TotpSecret = settingsRedactSentinel
cfg.Auth.Methods.PasswordAuth.AdminPassword = settingsRedactSentinel
cfg.Auth.Methods.PasswordAuth.Recaptcha.Secret = settingsRedactSentinel
cfg.Auth.Methods.OidcAuth.ClientSecret = settingsRedactSentinel
cfg.Auth.Methods.LdapAuth.UserPassword = settingsRedactSentinel
cfg.Auth.Methods.JwtAuth.Secret = settingsRedactSentinel
cfg.Integrations.OnlyOffice.Secret = settingsRedactSentinel
})
admin := &users.User{
FrontendUser: users.FrontendUser{
@@ -62,9 +72,9 @@ func TestSettingsGetHandler_redactsSecrets(t *testing.T) {
func TestSettingsGetHandler_doesNotMutateLiveConfig(t *testing.T) {
setupTestEnv(t)
prev := settings.Config
t.Cleanup(func() { settings.Config = prev })
settings.Config.Auth.Key = settingsRedactSentinel
swapSettingsConfigForTest(t, func(cfg *settings.Settings) {
cfg.Auth.Key = settingsRedactSentinel
})
admin := &users.User{
FrontendUser: users.FrontendUser{
@@ -92,10 +102,10 @@ func TestSettingsGetHandler_doesNotMutateLiveConfig(t *testing.T) {
func TestSettingsGetHandler_keepsNonSecretAuthFields(t *testing.T) {
setupTestEnv(t)
prev := settings.Config
t.Cleanup(func() { settings.Config = prev })
settings.Config.Auth.Key = settingsRedactSentinel
settings.Config.Auth.TokenExpirationHours = 7
swapSettingsConfigForTest(t, func(cfg *settings.Settings) {
cfg.Auth.Key = settingsRedactSentinel
cfg.Auth.TokenExpirationHours = 7
})
admin := &users.User{
FrontendUser: users.FrontendUser{
+53 -52
View File
@@ -151,7 +151,13 @@ func handleWithStaticData(w http.ResponseWriter, r *http.Request, d *requestCont
externalLinks = newExternalLinks
}
defaultThemeColor := settings.DefaultBackgroundColor()
// Instance default dark mode: htmlVars/defaultDark use a boolean for first paint;
// globalVars.darkMode carries the full preference (including "auto") for the SPA.
defaultDark := settings.DefaultDarkMode()
defaultThemeColor := settings.Config.Frontend.Styling.LightBackground
if defaultDark {
defaultThemeColor = settings.Config.Frontend.Styling.DarkBackground
}
staticURL := settings.Config.Http.BaseURL + "public/static"
description := settings.Config.Frontend.Description
title := settings.Config.Frontend.Name
@@ -251,6 +257,7 @@ func handleWithStaticData(w http.ResponseWriter, r *http.Request, d *requestCont
"customCSS": template.CSS(settings.Config.Frontend.Styling.CustomCSSRaw),
"userSelectedTheme": template.CSS(userSelectedTheme),
"defaultDarkMode": settings.DefaultDarkMode(),
"autoDarkMode": settings.DefaultDarkModeIsAuto(),
"lightBackground": settings.Config.Frontend.Styling.LightBackground,
"darkBackground": settings.Config.Frontend.Styling.DarkBackground,
"staticURL": staticURL,
@@ -275,57 +282,51 @@ func handleWithStaticData(w http.ResponseWriter, r *http.Request, d *requestCont
// only include constant values that don't change during a users session
data["globalVars"] = map[string]interface{}{
"name": settings.Config.Frontend.Name,
"minSearchLength": settings.Config.Server.MinSearchLength,
"searchResultsLimit": settings.Config.Server.SearchResultsLimit,
"disableExternal": settings.Config.Frontend.DisableDefaultLinks,
"darkMode": settings.Config.UserDefaults.UI.DarkMode,
"singleClick": settings.Config.UserDefaults.Listing.SingleClick,
"lightBackground": settings.Config.Frontend.Styling.LightBackground,
"darkBackground": settings.Config.Frontend.Styling.DarkBackground,
"baseURL": settings.Config.Http.BaseURL,
"version": versionString,
"commitSHA": commitSHAString,
"signup": settings.Config.Auth.Methods.PasswordAuth.Signup,
"noAuth": settings.Config.Auth.Methods.NoAuth,
"enableThumbs": !settings.Config.Server.DisablePreviews,
"externalLinks": externalLinks,
"externalUrl": strings.TrimSuffix(settings.Config.Http.ExternalUrl, "/"),
"onlyOfficeUrl": onlyOfficeUrlForSPA(),
"oidcAvailable": settings.Config.Auth.Methods.OidcAuth.Enabled,
"jwtAvailable": settings.Config.Auth.Methods.JwtAuth.Enabled,
"proxyAvailable": settings.Config.Auth.Methods.ProxyAuth.Enabled,
"passwordAvailable": settings.Config.Auth.Methods.PasswordAuth.Enabled,
"passwordMinLength": settings.PasswordMinLength(),
"ldapAvailable": settings.Config.Auth.Methods.LdapAuth.Enabled,
"ffmpegAvailable": settings.FFmpegEnabled(),
"hardwareAccelerationAvailable": settings.HardwareAccelerationSupported(),
"transcodeEnabled": settings.TranscodeEnabled(),
"transcodeProfiles": settings.EnabledTranscodeProfiles(),
"transcodeMaxPerUser": settings.TranscodeMaxConcurrentPerUser(),
"docPreviewExtensions": iteminfo.DocPreviewExtensionsForSPA(),
"updateAvailable": utils.GetUpdateAvailableUrl(),
"disableNavButtons": disableNavButtons,
"userSelectableThemes": settings.Config.Frontend.Styling.CustomThemeOptions,
"enableHeicConversion": settings.CanConvertImage("heic"),
"enableJxlConversion": settings.CanConvertImage("jxl"),
"eventBasedThemes": !settings.Config.Frontend.Styling.DisableEventBasedThemes,
"loginIcon": loginIcon,
"disableSidebar": disableSidebar,
"shareHash": shareHash,
"oidcLoginButtonText": settings.Config.Frontend.OIDCLoginButtonText,
"loginButtonText": settings.Config.Frontend.LoginButtonText,
"passkeyAvailable": settings.Config.Auth.Methods.PasskeyAuth.Enabled,
"passkeyLoginButtonText": settings.Config.Auth.Methods.PasskeyAuth.LoginButtonText,
"recaptcha": recaptchaEnabled,
"recaptchaHost": recaptchaHost,
"recaptchaKey": recaptchaKey,
"disablePWAInstall": settings.Config.Frontend.DisablePWAInstall,
"devMode": settings.Env.IsDevMode,
"playwrightTest": settings.Env.IsPlaywright,
"officeProduct": officeProductForSPA(),
"officeExtensions": wopiExtensionsForSPA(),
"textPreviewMimeTypes": iteminfo.TextPreviewMimeTypes(),
"name": settings.Config.Frontend.Name,
"minSearchLength": settings.Config.Server.MinSearchLength,
"searchResultsLimit": settings.Config.Server.SearchResultsLimit,
"disableExternal": settings.Config.Frontend.DisableDefaultLinks,
"darkMode": settings.DefaultDarkModePreference(),
"singleClick": settings.Config.UserDefaults.Listing.SingleClick,
"lightBackground": settings.Config.Frontend.Styling.LightBackground,
"darkBackground": settings.Config.Frontend.Styling.DarkBackground,
"baseURL": settings.Config.Http.BaseURL,
"version": versionString,
"commitSHA": commitSHAString,
"signup": settings.Config.Auth.Methods.PasswordAuth.Signup,
"noAuth": settings.Config.Auth.Methods.NoAuth,
"enableThumbs": !settings.Config.Server.DisablePreviews,
"externalLinks": externalLinks,
"externalUrl": strings.TrimSuffix(settings.Config.Http.ExternalUrl, "/"),
"onlyOfficeUrl": onlyOfficeUrlForSPA(),
"oidcAvailable": settings.Config.Auth.Methods.OidcAuth.Enabled,
"jwtAvailable": settings.Config.Auth.Methods.JwtAuth.Enabled,
"proxyAvailable": settings.Config.Auth.Methods.ProxyAuth.Enabled,
"passwordAvailable": settings.Config.Auth.Methods.PasswordAuth.Enabled,
"passwordMinLength": settings.PasswordMinLength(),
"ldapAvailable": settings.Config.Auth.Methods.LdapAuth.Enabled,
"ffmpegAvailable": settings.FFmpegEnabled(),
"docPreviewExtensions": iteminfo.DocPreviewExtensionsForSPA(),
"updateAvailable": utils.GetUpdateAvailableUrl(),
"disableNavButtons": disableNavButtons,
"userSelectableThemes": settings.Config.Frontend.Styling.CustomThemeOptions,
"eventBasedThemes": !settings.Config.Frontend.Styling.DisableEventBasedThemes,
"loginIcon": loginIcon,
"disableSidebar": disableSidebar,
"shareHash": shareHash,
"oidcLoginButtonText": settings.Config.Frontend.OIDCLoginButtonText,
"loginButtonText": settings.Config.Frontend.LoginButtonText,
"passkeyAvailable": settings.Config.Auth.Methods.PasskeyAuth.Enabled,
"passkeyLoginButtonText": settings.Config.Auth.Methods.PasskeyAuth.LoginButtonText,
"recaptcha": recaptchaEnabled,
"recaptchaHost": recaptchaHost,
"recaptchaKey": recaptchaKey,
"disablePWAInstall": settings.Config.Frontend.DisablePWAInstall,
"devMode": settings.Env.IsDevMode,
"playwrightTest": settings.Env.IsPlaywright,
"officeProduct": officeProductForSPA(),
"officeExtensions": wopiExtensionsForSPA(),
"textPreviewMimeTypes": iteminfo.TextPreviewMimeTypes(),
}
cspNonce, err := utils.CSPNonce()
+7 -1
View File
@@ -487,10 +487,16 @@ func ServeSingleFile(w http.ResponseWriter, r *http.Request, d *Context, source
bound = userScope
}
realPath, _, err := idx.GetRealPathScoped(bound, scopedFilePath)
realPath, isDir, err := idx.GetRealPathScoped(bound, scopedFilePath)
if err != nil {
return realPathErrStatus(err), err
}
if st, statErr := os.Stat(realPath); statErr == nil {
isDir = st.IsDir()
}
if !state.AccessPermittedAtRealPath(idx, permUser, realPath, isDir) {
return http.StatusForbidden, fmt.Errorf("access denied to path %s", scopedFilePath)
}
isOnlyOffice := IsOnlyOfficeCompatibleFile(displayFileName) && settings.Config.Integrations.OnlyOffice.Url != "" && !settings.Config.Integrations.OnlyOffice.IsCollabora()
var documentId string
+15
View File
@@ -0,0 +1,15 @@
package web
import (
"sync"
"testing"
)
// webGlobalTestMu serializes tests that mutate process-wide settings.Config or state.
var webGlobalTestMu sync.Mutex
func lockWebGlobals(t testing.TB) {
t.Helper()
webGlobalTestMu.Lock()
t.Cleanup(webGlobalTestMu.Unlock)
}
@@ -33,7 +33,7 @@ func TestTranscodeManagerIsWiredWithoutFFmpeg(t *testing.T) {
// Startup with ffmpeg missing and transcoding disabled.
settings.Env.FFmpegAvailable = false
disabled := originalMedia
disabled.Transcode = settings.MediaTranscode{Enabled: false}
disabled.Transcode = settings.FullAdminTranscode{UserTranscode: settings.UserTranscode{Enabled: false}}
settings.ApplyMediaSettings(disabled)
setupTestEnv(t)
+3 -1
View File
@@ -176,6 +176,8 @@ func usersPostHandler(w http.ResponseWriter, r *http.Request, d *Context) (int,
}
// Parse the JSON into the UserRequest struct
var req UserRequest
// Seed with defaults so the decode below only overrides fields the request sent.
state.PrefillUserProfileDefaults(&req.User)
if err = json.Unmarshal(body, &req); err != nil {
return http.StatusBadRequest, err
}
@@ -199,7 +201,7 @@ func usersPostHandler(w http.ResponseWriter, r *http.Request, d *Context) (int,
return status, err
}
plaintextPassword := req.User.Password
err = state.CreateUser(&req.User, plaintextPassword)
err = state.CreateUserKeepingProfile(&req.User, plaintextPassword)
if err != nil {
return http.StatusInternalServerError, err
}
@@ -0,0 +1,73 @@
package web
import (
"bytes"
"net/http"
"net/http/httptest"
"testing"
"github.com/gtsteffaniak/filebrowser/backend/internal/state"
"github.com/gtsteffaniak/filebrowser/backend/pkg/settings"
)
// Omitted permissions and omitted profile keys come from userDefaults; a partially
// specified fileLoading object only overrides the keys it contains.
func TestUsersPostOmittedFieldsFallBackToDefaults(t *testing.T) {
setupTestEnv(t)
defaults := settings.Config.UserDefaults
defaults.UI.Locale = "de"
defaults.Account.Permissions.Share = true
defaults.FileLoading.MaxConcurrent = 7
defaults.FileLoading.UploadChunkSize = 9
state.SetEffectiveUserDefaultsForTest(t, defaults)
admin := createPostTestAdmin(t)
body := []byte(`{"which":[],"data":{"username":"partial","password":"secret","loginMethod":"password",
"fileLoading":{"maxConcurrentUpload":3}}}`)
req := httptest.NewRequest(http.MethodPost, "/api/users", bytes.NewReader(body))
status, err := usersPostHandler(httptest.NewRecorder(), req, &Context{User: admin})
if err != nil || status != http.StatusCreated {
t.Fatalf("usersPostHandler() status=%d err=%v", status, err)
}
got, err := state.GetUserByUsername("partial")
if err != nil {
t.Fatal(err)
}
if got.Locale != "de" {
t.Errorf("locale = %q, want default de", got.Locale)
}
if !got.Permissions.Share {
t.Errorf("share = false, want true from default permissions")
}
if got.FileLoading.MaxConcurrent != 3 {
t.Errorf("maxConcurrentUpload = %d, want requested 3", got.FileLoading.MaxConcurrent)
}
if got.FileLoading.UploadChunkSize != 9 {
t.Errorf("uploadChunkSizeMb = %d, want default 9", got.FileLoading.UploadChunkSize)
}
}
// A create request must not mutate the shared in-memory userDefaults (json.Unmarshal
// reuses slice backing arrays).
func TestUsersPostDoesNotMutateUserDefaults(t *testing.T) {
setupTestEnv(t)
defaults := settings.Config.UserDefaults
defaults.Listing.NewFileTemplate = []string{"default.txt"}
state.SetEffectiveUserDefaultsForTest(t, defaults)
admin := createPostTestAdmin(t)
body := []byte(`{"which":[],"data":{"username":"nomutate","password":"secret","loginMethod":"password",
"newFileTemplate":["custom.txt"]}}`)
req := httptest.NewRequest(http.MethodPost, "/api/users", bytes.NewReader(body))
status, err := usersPostHandler(httptest.NewRecorder(), req, &Context{User: admin})
if err != nil || status != http.StatusCreated {
t.Fatalf("usersPostHandler() status=%d err=%v", status, err)
}
if got := state.EffectiveUserDefaults().Listing.NewFileTemplate; len(got) != 1 || got[0] != "default.txt" {
t.Errorf("userDefaults newFileTemplate = %v, want [default.txt]", got)
}
}
@@ -0,0 +1,139 @@
package web
import (
"bytes"
"net/http"
"net/http/httptest"
"testing"
"github.com/gtsteffaniak/filebrowser/backend/internal/database/users"
"github.com/gtsteffaniak/filebrowser/backend/internal/state"
"github.com/gtsteffaniak/filebrowser/backend/pkg/settings"
)
func createPostTestAdmin(t *testing.T) *users.User {
t.Helper()
admin := &users.User{
ID: 1,
FrontendUser: users.FrontendUser{
Username: "admin",
LoginMethod: users.LoginMethodOidc,
Permissions: users.Permissions{Admin: true},
},
}
if err := state.CreateUser(admin, ""); err != nil {
t.Fatal(err)
}
return admin
}
// Issue #3322: profile/account values in POST /api/users must not be replaced by userDefaults.
func TestUsersPostKeepsRequestedProfileValues(t *testing.T) {
setupTestEnv(t)
defaults := settings.Config.UserDefaults
defaults.UI.Locale = "en"
defaults.UI.ThemeColor = "var(--blue)"
defaults.UI.DarkMode = users.DarkModeDark
defaults.Account.LockPassword = false
defaults.Account.DisableSettings = false
defaults.Account.ShowAdvancedProfile = false
defaults.Listing.PromptRightCloseButton = false
defaults.Listing.NewFileTemplate = []string{""}
defaults.Listing.ShowHidden = true
state.SetEffectiveUserDefaultsForTest(t, defaults)
admin := createPostTestAdmin(t)
body := []byte(`{"which":[],"data":{
"username":"profiletest","password":"secret","loginMethod":"password",
"locale":"zhCN","themeColor":"var(--green)","darkMode":false,
"lockPassword":true,"disableSettings":true,"showAdvancedProfile":true,
"promptRightCloseButton":true,"newFileTemplate":["custom.txt"],
"preview":{"image":false},
"permissions":{"admin":false,"api":true,"share":true,"realtime":false},
"scopes":[{"name":"/","scope":"/","permissions":{"view":true,"download":true,"modify":true,"create":true,"delete":true}}]
}}`)
req := httptest.NewRequest(http.MethodPost, "/api/users", bytes.NewReader(body))
rec := httptest.NewRecorder()
status, err := usersPostHandler(rec, req, &Context{User: admin})
if err != nil || status != http.StatusCreated {
t.Fatalf("usersPostHandler() status=%d err=%v", status, err)
}
got, err := state.GetUserByUsername("profiletest")
if err != nil {
t.Fatal(err)
}
if got.Locale != "zhCN" {
t.Errorf("locale = %q, want zhCN", got.Locale)
}
if got.ThemeColor != "var(--green)" {
t.Errorf("themeColor = %q, want var(--green)", got.ThemeColor)
}
if got.DarkMode != users.DarkModeLight {
t.Errorf("darkMode = %q, want light", got.DarkMode)
}
if !got.LockPassword {
t.Errorf("lockPassword = false, want true")
}
if !got.DisableSettings {
t.Errorf("disableSettings = false, want true")
}
if !got.ShowAdvancedProfile {
t.Errorf("showAdvancedProfile = false, want true")
}
if !got.PromptRightCloseButton {
t.Errorf("promptRightCloseButton = false, want true")
}
if len(got.NewFileTemplate) != 1 || got.NewFileTemplate[0] != "custom.txt" {
t.Errorf("newFileTemplate = %v, want [custom.txt]", got.NewFileTemplate)
}
if got.Preview.Image {
t.Errorf("preview.image = true, want false")
}
// Omitted fields still come from userDefaults.
if !got.ShowHidden {
t.Errorf("showHidden = false, want true from userDefaults")
}
if !got.Preview.Video {
t.Errorf("preview.video = false, want true from userDefaults")
}
wantPerms := users.Permissions{Api: true, Share: true}
if got.Permissions.Admin != wantPerms.Admin || got.Permissions.Api != wantPerms.Api ||
got.Permissions.Share != wantPerms.Share || got.Permissions.Realtime != wantPerms.Realtime {
t.Errorf("permissions = %+v, want %+v", got.Permissions, wantPerms)
}
}
// Enforced userDefaults still win over request values on create.
func TestUsersPostEnforcedDefaultsWinOverRequest(t *testing.T) {
setupTestEnv(t)
defaults := settings.Config.UserDefaults
defaults.UI.Locale = "en"
state.SetEffectiveUserDefaultsForTest(t, defaults)
if err := state.PatchUserDefaultsEnforced([]byte(`{"ui":{"locale":true}}`)); err != nil {
t.Fatal(err)
}
admin := createPostTestAdmin(t)
body := []byte(`{"which":[],"data":{"username":"enforced","password":"secret","loginMethod":"password","locale":"zhCN","lockPassword":true}}`)
req := httptest.NewRequest(http.MethodPost, "/api/users", bytes.NewReader(body))
rec := httptest.NewRecorder()
status, err := usersPostHandler(rec, req, &Context{User: admin})
if err != nil || status != http.StatusCreated {
t.Fatalf("usersPostHandler() status=%d err=%v", status, err)
}
got, err := state.GetUserByUsername("enforced")
if err != nil {
t.Fatal(err)
}
if got.Locale != "en" {
t.Errorf("locale = %q, want enforced en", got.Locale)
}
if !got.LockPassword {
t.Errorf("lockPassword = false, want true (not enforced)")
}
}
+39
View File
@@ -19,6 +19,7 @@ import (
"github.com/gtsteffaniak/filebrowser/backend/internal/activity"
"github.com/gtsteffaniak/filebrowser/backend/internal/adapters/fs/files"
"github.com/gtsteffaniak/filebrowser/backend/internal/adapters/fs/fileutils"
"github.com/gtsteffaniak/filebrowser/backend/internal/database/access"
activitydb "github.com/gtsteffaniak/filebrowser/backend/internal/database/activity"
"github.com/gtsteffaniak/filebrowser/backend/internal/database/users"
commonerrors "github.com/gtsteffaniak/filebrowser/backend/internal/errors"
@@ -237,6 +238,35 @@ func (ffs *filteredFileSystem) checkScopePath(requestPath string, forRemoval boo
return err
}
// checkResolvedAccess enforces per-path DENY rules on the symlink-resolved target.
func (ffs *filteredFileSystem) checkResolvedAccess(requestPath string, forRemoval bool) error {
idx := indexing.GetIndex(ffs.source)
if idx == nil {
return fmt.Errorf("source not found")
}
userScope, err := ffs.user.GetScopeForSourceName(ffs.source)
if err != nil {
return err
}
fullIndexPath := utils.JoinScopedIndexPath(userScope, strings.TrimPrefix(requestPath, "/"))
var realPath string
if forRemoval {
realPath, err = idx.GetRealPathForRemoval(userScope, fullIndexPath)
} else {
realPath, err = idx.GetRealPathForMutation(userScope, fullIndexPath)
}
if err != nil {
if errors.Is(err, commonerrors.ErrPathEscapesScope) {
return os.ErrPermission
}
return err
}
if !state.AccessPermittedAtRealPath(idx, ffs.user.Username, realPath, access.IsDirAtRealPath(realPath)) {
return os.ErrPermission
}
return nil
}
func (ffs *filteredFileSystem) requireCreatePermissionForNewFile(requestPath string) error {
idx := indexing.GetIndex(ffs.source)
if idx == nil {
@@ -412,6 +442,9 @@ func (ffs *filteredFileSystem) Mkdir(ctx context.Context, name string, perm os.F
logger.Debugf("Mkdir: scope check denied for %s: %v", name, err)
return err
}
if err := ffs.checkResolvedAccess(name, false); err != nil {
return err
}
if err := ffs.fs.Mkdir(ctx, name, perm); err != nil {
return err
@@ -450,6 +483,9 @@ func (ffs *filteredFileSystem) OpenFile(ctx context.Context, requestPath string,
logger.Debugf("OpenFile: scope check denied for %s: %v", requestPath, err)
return nil, err
}
if err := ffs.checkResolvedAccess(requestPath, false); err != nil {
return nil, err
}
if (flag&os.O_CREATE) != 0 && !ffs.sourceFilePerms().Create {
if err := ffs.requireCreatePermissionForNewFile(requestPath); err != nil {
@@ -462,6 +498,9 @@ func (ffs *filteredFileSystem) OpenFile(ctx context.Context, requestPath string,
logger.Debugf("OpenFile: read denied by scope for %s", requestPath)
return nil, os.ErrPermission
}
if err := ffs.checkResolvedAccess(requestPath, false); err != nil {
return nil, err
}
}
var desiredMtime time.Time
+1
View File
@@ -30,6 +30,7 @@ import (
// setupWebDAVTestEnv sets up the test environment with multiple sources and users
func setupWebDAVTestEnv(t *testing.T) (string, string) {
lockWebGlobals(t)
// Create temp directories for test sources
tempDir := t.TempDir()
source1Path := filepath.Join(tempDir, "source1")
+77 -3
View File
@@ -8,6 +8,7 @@ import (
"runtime"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/gtsteffaniak/filebrowser/backend/internal/adapters/fs/fileutils"
@@ -24,8 +25,9 @@ import (
)
var (
RealPathCache = cache.NewCache[string](48*time.Hour, 72*time.Hour)
IsDirCache = cache.NewCache[bool](48*time.Hour, 72*time.Hour)
RealPathCache = cache.NewCache[string](48*time.Hour, 72*time.Hour)
IsDirCache = cache.NewCache[bool](48*time.Hour, 72*time.Hour)
realPathGeneration atomic.Uint64
)
// getDiskUsage returns the actual disk space used by a file in bytes
@@ -1222,6 +1224,7 @@ func (idx *Index) getRealPathInternal(boundIndexPath string, enforceScope bool,
if ok && cached != "" {
return cached, isDir, nil
}
generation := realPathGeneration.Load()
absolutePath, err := filepath.Abs(joinedPath)
if err != nil {
return absolutePath, false, fmt.Errorf("could not get real path: %v, %s", joinedPath, err)
@@ -1239,9 +1242,77 @@ func (idx *Index) getRealPathInternal(boundIndexPath string, enforceScope bool,
}
RealPathCache.Set(cacheKey, realPath)
IsDirCache.Set(cacheKey+":isdir", isDir)
if realPathGeneration.Load() != generation {
RealPathCache.Delete(cacheKey)
IsDirCache.Delete(cacheKey + ":isdir")
}
return realPath, isDir, nil
}
// realPathCacheJoinedPath builds the filesystem join used in RealPathCache keys.
func realPathCacheJoinedPath(sourceRoot, indexPath string) string {
seg := utils.CleanIndexPathSegment(indexPath)
if seg == "" {
return sourceRoot
}
return filepath.Join(sourceRoot, seg)
}
func realPathCacheIndexAncestors(indexPath string) []string {
rel := utils.CleanIndexPathSegment(indexPath)
if rel == "" {
return []string{"/"}
}
parts := strings.Split(rel, "/")
ancestors := []string{"/"}
for i := range parts {
ancestors = append(ancestors, "/"+strings.Join(parts[:i+1], "/"))
}
return ancestors
}
// ClearRealPathCaches drops every cached symlink resolution. The cache library has no
// prefix delete, so subtree invalidation uses a full clear; entries repopulate lazily.
func ClearRealPathCaches() {
realPathGeneration.Add(1)
RealPathCache.ClearAll()
IsDirCache.ClearAll()
}
// InvalidateRealPathCache drops cached symlink resolutions for indexPath and its
// ancestors. scopeBounds should include any caller scope prefixes used with
// GetRealPathScoped (e.g. user or share path).
func (idx *Index) InvalidateRealPathCache(indexPath string, scopeBounds ...string) {
realPathGeneration.Add(1)
boundSet := map[string]struct{}{"/": {}, "": {}}
for _, b := range scopeBounds {
b = strings.TrimRight(strings.TrimSpace(b), "/")
if b == "" || b == "/" {
boundSet["/"] = struct{}{}
boundSet[""] = struct{}{}
continue
}
if !strings.HasPrefix(b, "/") {
b = "/" + b
}
boundSet[b] = struct{}{}
}
bounds := make([]string, 0, len(boundSet))
for b := range boundSet {
bounds = append(bounds, b)
}
for _, ip := range realPathCacheIndexAncestors(indexPath) {
joined := realPathCacheJoinedPath(idx.Path, ip)
for _, enforceScope := range []bool{false, true} {
for _, bound := range bounds {
key := fmt.Sprintf("realpath|scoped=%t|bound=%q|path=%q", enforceScope, bound, joined)
RealPathCache.Delete(key)
IsDirCache.Delete(key + ":isdir")
}
}
}
}
// GetRealPathForMutation resolves an index path for a mutating filesystem
// operation (create/overwrite/mkdir/move-into). When the leaf exists it is fully
// resolved; otherwise the deepest existing ancestor is resolved and the missing
@@ -1603,7 +1674,10 @@ func setFilePreviewFlags(fileInfo *iteminfo.ItemInfo, realPath string) {
}
switch simpleType {
case "image":
if fileInfo.Size > iteminfo.LargeFileSizeThreshold {
// Camera RAW previews are the embedded JPEG, which is read without decoding the RAW data,
// so large RAW files (medium format, uncompressed) don't need the size limit.
isRaw := iteminfo.IsRawImage(strings.ToLower(filepath.Ext(fileInfo.Name)))
if fileInfo.Size > iteminfo.LargeFileSizeThreshold && !isRaw {
fileInfo.HasPreview = false
return
}
+34
View File
@@ -0,0 +1,34 @@
package indexing
import (
"testing"
"github.com/gtsteffaniak/filebrowser/backend/pkg/indexing/iteminfo"
)
func TestSetFilePreviewFlagsLargeImages(t *testing.T) {
large := iteminfo.LargeFileSizeThreshold + 1
tests := []struct {
name string
file string
typ string
size int64
want bool
}{
{"normal image under the limit", "a.jpg", "image/jpeg", 5 << 20, true},
{"normal image over the limit", "a.jpg", "image/jpeg", int64(large), false},
{"RAW under the limit", "a.raf", "image/x-fuji-raf", 26 << 20, true},
{"large RAW keeps its preview, the embedded JPEG is read without decoding", "a.RAF", "image/x-fuji-raf", int64(large), true},
{"large DNG keeps its preview", "a.dng", "image/x-adobe-dng", int64(large), true},
{"empty file has no preview", "a.raf", "image/x-fuji-raf", 0, false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
info := &iteminfo.ItemInfo{Name: tt.file, Type: tt.typ, Size: tt.size}
setFilePreviewFlags(info, "/nonexistent/"+tt.file)
if info.HasPreview != tt.want {
t.Fatalf("HasPreview = %v, want %v", info.HasPreview, tt.want)
}
})
}
}
@@ -2,6 +2,7 @@ package indexing
import (
"errors"
"fmt"
"os"
"path/filepath"
"testing"
@@ -163,6 +164,66 @@ func TestGetRealPathScoped_CacheKeyDoesNotCollideWithUnscoped(t *testing.T) {
}
}
func TestRealPathCacheGeneration_BumpsOnInvalidateAndClear(t *testing.T) {
idx := symlinkTestIndex(t.TempDir())
gen := realPathGeneration.Load()
idx.InvalidateRealPathCache("/any")
if realPathGeneration.Load() != gen+1 {
t.Fatalf("InvalidateRealPathCache: gen=%d want %d", realPathGeneration.Load(), gen+1)
}
gen = realPathGeneration.Load()
ClearRealPathCaches()
if realPathGeneration.Load() != gen+1 {
t.Fatalf("ClearRealPathCaches: gen=%d want %d", realPathGeneration.Load(), gen+1)
}
}
func TestClearRealPathCaches_ClearsWarmEntries(t *testing.T) {
sourceRoot, userScope := setupSymlinkEscapeFixture(t)
idx := symlinkTestIndex(sourceRoot)
userDir := filepath.Join(sourceRoot, "user1")
if err := os.WriteFile(filepath.Join(userDir, "foo"), []byte("x"), 0o644); err != nil {
t.Fatal(err)
}
if _, _, err := idx.GetRealPathScoped(userScope, "/user1/foo"); err != nil {
t.Fatalf("warm cache: %v", err)
}
joined := filepath.Join(sourceRoot, "user1", "foo")
key := fmt.Sprintf("realpath|scoped=%t|bound=%q|path=%q", true, userScope, joined)
if _, ok := RealPathCache.Get(key); !ok {
t.Fatalf("expected cache key %q", key)
}
ClearRealPathCaches()
if _, ok := RealPathCache.Get(key); ok {
t.Fatal("ClearRealPathCaches should drop warm entries")
}
}
func TestInvalidateRealPathCache_ClearsGetRealPathEntry(t *testing.T) {
sourceRoot, userScope := setupSymlinkEscapeFixture(t)
idx := symlinkTestIndex(sourceRoot)
userDir := filepath.Join(sourceRoot, "user1")
if err := os.WriteFile(filepath.Join(userDir, "foo"), []byte("x"), 0o644); err != nil {
t.Fatal(err)
}
if _, _, err := idx.GetRealPath("/user1/foo"); err != nil {
t.Fatalf("warm cache: %v", err)
}
joined := realPathCacheJoinedPath(sourceRoot, "/user1/foo")
key := fmt.Sprintf("realpath|scoped=%t|bound=%q|path=%q", false, "/", joined)
if _, ok := RealPathCache.Get(key); !ok {
t.Fatalf("expected cache key %q", key)
}
idx.InvalidateRealPathCache("/user1/foo", userScope)
if _, ok := RealPathCache.Get(key); ok {
t.Fatal("InvalidateRealPathCache should drop unscoped cache entry")
}
}
func TestGetRealPathForMutation_ExistingInScope(t *testing.T) {
sourceRoot, userScope := setupSymlinkEscapeFixture(t)
idx := symlinkTestIndex(sourceRoot)
Loaded 100 of 237 files, more files were not shown because too many files have changed in this diff. Show more