mirror of
https://github.com/gtsteffaniak/filebrowser.git
synced 2026-11-13 12:41:59 -05:00
Compare commits
5
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
fafb755576 | ||
|
|
3e05196256 | ||
|
|
4db06c9ab1 | ||
|
|
0d30433739 | ||
|
|
b45e3498b8 |
No files matched your search
+1
-3
@@ -10,9 +10,7 @@ backend/tmp
|
||||
backend/dist
|
||||
backend/backend*
|
||||
backend/vendor/**
|
||||
backend/internal/web/dist/*
|
||||
!backend/internal/web/dist/.gitkeep
|
||||
|
||||
|
||||
frontend/dist/**
|
||||
frontend/node_modules/**
|
||||
frontend/pkg/**
|
||||
@@ -42,7 +42,7 @@ jobs:
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: dist
|
||||
path: frontend/dist
|
||||
path: backend/internal/web/dist
|
||||
build_backend:
|
||||
name: Build backend
|
||||
runs-on: ubuntu-24.04-arm
|
||||
@@ -97,7 +97,7 @@ jobs:
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: dist
|
||||
path: frontend/dist
|
||||
path: backend/internal/web/dist
|
||||
- name: Download backend build
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
|
||||
@@ -12,11 +12,6 @@ All notable changes to this project will be documented in this file. For commit
|
||||
- Transcoding for media playback with quality presets -- requires ffmpeg.
|
||||
- Hardware Acceleration device selector (Software / detected GPU devices)
|
||||
- More operations use native api operations rather than ffmpeg, improving performance in many cases. Added temporary config option to force ffmpeg to be used
|
||||
- HLS keyframe indexes for MP4/MOV/M4V/MKV/WebM are read from the container index with go-media instead of demuxing every packet with ffprobe; ffprobe still runs when go-media cannot read the file or reports a keyframe gap that would rule out stream copy.
|
||||
- HEIC/AVIF rotation (irot) for the ffmpeg decode fallback is read with go-media instead of a full `ffmpeg -f null` decode.
|
||||
- AVIF images get previews through the HEIC preview path (go-media decode, ffmpeg fallback) when HEIC previews are enabled.
|
||||
- WMV/ASF video thumbnails use go-media for WMV3; WMV1/WMV2 still use ffmpeg.
|
||||
- ffmpeg and go-media share one media metadata cache keyed by path, size and modification time, so a probe, duration or subtitle result from either backend is reused by the other.
|
||||
- Document enhancements
|
||||
- Official Collabora support using WOPI config.
|
||||
- Document previews use a pure-go in-process package (all versions get it now).
|
||||
|
||||
+1
-1
@@ -39,7 +39,7 @@ COPY --from=base --chown=filebrowser:1000 [ "/app/filebrowser", "./" ]
|
||||
COPY --from=base --chown=filebrowser:1000 [ "/app/config.yaml", "./data/config.yaml" ]
|
||||
COPY --from=base --chown=filebrowser:1000 [ "/app/config.yaml", "./" ]
|
||||
COPY --from=base --chown=filebrowser:1000 [ "/app/reduce-rounded-corners.css", "./" ]
|
||||
COPY --from=nbuild --chown=filebrowser:1000 [ "/build/frontend/dist/", "./internal/web/dist/" ]
|
||||
COPY --from=nbuild --chown=filebrowser:1000 [ "/build/backend/internal/web/dist/", "./internal/web/dist/" ]
|
||||
## sanity checks
|
||||
RUN [ "filebrowser", "version" ]
|
||||
RUN [ "ffmpeg", "-version" ]
|
||||
|
||||
@@ -34,7 +34,7 @@ COPY --from=base --chown=filebrowser:1000 [ "/app/filebrowser", "./" ]
|
||||
COPY --from=base --chown=filebrowser:1000 [ "/app/config.yaml", "./data/config.yaml" ]
|
||||
COPY --from=base --chown=filebrowser:1000 [ "/app/config.yaml", "./" ]
|
||||
COPY --from=base --chown=filebrowser:1000 [ "/app/reduce-rounded-corners.css", "./" ]
|
||||
COPY --from=nbuild --chown=filebrowser:1000 [ "/build/frontend/dist/", "./internal/web/dist/" ]
|
||||
COPY --from=nbuild --chown=filebrowser:1000 [ "/build/backend/internal/web/dist/", "./internal/web/dist/" ]
|
||||
## sanity checks
|
||||
RUN [ "filebrowser", "version" ]
|
||||
# exposing default port for auto discovery.
|
||||
|
||||
@@ -68,7 +68,7 @@ COPY --from=base --chown=filebrowser:1000 [ "/app/filebrowser", "./" ]
|
||||
COPY --from=base --chown=filebrowser:1000 [ "/app/config.yaml", "./data/config.yaml" ]
|
||||
COPY --from=base --chown=filebrowser:1000 [ "/app/config.yaml", "./" ]
|
||||
COPY --from=base --chown=filebrowser:1000 [ "/app/reduce-rounded-corners.css", "./" ]
|
||||
COPY --from=nbuild --chown=filebrowser:1000 [ "/build/frontend/dist/", "./internal/web/dist/" ]
|
||||
COPY --from=nbuild --chown=filebrowser:1000 [ "/build/backend/internal/web/dist/", "./internal/web/dist/" ]
|
||||
|
||||
RUN filebrowser version && ffmpeg -version && ffprobe -version
|
||||
|
||||
|
||||
@@ -12,7 +12,7 @@ WORKDIR /app/backend/
|
||||
COPY [ "./backend/filebrowser", "./"]
|
||||
COPY [ "./backend/reduce-rounded-corners.css", "./no-rounded.css" ]
|
||||
RUN mkdir -p ./internal/web/dist
|
||||
COPY [ "./frontend/dist/", "./internal/web/dist/"]
|
||||
COPY [ "./backend/internal/web/dist/", "./internal/web/dist/"]
|
||||
|
||||
# generate duplicate 1.1MB files
|
||||
RUN dd if=/dev/zero of=/app/frontend/tests/playwright-files/1.1MB.bin bs=1024 count=1126 && dd if=/dev/zero of=/app/frontend/tests/playwright-files/1.1mb.bin bs=1024 count=1126
|
||||
|
||||
@@ -6,7 +6,7 @@ COPY [ "./frontend/tests", "./tests" ]
|
||||
WORKDIR /app/backend/
|
||||
COPY [ "./backend/filebrowser", "./"]
|
||||
RUN mkdir -p ./internal/web/dist
|
||||
COPY [ "./frontend/dist/", "./internal/web/dist/"]
|
||||
COPY [ "./backend/internal/web/dist/", "./internal/web/dist/"]
|
||||
RUN apt update && apt install nginx -y
|
||||
RUN mv default.conf /etc/nginx/conf.d/default.conf
|
||||
RUN sed -i 's/filebrowser/localhost/g' /etc/nginx/conf.d/default.conf
|
||||
|
||||
@@ -11,7 +11,7 @@ COPY [ "./_docker/src/settings/backend/config.yaml", "./"]
|
||||
COPY [ "./_docker/src/settings/backend/database.db.old", "./"]
|
||||
COPY [ "./backend/reduce-rounded-corners.css", "./no-rounded.css"]
|
||||
RUN mkdir -p ./internal/web/dist
|
||||
COPY [ "./frontend/dist/", "./internal/web/dist/"]
|
||||
COPY [ "./backend/internal/web/dist/", "./internal/web/dist/"]
|
||||
|
||||
ENV FILEBROWSER_PLAYWRIGHT_TEST=true
|
||||
RUN ./filebrowser user set testuser1 --password testuser1
|
||||
|
||||
@@ -11,7 +11,7 @@ COPY [ "./frontend/tests/", "./tests" ]
|
||||
WORKDIR /app/backend/
|
||||
COPY [ "./backend/filebrowser", "./"]
|
||||
RUN mkdir -p ./internal/web/dist
|
||||
COPY [ "./frontend/dist/", "./internal/web/dist/"]
|
||||
COPY [ "./backend/internal/web/dist/", "./internal/web/dist/"]
|
||||
|
||||
# Run tests
|
||||
ENV FILEBROWSER_PLAYWRIGHT_TEST=true
|
||||
|
||||
@@ -11,7 +11,7 @@ COPY [ "./frontend/tests/playwright-files", "./tests/playwright-files2" ]
|
||||
WORKDIR /app/backend/
|
||||
COPY [ "./backend/filebrowser", "./"]
|
||||
RUN mkdir -p ./internal/web/dist
|
||||
COPY [ "./frontend/dist/", "./internal/web/dist/"]
|
||||
COPY [ "./backend/internal/web/dist/", "./internal/web/dist/"]
|
||||
|
||||
# Run tests
|
||||
ENV FILEBROWSER_PLAYWRIGHT_TEST=true
|
||||
|
||||
@@ -7,7 +7,7 @@ WORKDIR /app/backend/
|
||||
RUN mkdir -p ./internal/web/dist
|
||||
COPY [ "./_docker/src/oidc/backend/", "./"]
|
||||
COPY ["./backend/filebrowser", "./"]
|
||||
COPY [ "./frontend/dist/", "./internal/web/dist/"]
|
||||
COPY [ "./backend/internal/web/dist/", "./internal/web/dist/"]
|
||||
|
||||
# Run tests
|
||||
WORKDIR /app/frontend
|
||||
|
||||
@@ -23,7 +23,7 @@ WORKDIR /app/backend/
|
||||
COPY [ "./backend/filebrowser", "./"]
|
||||
COPY [ "./backend/reduce-rounded-corners.css", "./no-rounded.css" ]
|
||||
RUN mkdir -p ./internal/web/dist
|
||||
COPY [ "./frontend/dist/", "./internal/web/dist/"]
|
||||
COPY [ "./backend/internal/web/dist/", "./internal/web/dist/"]
|
||||
|
||||
ENV FILEBROWSER_PLAYWRIGHT_TEST=true
|
||||
ENV CI=true
|
||||
|
||||
@@ -13,7 +13,7 @@ COPY [ "./frontend/tests", "./tests" ]
|
||||
WORKDIR /app/backend/
|
||||
COPY [ "./backend/filebrowser", "./"]
|
||||
RUN mkdir -p ./internal/web/dist
|
||||
COPY [ "./frontend/dist/", "./internal/web/dist/"]
|
||||
COPY [ "./backend/internal/web/dist/", "./internal/web/dist/"]
|
||||
|
||||
ENV FILEBROWSER_PLAYWRIGHT_TEST=true
|
||||
RUN ./filebrowser & sleep 2 && cd ../frontend && npx playwright test
|
||||
@@ -6,7 +6,7 @@ COPY [ "./frontend/tests", "./tests" ]
|
||||
WORKDIR /app/backend/
|
||||
COPY [ "./backend/filebrowser", "./"]
|
||||
RUN mkdir -p ./internal/web/dist
|
||||
COPY [ "./frontend/dist/", "./internal/web/dist/"]
|
||||
COPY [ "./backend/internal/web/dist/", "./internal/web/dist/"]
|
||||
RUN apt update && apt install nginx -y
|
||||
RUN mv default.conf /etc/nginx/conf.d/default.conf
|
||||
RUN mv .htpasswd /etc/nginx/conf.d/.htpasswd
|
||||
|
||||
@@ -10,7 +10,7 @@ COPY [ "./backend/filebrowser", "./"]
|
||||
COPY [ "./_docker/src/screenshots/backend/*", "./"]
|
||||
|
||||
RUN mkdir -p ./internal/web/dist
|
||||
COPY [ "./frontend/dist/", "./internal/web/dist/"]
|
||||
COPY [ "./backend/internal/web/dist/", "./internal/web/dist/"]
|
||||
|
||||
ENV FILEBROWSER_PLAYWRIGHT_TEST=true
|
||||
RUN ./filebrowser & sleep 2 && cd ../frontend && npx playwright test --project dark-screenshots
|
||||
@@ -13,7 +13,7 @@ WORKDIR /app/backend/
|
||||
COPY [ "./backend/filebrowser", "./"]
|
||||
COPY [ "./backend/reduce-rounded-corners.css", "./no-rounded.css"]
|
||||
RUN mkdir -p ./internal/web/dist
|
||||
COPY [ "./frontend/dist/", "./internal/web/dist/"]
|
||||
COPY [ "./backend/internal/web/dist/", "./internal/web/dist/"]
|
||||
|
||||
# Run tests
|
||||
ENV FILEBROWSER_PLAYWRIGHT_TEST=true
|
||||
|
||||
@@ -11,7 +11,7 @@ COPY [ "./frontend/tests/", "./tests" ]
|
||||
WORKDIR /app/backend/
|
||||
COPY [ "./backend/filebrowser", "./"]
|
||||
RUN mkdir -p ./internal/web/dist
|
||||
COPY [ "./frontend/dist/", "./internal/web/dist/"]
|
||||
COPY [ "./backend/internal/web/dist/", "./internal/web/dist/"]
|
||||
|
||||
# Run tests
|
||||
ENV FILEBROWSER_PLAYWRIGHT_TEST=true
|
||||
|
||||
@@ -33,7 +33,7 @@ COPY --from=base --chown=filebrowser:1000 [ "/app/filebrowser", "./" ]
|
||||
COPY --from=base --chown=filebrowser:1000 [ "/app/config.yaml", "./data/config.yaml" ]
|
||||
COPY --from=base --chown=filebrowser:1000 [ "/app/config.yaml", "./" ]
|
||||
COPY --from=base --chown=filebrowser:1000 [ "/app/reduce-rounded-corners.css", "./" ]
|
||||
COPY --from=nbuild --chown=filebrowser:1000 [ "/build/frontend/dist/", "./internal/web/dist/" ]
|
||||
COPY --from=nbuild --chown=filebrowser:1000 [ "/build/backend/internal/web/dist/", "./internal/web/dist/" ]
|
||||
## sanity checks
|
||||
RUN [ "filebrowser", "version" ]
|
||||
# exposing default port for auto discovery.
|
||||
|
||||
@@ -32,7 +32,7 @@ COPY --from=base --chown=filebrowser:1000 [ "/app/filebrowser", "./" ]
|
||||
COPY --from=base --chown=filebrowser:1000 [ "/app/config.yaml", "./data/config.yaml" ]
|
||||
COPY --from=base --chown=filebrowser:1000 [ "/app/config.yaml", "./" ]
|
||||
COPY --from=base --chown=filebrowser:1000 [ "/app/reduce-rounded-corners.css", "./" ]
|
||||
COPY --from=nbuild --chown=filebrowser:1000 [ "/build/frontend/dist/", "./internal/web/dist/" ]
|
||||
COPY --from=nbuild --chown=filebrowser:1000 [ "/build/backend/internal/web/dist/", "./internal/web/dist/" ]
|
||||
## sanity checks
|
||||
RUN [ "filebrowser", "version" ]
|
||||
# exposing default port for auto discovery.
|
||||
|
||||
+11
-11
@@ -5,7 +5,6 @@ go 1.27.1
|
||||
require (
|
||||
github.com/alecthomas/kong v1.16.1
|
||||
github.com/asdine/storm/v3 v3.2.1
|
||||
github.com/cespare/xxhash/v2 v2.3.0
|
||||
github.com/coreos/go-oidc/v3 v3.21.0
|
||||
github.com/coreos/go-systemd/v22 v22.7.0
|
||||
github.com/dhowden/tag v0.0.0-20240417053706-3d75831295e8
|
||||
@@ -22,21 +21,21 @@ require (
|
||||
github.com/gtsteffaniak/go-cache v1.1.0
|
||||
github.com/gtsteffaniak/go-ffmpeg v1.6.2
|
||||
github.com/gtsteffaniak/go-logger v1.1.0
|
||||
github.com/gtsteffaniak/go-media v1.2.2
|
||||
github.com/gtsteffaniak/go-media v1.2.0
|
||||
github.com/kovidgoyal/imaging v1.9.0
|
||||
github.com/pquerna/otp v1.5.0
|
||||
github.com/spf13/afero v1.15.0
|
||||
github.com/stretchr/testify v1.12.1
|
||||
github.com/swaggo/swag v1.16.6
|
||||
golang.org/x/crypto v0.58.0
|
||||
golang.org/x/image v0.48.0
|
||||
golang.org/x/image v0.47.0
|
||||
golang.org/x/mod v0.42.0
|
||||
golang.org/x/net v0.61.0
|
||||
golang.org/x/oauth2 v0.37.0
|
||||
golang.org/x/sync v0.25.0
|
||||
golang.org/x/sys v0.50.0
|
||||
golang.org/x/sync v0.24.0
|
||||
golang.org/x/sys v0.49.0
|
||||
golang.org/x/term v0.47.0
|
||||
golang.org/x/text v0.44.0
|
||||
golang.org/x/text v0.43.0
|
||||
golang.org/x/time v0.16.0
|
||||
golang.org/x/tools v0.52.0
|
||||
gopkg.in/yaml.v3 v3.0.1
|
||||
@@ -45,7 +44,7 @@ require (
|
||||
|
||||
require (
|
||||
github.com/Azure/go-ntlmssp v0.1.1 // indirect
|
||||
github.com/Eyevinn/mp4ff v0.62.1 // indirect
|
||||
github.com/Eyevinn/mp4ff v0.61.0 // indirect
|
||||
github.com/KyleBanks/depth v1.2.1 // indirect
|
||||
github.com/abema/go-mp4 v1.7.4 // indirect
|
||||
github.com/at-wat/ebml-go v0.19.4 // indirect
|
||||
@@ -55,10 +54,10 @@ require (
|
||||
github.com/dustin/go-humanize v1.1.0 // indirect
|
||||
github.com/fxamacker/cbor/v2 v2.9.6 // indirect
|
||||
github.com/gabriel-vasile/mimetype v1.4.16 // indirect
|
||||
github.com/gen2brain/gav1d v0.2.7 // indirect
|
||||
github.com/gen2brain/h265 v0.2.5 // indirect
|
||||
github.com/gen2brain/jxl v0.2.1 // indirect
|
||||
github.com/gen2brain/vpx v0.2.2 // indirect
|
||||
github.com/gen2brain/gav1d v0.2.6 // indirect
|
||||
github.com/gen2brain/h265 v0.2.4 // indirect
|
||||
github.com/gen2brain/jxl v0.2.0 // indirect
|
||||
github.com/gen2brain/vpx v0.2.1 // indirect
|
||||
github.com/go-asn1-ber/asn1-ber v1.5.8 // indirect
|
||||
github.com/go-errors/errors v1.5.1 // indirect
|
||||
github.com/go-jose/go-jose/v4 v4.1.6 // indirect
|
||||
@@ -81,6 +80,7 @@ require (
|
||||
github.com/golang/protobuf v1.5.4 // indirect
|
||||
github.com/golang/snappy v0.0.4 // indirect
|
||||
github.com/google/go-tpm v0.9.8 // indirect
|
||||
github.com/gtsteffaniak/go-av1 v1.0.0 // indirect
|
||||
github.com/gtsteffaniak/go-vpx v1.0.1 // indirect
|
||||
github.com/kovidgoyal/go-parallel v1.1.1 // indirect
|
||||
github.com/kovidgoyal/go-shm v1.0.0 // indirect
|
||||
|
||||
+22
-22
@@ -2,8 +2,8 @@ github.com/Azure/go-ntlmssp v0.1.1 h1:l+FM/EEMb0U9QZE7mKNEDw5Mu3mFiaa2GKOoTSsNDP
|
||||
github.com/Azure/go-ntlmssp v0.1.1/go.mod h1:NYqdhxd/8aAct/s4qSYZEerdPuH1liG2/X9DiVTbhpk=
|
||||
github.com/DataDog/zstd v1.4.1 h1:3oxKN3wbHibqx897utPC2LTQU4J+IHWWJO+glkAkpFM=
|
||||
github.com/DataDog/zstd v1.4.1/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo=
|
||||
github.com/Eyevinn/mp4ff v0.62.1 h1:EMuNIrmTCm5acOCVENca5Fzk+Mdek/l4bQZdk2EfReo=
|
||||
github.com/Eyevinn/mp4ff v0.62.1/go.mod h1:97zsDIJUiDTYqKpqq/b1K9foamxXmvsgE4GdPrPK3ks=
|
||||
github.com/Eyevinn/mp4ff v0.61.0 h1:uX3GBauEouxuGepRbv15JB7X2RyVyE2EZEfLpgimL6o=
|
||||
github.com/Eyevinn/mp4ff v0.61.0/go.mod h1:97zsDIJUiDTYqKpqq/b1K9foamxXmvsgE4GdPrPK3ks=
|
||||
github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc=
|
||||
github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE=
|
||||
github.com/Sereal/Sereal v0.0.0-20190618215532-0b8ac451a863 h1:BRrxwOZBolJN4gIwvZMJY1tzqBvQgpaZiQRuIDD40jM=
|
||||
@@ -25,8 +25,6 @@ github.com/at-wat/ebml-go v0.19.4/go.mod h1:wu3fQj+DWo6SOOl9c4yDi+gFs9Oa96Z2B5qx
|
||||
github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8=
|
||||
github.com/boombuler/barcode v1.1.0 h1:ChaYjBR63fr4LFyGn8E8nt7dBSt3MiU3zMOZqFvVkHo=
|
||||
github.com/boombuler/barcode v1.1.0/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8=
|
||||
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
|
||||
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/coreos/go-oidc/v3 v3.21.0 h1:wZo4Q9Pum8dYEj0eMUPrqR+kvuGkeUplbLpNCkBqoWM=
|
||||
github.com/coreos/go-oidc/v3 v3.21.0/go.mod h1:DYCf24+ncYi+XkIH97GY1+dqoRlbaSI26KVTCI9SrY4=
|
||||
github.com/coreos/go-systemd/v22 v22.7.0 h1:LAEzFkke61DFROc7zNLX/WA2i5J8gYqe0rSj9KI28KA=
|
||||
@@ -64,14 +62,14 @@ github.com/gabriel-vasile/mimetype v1.4.16 h1:5izNFuvA37st1dDRAcA7TtmngBmRgiIPDm
|
||||
github.com/gabriel-vasile/mimetype v1.4.16/go.mod h1:3oFFwxX8kTklZwzIl2aQEDZT0lKY8xH92ST88bAwba4=
|
||||
github.com/gen2brain/folio v0.2.8 h1:9iDI7h+zt4mjwEnzm1x8Pl9BrWqLwUwB74X/fnc1PZs=
|
||||
github.com/gen2brain/folio v0.2.8/go.mod h1:4/PcXfaXIYmW1nwQ4bJeE+aG5ybG58Aw9xrwODXaXII=
|
||||
github.com/gen2brain/gav1d v0.2.7 h1:ehu3jWt+0/46Is5llW/ZyX2OA3hTBJNukKnebzuk8KU=
|
||||
github.com/gen2brain/gav1d v0.2.7/go.mod h1:ReFKjHq7pvRhEk+pgRPAo9gOlcrf51sdzVcAbbX7e0E=
|
||||
github.com/gen2brain/h265 v0.2.5 h1:D3qCPx5GU+gHpinvLJtYMLSmQy64RGOm4fzMNGQcA1g=
|
||||
github.com/gen2brain/h265 v0.2.5/go.mod h1:kefpLxLhGpNCUG/q/aabJcD6GqB9Nhet3fNi9xX9Wzw=
|
||||
github.com/gen2brain/jxl v0.2.1 h1:74+8uw2RxUHGDVrDp+727PZ2hvAd5/psyV/mpsOQea0=
|
||||
github.com/gen2brain/jxl v0.2.1/go.mod h1:d7VMurg9ek62xfuyZyVAumFcuaQf9U8NlW57j/k5MQI=
|
||||
github.com/gen2brain/vpx v0.2.2 h1:kRCIx6Y2LsTe8hercAfhER2QbeHYnBOetovy4pkC4V0=
|
||||
github.com/gen2brain/vpx v0.2.2/go.mod h1:EVIdqguMSiapbtjDm4AWgFgYYJ4d+k8trXqjxYZ/4XE=
|
||||
github.com/gen2brain/gav1d v0.2.6 h1:7TSkrWTdE22k5Ul7vYZML7Nu80gNW6OovV7FSCITSE0=
|
||||
github.com/gen2brain/gav1d v0.2.6/go.mod h1:ReFKjHq7pvRhEk+pgRPAo9gOlcrf51sdzVcAbbX7e0E=
|
||||
github.com/gen2brain/h265 v0.2.4 h1:JUSbGkRAYZpd7ucCL192yx4ViEo75ZO6OEdm1TQt9qM=
|
||||
github.com/gen2brain/h265 v0.2.4/go.mod h1:kefpLxLhGpNCUG/q/aabJcD6GqB9Nhet3fNi9xX9Wzw=
|
||||
github.com/gen2brain/jxl v0.2.0 h1:cXSQnfbcC0JGFzaI344VzPMbe7mh6rYc0tjPcBUsE4U=
|
||||
github.com/gen2brain/jxl v0.2.0/go.mod h1:VjJHRai/8I8Es01mULqyiOihRBOjJr1Zo+NoN/MyIwo=
|
||||
github.com/gen2brain/vpx v0.2.1 h1:UD6tibcs/zlaIOKNFXeVyZuHBEaW9GYfxKSxvAN9cc8=
|
||||
github.com/gen2brain/vpx v0.2.1/go.mod h1:EVIdqguMSiapbtjDm4AWgFgYYJ4d+k8trXqjxYZ/4XE=
|
||||
github.com/go-asn1-ber/asn1-ber v1.5.8 h1:H9AZkK22UOmfX8J84ubyaZxKJZ3FMHVwn8swoMML7iQ=
|
||||
github.com/go-asn1-ber/asn1-ber v1.5.8/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
|
||||
github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q=
|
||||
@@ -159,14 +157,16 @@ github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3/go.mod h1:jl5iWTm0/hd
|
||||
github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/gtsteffaniak/go-av1 v1.0.0 h1:CILp67bcOtiDXvnrQubnH9ctub0OSKnUNA2JJuoIP+c=
|
||||
github.com/gtsteffaniak/go-av1 v1.0.0/go.mod h1:C/dIrjfdUcHgdyChr2YXwLY3pIVjENI9npSd98PU4lE=
|
||||
github.com/gtsteffaniak/go-cache v1.1.0 h1:ni/2cJpCxpGmaMZd92iTDIGOy+yUcMcxQuW7bSKhkCE=
|
||||
github.com/gtsteffaniak/go-cache v1.1.0/go.mod h1:rHUZvf4huhMX/bnNK4/oQ3sNAE2fYC2jFpAWZiBPWac=
|
||||
github.com/gtsteffaniak/go-ffmpeg v1.6.2 h1:gtvBg3G4PHO1Z/ZIPEK/XP4XT/g+l9S0Rpmmq6v5tnA=
|
||||
github.com/gtsteffaniak/go-ffmpeg v1.6.2/go.mod h1:FefhoD+KZzHWqBaUpntcKp6lXtUzdFW00F74axnENUQ=
|
||||
github.com/gtsteffaniak/go-logger v1.1.0 h1:baFds+iibuLLNpNLwygz32bDqdsunI6dGWcdxwMIVKI=
|
||||
github.com/gtsteffaniak/go-logger v1.1.0/go.mod h1:rapapTju4D7bwt6h4JOdZNUXDjnp0WkTBd6TN5yWglI=
|
||||
github.com/gtsteffaniak/go-media v1.2.2 h1:CgdXyqiZGa81vZUGL1AXGxQe+8owkUkX6roNACGQJ0w=
|
||||
github.com/gtsteffaniak/go-media v1.2.2/go.mod h1:x8k0z759GiawTtKBTXDNXF/pT1UQDhR4e70/eypXpOk=
|
||||
github.com/gtsteffaniak/go-media v1.2.0 h1:lbNL/G86uzzH8NaT45DBHKm1r0P7yndIo9pNC7gdqQs=
|
||||
github.com/gtsteffaniak/go-media v1.2.0/go.mod h1:EH+5KAVLWfFl0g4W0Xni0qJJH8MUDIhAw4wPcC0mhxY=
|
||||
github.com/gtsteffaniak/go-vpx v1.0.1 h1:9lSI8yoz6omDChaeV0AJrhdfe2YTrWvJrDENI/bnkAM=
|
||||
github.com/gtsteffaniak/go-vpx v1.0.1/go.mod h1:tRCdDZ15oZZTsiwmQAwyH56PTt1jn1C1O0P03oOkoJ8=
|
||||
github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8=
|
||||
@@ -256,8 +256,8 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk
|
||||
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||
golang.org/x/crypto v0.58.0 h1:COkYLr4k7nDI4r6QtMML2AfzeIswWHAJthLS6K/M/54=
|
||||
golang.org/x/crypto v0.58.0/go.mod h1:Xh+kl5A+M0gMiWDytH2Hhr5Z7WbNCpv/eB/EFgfSk+w=
|
||||
golang.org/x/image v0.48.0 h1:1AvuPcuoOTJpA7s4VZ5umrb90T3So0rkB/bSGmxkxik=
|
||||
golang.org/x/image v0.48.0/go.mod h1:mmtg64iqJKGqEgy+r13dfybl85tmxUd2lMnmaiSdWYY=
|
||||
golang.org/x/image v0.47.0 h1:za/8gVz++PPZqmZJBNSISICN5Rrj5zixP/8K++fgmqo=
|
||||
golang.org/x/image v0.47.0/go.mod h1:IkN5o4A7vq55vXN8FWjjnhg9Rs2GOtffqsSi9mWAKYE=
|
||||
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
|
||||
golang.org/x/mod v0.42.0 h1:ICeL3t2L30ljSPHHxDnR4fqVcBt/zR2zJAb9juavmQg=
|
||||
golang.org/x/mod v0.42.0/go.mod h1:u5fzIrRInkL8MaO1cbRkUHT64v303YIT5KEqpNr7jRs=
|
||||
@@ -277,8 +277,8 @@ golang.org/x/oauth2 v0.37.0 h1:JUlcxA8oAtauLfiH8FX2/FkAWHAdi0QtGCGc+hofE98=
|
||||
golang.org/x/oauth2 v0.37.0/go.mod h1:IxwZNxUULJmpBFf9K/9NTMSIfZZuvuTy1gGxhigP/58=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.25.0 h1:dJnifr0nMV8/SBnBvdo4eSGGnrc3O4FFx6IqvtLcmRs=
|
||||
golang.org/x/sync v0.25.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0=
|
||||
golang.org/x/sync v0.24.0 h1:iqiDrGoDgyuvAKrhLQthUEzU/R378N78gXbBjxed7oA=
|
||||
golang.org/x/sync v0.24.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
@@ -290,8 +290,8 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc
|
||||
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.50.0 h1:dkKZEFTEKaLr1sjhs0WiKqoi8Jj5nHbS3Njmx1UoMJs=
|
||||
golang.org/x/sys v0.50.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
|
||||
golang.org/x/sys v0.49.0 h1:XbzkgYJHdqh/8m2Uu0W/dQv8nktxx4BFHp1M0gROTXA=
|
||||
golang.org/x/sys v0.49.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
golang.org/x/term v0.47.0 h1:cpyOwQkr2Nzxe2p6KnrTG7Ei2wtzjsuSjhpXXepZ/BM=
|
||||
@@ -301,8 +301,8 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
|
||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
|
||||
golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
|
||||
golang.org/x/text v0.44.0 h1:YItdp/mQNGnsmrgpcdA5yQ3bnRxCfo/Tj/iByGW9GGo=
|
||||
golang.org/x/text v0.44.0/go.mod h1:S4FR1waQ9LbjQTls97SyRjDaUUVkaM+cXwwuG/G+MwY=
|
||||
golang.org/x/text v0.43.0 h1:1QivrlhwAsnMOcqgOdl9my7s9OyZl8O5/UGUTJVXoWc=
|
||||
golang.org/x/text v0.43.0/go.mod h1:hbSIYA/amXcRXTFSZqh8tWHEoqRvvbKPJ2uz/KU+a+s=
|
||||
golang.org/x/time v0.16.0 h1:vMb6ptszcQMkcwiRTAuNNU50gom6++Q/6gY2hDM6VDE=
|
||||
golang.org/x/time v0.16.0/go.mod h1:rVKOqvZeKvrDKTQiAHJ7wmwP0RzleSphoEA9RcdLA0s=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
|
||||
@@ -14,13 +14,13 @@ func TestUserUpdateChangesSkipsSensitiveFields(t *testing.T) {
|
||||
FrontendUser: users.FrontendUser{
|
||||
Username: "alice",
|
||||
NonAdminEditable: users.NonAdminEditable{
|
||||
DarkMode: users.DarkModeDark,
|
||||
DarkMode: true,
|
||||
},
|
||||
},
|
||||
TOTPSecret: "secret-value",
|
||||
}
|
||||
after := *before
|
||||
after.DarkMode = users.DarkModeLight
|
||||
after.DarkMode = false
|
||||
after.TOTPSecret = "new-secret"
|
||||
|
||||
changes := UserUpdateChanges(before, &after, []string{"totpSecret", "darkMode"}, false)
|
||||
@@ -36,7 +36,7 @@ func TestUserUpdateChangesFiltersUnchangedFields(t *testing.T) {
|
||||
FrontendUser: users.FrontendUser{
|
||||
Username: "akadmin",
|
||||
NonAdminEditable: users.NonAdminEditable{
|
||||
DarkMode: users.DarkModeDark,
|
||||
DarkMode: true,
|
||||
StickySidebar: true,
|
||||
Locale: "en",
|
||||
SingleClick: false,
|
||||
@@ -47,7 +47,7 @@ func TestUserUpdateChangesFiltersUnchangedFields(t *testing.T) {
|
||||
},
|
||||
}
|
||||
after := *before
|
||||
after.DarkMode = users.DarkModeLight
|
||||
after.DarkMode = false
|
||||
after.Sorting = users.Sorting{By: "name", Asc: true}
|
||||
|
||||
which := []string{
|
||||
@@ -60,7 +60,7 @@ func TestUserUpdateChangesFiltersUnchangedFields(t *testing.T) {
|
||||
if len(changes) != 3 {
|
||||
t.Fatalf("expected 3 changes, got %d: %#v", len(changes), changes)
|
||||
}
|
||||
if changes[0].Field != "darkMode" || changes[0].From != "dark" || changes[0].To != "light" {
|
||||
if changes[0].Field != "darkMode" || changes[0].From != "true" || changes[0].To != "false" {
|
||||
t.Fatalf("unexpected darkMode change: %#v", changes[0])
|
||||
}
|
||||
if changes[1].Field != "sorting.asc" || changes[1].From != "false" || changes[1].To != "true" {
|
||||
|
||||
@@ -17,7 +17,6 @@ import (
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/errors"
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/ffmpeg"
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/gomedia"
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/mediacache"
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/utils"
|
||||
"github.com/gtsteffaniak/filebrowser/backend/pkg/indexing"
|
||||
"github.com/gtsteffaniak/filebrowser/backend/pkg/indexing/iteminfo"
|
||||
@@ -394,8 +393,7 @@ func processContent(info *iteminfo.ExtendedFileInfo, idx *indexing.Index, opts u
|
||||
info.GetSubtitles(parentInfo)
|
||||
}
|
||||
subtitles, ok := gomedia.EmbeddedSubtitles(info.RealPath, info.ModTime, func() ([]utils.SubtitleTrack, bool) {
|
||||
// Shadow run: bypass the shared cache so ffprobe is compared fresh.
|
||||
return ffmpeg.DetectEmbeddedSubtitlesContext(mediacache.WithoutCache(context.Background()), info.RealPath), true
|
||||
return ffmpeg.DetectEmbeddedSubtitles(info.RealPath, info.ModTime), true
|
||||
})
|
||||
if !ok {
|
||||
subtitles = ffmpeg.DetectEmbeddedSubtitles(info.RealPath, info.ModTime)
|
||||
@@ -543,9 +541,8 @@ func audioDuration(ctx context.Context, item *iteminfo.ExtendedItemInfo, realPat
|
||||
if service == nil {
|
||||
service = ffmpeg.Get()
|
||||
}
|
||||
// The shadow run can outlive the request, so it uses a context that is not
|
||||
// cancelled with it. It bypasses the shared cache so ffmpeg is compared fresh.
|
||||
shadowCtx := mediacache.WithoutCache(context.WithoutCancel(ctx))
|
||||
// The shadow run can outlive the request, so it uses a context that is not cancelled with it.
|
||||
shadowCtx := context.WithoutCancel(ctx)
|
||||
legacy := func() (float64, bool) {
|
||||
d, err := service.GetMediaDuration(shadowCtx, realPath)
|
||||
return d, err == nil
|
||||
@@ -716,9 +713,8 @@ func extractVideoMetadata(ctx context.Context, item *iteminfo.ExtendedItemInfo,
|
||||
if probeService == nil {
|
||||
probeService = ffmpeg.Get()
|
||||
}
|
||||
// The shadow run can outlive the request, so it uses a context that is not
|
||||
// cancelled with it. It bypasses the shared cache so ffprobe is compared fresh.
|
||||
shadowCtx := mediacache.WithoutCache(context.WithoutCancel(ctx))
|
||||
// The shadow run can outlive the request, so it uses a context that is not cancelled with it.
|
||||
shadowCtx := context.WithoutCancel(ctx)
|
||||
legacyProbe := func() (gomedia.ProbeResult, bool) {
|
||||
info, err := probeService.ProbeFile(shadowCtx, realPath)
|
||||
if err != nil || info == nil {
|
||||
|
||||
@@ -25,8 +25,8 @@ func TestFinishUserLoad_legacyProfileRowPreservesAbsentSettingsFields(t *testing
|
||||
if !loaded.ShowFirstLogin {
|
||||
t.Fatal("expected top-level showFirstLogin to be preserved when settings omits it")
|
||||
}
|
||||
if !loaded.DarkMode.IsDark() {
|
||||
t.Fatalf("expected profile darkMode=true, got %q", loaded.DarkMode)
|
||||
if !loaded.DarkMode {
|
||||
t.Fatal("expected profile darkMode=true")
|
||||
}
|
||||
if len(loaded.PasskeyCredentials) != 0 {
|
||||
t.Fatalf("expected no passkeys when settings omits passkeyCredentials, got %#v", loaded.PasskeyCredentials)
|
||||
@@ -95,7 +95,7 @@ func TestUserDataRoundTrip_profileSettingsOverlay(t *testing.T) {
|
||||
LoginMethod: users.LoginMethodPassword,
|
||||
ShowFirstLogin: true,
|
||||
NonAdminEditable: users.NonAdminEditable{
|
||||
DarkMode: users.DarkModeDark,
|
||||
DarkMode: true,
|
||||
SidebarLinks: []users.SidebarLink{
|
||||
{Name: "Home", Target: "/files"},
|
||||
},
|
||||
|
||||
@@ -1,73 +0,0 @@
|
||||
package users
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// DarkMode is the user's theme preference: "auto" (follow the client's
|
||||
// system preference), "dark", or "light". It unmarshals from legacy JSON
|
||||
// booleans for backward compatibility with existing stored users.
|
||||
type DarkMode string
|
||||
|
||||
const (
|
||||
DarkModeAuto DarkMode = "auto"
|
||||
DarkModeDark DarkMode = "dark"
|
||||
DarkModeLight DarkMode = "light"
|
||||
)
|
||||
|
||||
// UnmarshalJSON accepts legacy boolean values (true -> dark, false -> light),
|
||||
// the tri-state strings ("auto", "dark", "light"), and null (unset).
|
||||
func (d *DarkMode) UnmarshalJSON(data []byte) error {
|
||||
var v interface{}
|
||||
if err := json.Unmarshal(data, &v); err != nil {
|
||||
return err
|
||||
}
|
||||
switch val := v.(type) {
|
||||
case nil:
|
||||
*d = ""
|
||||
case bool:
|
||||
if val {
|
||||
*d = DarkModeDark
|
||||
} else {
|
||||
*d = DarkModeLight
|
||||
}
|
||||
case string:
|
||||
switch DarkMode(strings.ToLower(val)) {
|
||||
case "":
|
||||
*d = ""
|
||||
case "true":
|
||||
// Some YAML decoders cast scalar booleans to strings.
|
||||
*d = DarkModeDark
|
||||
case "false":
|
||||
*d = DarkModeLight
|
||||
case DarkModeAuto:
|
||||
*d = DarkModeAuto
|
||||
case DarkModeDark:
|
||||
*d = DarkModeDark
|
||||
case DarkModeLight:
|
||||
*d = DarkModeLight
|
||||
default:
|
||||
return fmt.Errorf("invalid darkMode value %q: must be \"auto\", \"dark\" or \"light\"", val)
|
||||
}
|
||||
default:
|
||||
return fmt.Errorf("invalid darkMode value: must be a boolean or one of \"auto\", \"dark\", \"light\"")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// IsDark reports the effective boolean value for server-side rendering
|
||||
// contexts. "auto" cannot be resolved server-side; callers that can know the
|
||||
// client preference should handle DarkModeAuto explicitly.
|
||||
func (d DarkMode) IsDark() bool {
|
||||
return d == DarkModeDark
|
||||
}
|
||||
|
||||
// Or returns d unless it is unset (""), in which case it returns def.
|
||||
func (d DarkMode) Or(def DarkMode) DarkMode {
|
||||
if d == "" {
|
||||
return def
|
||||
}
|
||||
return d
|
||||
}
|
||||
@@ -308,7 +308,7 @@ type NonAdminEditable struct {
|
||||
DeleteWithoutConfirming bool `json:"deleteWithoutConfirming"` // delete files without confirmation
|
||||
Preview Preview `json:"preview"`
|
||||
StickySidebar bool `json:"stickySidebar"` // keep sidebar open when navigating
|
||||
DarkMode DarkMode `json:"darkMode"` // theme preference: "auto" (follow system), "dark", or "light"
|
||||
DarkMode bool `json:"darkMode"` // should dark mode be enabled
|
||||
Password string `json:"password,omitempty"`
|
||||
Locale string `json:"locale"` // language to use: eg. de, en, or fr
|
||||
ViewMode string `json:"viewMode"` // view mode to use: eg. normal, list, grid, or compact
|
||||
|
||||
@@ -0,0 +1,14 @@
|
||||
package ffmpeg
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/utils"
|
||||
"github.com/gtsteffaniak/go-cache/cache"
|
||||
)
|
||||
|
||||
var (
|
||||
MediaCache = cache.NewCache[[]utils.SubtitleTrack](24 * time.Hour) // subtitle track lists from ffprobe
|
||||
SubtitleContentCache = cache.NewCache[string](24 * time.Hour) // extracted embedded subtitle content
|
||||
MetadataCache = cache.NewCache[float64](24 * time.Hour) // media duration from ffprobe
|
||||
)
|
||||
@@ -9,51 +9,13 @@ import (
|
||||
"os/exec"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
|
||||
"github.com/kovidgoyal/imaging"
|
||||
)
|
||||
|
||||
// HEICDisplayRotationReader reads the display rotation that a HEIC/HEIF/AVIF
|
||||
// file's item properties (irot) ask for, without decoding it. The rotation is
|
||||
// in degrees counter-clockwise, normalized to (-180, 180], which is the value
|
||||
// ffmpeg prints as "Display Matrix: rotation of ..." for the same file. ok is
|
||||
// false when the reader could not tell; the ffmpeg probe is used instead.
|
||||
type HEICDisplayRotationReader func(path string) (rotation float64, ok bool)
|
||||
|
||||
// heicDisplayRotationReader is the header reader set by
|
||||
// SetHEICDisplayRotationReader, or nil.
|
||||
var heicDisplayRotationReader atomic.Pointer[HEICDisplayRotationReader]
|
||||
|
||||
// SetHEICDisplayRotationReader installs the reader HEICDisplayRotation tries
|
||||
// before running ffmpeg. The gomedia package installs go-media's reader when it
|
||||
// is loaded; it cannot be called from here directly because gomedia imports
|
||||
// this package. Passing nil removes the reader.
|
||||
func SetHEICDisplayRotationReader(fn HEICDisplayRotationReader) {
|
||||
if fn == nil {
|
||||
heicDisplayRotationReader.Store(nil)
|
||||
return
|
||||
}
|
||||
heicDisplayRotationReader.Store(&fn)
|
||||
}
|
||||
|
||||
// HEICDisplayRotation returns the display rotation of heicPath and whether it is
|
||||
// known. The installed header reader (go-media) answers first; the
|
||||
// "ffmpeg -f null" stderr probe runs only when no reader is installed or the
|
||||
// reader could not read the file.
|
||||
func (s *Service) HEICDisplayRotation(ctx context.Context, heicPath string) (float64, bool) {
|
||||
if p := heicDisplayRotationReader.Load(); p != nil {
|
||||
if rotation, ok := (*p)(heicPath); ok {
|
||||
return rotation, true
|
||||
}
|
||||
}
|
||||
return s.GetHEICDisplayMatrixRotation(ctx, heicPath)
|
||||
}
|
||||
|
||||
// GetHEICDisplayMatrixRotation returns the largest rotation (degrees) ffmpeg reports for a
|
||||
// HEIC display matrix. The second return value is false when the probe could not run or
|
||||
// parse stderr — callers must not treat that as “no rotation was applied”.
|
||||
// HEICDisplayRotation is preferred; it only runs this probe as a fallback.
|
||||
func (s *Service) GetHEICDisplayMatrixRotation(ctx context.Context, heicPath string) (float64, bool) {
|
||||
if s == nil {
|
||||
return 0, false
|
||||
@@ -74,8 +36,6 @@ func (s *Service) GetHEICDisplayMatrixRotation(ctx context.Context, heicPath str
|
||||
return rotation, ok
|
||||
}
|
||||
|
||||
// parseDisplayMatrixRotation returns the largest-magnitude "Display Matrix"
|
||||
// rotation in ffmpeg stderr, and whether any was found.
|
||||
func parseDisplayMatrixRotation(stderr string) (float64, bool) {
|
||||
var best float64
|
||||
var found bool
|
||||
@@ -105,7 +65,6 @@ func parseDisplayMatrixRotation(stderr string) (float64, bool) {
|
||||
return best, found
|
||||
}
|
||||
|
||||
// isNormalOrientation reports whether an EXIF orientation needs no change.
|
||||
func isNormalOrientation(orientation string) bool {
|
||||
switch orientation {
|
||||
case "", "Horizontal (normal)", "Top-left", "1":
|
||||
@@ -115,8 +74,6 @@ func isNormalOrientation(orientation string) bool {
|
||||
}
|
||||
}
|
||||
|
||||
// isPureRotationOrientation reports whether an EXIF orientation is a rotation
|
||||
// without a mirror.
|
||||
func isPureRotationOrientation(orientation string) bool {
|
||||
switch orientation {
|
||||
case "Rotate 90 CW", "Right-top", "6",
|
||||
@@ -143,8 +100,6 @@ func orientationNeedsPostCorrection(orientation string, displayRotation float64,
|
||||
return true
|
||||
}
|
||||
|
||||
// applyOrientationToJPEG rotates or flips JPEG bytes for an EXIF orientation.
|
||||
// The input is returned unchanged when it cannot be decoded or re-encoded.
|
||||
func applyOrientationToJPEG(jpegBytes []byte, orientation string) []byte {
|
||||
if len(jpegBytes) < 100 || isNormalOrientation(orientation) {
|
||||
return jpegBytes
|
||||
|
||||
@@ -50,35 +50,3 @@ func TestOrientationNeedsPostCorrection(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestHEICDisplayRotationPrefersReader(t *testing.T) {
|
||||
prev := heicDisplayRotationReader.Load()
|
||||
t.Cleanup(func() { heicDisplayRotationReader.Store(prev) })
|
||||
|
||||
var calls int
|
||||
SetHEICDisplayRotationReader(func(path string) (float64, bool) {
|
||||
calls++
|
||||
if path == "known.heic" {
|
||||
return -90, true
|
||||
}
|
||||
return 0, false
|
||||
})
|
||||
var svc *Service // nil: the ffmpeg probe cannot run
|
||||
got, ok := svc.HEICDisplayRotation(t.Context(), "known.heic")
|
||||
if !ok || got != -90 {
|
||||
t.Fatalf("HEICDisplayRotation(known) = %v, %v; want -90, true", got, ok)
|
||||
}
|
||||
// The reader cannot tell: the ffmpeg probe runs, and with no service it
|
||||
// reports unknown rather than "no rotation".
|
||||
if _, ok := svc.HEICDisplayRotation(t.Context(), "other.heic"); ok {
|
||||
t.Fatal("HEICDisplayRotation(other) ok = true, want the ffmpeg probe's false")
|
||||
}
|
||||
if calls != 2 {
|
||||
t.Fatalf("reader calls = %d, want 2", calls)
|
||||
}
|
||||
|
||||
SetHEICDisplayRotationReader(nil)
|
||||
if _, ok := svc.HEICDisplayRotation(t.Context(), "known.heic"); ok {
|
||||
t.Fatal("with no reader, HEICDisplayRotation must use the ffmpeg probe")
|
||||
}
|
||||
}
|
||||
@@ -81,16 +81,14 @@ func (s *Service) ConvertHEICToJPEG(ctx context.Context, heicPath string, target
|
||||
return s.CorrectHEICJPEGOrientation(ctx, heicPath, jpegBytes), nil
|
||||
}
|
||||
|
||||
// CorrectHEICJPEGOrientation applies EXIF orientation after a filter-free ffmpeg HEIC decode
|
||||
// when the pixels are still in coded orientation. The display rotation (irot) comes from
|
||||
// HEICDisplayRotation: go-media reads it from the item properties, and the ffmpeg stderr probe
|
||||
// is only a fallback.
|
||||
// CorrectHEICJPEGOrientation applies irot / display-matrix correction after a full-resolution
|
||||
// HEIC decode when pixels are still in coded orientation (go-media and ffmpeg filter-free paths).
|
||||
func (s *Service) CorrectHEICJPEGOrientation(ctx context.Context, heicPath string, jpegBytes []byte) []byte {
|
||||
orientation, err := s.GetHEICOrientation(heicPath)
|
||||
if err != nil {
|
||||
orientation = "Horizontal (normal)"
|
||||
}
|
||||
displayRotation, displayKnown := s.HEICDisplayRotation(ctx, heicPath)
|
||||
displayRotation, displayKnown := s.GetHEICDisplayMatrixRotation(ctx, heicPath)
|
||||
if orientationNeedsPostCorrection(orientation, displayRotation, displayKnown) {
|
||||
return applyOrientationToJPEG(jpegBytes, orientation)
|
||||
}
|
||||
|
||||
@@ -1,115 +0,0 @@
|
||||
package ffmpeg
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/mediacache"
|
||||
)
|
||||
|
||||
const cacheFixtureMP4 = "../../../frontend/tests/playwright-files/myfolder/testdata/sample-640x480.mp4"
|
||||
|
||||
var (
|
||||
testServiceOnce sync.Once
|
||||
testServiceErr error
|
||||
)
|
||||
|
||||
// initTestService starts the ffmpeg service once, without hardware probes, or
|
||||
// skips.
|
||||
func initTestService(t *testing.T) *Service {
|
||||
t.Helper()
|
||||
if testing.Short() {
|
||||
t.Skip("starts ffmpeg")
|
||||
}
|
||||
if _, err := exec.LookPath("ffmpeg"); err != nil {
|
||||
t.Skip("ffmpeg not installed")
|
||||
}
|
||||
if _, err := exec.LookPath("ffprobe"); err != nil {
|
||||
t.Skip("ffprobe not installed")
|
||||
}
|
||||
testServiceOnce.Do(func() {
|
||||
testServiceErr = Initialize(context.Background(), InitOptions{SkipHWTests: true, CacheDir: os.TempDir()})
|
||||
})
|
||||
if testServiceErr != nil {
|
||||
t.Skipf("ffmpeg init failed: %v", testServiceErr)
|
||||
}
|
||||
svc := Get()
|
||||
if svc == nil {
|
||||
t.Skip("ffmpeg service unavailable")
|
||||
}
|
||||
return svc
|
||||
}
|
||||
|
||||
// copyFixture copies the MP4 fixture so the test owns its cache key.
|
||||
func copyFixture(t *testing.T) (string, mediacache.Key) {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile(cacheFixtureMP4)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
path := filepath.Join(t.TempDir(), "sample.mp4")
|
||||
if err = os.WriteFile(path, data, 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
key, err := mediacache.Stat(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
mediacache.Probes.Delete(key.String())
|
||||
mediacache.Durations.Delete(key.String())
|
||||
})
|
||||
return path, key
|
||||
}
|
||||
|
||||
// TestProbeFileUsesSharedCache checks that ffprobe results go into the shared
|
||||
// cache, cached values are reused, and WithoutCache bypasses both.
|
||||
func TestProbeFileUsesSharedCache(t *testing.T) {
|
||||
svc := initTestService(t)
|
||||
path, key := copyFixture(t)
|
||||
|
||||
got, err := svc.ProbeFile(context.Background(), path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cached, ok := mediacache.Probes.Get(key.String())
|
||||
if !ok || cached != *got {
|
||||
t.Fatalf("ProbeFile must store its result in the shared cache: cached=%+v ok=%v got=%+v", cached, ok, *got)
|
||||
}
|
||||
|
||||
sentinel := FileProbeInfo{Duration: 1, VideoCodec: "sentinel"}
|
||||
mediacache.Probes.Set(key.String(), sentinel)
|
||||
got, err = svc.ProbeFile(context.Background(), path)
|
||||
if err != nil || *got != sentinel {
|
||||
t.Fatalf("ProbeFile must reuse a cached value (e.g. from go-media): got=%+v err=%v", got, err)
|
||||
}
|
||||
|
||||
fresh, err := svc.ProbeFile(mediacache.WithoutCache(context.Background()), path)
|
||||
if err != nil || *fresh == sentinel {
|
||||
t.Fatalf("WithoutCache must probe fresh: got=%+v err=%v", fresh, err)
|
||||
}
|
||||
if cached, _ := mediacache.Probes.Get(key.String()); cached != sentinel {
|
||||
t.Fatal("WithoutCache must not overwrite the shared cache")
|
||||
}
|
||||
}
|
||||
|
||||
// TestGetMediaDurationUsesSharedCache checks that cached durations are reused
|
||||
// and WithoutCache probes fresh.
|
||||
func TestGetMediaDurationUsesSharedCache(t *testing.T) {
|
||||
svc := initTestService(t)
|
||||
path, key := copyFixture(t)
|
||||
|
||||
mediacache.Durations.Set(key.String(), 4242)
|
||||
d, err := svc.GetMediaDuration(context.Background(), path)
|
||||
if err != nil || d != 4242 {
|
||||
t.Fatalf("GetMediaDuration must reuse a cached value: d=%v err=%v", d, err)
|
||||
}
|
||||
d, err = svc.GetMediaDuration(mediacache.WithoutCache(context.Background()), path)
|
||||
if err != nil || d == 4242 || d <= 0 {
|
||||
t.Fatalf("WithoutCache must probe fresh: d=%v err=%v", d, err)
|
||||
}
|
||||
}
|
||||
@@ -8,7 +8,6 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/mediacache"
|
||||
goffmpeg "github.com/gtsteffaniak/go-ffmpeg"
|
||||
)
|
||||
|
||||
@@ -50,29 +49,24 @@ func resolveLocalMediaPath(mediaPath string) (string, os.FileInfo, error) {
|
||||
return clean, info, nil
|
||||
}
|
||||
|
||||
// FileProbeInfo holds probe results for a local media file. It is the shared
|
||||
// cache type, so go-media and ffprobe results are interchangeable.
|
||||
type FileProbeInfo = mediacache.ProbeInfo
|
||||
// FileProbeInfo holds ffprobe results for a local media file.
|
||||
type FileProbeInfo struct {
|
||||
Duration float64
|
||||
VideoCodec string
|
||||
AudioCodec string
|
||||
FormatName string
|
||||
}
|
||||
|
||||
// ProbeFile extracts duration and codec info from a media file. Results are
|
||||
// stored in the shared media cache, which the go-media probe also reads, unless
|
||||
// ctx is mediacache.WithoutCache.
|
||||
// ProbeFile extracts duration and codec info from a media file.
|
||||
func (s *Service) ProbeFile(ctx context.Context, mediaPath string) (*FileProbeInfo, error) {
|
||||
if s == nil || s.inner == nil {
|
||||
return nil, fmt.Errorf("ffmpeg service not available")
|
||||
}
|
||||
|
||||
localPath, fileInfo, err := resolveLocalMediaPath(mediaPath)
|
||||
localPath, _, err := resolveLocalMediaPath(mediaPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
useCache := !mediacache.Bypassed(ctx)
|
||||
cacheKey := mediacache.FromFileInfo(localPath, fileInfo).String()
|
||||
if useCache {
|
||||
if cached, ok := mediacache.Probes.Get(cacheKey); ok {
|
||||
return &cached, nil
|
||||
}
|
||||
}
|
||||
|
||||
info, err := s.inner.ProbeStream(ctx, goffmpeg.ProbeStreamOptions{
|
||||
URL: localPath,
|
||||
@@ -83,20 +77,15 @@ func (s *Service) ProbeFile(ctx context.Context, mediaPath string) (*FileProbeIn
|
||||
return nil, err
|
||||
}
|
||||
|
||||
res := FileProbeInfo{
|
||||
return &FileProbeInfo{
|
||||
Duration: info.Duration,
|
||||
VideoCodec: info.VideoCodec,
|
||||
AudioCodec: info.AudioCodec,
|
||||
FormatName: info.FormatName,
|
||||
}
|
||||
if useCache {
|
||||
mediacache.Probes.Set(cacheKey, res)
|
||||
}
|
||||
return &res, nil
|
||||
}, nil
|
||||
}
|
||||
|
||||
// GetMediaDuration extracts the duration of a media file in seconds. Results
|
||||
// are stored in the shared media cache unless ctx is mediacache.WithoutCache.
|
||||
// GetMediaDuration extracts the duration of a media file in seconds.
|
||||
func (s *Service) GetMediaDuration(ctx context.Context, mediaPath string) (float64, error) {
|
||||
if s == nil || s.inner == nil {
|
||||
return 0, fmt.Errorf("ffmpeg service not available")
|
||||
@@ -106,12 +95,9 @@ func (s *Service) GetMediaDuration(ctx context.Context, mediaPath string) (float
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
useCache := !mediacache.Bypassed(ctx)
|
||||
cacheKey := mediacache.FromFileInfo(localPath, fileInfo).String()
|
||||
if useCache {
|
||||
if duration, found := mediacache.Durations.Get(cacheKey); found {
|
||||
return duration, nil
|
||||
}
|
||||
cacheKey := fmt.Sprintf("%s:%d", localPath, fileInfo.ModTime().Unix())
|
||||
if duration, found := MetadataCache.Get(cacheKey); found {
|
||||
return duration, nil
|
||||
}
|
||||
|
||||
duration, err := s.inner.GetMediaDuration(ctx, localPath)
|
||||
@@ -119,8 +105,6 @@ func (s *Service) GetMediaDuration(ctx context.Context, mediaPath string) (float
|
||||
return 0, err
|
||||
}
|
||||
|
||||
if useCache {
|
||||
mediacache.Durations.Set(cacheKey, duration)
|
||||
}
|
||||
MetadataCache.Set(cacheKey, duration)
|
||||
return duration, nil
|
||||
}
|
||||
@@ -9,7 +9,6 @@ import (
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/mediacache"
|
||||
goffmpeg "github.com/gtsteffaniak/go-ffmpeg"
|
||||
"github.com/gtsteffaniak/go-ffmpeg/capabilities"
|
||||
"github.com/gtsteffaniak/go-ffmpeg/ops"
|
||||
@@ -461,8 +460,7 @@ func (s *Service) VideoPreview(ctx context.Context, w io.Writer, opts ops.Previe
|
||||
return s.inner.CaptureFrame(ctx, w, frame)
|
||||
}
|
||||
|
||||
// ExtractSubtitle returns embedded subtitle content as WebVTT. Results are
|
||||
// stored in the shared media cache unless ctx is mediacache.WithoutCache.
|
||||
// ExtractSubtitle returns embedded subtitle content as WebVTT.
|
||||
func (s *Service) ExtractSubtitle(ctx context.Context, videoPath string, streamIndex int) (string, error) {
|
||||
if s == nil || s.inner == nil {
|
||||
return "", fmt.Errorf("ffmpeg service not available")
|
||||
@@ -472,12 +470,9 @@ func (s *Service) ExtractSubtitle(ctx context.Context, videoPath string, streamI
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to stat file: %w", err)
|
||||
}
|
||||
useCache := !mediacache.Bypassed(ctx)
|
||||
cacheKey := mediacache.FromFileInfo(videoPath, fileInfo).Stream(streamIndex)
|
||||
if useCache {
|
||||
if content, ok := mediacache.SubtitleContent.Get(cacheKey); ok {
|
||||
return content, nil
|
||||
}
|
||||
cacheKey := fmt.Sprintf("subtitle_content:%s:%d:%d", videoPath, streamIndex, fileInfo.ModTime().Unix())
|
||||
if content, ok := SubtitleContentCache.Get(cacheKey); ok {
|
||||
return content, nil
|
||||
}
|
||||
|
||||
content, err := s.inner.ExtractSubtitle(ctx, videoPath, streamIndex)
|
||||
@@ -485,9 +480,7 @@ func (s *Service) ExtractSubtitle(ctx context.Context, videoPath string, streamI
|
||||
return "", err
|
||||
}
|
||||
|
||||
if useCache {
|
||||
mediacache.SubtitleContent.Set(cacheKey, content)
|
||||
}
|
||||
SubtitleContentCache.Set(cacheKey, content)
|
||||
return content, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -7,30 +7,16 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/mediacache"
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/utils"
|
||||
goffmpeg "github.com/gtsteffaniak/go-ffmpeg"
|
||||
"github.com/gtsteffaniak/go-logger/logger"
|
||||
)
|
||||
|
||||
// DetectEmbeddedSubtitles detects embedded subtitle streams. Results are stored
|
||||
// in the shared media cache, keyed by the file's current size and modification
|
||||
// time; modtime is kept for API compatibility and is not used for the key.
|
||||
// DetectEmbeddedSubtitles detects embedded subtitle streams.
|
||||
func DetectEmbeddedSubtitles(videoPath string, modtime time.Time) []utils.SubtitleTrack {
|
||||
_ = modtime
|
||||
return DetectEmbeddedSubtitlesContext(context.Background(), videoPath)
|
||||
}
|
||||
|
||||
// DetectEmbeddedSubtitlesContext is DetectEmbeddedSubtitles with a context. The
|
||||
// shared media cache is skipped when ctx is mediacache.WithoutCache.
|
||||
func DetectEmbeddedSubtitlesContext(ctx context.Context, videoPath string) []utils.SubtitleTrack {
|
||||
useCache := !mediacache.Bypassed(ctx)
|
||||
key, statErr := mediacache.Stat(videoPath)
|
||||
useCache = useCache && statErr == nil
|
||||
if useCache {
|
||||
if cached, ok := mediacache.SubtitleTracks.Get(key.String()); ok {
|
||||
return cached
|
||||
}
|
||||
key := "embedded_subtitles:" + videoPath + ":" + modtime.Format(time.RFC3339)
|
||||
if cached, ok := MediaCache.Get(key); ok {
|
||||
return cached
|
||||
}
|
||||
|
||||
svc := Get()
|
||||
@@ -38,16 +24,14 @@ func DetectEmbeddedSubtitlesContext(ctx context.Context, videoPath string) []uti
|
||||
return nil
|
||||
}
|
||||
|
||||
tracks, err := svc.inner.DetectSubtitles(ctx, videoPath)
|
||||
tracks, err := svc.inner.DetectSubtitles(context.Background(), videoPath)
|
||||
if err != nil {
|
||||
logger.Debug("ffprobe failed for file: " + videoPath + ", error: " + err.Error())
|
||||
return nil
|
||||
}
|
||||
|
||||
subtitles := mapSubtitleTracks(tracks)
|
||||
if useCache {
|
||||
mediacache.SubtitleTracks.Set(key.String(), subtitles)
|
||||
}
|
||||
MediaCache.Set(key, subtitles)
|
||||
return subtitles
|
||||
}
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
package preview
|
||||
package folio
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
@@ -17,9 +17,9 @@ const folioTextCheckWindow = 4096 // html.readsAsText validation window
|
||||
// maxTextPreviewLoad is the most plain-text we read for a first-page thumbnail.
|
||||
const maxTextPreviewLoad = 32 << 10 // 32 KiB
|
||||
|
||||
// folioSupportsPreview reports whether folio should attempt this file (name + MIME).
|
||||
// SupportsPreview reports whether folio should attempt this file (name + MIME).
|
||||
// doc.Open sniffs content; extensions like .css are not in FolioConvertable but still render.
|
||||
func folioSupportsPreview(name, mimetype string) bool {
|
||||
func SupportsPreview(name, mimetype string) bool {
|
||||
ext := strings.ToLower(filepath.Ext(name))
|
||||
if iteminfo.FolioConvertable[ext] {
|
||||
return true
|
||||
@@ -0,0 +1,167 @@
|
||||
// Package folio is the pure-Go document preview fast path using gen2brain/folio.
|
||||
// When folio cannot handle a file, callers fall back to the office converter (when configured).
|
||||
package folio
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"image"
|
||||
"image/jpeg"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/gen2brain/folio/doc"
|
||||
"github.com/gtsteffaniak/filebrowser/backend/pkg/indexing/iteminfo"
|
||||
"github.com/gtsteffaniak/filebrowser/backend/pkg/settings"
|
||||
"github.com/gtsteffaniak/go-logger/logger"
|
||||
"github.com/kovidgoyal/imaging"
|
||||
)
|
||||
|
||||
var renderMu sync.Mutex
|
||||
|
||||
// RenderOptions controls thumbnail sizing for one rendered page.
|
||||
type RenderOptions struct {
|
||||
MaxWidth int
|
||||
MaxHeight int
|
||||
Fill bool
|
||||
JpegQuality int
|
||||
Original bool
|
||||
}
|
||||
|
||||
// Enabled reports whether the folio fast path may run.
|
||||
func Enabled() bool {
|
||||
return !settings.AlwaysUseOnlyOfficeEnabled()
|
||||
}
|
||||
|
||||
// SupportsExtension reports whether folio targets the file extension (including leading dot).
|
||||
func SupportsExtension(ext string) bool {
|
||||
ext = strings.ToLower(ext)
|
||||
if !strings.HasPrefix(ext, ".") {
|
||||
ext = "." + ext
|
||||
}
|
||||
return iteminfo.FolioConvertable[ext]
|
||||
}
|
||||
|
||||
func logFallback(op, path string, err error) {
|
||||
if errors.Is(err, doc.ErrUnsupported) {
|
||||
logger.Warningf("folio does not support %s for %s (%v); using office converter if configured", op, path, err)
|
||||
return
|
||||
}
|
||||
logger.Errorf("folio %s failed for %s: %v; falling back to office converter if configured", op, path, err)
|
||||
}
|
||||
|
||||
// RenderPageJPEG renders one document page to JPEG. ok is true when folio returned bytes.
|
||||
// name and mimetype should match the file being previewed; they control eligibility for
|
||||
// folio (content sniffing does not depend on the path extension).
|
||||
func RenderPageJPEG(ctx context.Context, path string, pageNumber int, opts RenderOptions, name, mimetype string) (data []byte, ok bool, err error) {
|
||||
if !Enabled() {
|
||||
return nil, false, nil
|
||||
}
|
||||
if name == "" {
|
||||
name = filepath.Base(path)
|
||||
}
|
||||
if !SupportsPreview(name, mimetype) {
|
||||
return nil, false, nil
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
return nil, false, ctx.Err()
|
||||
}
|
||||
|
||||
type result struct {
|
||||
data []byte
|
||||
err error
|
||||
}
|
||||
done := make(chan result, 1)
|
||||
go func() {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
done <- result{err: fmt.Errorf("folio render panicked: %v", r)}
|
||||
}
|
||||
}()
|
||||
data, err := renderPageJPEG(ctx, path, name, mimetype, pageNumber, opts)
|
||||
done <- result{data: data, err: err}
|
||||
}()
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, false, ctx.Err()
|
||||
case res := <-done:
|
||||
if res.err != nil {
|
||||
if !errors.Is(res.err, context.Canceled) && !errors.Is(res.err, context.DeadlineExceeded) {
|
||||
logFallback("render", path, res.err)
|
||||
}
|
||||
return nil, false, res.err
|
||||
}
|
||||
if len(res.data) == 0 {
|
||||
return nil, false, fmt.Errorf("folio render returned empty jpeg")
|
||||
}
|
||||
return res.data, true, nil
|
||||
}
|
||||
}
|
||||
|
||||
func renderPageJPEG(ctx context.Context, path, name, mimetype string, pageNumber int, opts RenderOptions) ([]byte, error) {
|
||||
renderMu.Lock()
|
||||
defer renderMu.Unlock()
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
d, err := openDocument(path, name, mimetype)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer d.Close()
|
||||
|
||||
if pageNumber < 0 || pageNumber >= d.NumPages() {
|
||||
return nil, fmt.Errorf("invalid page number %d (pages: %d)", pageNumber, d.NumPages())
|
||||
}
|
||||
|
||||
p, err := d.Page(pageNumber)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var img image.Image
|
||||
if opts.Original || opts.MaxWidth <= 0 || opts.MaxHeight <= 0 {
|
||||
const maxEdge = 4096
|
||||
rgba, err := p.ImageDPI(150)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
img = rgba
|
||||
w, h := rgba.Bounds().Dx(), rgba.Bounds().Dy()
|
||||
if w > maxEdge || h > maxEdge {
|
||||
scale := min(float64(maxEdge)/float64(w), float64(maxEdge)/float64(h))
|
||||
newW := max(1, int(float64(w)*scale))
|
||||
newH := max(1, int(float64(h)*scale))
|
||||
img = imaging.Resize(img, newW, newH, imaging.Lanczos)
|
||||
}
|
||||
} else {
|
||||
rgba, err := p.ImageDPI(150)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
img = rgba
|
||||
if opts.Fill {
|
||||
img = imaging.Fill(img, opts.MaxWidth, opts.MaxHeight, imaging.Center, imaging.Lanczos)
|
||||
} else {
|
||||
b := img.Bounds()
|
||||
if b.Dx() > opts.MaxWidth || b.Dy() > opts.MaxHeight {
|
||||
img = imaging.Fit(img, opts.MaxWidth, opts.MaxHeight, imaging.Lanczos)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
q := opts.JpegQuality
|
||||
if q <= 0 {
|
||||
q = 85
|
||||
}
|
||||
var buf bytes.Buffer
|
||||
if err := jpeg.Encode(&buf, img, &jpeg.Options{Quality: q}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return buf.Bytes(), nil
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
package preview
|
||||
package folio
|
||||
|
||||
import (
|
||||
"os"
|
||||
@@ -8,12 +8,12 @@ import (
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestRenderDocPageJPEG_txt(t *testing.T) {
|
||||
func TestRenderPageJPEG_txt(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "hello.txt")
|
||||
require.NoError(t, os.WriteFile(path, []byte("Hello folio preview"), 0o644))
|
||||
|
||||
data, ok, err := renderDocPageJPEG(t.Context(), path, 0, docRenderOptions{
|
||||
data, ok, err := RenderPageJPEG(t.Context(), path, 0, RenderOptions{
|
||||
MaxWidth: 256,
|
||||
MaxHeight: 256,
|
||||
Fill: true,
|
||||
@@ -26,8 +26,8 @@ func TestRenderDocPageJPEG_txt(t *testing.T) {
|
||||
require.Equal(t, byte(0xd8), data[1])
|
||||
}
|
||||
|
||||
func TestRenderDocPageJPEG_unsupportedExtension(t *testing.T) {
|
||||
data, ok, err := renderDocPageJPEG(t.Context(), "/nonexistent/file.xyz", 0, docRenderOptions{}, "file.xyz", "application/octet-stream")
|
||||
func TestRenderPageJPEG_unsupportedExtension(t *testing.T) {
|
||||
data, ok, err := RenderPageJPEG(t.Context(), "/nonexistent/file.xyz", 0, RenderOptions{}, "file.xyz", "application/octet-stream")
|
||||
require.NoError(t, err)
|
||||
require.False(t, ok)
|
||||
require.Nil(t, data)
|
||||
+8
-3
@@ -1,4 +1,4 @@
|
||||
package preview
|
||||
package folio
|
||||
|
||||
import (
|
||||
"os"
|
||||
@@ -37,17 +37,22 @@ func TestSanitizeFolioText_stripsNUL(t *testing.T) {
|
||||
raw := []byte("hello\x00world")
|
||||
got := sanitizeFolioText(raw)
|
||||
require.Equal(t, []byte("hello world"), got)
|
||||
opts, err := folioRenderOptionsForTest()
|
||||
require.NoError(t, err)
|
||||
|
||||
opts := docRenderOptions{MaxWidth: 256, MaxHeight: 256, Fill: true, JpegQuality: 80}
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "nul.txt")
|
||||
require.NoError(t, os.WriteFile(path, raw, 0o600))
|
||||
data, ok, err := renderDocPageJPEG(t.Context(), path, 0, opts, "nul.txt", "text/plain")
|
||||
data, ok, err := RenderPageJPEG(t.Context(), path, 0, opts, "nul.txt", "text/plain")
|
||||
require.NoError(t, err)
|
||||
require.True(t, ok, "err=%v", err)
|
||||
require.NotEmpty(t, data)
|
||||
}
|
||||
|
||||
func folioRenderOptionsForTest() (RenderOptions, error) {
|
||||
return RenderOptions{MaxWidth: 256, MaxHeight: 256, Fill: true, JpegQuality: 80}, nil
|
||||
}
|
||||
|
||||
func TestReadTextPreviewPrefix_cap(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "big.log")
|
||||
@@ -1,73 +0,0 @@
|
||||
package gomedia
|
||||
|
||||
import (
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
const fixtureWMV2 = fixtureDir + "/file_example_WMV_480_1_2MB.wmv"
|
||||
|
||||
// TestContainerGates pins which extensions go-media handles for probes and
|
||||
// for thumbnails.
|
||||
func TestContainerGates(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
probe, thumbnail bool
|
||||
}{
|
||||
{"a.mp4", true, true},
|
||||
{"a.M4V", true, true},
|
||||
{"a.mov", true, true},
|
||||
{"a.mkv", true, true},
|
||||
{"a.webm", true, true},
|
||||
{"a.wmv", false, true},
|
||||
{"a.ASF", false, true},
|
||||
{"a.avi", false, false},
|
||||
{"a.ts", false, false},
|
||||
{"a.m2ts", false, false},
|
||||
{"a.mts", false, false},
|
||||
{"a.mpg", false, false},
|
||||
{"a.mpeg", false, false},
|
||||
{"a.vob", false, false},
|
||||
{"a.flv", false, false},
|
||||
}
|
||||
for _, c := range cases {
|
||||
require.Equal(t, c.probe, supportsVideoContainer(c.name), "probe gate for %s", c.name)
|
||||
require.Equal(t, c.thumbnail, supportsVideoThumbnail(c.name), "thumbnail gate for %s", c.name)
|
||||
}
|
||||
for _, name := range []string{"a.heic", "a.HEIF", "a.heics", "a.avif"} {
|
||||
require.True(t, supportsHEIFImage(name), name)
|
||||
}
|
||||
require.False(t, supportsHEIFImage("a.jpg"))
|
||||
}
|
||||
|
||||
// TestWMV2FallsBackToFFmpeg checks that a WMV2 file is not probed by go-media
|
||||
// and that its thumbnail falls back to ffmpeg (go-media has no WMV1/WMV2
|
||||
// decoder).
|
||||
func TestWMV2FallsBackToFFmpeg(t *testing.T) {
|
||||
withGate(t, false)
|
||||
_, ok := Probe(fixtureWMV2, nil)
|
||||
require.False(t, ok, "WMV probes stay on ffprobe")
|
||||
_, ok = VideoThumbnail(fixtureWMV2, 320, 240, 80, 10, nil)
|
||||
require.False(t, ok, "WMV2 thumbnails must fall back to ffmpeg")
|
||||
}
|
||||
|
||||
// TestWMV1FallsBackToFFmpeg is TestWMV2FallsBackToFFmpeg for a WMV1 stream
|
||||
// encoded on the fly.
|
||||
func TestWMV1FallsBackToFFmpeg(t *testing.T) {
|
||||
withGate(t, false)
|
||||
requireTool(t, "ffmpeg")
|
||||
path := filepath.Join(t.TempDir(), "wmv1.wmv")
|
||||
out, err := exec.Command("ffmpeg", "-v", "error", "-y", "-f", "lavfi", "-i", "testsrc2=size=160x120:rate=25:duration=2",
|
||||
"-c:v", "wmv1", path).CombinedOutput()
|
||||
if err != nil {
|
||||
t.Skipf("ffmpeg cannot encode wmv1: %v: %s", err, out)
|
||||
}
|
||||
_, err = os.Stat(path)
|
||||
require.NoError(t, err)
|
||||
_, ok := VideoThumbnail(path, 160, 120, 80, 10, nil)
|
||||
require.False(t, ok, "WMV1 thumbnails must fall back to ffmpeg")
|
||||
}
|
||||
@@ -20,30 +20,28 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/ffmpeg"
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/mediacache"
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/utils"
|
||||
"github.com/gtsteffaniak/filebrowser/backend/pkg/settings"
|
||||
"github.com/gtsteffaniak/go-cache/cache"
|
||||
"github.com/gtsteffaniak/go-logger/logger"
|
||||
"github.com/gtsteffaniak/go-media/pkg/media"
|
||||
"golang.org/x/sync/singleflight"
|
||||
)
|
||||
|
||||
// Probe, duration, and subtitle results are cached in the mediacache package,
|
||||
// which the ffmpeg code path shares. A directory listing probes every video
|
||||
// child, and the preview flow probes the focused file again; the cache keeps
|
||||
// each request from re-reading the same containers. Keys include size and
|
||||
// modification time, so an edited file is read again.
|
||||
var (
|
||||
subtitleListCache = cache.NewCache[[]utils.SubtitleTrack](24 * time.Hour)
|
||||
subtitleContentCache = cache.NewCache[string](24 * time.Hour)
|
||||
|
||||
// probeGroup coalesces concurrent probes of the same file so a burst of
|
||||
// requests for one container reads it once.
|
||||
var probeGroup singleflight.Group
|
||||
// probeCache caches successful probe results by file identity (path, modtime,
|
||||
// size). A directory listing probes every video child, and the preview flow
|
||||
// probes the focused file again; without this each request re-read the same
|
||||
// containers. Keying on modtime+size means an edited file is re-probed.
|
||||
probeCache = cache.NewCache[ProbeResult](24 * time.Hour)
|
||||
|
||||
func init() {
|
||||
// The ffmpeg HEIC decode path corrects orientation after decoding and needs
|
||||
// the file's irot. internal/ffmpeg cannot import this package, so go-media's
|
||||
// header reader is installed here.
|
||||
ffmpeg.SetHEICDisplayRotationReader(HEICDisplayRotation)
|
||||
}
|
||||
// probeGroup coalesces concurrent probes of the same file so a burst of
|
||||
// requests for one container reads it once.
|
||||
probeGroup singleflight.Group
|
||||
)
|
||||
|
||||
// Enabled reports whether the go-media fast path may run. The admin
|
||||
// "Always use ffmpeg" setting turns it off.
|
||||
@@ -62,15 +60,9 @@ func videoDimensions(info *media.ProbeInfo) (w, h int, ok bool) {
|
||||
}
|
||||
|
||||
// videoContainerExts are the containers go-media parses for video probes,
|
||||
// thumbnails, embedded subtitles, and keyframe indexes. go-media reads MOV as
|
||||
// MP4.
|
||||
//
|
||||
// go-media also demuxes AVI, ASF/WMV, MPEG-TS/M2TS, and MPEG-PS (VOB), but their
|
||||
// probes are left to ffmpeg on purpose: go-media's results for those containers
|
||||
// do not yet match ffprobe (TS/M2TS and PS list no audio streams, PS durations
|
||||
// are missing or wrong, AVI misnames MPEG-4 Part 2 and AAC, ASF reports WMA Pro
|
||||
// as wmav2). Those files go straight to ffmpeg with no file open and no log
|
||||
// line. See videoThumbnailExts for the thumbnail exception.
|
||||
// thumbnails, and embedded subtitles. Other video containers (AVI, WMV, TS, ...)
|
||||
// are not go-media targets, so they go straight to ffmpeg with no file open and
|
||||
// no log line. go-media reads MOV as MP4.
|
||||
var videoContainerExts = map[string]bool{
|
||||
".mp4": true,
|
||||
".m4v": true,
|
||||
@@ -79,40 +71,15 @@ var videoContainerExts = map[string]bool{
|
||||
".webm": true,
|
||||
}
|
||||
|
||||
// videoThumbnailExts are the containers go-media decodes video thumbnails from:
|
||||
// videoContainerExts plus ASF/WMV, whose WMV3 keyframes go-media decodes
|
||||
// bit-exact with ffmpeg. WMV1, WMV2, MS-MPEG4, and WVC1 Advanced Profile return
|
||||
// an unsupported error and fall back to ffmpeg. AVI, TS/M2TS, and PS stay on
|
||||
// ffmpeg: go-media's TS and PS thumbnails scan the whole file for the duration
|
||||
// (several times slower than ffmpeg on large files), and AVI seeks by a
|
||||
// duration that can be wrong.
|
||||
var videoThumbnailExts = map[string]bool{
|
||||
".mp4": true,
|
||||
".m4v": true,
|
||||
".mov": true,
|
||||
".mkv": true,
|
||||
".webm": true,
|
||||
".wmv": true,
|
||||
".asf": true,
|
||||
}
|
||||
|
||||
// heifImageExts are still-image containers go-media decodes for HEIC preview.
|
||||
// AVIF is a HEIF container with an AV1 image item; go-media decodes it in pure
|
||||
// Go and applies its irot/imir, like HEIC.
|
||||
var heifImageExts = map[string]bool{
|
||||
".heic": true,
|
||||
".heif": true,
|
||||
".heics": true,
|
||||
".avif": true,
|
||||
}
|
||||
|
||||
const heicDecodeMaxEdge = 16384
|
||||
|
||||
// stillImageMaxSourcePixels caps the original image area (width×height) passed to
|
||||
// go-media for HEIC, JPEG XL, and camera RAW decode. JPEG XL honors it today
|
||||
// (~50 MiB of 8-bit RGB at 4096×4096); other still formats ignore the field.
|
||||
const stillImageMaxSourcePixels = 4096 * 4096
|
||||
|
||||
// cameraRawExts match go-media rawpreview formats (also iteminfo raw extensions).
|
||||
var cameraRawExts = map[string]bool{
|
||||
".cr2": true, ".cr3": true, ".nef": true, ".nrw": true, ".arw": true, ".srf": true, ".sr2": true,
|
||||
@@ -121,27 +88,15 @@ var cameraRawExts = map[string]bool{
|
||||
".dc2": true, ".x3f": true, ".iiq": true, ".nkc": true, ".r3d": true,
|
||||
}
|
||||
|
||||
// supportsVideoContainer reports whether go-media targets path's container for
|
||||
// probes, embedded subtitles, and keyframe indexes.
|
||||
// supportsVideoContainer reports whether go-media targets path's container.
|
||||
func supportsVideoContainer(path string) bool {
|
||||
return videoContainerExts[strings.ToLower(filepath.Ext(path))]
|
||||
}
|
||||
|
||||
// supportsVideoThumbnail reports whether go-media targets path's container for
|
||||
// video thumbnails.
|
||||
func supportsVideoThumbnail(path string) bool {
|
||||
return videoThumbnailExts[strings.ToLower(filepath.Ext(path))]
|
||||
}
|
||||
|
||||
// supportsHEIFImage reports whether path is a HEIF-family still image
|
||||
// (HEIC/HEIF/AVIF) go-media decodes.
|
||||
|
||||
func supportsHEIFImage(path string) bool {
|
||||
return heifImageExts[strings.ToLower(filepath.Ext(path))]
|
||||
}
|
||||
|
||||
// supportsCameraRAW reports whether path is a camera RAW format go-media
|
||||
// extracts a preview from.
|
||||
func supportsCameraRAW(path string) bool {
|
||||
return cameraRawExts[strings.ToLower(filepath.Ext(path))]
|
||||
}
|
||||
@@ -173,26 +128,16 @@ func logUnsupported(op, path, reason string) {
|
||||
|
||||
// openFile opens path for go-media, which reads through io.ReaderAt.
|
||||
func openFile(path string) (*os.File, int64, error) {
|
||||
f, info, err := openFileInfo(path)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
return f, info.Size(), nil
|
||||
}
|
||||
|
||||
// openFileInfo opens path and returns its FileInfo, which also gives the
|
||||
// shared cache key.
|
||||
func openFileInfo(path string) (*os.File, os.FileInfo, error) {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
return nil, 0, err
|
||||
}
|
||||
info, err := f.Stat()
|
||||
if err != nil {
|
||||
_ = f.Close()
|
||||
return nil, nil, err
|
||||
return nil, 0, err
|
||||
}
|
||||
return f, info, nil
|
||||
return f, info.Size(), nil
|
||||
}
|
||||
|
||||
// ffprobeFormatNames maps go-media containers to the format_name strings ffprobe
|
||||
@@ -229,25 +174,30 @@ func probeFast(path string) (ProbeResult, bool) {
|
||||
return ProbeResult{}, false
|
||||
}
|
||||
|
||||
f, info, err := openFileInfo(path)
|
||||
f, size, err := openFile(path)
|
||||
if err != nil {
|
||||
logFallback("probe", path, err)
|
||||
return ProbeResult{}, false
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
cacheKey := mediacache.FromFileInfo(path, info).String()
|
||||
if res, ok := mediacache.Probes.Get(cacheKey); ok {
|
||||
info, err := f.Stat()
|
||||
if err != nil {
|
||||
logFallback("probe", path, err)
|
||||
return ProbeResult{}, false
|
||||
}
|
||||
cacheKey := probeCacheKey(path, info.ModTime().UnixNano(), size)
|
||||
if res, ok := probeCache.Get(cacheKey); ok {
|
||||
return res, true
|
||||
}
|
||||
|
||||
// Coalesce concurrent probes of the same file into one container read.
|
||||
v, err, _ := probeGroup.Do(cacheKey, func() (any, error) {
|
||||
res, ok := parseProbe(path, f, info.Size())
|
||||
res, ok := parseProbe(path, f, size)
|
||||
if !ok {
|
||||
return nil, errProbeUnsupported
|
||||
}
|
||||
mediacache.Probes.Set(cacheKey, res)
|
||||
probeCache.Set(cacheKey, res)
|
||||
return res, nil
|
||||
})
|
||||
if err != nil {
|
||||
@@ -264,6 +214,12 @@ func probeFast(path string) (ProbeResult, bool) {
|
||||
// no detail because parseProbe already logged the specific reason.
|
||||
var errProbeUnsupported = errors.New("probe unsupported")
|
||||
|
||||
// probeCacheKey identifies a file version for caching: path plus modtime and size.
|
||||
// An edited file gets a new key, so stale metadata is never served.
|
||||
func probeCacheKey(path string, modTimeNanos, size int64) string {
|
||||
return fmt.Sprintf("probe:%s:%d:%d", path, modTimeNanos, size)
|
||||
}
|
||||
|
||||
// parseProbe reads a container and builds a ProbeResult. It reports false when
|
||||
// go-media should defer to ffmpeg, having logged why.
|
||||
func parseProbe(path string, f *os.File, size int64) (ProbeResult, bool) {
|
||||
@@ -319,18 +275,14 @@ func durationFast(path string) (float64, bool) {
|
||||
if !Enabled() {
|
||||
return 0, false
|
||||
}
|
||||
f, info, err := openFileInfo(path)
|
||||
f, size, err := openFile(path)
|
||||
if err != nil {
|
||||
logFallback("duration", path, err)
|
||||
return 0, false
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
cacheKey := mediacache.FromFileInfo(path, info).String()
|
||||
if d, ok := mediacache.Durations.Get(cacheKey); ok {
|
||||
return d, true
|
||||
}
|
||||
res, err := media.Duration(f, info.Size())
|
||||
res, err := media.Duration(f, size)
|
||||
if err != nil {
|
||||
logFallback("duration", path, err)
|
||||
return 0, false
|
||||
@@ -339,9 +291,7 @@ func durationFast(path string) (float64, bool) {
|
||||
// Unknown duration is not an error; ffmpeg may still find one.
|
||||
return 0, false
|
||||
}
|
||||
d := res.Duration.Seconds()
|
||||
mediacache.Durations.Set(cacheKey, d)
|
||||
return d, true
|
||||
return res.Duration.Seconds(), true
|
||||
}
|
||||
|
||||
// AudioInfo is the tag, lyrics availability, and cover art FileBrowser reads
|
||||
@@ -429,9 +379,7 @@ func lyricsFast(path string) (text, format string, ok bool) {
|
||||
|
||||
// EmbeddedSubtitles lists embedded subtitle tracks. go-media supports MKV, WebM,
|
||||
// and MP4 only. Other containers are reported as unsupported, not as "no
|
||||
// subtitles", because go-media returns an empty list for them. modtime is kept
|
||||
// for API compatibility; the cache key comes from the file's current size and
|
||||
// modification time.
|
||||
// subtitles", because go-media returns an empty list for them.
|
||||
// legacy is the ffmpeg subtitle detection, run only in shadow mode.
|
||||
func EmbeddedSubtitles(path string, modtime time.Time, legacy Legacy[[]utils.SubtitleTrack]) ([]utils.SubtitleTrack, bool) {
|
||||
return timed("subtitle list", path, legacy, subtitleListDiff, func() ([]utils.SubtitleTrack, bool) {
|
||||
@@ -446,20 +394,19 @@ func embeddedSubtitlesFast(path string, modtime time.Time) ([]utils.SubtitleTrac
|
||||
if !supportsVideoContainer(path) {
|
||||
return nil, false
|
||||
}
|
||||
_ = modtime
|
||||
f, st, err := openFileInfo(path)
|
||||
key := fmt.Sprintf("embedded_subtitles:%s:%d", path, modtime.Unix())
|
||||
if cached, ok := subtitleListCache.Get(key); ok {
|
||||
return cached, true
|
||||
}
|
||||
|
||||
f, size, err := openFile(path)
|
||||
if err != nil {
|
||||
logFallback("subtitle list", path, err)
|
||||
return nil, false
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
key := mediacache.FromFileInfo(path, st).String()
|
||||
if cached, ok := mediacache.SubtitleTracks.Get(key); ok {
|
||||
return cached, true
|
||||
}
|
||||
|
||||
info, err := media.Probe(f, st.Size())
|
||||
info, err := media.Probe(f, size)
|
||||
if err != nil {
|
||||
logFallback("subtitle list", path, err)
|
||||
return nil, false
|
||||
@@ -481,7 +428,7 @@ func embeddedSubtitlesFast(path string, modtime time.Time) ([]utils.SubtitleTrac
|
||||
})
|
||||
}
|
||||
tracks := ffmpeg.BuildSubtitleTracks(streams)
|
||||
mediacache.SubtitleTracks.Set(key, tracks)
|
||||
subtitleListCache.Set(key, tracks)
|
||||
return tracks, true
|
||||
}
|
||||
|
||||
@@ -501,19 +448,24 @@ func extractSubtitleFast(path string, streamIndex int) (string, bool) {
|
||||
if !supportsVideoContainer(path) {
|
||||
return "", false
|
||||
}
|
||||
f, st, err := openFileInfo(path)
|
||||
st, err := os.Stat(path)
|
||||
if err != nil {
|
||||
logFallback("subtitle extract", path, err)
|
||||
return "", false
|
||||
}
|
||||
key := fmt.Sprintf("subtitle_content:%s:%d:%d", path, streamIndex, st.ModTime().Unix())
|
||||
if content, ok := subtitleContentCache.Get(key); ok {
|
||||
return content, true
|
||||
}
|
||||
|
||||
f, size, err := openFile(path)
|
||||
if err != nil {
|
||||
logFallback("subtitle extract", path, err)
|
||||
return "", false
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
key := mediacache.FromFileInfo(path, st).Stream(streamIndex)
|
||||
if content, ok := mediacache.SubtitleContent.Get(key); ok {
|
||||
return content, true
|
||||
}
|
||||
|
||||
vtt, err := media.ExtractEmbeddedSubtitle(f, st.Size(), streamIndex)
|
||||
vtt, err := media.ExtractEmbeddedSubtitle(f, size, streamIndex)
|
||||
if err != nil {
|
||||
logFallback("subtitle extract", path, err)
|
||||
return "", false
|
||||
@@ -522,7 +474,7 @@ func extractSubtitleFast(path string, streamIndex int) (string, bool) {
|
||||
logFallback("subtitle extract", path, errors.New("empty subtitle output"))
|
||||
return "", false
|
||||
}
|
||||
mediacache.SubtitleContent.Set(key, vtt)
|
||||
subtitleContentCache.Set(key, vtt)
|
||||
return vtt, true
|
||||
}
|
||||
|
||||
@@ -544,7 +496,7 @@ func videoThumbnailFast(path string, width, height, quality int, seekPercent flo
|
||||
if !Enabled() {
|
||||
return nil, false
|
||||
}
|
||||
if !supportsVideoThumbnail(path) {
|
||||
if !supportsVideoContainer(path) {
|
||||
return nil, false
|
||||
}
|
||||
f, size, err := openFile(path)
|
||||
@@ -676,11 +628,10 @@ func stillImageJPEG(path string, maxW, maxH, quality int, op string, supported f
|
||||
quality = 85
|
||||
}
|
||||
res, err := media.Thumbnail(f, size, media.ThumbnailOptions{
|
||||
MaxWidth: maxW,
|
||||
MaxHeight: maxH,
|
||||
Format: "jpeg",
|
||||
Quality: quality,
|
||||
MaxSourcePixels: stillImageMaxSourcePixels,
|
||||
MaxWidth: maxW,
|
||||
MaxHeight: maxH,
|
||||
Format: "jpeg",
|
||||
Quality: quality,
|
||||
})
|
||||
if err != nil {
|
||||
logFail(err)
|
||||
|
||||
@@ -11,8 +11,6 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/mediacache"
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/utils"
|
||||
"github.com/gtsteffaniak/filebrowser/backend/pkg/settings"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
@@ -133,78 +131,27 @@ func TestProbeUsesCache(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, os.WriteFile(path, data, 0o644))
|
||||
|
||||
key, err := mediacache.Stat(path)
|
||||
st, err := os.Stat(path)
|
||||
require.NoError(t, err)
|
||||
key := probeCacheKey(path, st.ModTime().UnixNano(), st.Size())
|
||||
sentinel := ProbeResult{Duration: 123, VideoCodec: "sentinel", AudioCodec: "sentinel", FormatName: "sentinel"}
|
||||
// The ffmpeg probe stores into the same shared cache, so this also stands in
|
||||
// for a value ffprobe produced earlier.
|
||||
mediacache.Probes.Set(key.String(), sentinel)
|
||||
t.Cleanup(func() { mediacache.Probes.Delete(key.String()) })
|
||||
probeCache.Set(key, sentinel)
|
||||
t.Cleanup(func() { probeCache.Delete(key) })
|
||||
|
||||
got, ok := Probe(path, nil)
|
||||
require.True(t, ok)
|
||||
require.Equal(t, sentinel, got, "probe must return the cached value for an unchanged file")
|
||||
|
||||
// An edit (new size and mtime) must miss the cached value.
|
||||
require.NoError(t, os.WriteFile(path, append(data, 0), 0o644))
|
||||
future := time.Now().Add(time.Hour)
|
||||
require.NoError(t, os.Chtimes(path, future, future))
|
||||
got, ok = Probe(path, nil)
|
||||
require.True(t, ok)
|
||||
require.NotEqual(t, sentinel, got, "an edited file must be probed again")
|
||||
edited, err := mediacache.Stat(path)
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { mediacache.Probes.Delete(edited.String()) })
|
||||
}
|
||||
|
||||
// TestDurationUsesSharedCache checks that Duration reads and fills the shared
|
||||
// cache that ffmpeg's GetMediaDuration also uses.
|
||||
func TestDurationUsesSharedCache(t *testing.T) {
|
||||
withGate(t, false)
|
||||
path := filepath.Join(t.TempDir(), "sample.mp4")
|
||||
data, err := os.ReadFile(fixtureMP4)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, os.WriteFile(path, data, 0o644))
|
||||
key, err := mediacache.Stat(path)
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { mediacache.Durations.Delete(key.String()) })
|
||||
|
||||
d, ok := Duration(path, nil)
|
||||
require.True(t, ok)
|
||||
cached, found := mediacache.Durations.Get(key.String())
|
||||
require.True(t, found, "go-media duration must be stored in the shared cache")
|
||||
require.Equal(t, d, cached)
|
||||
|
||||
mediacache.Durations.Set(key.String(), 4242)
|
||||
d, ok = Duration(path, nil)
|
||||
require.True(t, ok)
|
||||
require.Equal(t, 4242.0, d, "a cached duration (from either backend) must be reused")
|
||||
}
|
||||
|
||||
// TestSubtitleCachesUseFileIdentity checks the subtitle list cache is keyed by
|
||||
// the file's real size and mtime, not by the caller's modtime argument.
|
||||
func TestSubtitleCachesUseFileIdentity(t *testing.T) {
|
||||
withGate(t, false)
|
||||
path := filepath.Join(t.TempDir(), "sample.mp4")
|
||||
data, err := os.ReadFile(fixtureMP4)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, os.WriteFile(path, data, 0o644))
|
||||
key, err := mediacache.Stat(path)
|
||||
require.NoError(t, err)
|
||||
sentinel := []utils.SubtitleTrack{{Name: "sentinel"}}
|
||||
mediacache.SubtitleTracks.Set(key.String(), sentinel)
|
||||
t.Cleanup(func() { mediacache.SubtitleTracks.Delete(key.String()) })
|
||||
|
||||
// The modtime argument is ignored for the key.
|
||||
got, ok := EmbeddedSubtitles(path, time.Unix(1, 0), nil)
|
||||
require.True(t, ok)
|
||||
require.Equal(t, sentinel, got)
|
||||
|
||||
mediacache.SubtitleContent.Set(key.Stream(7), "WEBVTT\n\nsentinel")
|
||||
t.Cleanup(func() { mediacache.SubtitleContent.Delete(key.Stream(7)) })
|
||||
vtt, ok := ExtractSubtitle(path, 7, nil)
|
||||
require.True(t, ok)
|
||||
require.Equal(t, "WEBVTT\n\nsentinel", vtt)
|
||||
// TestProbeCacheKeyTracksFileIdentity checks that path, modtime, and size all
|
||||
// participate in the cache key, so an edited file is never served stale metadata.
|
||||
func TestProbeCacheKeyTracksFileIdentity(t *testing.T) {
|
||||
base := probeCacheKey("f.mp4", 100, 200)
|
||||
require.Equal(t, base, probeCacheKey("f.mp4", 100, 200), "identical identity must hash the same")
|
||||
require.NotEqual(t, base, probeCacheKey("f.mp4", 101, 200), "modtime change must change the key")
|
||||
require.NotEqual(t, base, probeCacheKey("f.mp4", 100, 201), "size change must change the key")
|
||||
require.NotEqual(t, base, probeCacheKey("g.mp4", 100, 200), "path change must change the key")
|
||||
require.NotEqual(t, base, probeCacheKey("f.mp4", 100, 201), "key must not collide across fields")
|
||||
}
|
||||
|
||||
func TestProbeWebMMatchesFFprobe(t *testing.T) {
|
||||
|
||||
@@ -1,101 +0,0 @@
|
||||
package gomedia
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"math"
|
||||
"time"
|
||||
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/ffmpeg"
|
||||
"github.com/gtsteffaniak/go-media/pkg/media"
|
||||
)
|
||||
|
||||
// heicRotationShadowTimeout bounds the dev-mode ffmpeg comparison probe.
|
||||
const heicRotationShadowTimeout = time.Minute
|
||||
|
||||
// HEICDisplayRotation returns the display rotation that a HEIC/HEIF/AVIF
|
||||
// file's irot property asks for, read from the item properties without
|
||||
// decoding. The value is degrees counter-clockwise in (-180, 180], the same
|
||||
// number ffmpeg prints as "Display Matrix: rotation of ..." for the file. A
|
||||
// file with no irot returns 0 and ok=true. ok is false when go-media is
|
||||
// disabled or cannot read the item properties; the caller then runs the ffmpeg
|
||||
// probe.
|
||||
//
|
||||
// It is installed as ffmpeg's HEIC display rotation reader. In shadow mode the
|
||||
// ffmpeg stderr probe runs for comparison.
|
||||
func HEICDisplayRotation(path string) (float64, bool) {
|
||||
return timed("heic rotation", path, heicRotationLegacy(path), heicRotationDiff, func() (float64, bool) {
|
||||
return heicDisplayRotationFast(path)
|
||||
})
|
||||
}
|
||||
|
||||
// heicRotationLegacy returns the ffmpeg stderr probe for shadow mode, or nil
|
||||
// when ffmpeg is not available.
|
||||
func heicRotationLegacy(path string) Legacy[float64] {
|
||||
if !ShadowEnabled() {
|
||||
return nil
|
||||
}
|
||||
svc := ffmpeg.Get()
|
||||
if svc == nil {
|
||||
return nil
|
||||
}
|
||||
return func() (float64, bool) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), heicRotationShadowTimeout)
|
||||
defer cancel()
|
||||
return svc.GetHEICDisplayMatrixRotation(ctx, path)
|
||||
}
|
||||
}
|
||||
|
||||
// heicDisplayRotationFast reads the primary item's irot with go-media and
|
||||
// converts it to ffmpeg's display rotation.
|
||||
func heicDisplayRotationFast(path string) (float64, bool) {
|
||||
if !Enabled() {
|
||||
return 0, false
|
||||
}
|
||||
// No extension check: the resolved path can have another name (symlink
|
||||
// target). go-media sniffs the content and reports other files unsupported.
|
||||
f, size, err := openFile(path)
|
||||
if err != nil {
|
||||
logFallback("heic rotation", path, err)
|
||||
return 0, false
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
t, err := media.HEIFTransform(f, size)
|
||||
if err != nil {
|
||||
logFallback("heic rotation", path, err)
|
||||
return 0, false
|
||||
}
|
||||
return displayRotation(t), true
|
||||
}
|
||||
|
||||
// displayRotation converts go-media's counter-clockwise irot angle to ffmpeg's
|
||||
// display matrix convention: counter-clockwise degrees normalized to
|
||||
// (-180, 180]. ffmpeg builds the matrix with av_display_rotation_set(-irot)
|
||||
// and prints av_display_rotation_get, so irot 270 is reported as -90. A mirror
|
||||
// does not change the rotation value.
|
||||
func displayRotation(t media.ImageTransform) float64 {
|
||||
r := ((t.Rotation % 360) + 360) % 360
|
||||
if r > 180 {
|
||||
r -= 360
|
||||
}
|
||||
return float64(r)
|
||||
}
|
||||
|
||||
// heicRotationDiff compares go-media's display rotation with ffmpeg's. 180 and
|
||||
// -180 are the same rotation.
|
||||
func heicRotationDiff(fast, legacy float64) []string {
|
||||
norm := func(v float64) float64 {
|
||||
v = math.Mod(v, 360)
|
||||
if v <= -180 {
|
||||
v += 360
|
||||
} else if v > 180 {
|
||||
v -= 360
|
||||
}
|
||||
return v
|
||||
}
|
||||
if math.Abs(norm(fast)-norm(legacy)) > 0.5 {
|
||||
return []string{fmt.Sprintf("display rotation go-media=%.2f ffmpeg=%.2f", fast, legacy)}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -1,82 +0,0 @@
|
||||
package gomedia
|
||||
|
||||
import (
|
||||
"math"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/ffmpeg"
|
||||
"github.com/gtsteffaniak/go-media/pkg/media"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// fixtureAVIFRotated has irot=1 (90° counter-clockwise) and imir axis 0.
|
||||
const fixtureAVIFRotated = "testdata/irot1_imir0.avif"
|
||||
|
||||
// TestDisplayRotation checks the irot to ffmpeg display rotation mapping.
|
||||
func TestDisplayRotation(t *testing.T) {
|
||||
cases := []struct {
|
||||
rotation int
|
||||
want float64
|
||||
}{
|
||||
{0, 0}, {90, 90}, {180, 180}, {270, -90}, {-90, -90}, {360, 0}, {450, 90},
|
||||
}
|
||||
for _, c := range cases {
|
||||
require.Equal(t, c.want, displayRotation(media.ImageTransform{Rotation: c.rotation}), "rotation %d", c.rotation)
|
||||
}
|
||||
require.Equal(t, 90.0, displayRotation(media.ImageTransform{Rotation: 90, Mirror: true}), "a mirror does not change the rotation")
|
||||
}
|
||||
|
||||
// TestHEICDisplayRotationReadsIrot checks the rotation read from a fixture
|
||||
// with irot=1.
|
||||
func TestHEICDisplayRotationReadsIrot(t *testing.T) {
|
||||
withGate(t, false)
|
||||
got, ok := HEICDisplayRotation(fixtureAVIFRotated)
|
||||
require.True(t, ok)
|
||||
require.Equal(t, 90.0, got)
|
||||
}
|
||||
|
||||
// TestHEICDisplayRotationFallsBack checks that non-HEIF input, broken input,
|
||||
// and the Always use ffmpeg setting leave the answer to ffmpeg.
|
||||
func TestHEICDisplayRotationFallsBack(t *testing.T) {
|
||||
withGate(t, false)
|
||||
_, ok := HEICDisplayRotation(fixtureMP4)
|
||||
require.False(t, ok, "a non-HEIF file must fall back to the ffmpeg probe")
|
||||
|
||||
junk := filepath.Join(t.TempDir(), "junk.heic")
|
||||
require.NoError(t, os.WriteFile(junk, []byte("not heif"), 0o644))
|
||||
_, ok = HEICDisplayRotation(junk)
|
||||
require.False(t, ok)
|
||||
|
||||
withGate(t, true)
|
||||
_, ok = HEICDisplayRotation(fixtureAVIFRotated)
|
||||
require.False(t, ok, "the Always use ffmpeg setting must skip go-media")
|
||||
}
|
||||
|
||||
// TestHEICDisplayRotationInstalled checks that loading this package installs
|
||||
// go-media as the ffmpeg package's rotation reader.
|
||||
func TestHEICDisplayRotationInstalled(t *testing.T) {
|
||||
withGate(t, false)
|
||||
var svc *ffmpeg.Service // nil: the ffmpeg probe cannot run, so a result proves the reader answered
|
||||
got, ok := svc.HEICDisplayRotation(t.Context(), fixtureAVIFRotated)
|
||||
require.True(t, ok)
|
||||
require.Equal(t, 90.0, got)
|
||||
}
|
||||
|
||||
// TestHEICRotationDiff checks the shadow comparison of display rotations.
|
||||
func TestHEICRotationDiff(t *testing.T) {
|
||||
require.Empty(t, heicRotationDiff(-90, -90))
|
||||
require.Empty(t, heicRotationDiff(180, -180))
|
||||
require.Empty(t, heicRotationDiff(0, math.Copysign(0, -1)))
|
||||
require.Len(t, heicRotationDiff(90, -90), 1)
|
||||
}
|
||||
|
||||
// TestAVIFDecodesThroughHEICPath checks that an AVIF file is accepted by the
|
||||
// HEIC decode path.
|
||||
func TestAVIFDecodesThroughHEICPath(t *testing.T) {
|
||||
withGate(t, false)
|
||||
data, ok := HEICJPEG(fixtureAVIFRotated, 80, nil)
|
||||
require.True(t, ok)
|
||||
require.NotEmpty(t, data)
|
||||
}
|
||||
@@ -2,13 +2,11 @@ package gomedia
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"image/jpeg"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/gtsteffaniak/go-media/pkg/media"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
@@ -68,24 +66,3 @@ func TestJXLJPEGMissingFileReturnsFalse(t *testing.T) {
|
||||
require.False(t, ok)
|
||||
require.Empty(t, data)
|
||||
}
|
||||
|
||||
// JPEG XL above stillImageMaxSourcePixels must fail with go-media's ErrUnsupported
|
||||
// and return ok=false (no ffmpeg fallback).
|
||||
func TestJXLJPEGRejectsOversizedSource(t *testing.T) {
|
||||
withGate(t, false)
|
||||
f, err := os.Open(fixtureJXL)
|
||||
require.NoError(t, err)
|
||||
defer f.Close()
|
||||
info, err := f.Stat()
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = media.Thumbnail(f, info.Size(), media.ThumbnailOptions{
|
||||
MaxWidth: 256,
|
||||
MaxHeight: 256,
|
||||
Format: "jpeg",
|
||||
Quality: 85,
|
||||
MaxSourcePixels: 1,
|
||||
})
|
||||
require.Error(t, err)
|
||||
require.True(t, errors.Is(err, media.ErrUnsupported), "oversized JXL must surface ErrUnsupported, got %v", err)
|
||||
}
|
||||
@@ -1,145 +0,0 @@
|
||||
package gomedia
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"math"
|
||||
"sort"
|
||||
|
||||
"github.com/gtsteffaniak/go-media/pkg/media"
|
||||
)
|
||||
|
||||
// keyframeTimeDedupeSec matches go-ffmpeg's ffprobe keyframe probe, which
|
||||
// drops keyframes within 1 ms of the previous one.
|
||||
const keyframeTimeDedupeSec = 0.001
|
||||
|
||||
// keyframeShadowTolerance is how far apart two backends' keyframe times can be
|
||||
// before the shadow comparison reports them. ffprobe prints six decimals.
|
||||
const keyframeShadowTolerance = 0.002
|
||||
|
||||
// KeyframeTimes returns the presentation times, in seconds, of the first video
|
||||
// stream's keyframes, for HLS segment planning. The result is sorted, has no
|
||||
// negative times, and drops keyframes within 1 ms of the previous one, the same
|
||||
// as go-ffmpeg's ffprobe probe. go-media reads only the container index (MP4
|
||||
// stss/stts/ctts, Matroska Cues or block headers), so it is much faster than
|
||||
// demuxing every packet with ffprobe.
|
||||
//
|
||||
// ok is false when go-media is disabled, the container is not one go-media
|
||||
// indexes, or the read fails; the caller then runs ffprobe. ctx cancels the
|
||||
// file reads. legacy is the ffprobe probe, run only in shadow mode.
|
||||
//
|
||||
// go-media can report fewer keyframes than ffprobe (ffprobe's parsers also flag
|
||||
// IDR frames the container index omits, and some Matroska muxers index only
|
||||
// part of the keyframes in Cues). It never reports a keyframe ffprobe would not.
|
||||
func KeyframeTimes(ctx context.Context, path string, legacy Legacy[[]float64]) ([]float64, bool) {
|
||||
return timed("keyframe index", path, legacy, keyframeDiff, func() ([]float64, bool) {
|
||||
return keyframeTimesFast(ctx, path, 0)
|
||||
})
|
||||
}
|
||||
|
||||
// KeyframeTimesWithin is KeyframeTimes limited to keyframes within windowSec
|
||||
// seconds of the first keyframe. It is the pre-flight check that can rule out
|
||||
// stream copy without reading the whole index (Matroska files without Cues).
|
||||
// It has no shadow comparison.
|
||||
func KeyframeTimesWithin(ctx context.Context, path string, windowSec float64) ([]float64, bool) {
|
||||
return timed("keyframe preflight", path, nil, nil, func() ([]float64, bool) {
|
||||
return keyframeTimesFast(ctx, path, windowSec)
|
||||
})
|
||||
}
|
||||
|
||||
// keyframeTimesFast reads the keyframe index with go-media. windowSec > 0
|
||||
// limits it to keyframes within that many seconds of the first one.
|
||||
func keyframeTimesFast(ctx context.Context, path string, windowSec float64) ([]float64, bool) {
|
||||
if !Enabled() {
|
||||
return nil, false
|
||||
}
|
||||
if !supportsVideoContainer(path) {
|
||||
return nil, false
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, false
|
||||
}
|
||||
f, size, err := openFile(path)
|
||||
if err != nil {
|
||||
logFallback("keyframe index", path, err)
|
||||
return nil, false
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
r := ctxReaderAt{ctx: ctx, r: f}
|
||||
var times []float64
|
||||
if windowSec > 0 {
|
||||
times, err = media.KeyframeTimesWithin(r, size, windowSec)
|
||||
} else {
|
||||
times, err = media.KeyframeTimes(r, size)
|
||||
}
|
||||
if err != nil {
|
||||
if ctx.Err() == nil {
|
||||
logFallback("keyframe index", path, err)
|
||||
}
|
||||
return nil, false
|
||||
}
|
||||
times = dedupeKeyframeTimes(times)
|
||||
if len(times) == 0 {
|
||||
logUnsupported("keyframe index", path, "no keyframes in container index")
|
||||
return nil, false
|
||||
}
|
||||
return times, true
|
||||
}
|
||||
|
||||
// dedupeKeyframeTimes sorts times, drops negative values, and drops times
|
||||
// within keyframeTimeDedupeSec of the previous kept time, as go-ffmpeg does for
|
||||
// ffprobe output.
|
||||
func dedupeKeyframeTimes(times []float64) []float64 {
|
||||
sorted := append([]float64(nil), times...)
|
||||
sort.Float64s(sorted)
|
||||
out := make([]float64, 0, len(sorted))
|
||||
for _, t := range sorted {
|
||||
if t < 0 || math.IsNaN(t) || math.IsInf(t, 0) {
|
||||
continue
|
||||
}
|
||||
if len(out) == 0 || t > out[len(out)-1]+keyframeTimeDedupeSec {
|
||||
out = append(out, t)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// ctxReaderAt fails reads once ctx is done, so a cancelled request or the plan
|
||||
// timeout stops a long index walk on a slow share.
|
||||
type ctxReaderAt struct {
|
||||
ctx context.Context
|
||||
r io.ReaderAt
|
||||
}
|
||||
|
||||
// ReadAt reads from the wrapped reader unless the context is done.
|
||||
func (c ctxReaderAt) ReadAt(p []byte, off int64) (int, error) {
|
||||
if err := c.ctx.Err(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return c.r.ReadAt(p, off)
|
||||
}
|
||||
|
||||
// keyframeDiff compares go-media's keyframe index with ffprobe's. go-media
|
||||
// reporting a subset of ffprobe's keyframes is expected (parser-flagged IDR
|
||||
// frames, partial Cues) and is logged at info level.
|
||||
func keyframeDiff(fast, legacy []float64) []string {
|
||||
missing := 0
|
||||
j := 0
|
||||
for _, t := range fast {
|
||||
for j < len(legacy) && legacy[j] < t-keyframeShadowTolerance {
|
||||
j++
|
||||
missing++
|
||||
}
|
||||
if j >= len(legacy) || math.Abs(legacy[j]-t) > keyframeShadowTolerance {
|
||||
return []string{fmt.Sprintf("keyframe at %.6fs from go-media not in ffmpeg index (go-media=%d ffmpeg=%d keyframes)", t, len(fast), len(legacy))}
|
||||
}
|
||||
j++
|
||||
}
|
||||
missing += len(legacy) - j
|
||||
if missing > 0 {
|
||||
return []string{fmt.Sprintf("%sgo-media found %d of %d ffmpeg keyframes (keyframes outside the container index)", expectedDiffPrefix, len(fast), len(legacy))}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -1,176 +0,0 @@
|
||||
package gomedia
|
||||
|
||||
import (
|
||||
"context"
|
||||
"math"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// ffprobeKeyframes runs the same packet probe as go-ffmpeg's
|
||||
// ProbeVideoKeyframeTimes: key packets of v:0, pts_time falling back to
|
||||
// dts_time, sorted and de-duplicated within 1 ms.
|
||||
func ffprobeKeyframes(t *testing.T, path string) []float64 {
|
||||
t.Helper()
|
||||
requireTool(t, "ffprobe")
|
||||
out, err := exec.Command("ffprobe", "-v", "error", "-select_streams", "v:0",
|
||||
"-show_entries", "packet=pts_time,dts_time,flags", "-of", "csv=p=0", path).Output()
|
||||
require.NoError(t, err)
|
||||
var times []float64
|
||||
for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") {
|
||||
f := strings.Split(strings.TrimSpace(line), ",")
|
||||
if len(f) < 3 || !strings.Contains(f[2], "K") {
|
||||
continue
|
||||
}
|
||||
v, perr := strconv.ParseFloat(f[0], 64)
|
||||
if perr != nil {
|
||||
v, perr = strconv.ParseFloat(f[1], 64)
|
||||
}
|
||||
if perr == nil {
|
||||
times = append(times, v)
|
||||
}
|
||||
}
|
||||
sort.Float64s(times)
|
||||
return dedupeKeyframeTimes(times)
|
||||
}
|
||||
|
||||
// encodeKeyframeClip makes a short clip with a fixed GOP. B-frames give MP4
|
||||
// files a ctts table and an edit list, the hard case for the timestamp model.
|
||||
func encodeKeyframeClip(t *testing.T, name string, args ...string) string {
|
||||
t.Helper()
|
||||
requireTool(t, "ffmpeg")
|
||||
path := filepath.Join(t.TempDir(), name)
|
||||
full := []string{"-v", "error", "-y", "-f", "lavfi", "-i", "testsrc2=size=160x120:rate=25:duration=6"}
|
||||
full = append(full, args...)
|
||||
full = append(full, path)
|
||||
if out, err := exec.Command("ffmpeg", full...).CombinedOutput(); err != nil {
|
||||
t.Skipf("ffmpeg cannot encode %s: %v: %s", name, err, out)
|
||||
}
|
||||
return path
|
||||
}
|
||||
|
||||
// requireSameKeyframes fails unless got matches want within the shadow
|
||||
// tolerance.
|
||||
func requireSameKeyframes(t *testing.T, want, got []float64) {
|
||||
t.Helper()
|
||||
require.Equal(t, len(want), len(got), "keyframe count: ffprobe=%v go-media=%v", want, got)
|
||||
for i := range want {
|
||||
require.InDelta(t, want[i], got[i], keyframeShadowTolerance, "keyframe %d: ffprobe=%v go-media=%v", i, want, got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestKeyframeTimesMatchFFprobe compares go-media keyframe indexes with
|
||||
// ffprobe for fixtures and encoded clips, including B-frame MP4/MOV.
|
||||
func TestKeyframeTimesMatchFFprobe(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("encodes clips with ffmpeg")
|
||||
}
|
||||
withGate(t, false)
|
||||
x264 := []string{"-c:v", "libx264", "-g", "50", "-keyint_min", "50", "-sc_threshold", "0", "-bf", "2", "-pix_fmt", "yuv420p"}
|
||||
cases := map[string]string{
|
||||
"fixture.mp4": fixtureMP4,
|
||||
"fixture.webm": fixtureWebM,
|
||||
"bframes.mp4": encodeKeyframeClip(t, "bframes.mp4", x264...),
|
||||
"bframes.mov": encodeKeyframeClip(t, "bframes.mov", x264...),
|
||||
"bframes.mkv": encodeKeyframeClip(t, "bframes.mkv", x264...),
|
||||
"mpeg4.m4v": encodeKeyframeClip(t, "mpeg4.m4v", "-c:v", "mpeg4", "-g", "30", "-f", "mp4"),
|
||||
}
|
||||
for name, path := range cases {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
want := ffprobeKeyframes(t, path)
|
||||
got, ok := KeyframeTimes(context.Background(), path, nil)
|
||||
require.True(t, ok)
|
||||
requireSameKeyframes(t, want, got)
|
||||
|
||||
within, ok := KeyframeTimesWithin(context.Background(), path, 2.5)
|
||||
require.True(t, ok)
|
||||
require.NotEmpty(t, within)
|
||||
require.LessOrEqual(t, len(within), len(got))
|
||||
for i := range within {
|
||||
require.Equal(t, got[i], within[i])
|
||||
require.LessOrEqual(t, within[i]-got[0], 2.5+1e-9)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestKeyframeTimesFallsBack checks the cases that must leave the index to
|
||||
// ffprobe.
|
||||
func TestKeyframeTimesFallsBack(t *testing.T) {
|
||||
withGate(t, false)
|
||||
dir := t.TempDir()
|
||||
|
||||
// Not an indexed container: no file open, ffprobe answers.
|
||||
ts := filepath.Join(dir, "clip.ts")
|
||||
require.NoError(t, os.WriteFile(ts, []byte{0x47}, 0o644))
|
||||
_, ok := KeyframeTimes(context.Background(), ts, nil)
|
||||
require.False(t, ok)
|
||||
|
||||
// Indexed extension but not a real container.
|
||||
junk := filepath.Join(dir, "junk.mp4")
|
||||
require.NoError(t, os.WriteFile(junk, []byte("not a video"), 0o644))
|
||||
_, ok = KeyframeTimes(context.Background(), junk, nil)
|
||||
require.False(t, ok)
|
||||
|
||||
// Missing file.
|
||||
_, ok = KeyframeTimes(context.Background(), filepath.Join(dir, "missing.mp4"), nil)
|
||||
require.False(t, ok)
|
||||
|
||||
// Cancelled context.
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
_, ok = KeyframeTimes(ctx, fixtureMP4, nil)
|
||||
require.False(t, ok)
|
||||
}
|
||||
|
||||
// TestKeyframeTimesGate checks that the Always use ffmpeg setting disables
|
||||
// the go-media keyframe index.
|
||||
func TestKeyframeTimesGate(t *testing.T) {
|
||||
withGate(t, true)
|
||||
_, ok := KeyframeTimes(context.Background(), fixtureMP4, nil)
|
||||
require.False(t, ok, "KeyframeTimes must skip go-media when the gate is on")
|
||||
_, ok = KeyframeTimesWithin(context.Background(), fixtureMP4, 10)
|
||||
require.False(t, ok)
|
||||
}
|
||||
|
||||
// TestDedupeKeyframeTimes checks sorting, filtering, and the 1 ms dedupe.
|
||||
func TestDedupeKeyframeTimes(t *testing.T) {
|
||||
got := dedupeKeyframeTimes([]float64{2, -1, 0, 0.0005, 1, 1.001, 1.0021, math.NaN(), math.Inf(1)})
|
||||
require.Equal(t, []float64{0, 1, 1.0021, 2}, got)
|
||||
require.Empty(t, dedupeKeyframeTimes(nil))
|
||||
}
|
||||
|
||||
// TestCtxReaderAt checks that reads stop once the context is cancelled.
|
||||
func TestCtxReaderAt(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
r := ctxReaderAt{ctx: ctx, r: strings.NewReader("abcdef")}
|
||||
buf := make([]byte, 3)
|
||||
n, err := r.ReadAt(buf, 1)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "bcd", string(buf[:n]))
|
||||
cancel()
|
||||
_, err = r.ReadAt(buf, 0)
|
||||
require.ErrorIs(t, err, context.Canceled)
|
||||
}
|
||||
|
||||
// TestKeyframeDiff checks the shadow comparison: a subset is expected, an
|
||||
// extra keyframe is a real difference.
|
||||
func TestKeyframeDiff(t *testing.T) {
|
||||
require.Empty(t, keyframeDiff([]float64{0, 2, 4}, []float64{0, 2.001, 4}))
|
||||
|
||||
subset := keyframeDiff([]float64{0, 4}, []float64{0, 2, 4, 6})
|
||||
require.Len(t, subset, 1)
|
||||
require.True(t, strings.HasPrefix(subset[0], expectedDiffPrefix), subset[0])
|
||||
require.Contains(t, subset[0], "2 of 4")
|
||||
|
||||
extra := keyframeDiff([]float64{0, 1, 2}, []float64{0, 2})
|
||||
require.Len(t, extra, 1)
|
||||
require.False(t, strings.HasPrefix(extra[0], expectedDiffPrefix), extra[0])
|
||||
}
|
||||
@@ -59,8 +59,8 @@ func runShadow[T any](op, path string, goLatency time.Duration, fast T, legacy L
|
||||
ffStart := time.Now()
|
||||
legacyVal, ffOK := legacy()
|
||||
ffLatency := time.Since(ffStart)
|
||||
shadowLogf("go-media shadow %s for %s: go-media=%.2fms legacy(ffmpeg/dhowden)=%.2fms diff(go-legacy)=%+.2fms",
|
||||
op, path, ms(goLatency), ms(ffLatency), ms(goLatency)-ms(ffLatency))
|
||||
shadowLogf("go-media shadow %s for %s: go-media=%.2fms legacy(ffmpeg/dhowden)=%.2fms diff(legacy-go)=%+.2fms legacy_ok=%t",
|
||||
op, path, ms(goLatency), ms(ffLatency), ms(ffLatency)-ms(goLatency), ffOK)
|
||||
|
||||
if !ffOK {
|
||||
// go-media read the file but the legacy path failed. That is an outcome
|
||||
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 4.7 KiB |
@@ -9,7 +9,6 @@ import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/database/users"
|
||||
"github.com/gtsteffaniak/filebrowser/backend/pkg/settings"
|
||||
"github.com/kovidgoyal/imaging"
|
||||
)
|
||||
@@ -123,21 +122,23 @@ func TestDefaultBackgroundColor(t *testing.T) {
|
||||
saved := settings.Config
|
||||
defer func() { settings.Config = saved }()
|
||||
|
||||
dark := true
|
||||
light := false
|
||||
|
||||
settings.Config.Frontend.Styling.DarkBackground = "#111111"
|
||||
settings.Config.Frontend.Styling.LightBackground = "#ffffff"
|
||||
|
||||
settings.Config.UserDefaults.UI.DarkMode = users.DarkModeDark
|
||||
settings.Config.UserDefaults.UI.DarkMode = &dark
|
||||
if got := settings.DefaultBackgroundColor(); got != "#111111" {
|
||||
t.Fatalf("dark branch: got %q, want %q", got, "#111111")
|
||||
}
|
||||
|
||||
settings.Config.UserDefaults.UI.DarkMode = users.DarkModeLight
|
||||
settings.Config.UserDefaults.UI.DarkMode = &light
|
||||
if got := settings.DefaultBackgroundColor(); got != "#ffffff" {
|
||||
t.Fatalf("light branch (dark disabled): got %q, want %q", got, "#ffffff")
|
||||
}
|
||||
|
||||
settings.Config.UserDefaults.UI.DarkMode = ""
|
||||
settings.Config.UserDefaults.UI.DarkMode = nil
|
||||
if got := settings.DefaultBackgroundColor(); got != "#ffffff" {
|
||||
t.Fatalf("light branch (unset): got %q, want %q", got, "#ffffff")
|
||||
}
|
||||
|
||||
@@ -1,109 +0,0 @@
|
||||
// Package mediacache holds the media metadata caches shared by the ffmpeg and
|
||||
// go-media code paths. Both backends read and write the same entries, so a
|
||||
// value found by one is reused by the other instead of being probed again.
|
||||
//
|
||||
// Every entry is keyed by file identity: path, size, and modification time in
|
||||
// nanoseconds. An edited file gets a new key, so a stale value is never served
|
||||
// for it; the old entry ages out after TTL.
|
||||
package mediacache
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/utils"
|
||||
"github.com/gtsteffaniak/go-cache/cache"
|
||||
)
|
||||
|
||||
// TTL is how long a cached media value is kept after it is stored.
|
||||
const TTL = 24 * time.Hour
|
||||
|
||||
// ProbeInfo is the container summary both backends produce: ffprobe through
|
||||
// go-ffmpeg, and go-media. FormatName uses ffprobe's format_name strings.
|
||||
type ProbeInfo struct {
|
||||
Duration float64
|
||||
VideoCodec string
|
||||
AudioCodec string
|
||||
FormatName string
|
||||
}
|
||||
|
||||
var (
|
||||
// Probes caches container probes (duration, first video and audio codec,
|
||||
// format name).
|
||||
Probes = cache.NewCache[ProbeInfo](TTL)
|
||||
// Durations caches media length in seconds.
|
||||
Durations = cache.NewCache[float64](TTL)
|
||||
// SubtitleTracks caches the embedded subtitle track list of a video.
|
||||
SubtitleTracks = cache.NewCache[[]utils.SubtitleTrack](TTL)
|
||||
// SubtitleContent caches one embedded subtitle stream converted to WebVTT.
|
||||
// Use Key.Stream for its key.
|
||||
SubtitleContent = cache.NewCache[string](TTL)
|
||||
)
|
||||
|
||||
// Key identifies one version of a file. The zero Key is never produced for a
|
||||
// real file, because Path is always set.
|
||||
type Key struct {
|
||||
Path string
|
||||
Size int64
|
||||
ModTime int64 // modification time in Unix nanoseconds
|
||||
}
|
||||
|
||||
// FromFileInfo returns the Key for path using an already-read FileInfo, so
|
||||
// callers that stat or open the file anyway do not stat it twice.
|
||||
func FromFileInfo(path string, info fs.FileInfo) Key {
|
||||
return Key{Path: path, Size: info.Size(), ModTime: info.ModTime().UnixNano()}
|
||||
}
|
||||
|
||||
// Stat returns the Key for path. It follows symlinks, like os.Stat.
|
||||
func Stat(path string) (Key, error) {
|
||||
info, err := os.Stat(path)
|
||||
if err != nil {
|
||||
return Key{}, err
|
||||
}
|
||||
return FromFileInfo(path, info), nil
|
||||
}
|
||||
|
||||
// String returns the cache key for k. The path is quoted so a path containing
|
||||
// the separator cannot collide with another file's key.
|
||||
func (k Key) String() string {
|
||||
return fmt.Sprintf("%q:%d:%d", k.Path, k.Size, k.ModTime)
|
||||
}
|
||||
|
||||
// Stream returns the cache key for one stream of k, used by SubtitleContent.
|
||||
func (k Key) Stream(index int) string {
|
||||
return fmt.Sprintf("%s:%d", k.String(), index)
|
||||
}
|
||||
|
||||
// ClearAll drops every cached media value. It is used when the backend that
|
||||
// produced the values is switched (the admin "Always use ffmpeg" setting), so
|
||||
// the newly selected backend answers fresh.
|
||||
func ClearAll() {
|
||||
Probes.ClearAll()
|
||||
Durations.ClearAll()
|
||||
SubtitleTracks.ClearAll()
|
||||
SubtitleContent.ClearAll()
|
||||
}
|
||||
|
||||
// bypassKey is the context key set by WithoutCache.
|
||||
type bypassKey struct{}
|
||||
|
||||
// WithoutCache returns a context under which the ffmpeg code paths neither read
|
||||
// nor write the shared caches. Dev-mode shadow runs use it, so the ffmpeg result
|
||||
// they compare against go-media is computed fresh rather than read back from the
|
||||
// value go-media just stored.
|
||||
func WithoutCache(ctx context.Context) context.Context {
|
||||
return context.WithValue(ctx, bypassKey{}, true)
|
||||
}
|
||||
|
||||
// Bypassed reports whether ctx was returned by WithoutCache. A nil context is
|
||||
// not bypassed.
|
||||
func Bypassed(ctx context.Context) bool {
|
||||
if ctx == nil {
|
||||
return false
|
||||
}
|
||||
v, _ := ctx.Value(bypassKey{}).(bool)
|
||||
return v
|
||||
}
|
||||
@@ -1,157 +0,0 @@
|
||||
package mediacache
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// TestStatKeyChangesWithSizeAndModTime checks on disk that a size change
|
||||
// or an mtime change gives a new key. The mtimes differ by whole seconds so
|
||||
// the check holds on filesystems with coarse timestamps; the 1 ns case is
|
||||
// covered by TestFromFileInfoNanosecondModTime.
|
||||
func TestStatKeyChangesWithSizeAndModTime(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "a.mp4")
|
||||
if err := os.WriteFile(path, []byte("one"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
k1, err := Stat(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if k1.Path != path || k1.Size != 3 || k1.ModTime == 0 {
|
||||
t.Fatalf("unexpected key %+v", k1)
|
||||
}
|
||||
|
||||
// Same size, different mtime.
|
||||
mt := time.Unix(1_700_000_000, 0)
|
||||
if err = os.Chtimes(path, mt, mt); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
k2, err := Stat(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if k2.String() == k1.String() {
|
||||
t.Fatalf("mtime change kept key %s", k2)
|
||||
}
|
||||
mt2 := mt.Add(time.Second)
|
||||
if err = os.Chtimes(path, mt2, mt2); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
k3, _ := Stat(path)
|
||||
if k3.String() == k2.String() {
|
||||
t.Fatalf("1s mtime change kept key %s", k3)
|
||||
}
|
||||
|
||||
// Same mtime, different size.
|
||||
if err = os.WriteFile(path, []byte("three"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = os.Chtimes(path, mt2, mt2); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
k4, _ := Stat(path)
|
||||
if k4.String() == k3.String() {
|
||||
t.Fatalf("size change kept key %s", k4)
|
||||
}
|
||||
}
|
||||
|
||||
// stubFileInfo is an fs.FileInfo with a fixed size and mtime.
|
||||
type stubFileInfo struct {
|
||||
size int64
|
||||
modTime time.Time
|
||||
}
|
||||
|
||||
// Name implements fs.FileInfo.
|
||||
func (s stubFileInfo) Name() string { return "a.mp4" }
|
||||
|
||||
// Size implements fs.FileInfo.
|
||||
func (s stubFileInfo) Size() int64 { return s.size }
|
||||
|
||||
// Mode implements fs.FileInfo.
|
||||
func (s stubFileInfo) Mode() fs.FileMode { return 0o600 }
|
||||
|
||||
// ModTime implements fs.FileInfo.
|
||||
func (s stubFileInfo) ModTime() time.Time { return s.modTime }
|
||||
|
||||
// IsDir implements fs.FileInfo.
|
||||
func (s stubFileInfo) IsDir() bool { return false }
|
||||
|
||||
// Sys implements fs.FileInfo.
|
||||
func (s stubFileInfo) Sys() any { return nil }
|
||||
|
||||
// TestFromFileInfoNanosecondModTime checks that mtimes 1 ns apart give
|
||||
// different keys, independent of what a filesystem can store.
|
||||
func TestFromFileInfoNanosecondModTime(t *testing.T) {
|
||||
a := FromFileInfo("/m/a.mp4", stubFileInfo{size: 3, modTime: time.Unix(1_700_000_000, 123)})
|
||||
b := FromFileInfo("/m/a.mp4", stubFileInfo{size: 3, modTime: time.Unix(1_700_000_000, 124)})
|
||||
if a.String() == b.String() {
|
||||
t.Fatalf("1ns mtime change kept key %s", a)
|
||||
}
|
||||
if a.ModTime != time.Unix(1_700_000_000, 123).UnixNano() {
|
||||
t.Fatalf("ModTime = %d, want nanoseconds", a.ModTime)
|
||||
}
|
||||
}
|
||||
|
||||
// TestStatMissingFile checks that Stat reports a missing file.
|
||||
func TestStatMissingFile(t *testing.T) {
|
||||
if _, err := Stat(filepath.Join(t.TempDir(), "missing")); err == nil {
|
||||
t.Fatal("expected an error for a missing file")
|
||||
}
|
||||
}
|
||||
|
||||
// TestKeyStringDoesNotCollide checks that keys of different files, and of
|
||||
// different streams of one file, never collide.
|
||||
func TestKeyStringDoesNotCollide(t *testing.T) {
|
||||
a := Key{Path: "a:1", Size: 2, ModTime: 3}
|
||||
b := Key{Path: "a", Size: 1, ModTime: 2}
|
||||
if a.String() == b.String() {
|
||||
t.Fatalf("keys collide: %s", a)
|
||||
}
|
||||
if a.Stream(1) == a.Stream(2) || a.Stream(1) == a.String() {
|
||||
t.Fatalf("stream keys collide: %s %s", a.Stream(1), a.Stream(2))
|
||||
}
|
||||
}
|
||||
|
||||
// TestClearAll checks that ClearAll empties every shared cache.
|
||||
func TestClearAll(t *testing.T) {
|
||||
k := Key{Path: "/clear", Size: 1, ModTime: 1}
|
||||
Probes.Set(k.String(), ProbeInfo{Duration: 1})
|
||||
Durations.Set(k.String(), 1)
|
||||
SubtitleContent.Set(k.Stream(0), "x")
|
||||
ClearAll()
|
||||
if _, ok := Probes.Get(k.String()); ok {
|
||||
t.Fatal("probe survived ClearAll")
|
||||
}
|
||||
if _, ok := Durations.Get(k.String()); ok {
|
||||
t.Fatal("duration survived ClearAll")
|
||||
}
|
||||
if _, ok := SubtitleContent.Get(k.Stream(0)); ok {
|
||||
t.Fatal("subtitle content survived ClearAll")
|
||||
}
|
||||
}
|
||||
|
||||
// TestWithoutCache checks the cache bypass marker and that derived contexts
|
||||
// keep it.
|
||||
func TestWithoutCache(t *testing.T) {
|
||||
if Bypassed(context.Background()) {
|
||||
t.Fatal("background context must not bypass")
|
||||
}
|
||||
//nolint:staticcheck // nil is part of the documented contract.
|
||||
if Bypassed(nil) {
|
||||
t.Fatal("nil context must not bypass")
|
||||
}
|
||||
ctx := WithoutCache(context.Background())
|
||||
if !Bypassed(ctx) {
|
||||
t.Fatal("WithoutCache context must bypass")
|
||||
}
|
||||
child, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
if !Bypassed(child) {
|
||||
t.Fatal("bypass must survive derived contexts")
|
||||
}
|
||||
}
|
||||
@@ -5,26 +5,16 @@ import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/folio"
|
||||
"github.com/gtsteffaniak/filebrowser/backend/pkg/indexing/iteminfo"
|
||||
)
|
||||
|
||||
const (
|
||||
// docPreviewTimeout bounds small, large and xlarge page renders.
|
||||
docPreviewTimeout = 2 * time.Second
|
||||
// docOriginalTimeout bounds "original" renders, which are higher resolution.
|
||||
docOriginalTimeout = 10 * time.Second
|
||||
)
|
||||
|
||||
func (s *Service) GenerateImageFromDoc(ctx context.Context, file iteminfo.ExtendedFileInfo, pageNumber int, previewSize string) ([]byte, error) {
|
||||
if ctx.Err() != nil {
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
|
||||
timeout := docPreviewTimeout
|
||||
if previewSize == "original" {
|
||||
timeout = docOriginalTimeout
|
||||
}
|
||||
timeoutCtx, cancel := context.WithTimeout(ctx, timeout)
|
||||
timeoutCtx, cancel := context.WithTimeout(ctx, 2*time.Second)
|
||||
defer cancel()
|
||||
|
||||
opts, err := folioRenderOptions(previewSize)
|
||||
@@ -32,7 +22,7 @@ func (s *Service) GenerateImageFromDoc(ctx context.Context, file iteminfo.Extend
|
||||
return nil, err
|
||||
}
|
||||
|
||||
imageBytes, ok, err := renderDocPageJPEG(timeoutCtx, file.RealPath, pageNumber, opts, file.Name, file.Type)
|
||||
imageBytes, ok, err := folio.RenderPageJPEG(timeoutCtx, file.RealPath, pageNumber, opts, file.Name, file.Type)
|
||||
if !ok {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -40,22 +30,20 @@ func (s *Service) GenerateImageFromDoc(ctx context.Context, file iteminfo.Extend
|
||||
return nil, nil
|
||||
}
|
||||
if timeoutCtx.Err() != nil {
|
||||
return nil, fmt.Errorf("document preview generation timed out after %s for '%s'", timeout, file.Name)
|
||||
return nil, fmt.Errorf("document preview generation timed out after 2 seconds for '%s'", file.Name)
|
||||
}
|
||||
return imageBytes, nil
|
||||
}
|
||||
|
||||
// folioRenderOptions maps a preview size onto the same dimensions, fill mode and JPEG quality
|
||||
// the image pipeline uses (see getPreviewOptions); "original" renders at full resolution.
|
||||
func folioRenderOptions(previewSize string) (docRenderOptions, error) {
|
||||
func folioRenderOptions(previewSize string) (folio.RenderOptions, error) {
|
||||
if previewSize == "original" {
|
||||
return docRenderOptions{Original: true, JpegQuality: QualityHigh.jpegQuality()}, nil
|
||||
return folio.RenderOptions{Original: true, JpegQuality: QualityHigh.jpegQuality()}, nil
|
||||
}
|
||||
opts, err := getPreviewOptions(previewSize)
|
||||
if err != nil {
|
||||
return docRenderOptions{}, err
|
||||
return folio.RenderOptions{}, err
|
||||
}
|
||||
return docRenderOptions{
|
||||
return folio.RenderOptions{
|
||||
MaxWidth: opts.Width,
|
||||
MaxHeight: opts.Height,
|
||||
Fill: opts.ResizeMode == ResizeModeFill,
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/folio"
|
||||
"github.com/gtsteffaniak/filebrowser/backend/pkg/indexing/iteminfo"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
@@ -16,7 +17,7 @@ func TestFolioRender_utf8TruncatedPlaywrightFile(t *testing.T) {
|
||||
}
|
||||
opts, err := folioRenderOptions("small")
|
||||
require.NoError(t, err)
|
||||
data, ok, err := renderDocPageJPEG(t.Context(), realPath, 0, opts, "utf8-truncated.txt", "text/plain")
|
||||
data, ok, err := folio.RenderPageJPEG(t.Context(), realPath, 0, opts, "utf8-truncated.txt", "text/plain")
|
||||
require.NoError(t, err)
|
||||
require.True(t, ok, "err=%v", err)
|
||||
require.NotEmpty(t, data)
|
||||
@@ -28,7 +29,7 @@ func TestFolioRender_cssAsTxt(t *testing.T) {
|
||||
require.NoError(t, os.WriteFile(cssPath, []byte("body { color: red; }\n"), 0o644))
|
||||
opts, err := folioRenderOptions("small")
|
||||
require.NoError(t, err)
|
||||
data, ok, err := renderDocPageJPEG(t.Context(), cssPath, 0, opts, "styles.css", "text/css")
|
||||
data, ok, err := folio.RenderPageJPEG(t.Context(), cssPath, 0, opts, "styles.css", "text/css")
|
||||
require.NoError(t, err)
|
||||
require.True(t, ok, "err=%v", err)
|
||||
require.NotEmpty(t, data)
|
||||
|
||||
@@ -1,208 +0,0 @@
|
||||
// folio.go is the pure-Go document preview fast path using gen2brain/folio.
|
||||
// When folio cannot handle a file, callers fall back to the office converter (when configured).
|
||||
package preview
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"image"
|
||||
"image/jpeg"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
|
||||
"github.com/gen2brain/folio/doc"
|
||||
"github.com/gtsteffaniak/filebrowser/backend/pkg/settings"
|
||||
"github.com/gtsteffaniak/go-logger/logger"
|
||||
"github.com/kovidgoyal/imaging"
|
||||
)
|
||||
|
||||
const (
|
||||
// docSupersample renders a page at this multiple of the target box before the
|
||||
// final downscale. Rasterized text needs the extra pixels to stay crisp.
|
||||
docSupersample = 2
|
||||
// docOriginalDPI is the resolution used for "original" page previews.
|
||||
docOriginalDPI = 200
|
||||
// docMaxRenderEdge caps the longest edge of a rendered page, in pixels.
|
||||
docMaxRenderEdge = 4096
|
||||
// docMinDPI and docMaxDPI bound the computed render resolution.
|
||||
docMinDPI = 36.0
|
||||
docMaxDPI = 600.0
|
||||
)
|
||||
|
||||
var renderMu sync.Mutex
|
||||
|
||||
// docRenderOptions controls the size and quality of one rendered document page.
|
||||
type docRenderOptions struct {
|
||||
MaxWidth int
|
||||
MaxHeight int
|
||||
Fill bool
|
||||
JpegQuality int
|
||||
Original bool
|
||||
}
|
||||
|
||||
// folioEnabled reports whether the folio fast path may run.
|
||||
func folioEnabled() bool {
|
||||
return !settings.AlwaysUseOnlyOfficeEnabled()
|
||||
}
|
||||
|
||||
func logFolioFallback(op, path string, err error) {
|
||||
if errors.Is(err, doc.ErrUnsupported) {
|
||||
logger.Warningf("folio does not support %s for %s (%v); using office converter if configured", op, path, err)
|
||||
return
|
||||
}
|
||||
logger.Errorf("folio %s failed for %s: %v; falling back to office converter if configured", op, path, err)
|
||||
}
|
||||
|
||||
// renderDocPageJPEG renders one document page to JPEG. ok is true when folio returned bytes.
|
||||
// name and mimetype should match the file being previewed; they control eligibility for
|
||||
// folio (content sniffing does not depend on the path extension).
|
||||
func renderDocPageJPEG(ctx context.Context, path string, pageNumber int, opts docRenderOptions, name, mimetype string) (data []byte, ok bool, err error) {
|
||||
if !folioEnabled() {
|
||||
return nil, false, nil
|
||||
}
|
||||
if name == "" {
|
||||
name = filepath.Base(path)
|
||||
}
|
||||
if !folioSupportsPreview(name, mimetype) {
|
||||
return nil, false, nil
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
return nil, false, ctx.Err()
|
||||
}
|
||||
|
||||
type result struct {
|
||||
data []byte
|
||||
err error
|
||||
}
|
||||
done := make(chan result, 1)
|
||||
go func() {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
done <- result{err: fmt.Errorf("folio render panicked: %v", r)}
|
||||
}
|
||||
}()
|
||||
data, err := renderPageJPEG(ctx, path, name, mimetype, pageNumber, opts)
|
||||
done <- result{data: data, err: err}
|
||||
}()
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, false, ctx.Err()
|
||||
case res := <-done:
|
||||
if res.err != nil {
|
||||
if !errors.Is(res.err, context.Canceled) && !errors.Is(res.err, context.DeadlineExceeded) {
|
||||
logFolioFallback("render", path, res.err)
|
||||
}
|
||||
return nil, false, res.err
|
||||
}
|
||||
if len(res.data) == 0 {
|
||||
return nil, false, fmt.Errorf("folio render returned empty jpeg")
|
||||
}
|
||||
return res.data, true, nil
|
||||
}
|
||||
}
|
||||
|
||||
func renderPageJPEG(ctx context.Context, path, name, mimetype string, pageNumber int, opts docRenderOptions) ([]byte, error) {
|
||||
renderMu.Lock()
|
||||
defer renderMu.Unlock()
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
d, err := openDocument(path, name, mimetype)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer d.Close()
|
||||
|
||||
if pageNumber < 0 || pageNumber >= d.NumPages() {
|
||||
return nil, fmt.Errorf("invalid page number %d (pages: %d)", pageNumber, d.NumPages())
|
||||
}
|
||||
|
||||
p, err := d.Page(pageNumber)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
boxed := !opts.Original && opts.MaxWidth > 0 && opts.MaxHeight > 0
|
||||
var dpi float64
|
||||
if boxed {
|
||||
dpi = docBoxRenderDPI(p, d.Kind(), opts.MaxWidth, opts.MaxHeight, opts.Fill)
|
||||
} else {
|
||||
dpi = docOriginalRenderDPI(p, d.Kind())
|
||||
}
|
||||
|
||||
rendered, err := p.ImageDPI(dpi)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if rendered == nil {
|
||||
return nil, fmt.Errorf("folio rendered no image for page %d", pageNumber)
|
||||
}
|
||||
|
||||
var img image.Image = rendered
|
||||
if boxed {
|
||||
// The page was rendered at docSupersample times the target, so this
|
||||
// single high-quality downscale produces the anti-aliased final image.
|
||||
if opts.Fill {
|
||||
img = imaging.Fill(img, opts.MaxWidth, opts.MaxHeight, imaging.Center, imaging.Lanczos)
|
||||
} else {
|
||||
img = imaging.Fit(img, opts.MaxWidth, opts.MaxHeight, imaging.Lanczos)
|
||||
}
|
||||
}
|
||||
|
||||
q := opts.JpegQuality
|
||||
if q <= 0 {
|
||||
q = 85
|
||||
}
|
||||
var buf bytes.Buffer
|
||||
if err := jpeg.Encode(&buf, img, &jpeg.Options{Quality: q}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return buf.Bytes(), nil
|
||||
}
|
||||
|
||||
// docUnitDPI is the resolution at which folio's ImageDPI renders page units one to one:
|
||||
// 72 for PDF and book pages, 96 for drawings (SVG).
|
||||
func docUnitDPI(kind doc.Kind) float64 {
|
||||
if kind == doc.KindSVG {
|
||||
return 96
|
||||
}
|
||||
return 72
|
||||
}
|
||||
|
||||
// docMaxEdgeDPI is the highest resolution at which the longest page edge stays within docMaxRenderEdge.
|
||||
func docMaxEdgeDPI(p doc.Page, kind doc.Kind) float64 {
|
||||
b := p.Bounds()
|
||||
longest := float64(max(b.X1-b.X0, b.Y1-b.Y0))
|
||||
if longest <= 0 {
|
||||
return docMinDPI
|
||||
}
|
||||
return docUnitDPI(kind) * docMaxRenderEdge / longest
|
||||
}
|
||||
|
||||
// docBoxRenderDPI picks the resolution for a page that will be scaled into a boxW x boxH target.
|
||||
// Fill covers the box; fit stays inside it. The page is rendered at docSupersample times the target.
|
||||
func docBoxRenderDPI(p doc.Page, kind doc.Kind, boxW, boxH int, fill bool) float64 {
|
||||
b := p.Bounds()
|
||||
pw, ph := float64(b.X1-b.X0), float64(b.Y1-b.Y0)
|
||||
if pw <= 0 || ph <= 0 {
|
||||
return docMinDPI
|
||||
}
|
||||
// Scale (pixels per page unit) needed for the supersampled render to reach the box on each axis.
|
||||
sx := float64(boxW*docSupersample) / pw
|
||||
sy := float64(boxH*docSupersample) / ph
|
||||
s := min(sx, sy)
|
||||
if fill {
|
||||
s = max(sx, sy)
|
||||
}
|
||||
dpi := min(s*docUnitDPI(kind), docMaxEdgeDPI(p, kind))
|
||||
return min(max(dpi, docMinDPI), docMaxDPI)
|
||||
}
|
||||
|
||||
// docOriginalRenderDPI picks the resolution for "original" previews: docOriginalDPI, capped by docMaxRenderEdge.
|
||||
func docOriginalRenderDPI(p doc.Page, kind doc.Kind) float64 {
|
||||
return min(docOriginalDPI, docMaxEdgeDPI(p, kind))
|
||||
}
|
||||
@@ -18,6 +18,7 @@ import (
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/adapters/fs/fileutils"
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/embeddedpreview"
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/ffmpeg"
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/folio"
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/imagemeta"
|
||||
"github.com/gtsteffaniak/filebrowser/backend/pkg/indexing/iteminfo"
|
||||
"github.com/gtsteffaniak/filebrowser/backend/pkg/settings"
|
||||
@@ -272,9 +273,8 @@ func determinePreviewType(file iteminfo.ExtendedFileInfo) filePreviewType {
|
||||
return previewTypeOffice
|
||||
}
|
||||
|
||||
// Check HEIC with conversion enabled. AVIF is a HEIF container too and uses
|
||||
// the same path (go-media decode, ffmpeg fallback) under the HEIC setting.
|
||||
if (strings.HasPrefix(file.Type, "image/heic") || file.Type == "image/avif") &&
|
||||
// Check HEIC with conversion enabled
|
||||
if strings.HasPrefix(file.Type, "image/heic") &&
|
||||
settings.CanConvertImage(string(settings.HEICImagePreview)) {
|
||||
return previewTypeHEIC
|
||||
}
|
||||
@@ -481,7 +481,7 @@ func GeneratePreviewWithMD5(ctx context.Context, file iteminfo.ExtendedFileInfo,
|
||||
|
||||
// Folio document previews render at target size in-process; they skip imageSem.
|
||||
previewTypeEarly := determinePreviewType(file)
|
||||
skipImageSem := previewTypeEarly == previewTypeDocument && folioEnabled() && iteminfo.HasDocConvertableExtension(file.Name, file.Type)
|
||||
skipImageSem := previewTypeEarly == previewTypeDocument && folio.Enabled() && iteminfo.HasDocConvertableExtension(file.Name, file.Type)
|
||||
// True when the semaphore block below holds a permit for this whole function.
|
||||
imagePermitHeld := !skipImageSem
|
||||
if !skipImageSem {
|
||||
@@ -576,7 +576,7 @@ func GeneratePreviewWithMD5(ctx context.Context, file iteminfo.ExtendedFileInfo,
|
||||
// Video thumbnails come from go-media or ffmpeg already sized to the preview
|
||||
// box, so they are final and need no ffmpeg gate here.
|
||||
if !fromEmbeddedPreview && (previewType == previewTypeImage ||
|
||||
(previewType == previewTypeDocument && folioEnabled() && iteminfo.HasDocConvertableExtension(file.Name, file.Type)) ||
|
||||
(previewType == previewTypeDocument && folio.Enabled() && iteminfo.HasDocConvertableExtension(file.Name, file.Type)) ||
|
||||
previewType == previewTypeVideo) {
|
||||
if err := service.fileCache.Store(ctx, cacheKey, imageBytes); err != nil {
|
||||
logger.Errorf("failed to cache image: %v", err)
|
||||
|
||||
@@ -1,49 +0,0 @@
|
||||
package preview
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/gtsteffaniak/filebrowser/backend/pkg/indexing/iteminfo"
|
||||
"github.com/gtsteffaniak/filebrowser/backend/pkg/settings"
|
||||
)
|
||||
|
||||
// withHEICConversion sets ffmpeg availability and the HEIC preview setting for
|
||||
// one test.
|
||||
func withHEICConversion(t *testing.T, enabled bool) {
|
||||
t.Helper()
|
||||
prevAvail := settings.Env.FFmpegAvailable
|
||||
prevPreview := settings.Config.Integrations.Media.Convert.ImagePreview
|
||||
t.Cleanup(func() {
|
||||
settings.Env.FFmpegAvailable = prevAvail
|
||||
settings.Config.Integrations.Media.Convert.ImagePreview = prevPreview
|
||||
})
|
||||
settings.Env.FFmpegAvailable = true
|
||||
settings.Config.Integrations.Media.Convert.ImagePreview = map[settings.ImagePreviewType]*bool{
|
||||
settings.HEICImagePreview: &enabled,
|
||||
}
|
||||
}
|
||||
|
||||
// previewFile returns a file info with only a name and MIME type set.
|
||||
func previewFile(name, mimeType string) iteminfo.ExtendedFileInfo {
|
||||
var f iteminfo.ExtendedFileInfo
|
||||
f.Name = name
|
||||
f.Type = mimeType
|
||||
return f
|
||||
}
|
||||
|
||||
// TestDeterminePreviewTypeAVIF checks that AVIF uses the HEIF decode path when
|
||||
// HEIC conversion is enabled, and the plain image path otherwise.
|
||||
func TestDeterminePreviewTypeAVIF(t *testing.T) {
|
||||
withHEICConversion(t, true)
|
||||
if got := determinePreviewType(previewFile("a.avif", "image/avif")); got != previewTypeHEIC {
|
||||
t.Fatalf("avif with HEIC conversion = %v, want previewTypeHEIC", got)
|
||||
}
|
||||
if got := determinePreviewType(previewFile("a.heic", "image/heic")); got != previewTypeHEIC {
|
||||
t.Fatalf("heic with HEIC conversion = %v, want previewTypeHEIC", got)
|
||||
}
|
||||
|
||||
withHEICConversion(t, false)
|
||||
if got := determinePreviewType(previewFile("a.avif", "image/avif")); got != previewTypeImage {
|
||||
t.Fatalf("avif without HEIC conversion = %v, want previewTypeImage", got)
|
||||
}
|
||||
}
|
||||
@@ -28,11 +28,8 @@ func (s *Service) GenerateVideoPreview(ctx context.Context, videoPath, previewSi
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// go-media fast path: fit within the box, no crop. go-media decodes H.264,
|
||||
// HEVC, VP8, VP9, AV1, MPEG-4 Part 2, and WMV3 keyframes in pure Go. It falls
|
||||
// back to ffmpeg when the container is not a go-media target (AVI, MPEG-TS,
|
||||
// MPEG-PS, ...), the codec or profile is unsupported (WMV1/WMV2, VC-1
|
||||
// Advanced, AV1 above 4K, ...), or the decode fails.
|
||||
// go-media fast path: fit within the box, no crop. Falls back to ffmpeg when
|
||||
// unsupported (e.g. HEVC, VP9, AV1) or failed.
|
||||
// In shadow mode (dev only) the ffmpeg preview also runs, for comparison.
|
||||
// The shadow run can outlive the request, so it uses a context that is not cancelled with it.
|
||||
shadowCtx := context.WithoutCancel(ctx)
|
||||
|
||||
@@ -14,21 +14,14 @@ const (
|
||||
|
||||
var (
|
||||
mediaSettingsMu sync.RWMutex
|
||||
mediaSettingsDefault settings.FullAdminMedia
|
||||
mediaSettingsDefault settings.MediaSettings
|
||||
)
|
||||
|
||||
// saveMediaSettings is the only write path for media settings. It clears derived
|
||||
// fields (EnabledProfiles, ImagePreview) so read-time values are never persisted.
|
||||
func saveMediaSettings(key string, ms settings.FullAdminMedia) error {
|
||||
settings.ClearMediaDerived(&ms)
|
||||
return sqlDb.SaveSetting(key, ms)
|
||||
}
|
||||
|
||||
// InitMediaSettings loads persisted runtime media settings from SQLite and seeds from config when missing.
|
||||
func InitMediaSettings() error {
|
||||
if _, err := sqlDb.GetSetting(mediaSettingsDefaultKey); err != nil {
|
||||
seed := settings.SanitizeMediaSettings(settings.CurrentMediaSettings())
|
||||
if saveErr := saveMediaSettings(mediaSettingsDefaultKey, seed); saveErr != nil {
|
||||
if saveErr := sqlDb.SaveSetting(mediaSettingsDefaultKey, seed); saveErr != nil {
|
||||
return fmt.Errorf("seed media settings: %w", saveErr)
|
||||
}
|
||||
}
|
||||
@@ -50,7 +43,7 @@ func InitMediaSettings() error {
|
||||
// sanitize, so clamps (e.g. the readRate floor) must run again.
|
||||
overlaid = settings.SanitizeMediaSettings(overlaid)
|
||||
if !settings.MediaSettingsEqual(loaded, overlaid) {
|
||||
if saveErr := saveMediaSettings(mediaSettingsDefaultKey, overlaid); saveErr != nil {
|
||||
if saveErr := sqlDb.SaveSetting(mediaSettingsDefaultKey, overlaid); saveErr != nil {
|
||||
return fmt.Errorf("save config-synced media settings: %w", saveErr)
|
||||
}
|
||||
loaded = overlaid
|
||||
@@ -64,22 +57,22 @@ func InitMediaSettings() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func loadMediaSettingsSetting(key string) (settings.FullAdminMedia, error) {
|
||||
func loadMediaSettingsSetting(key string) (settings.MediaSettings, error) {
|
||||
raw, err := sqlDb.GetSetting(key)
|
||||
if err != nil {
|
||||
return settings.FullAdminMedia{}, err
|
||||
return settings.MediaSettings{}, err
|
||||
}
|
||||
var ms settings.FullAdminMedia
|
||||
var ms settings.MediaSettings
|
||||
if err := json.Unmarshal(raw, &ms); err != nil {
|
||||
return settings.FullAdminMedia{}, fmt.Errorf("parse %s: %w", key, err)
|
||||
return settings.MediaSettings{}, fmt.Errorf("parse %s: %w", key, err)
|
||||
}
|
||||
return ms, nil
|
||||
}
|
||||
|
||||
// GetMediaSettings returns the stored runtime media settings. The value is deep-copied
|
||||
// GetMediaSettings returns the runtime media settings. The value is deep-copied
|
||||
// so callers cannot mutate the stored settings (or the live config they are
|
||||
// applied to) through the conversion maps.
|
||||
func GetMediaSettings() settings.FullAdminMedia {
|
||||
func GetMediaSettings() settings.MediaSettings {
|
||||
mediaSettingsMu.RLock()
|
||||
defer mediaSettingsMu.RUnlock()
|
||||
return settings.CloneMediaSettings(mediaSettingsDefault)
|
||||
@@ -97,11 +90,8 @@ func PatchMediaSettings(patchJSON []byte) error {
|
||||
return mergeErr
|
||||
}
|
||||
merged = settings.SanitizeMediaSettings(merged)
|
||||
// A patch may echo derived fields back (e.g. imagePreview from the admin UI).
|
||||
// Clear them before the value reaches memory or the database.
|
||||
settings.ClearMediaDerived(&merged)
|
||||
|
||||
if saveErr := saveMediaSettings(mediaSettingsDefaultKey, merged); saveErr != nil {
|
||||
if saveErr := sqlDb.SaveSetting(mediaSettingsDefaultKey, merged); saveErr != nil {
|
||||
mediaSettingsMu.Unlock()
|
||||
return fmt.Errorf("save media settings: %w", saveErr)
|
||||
}
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
package state
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
|
||||
"github.com/gtsteffaniak/filebrowser/backend/pkg/settings"
|
||||
@@ -35,56 +34,6 @@ func TestGetMediaSettingsReturnsIndependentCopy(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// Derived read-time values must never reach the database, even when a caller
|
||||
// passes a struct that has them populated.
|
||||
func TestSaveMediaSettingsNeverPersistsDerivedFields(t *testing.T) {
|
||||
initSidebarLinkTestDB(t)
|
||||
|
||||
ms := GetMediaSettings()
|
||||
ms.Transcode.EnabledProfiles = []string{"quality"}
|
||||
ms.ImagePreview = map[settings.ImagePreviewType]bool{settings.JPEGImagePreview: true}
|
||||
if err := saveMediaSettings(mediaSettingsDefaultKey, ms); err != nil {
|
||||
t.Fatalf("saveMediaSettings: %v", err)
|
||||
}
|
||||
|
||||
raw, err := sqlDb.GetSetting(mediaSettingsDefaultKey)
|
||||
if err != nil {
|
||||
t.Fatalf("GetSetting: %v", err)
|
||||
}
|
||||
var stored map[string]any
|
||||
if err := json.Unmarshal(raw, &stored); err != nil {
|
||||
t.Fatalf("decode stored settings: %v", err)
|
||||
}
|
||||
transcode, ok := stored["transcode"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("stored transcode = %T, want object", stored["transcode"])
|
||||
}
|
||||
if _, ok := transcode["enabledProfiles"]; ok {
|
||||
t.Fatal("enabledProfiles (derived) was persisted")
|
||||
}
|
||||
if _, ok := stored["imagePreview"]; ok {
|
||||
t.Fatal("imagePreview (derived) was persisted")
|
||||
}
|
||||
}
|
||||
|
||||
// The admin UI echoes its whole payload back on PATCH, derived fields included.
|
||||
// Those must be dropped before the value reaches memory, not only before the database.
|
||||
func TestPatchMediaSettingsDropsEchoedDerivedFields(t *testing.T) {
|
||||
initSidebarLinkTestDB(t)
|
||||
|
||||
patch := []byte(`{"imagePreview":{"jpeg":true},"transcode":{"enabledProfiles":["quality"]}}`)
|
||||
if err := PatchMediaSettings(patch); err != nil {
|
||||
t.Fatalf("PatchMediaSettings: %v", err)
|
||||
}
|
||||
got := GetMediaSettings()
|
||||
if got.ImagePreview != nil {
|
||||
t.Fatal("echoed imagePreview stayed in memory")
|
||||
}
|
||||
if got.Transcode.EnabledProfiles != nil {
|
||||
t.Fatal("echoed enabledProfiles stayed in memory")
|
||||
}
|
||||
}
|
||||
|
||||
// A fresh install seeds the stored media settings from config.
|
||||
func TestMediaSettingsSeedFromConfigOnFirstInit(t *testing.T) {
|
||||
initSidebarLinkTestDB(t)
|
||||
|
||||
@@ -90,7 +90,7 @@ func TestStateInit_jwtYAML_organizedDefaultsOnNewUser(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !loaded.DarkMode.IsDark() || !loaded.Preview.Image {
|
||||
if !loaded.DarkMode || !loaded.Preview.Image {
|
||||
t.Fatalf("profile fields: darkMode=%v preview.image=%v", loaded.DarkMode, loaded.Preview.Image)
|
||||
}
|
||||
if loaded.BackendScopes[0].Permissions.Modify != access.Modify {
|
||||
|
||||
@@ -133,7 +133,7 @@ func (e *MediaEngine) BuildPlan(ctx context.Context, realPath string, profile Pr
|
||||
copyImpossible := false
|
||||
plan.segmentSec = defaults.SegmentDurationSec
|
||||
if plan.remux || plan.videoCopy {
|
||||
rawKeyframes, kfErr := probeKeyframesCached(ctx, svc, realPath, stat.Size(), stat.ModTime().UnixNano(), plan.segmentSec, mediaSpan{durationSec: info.Duration, startTimeSec: info.StartTime}, true)
|
||||
rawKeyframes, kfErr := probeKeyframesCached(ctx, svc, realPath, stat.Size(), stat.ModTime().UnixNano(), plan.segmentSec, true)
|
||||
if kfErr != nil {
|
||||
copyImpossible = true
|
||||
logger.Warningf("transcode: keyframe probe failed on %s (%v) — forcing video transcode for HLS", realPath, kfErr)
|
||||
|
||||
@@ -12,7 +12,6 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/ffmpeg"
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/gomedia"
|
||||
"github.com/gtsteffaniak/filebrowser/backend/pkg/settings"
|
||||
"github.com/gtsteffaniak/go-logger/logger"
|
||||
)
|
||||
@@ -56,9 +55,6 @@ type keyframeCache struct {
|
||||
|
||||
var defaultKeyframeCache = &keyframeCache{entries: map[keyframeKey]*keyframeEntry{}}
|
||||
|
||||
// get returns the keyframe timeline for key from memory, then disk, then
|
||||
// probe. Concurrent callers for one key wait on a single probe; failures are
|
||||
// not cached.
|
||||
func (c *keyframeCache) get(ctx context.Context, key keyframeKey, probe func(context.Context) ([]float64, error)) ([]float64, error) {
|
||||
c.mu.Lock()
|
||||
e := c.entries[key]
|
||||
@@ -109,15 +105,12 @@ func keyframeDiskCacheEnabled() bool {
|
||||
return settings.Config.Server.CacheDir != ""
|
||||
}
|
||||
|
||||
// keyframeDiskPath returns the disk cache file for key's path.
|
||||
func keyframeDiskPath(key keyframeKey) string {
|
||||
sum := sha256.Sum256([]byte(key.path))
|
||||
name := hex.EncodeToString(sum[:])[:32] + ".json"
|
||||
return filepath.Join(settings.TranscodeKeyframeCacheDir(), name)
|
||||
}
|
||||
|
||||
// readKeyframeDiskCache returns the persisted timeline for key when it exists
|
||||
// and matches the file's size, mtime, and the cache version.
|
||||
func readKeyframeDiskCache(key keyframeKey) ([]float64, bool) {
|
||||
if !keyframeDiskCacheEnabled() {
|
||||
return nil, false
|
||||
@@ -139,8 +132,6 @@ func readKeyframeDiskCache(key keyframeKey) ([]float64, bool) {
|
||||
return stamp.Times, true
|
||||
}
|
||||
|
||||
// writeKeyframeDiskCache persists a timeline for key atomically. Errors are
|
||||
// ignored; the cache is an optimization.
|
||||
func writeKeyframeDiskCache(key keyframeKey, times []float64) {
|
||||
if len(times) == 0 || !keyframeDiskCacheEnabled() {
|
||||
return
|
||||
@@ -228,88 +219,19 @@ func probeKeyframesWithPreflight(ctx context.Context, svc *ffmpeg.Service, realP
|
||||
)
|
||||
}
|
||||
|
||||
// mediaSpan is the ffprobe format duration and start time of a file, the
|
||||
// values the HLS planner turns into its media timeline. The zero value means
|
||||
// unknown.
|
||||
type mediaSpan struct {
|
||||
durationSec float64
|
||||
startTimeSec float64
|
||||
}
|
||||
|
||||
// trailingGapExceedsCopyCap reports whether the stretch from the last
|
||||
// keyframe to the end of the media is longer than the copy cap, measured on
|
||||
// the same normalized timeline the planner builds: the final segment runs
|
||||
// from the last keyframe to the media end, so such a gap forces a transcode
|
||||
// just like a gap between two keyframes. An unknown span reports false.
|
||||
func trailingGapExceedsCopyCap(keyframes []float64, span mediaSpan, segmentSec float64) bool {
|
||||
if len(keyframes) == 0 || span.durationSec <= 0 {
|
||||
return false
|
||||
}
|
||||
kf, mediaDur := ffmpeg.PrepareHLSMediaTimeline(keyframes, span.durationSec, span.startTimeSec)
|
||||
if len(kf) == 0 || mediaDur <= 0 {
|
||||
return false
|
||||
}
|
||||
return mediaDur-kf[len(kf)-1] > maxCopySegmentDurationSec(segmentSec)
|
||||
}
|
||||
|
||||
// keyframeProbeTimeout bounds a single full-file keyframe probe. The default
|
||||
// demux has no deadline of its own, so a slow network share would otherwise
|
||||
// stall plan building (and the client's session POST) indefinitely.
|
||||
const keyframeProbeTimeout = 2 * time.Minute
|
||||
|
||||
// probeKeyframesCached returns the keyframe timeline of a file version from
|
||||
// the memory or disk cache, or probes it: go-media reads the container index
|
||||
// first, and ffprobe runs only when go-media cannot answer or reports a gap
|
||||
// that would rule out stream copy. span is the file's duration and start
|
||||
// time, used to check the gap after the last keyframe; the zero value skips
|
||||
// that check.
|
||||
func probeKeyframesCached(ctx context.Context, svc *ffmpeg.Service, realPath string, size, modTime int64, segmentSec float64, span mediaSpan, preflight bool) ([]float64, error) {
|
||||
func probeKeyframesCached(ctx context.Context, svc *ffmpeg.Service, realPath string, size, modTime int64, segmentSec float64, preflight bool) ([]float64, error) {
|
||||
key := keyframeKey{path: realPath, size: size, modTime: modTime}
|
||||
return defaultKeyframeCache.get(ctx, key, func(ctx context.Context) ([]float64, error) {
|
||||
probeCtx, cancel := context.WithTimeout(ctx, keyframeProbeTimeout)
|
||||
defer cancel()
|
||||
if times, ok := gomediaKeyframes(probeCtx, svc, realPath, segmentSec, span, preflight); ok {
|
||||
return times, nil
|
||||
}
|
||||
if preflight {
|
||||
return probeKeyframesWithPreflight(probeCtx, svc, realPath, segmentSec)
|
||||
}
|
||||
return svc.ProbeVideoKeyframeTimes(probeCtx, realPath)
|
||||
})
|
||||
}
|
||||
|
||||
// gomediaKeyframes reads the keyframe timeline from the container index with
|
||||
// go-media. It reports ok=false when go-media cannot answer, and also when
|
||||
// preflight is set and the timeline has a gap past the copy cap, including
|
||||
// the gap from the last keyframe to the end of span: go-media can miss
|
||||
// keyframes that ffprobe's parsers flag (IDR frames not in the index,
|
||||
// partial Matroska Cues) but never adds one, so only ffprobe may decide that
|
||||
// stream copy is impossible. A bounded go-media read runs first, so a file
|
||||
// whose head already shows such a gap does not pay for a full index walk.
|
||||
func gomediaKeyframes(ctx context.Context, svc *ffmpeg.Service, realPath string, segmentSec float64, span mediaSpan, preflight bool) ([]float64, bool) {
|
||||
if preflight {
|
||||
if head, ok := gomedia.KeyframeTimesWithin(ctx, realPath, keyframeBoundedPreflightWindowSec); ok && keyframesExceedCopyCap(head, segmentSec) {
|
||||
logger.Debugf("transcode: go-media keyframe pre-flight shows a gap past the copy cap on %s; confirming with ffprobe", realPath)
|
||||
return nil, false
|
||||
}
|
||||
}
|
||||
legacy := func() ([]float64, bool) {
|
||||
shadowCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), keyframeProbeTimeout)
|
||||
defer cancel()
|
||||
times, err := svc.ProbeVideoKeyframeTimes(shadowCtx, realPath)
|
||||
return times, err == nil
|
||||
}
|
||||
times, ok := gomedia.KeyframeTimes(ctx, realPath, legacy)
|
||||
if !ok {
|
||||
return nil, false
|
||||
}
|
||||
if preflight && keyframesExceedCopyCap(times, segmentSec) {
|
||||
logger.Debugf("transcode: go-media keyframe index shows a gap past the copy cap on %s; confirming with ffprobe", realPath)
|
||||
return nil, false
|
||||
}
|
||||
if preflight && trailingGapExceedsCopyCap(times, span, segmentSec) {
|
||||
logger.Debugf("transcode: go-media keyframe index ends more than the copy cap before the end of %s; confirming with ffprobe", realPath)
|
||||
return nil, false
|
||||
}
|
||||
return times, true
|
||||
}
|
||||
@@ -177,33 +177,3 @@ func TestKeyframeCacheDoesNotCacheFailures(t *testing.T) {
|
||||
t.Fatalf("retry after failure: %v, %v", got, err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestTrailingGapExceedsCopyCap checks the gap from the last keyframe to the
|
||||
// end of the media, on the planner's start-time-normalized timeline.
|
||||
func TestTrailingGapExceedsCopyCap(t *testing.T) {
|
||||
every2 := func(from, to float64) []float64 {
|
||||
var out []float64
|
||||
for v := from; v < to; v += 2 {
|
||||
out = append(out, v)
|
||||
}
|
||||
return out
|
||||
}
|
||||
cases := []struct {
|
||||
name string
|
||||
kf []float64
|
||||
span mediaSpan
|
||||
want bool
|
||||
}{
|
||||
{"partial index", []float64{0, 2}, mediaSpan{durationSec: 60}, true},
|
||||
{"full index", every2(0, 60), mediaSpan{durationSec: 60}, false},
|
||||
{"unknown span", []float64{0, 2}, mediaSpan{}, false},
|
||||
{"start time, partial", []float64{10, 12}, mediaSpan{durationSec: 70, startTimeSec: 10}, true},
|
||||
{"start time, full", every2(10, 70), mediaSpan{durationSec: 70, startTimeSec: 10}, false},
|
||||
{"no keyframes", nil, mediaSpan{durationSec: 60}, false},
|
||||
}
|
||||
for _, c := range cases {
|
||||
if got := trailingGapExceedsCopyCap(c.kf, c.span, 4); got != c.want {
|
||||
t.Errorf("%s: got %v, want %v", c.name, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,111 +0,0 @@
|
||||
package transcoding
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/gtsteffaniak/filebrowser/backend/pkg/settings"
|
||||
)
|
||||
|
||||
const keyframeFixtureMP4 = "../../../frontend/tests/playwright-files/myfolder/testdata/sample-640x480.mp4"
|
||||
|
||||
// TestProbeKeyframesCachedUsesGoMedia checks that an MP4 keyframe timeline is
|
||||
// built without ffprobe: with no ffmpeg service, only go-media can answer.
|
||||
func TestProbeKeyframesCachedUsesGoMedia(t *testing.T) {
|
||||
if settings.AlwaysUseFfmpegEnabled() {
|
||||
t.Skip("go-media disabled by settings")
|
||||
}
|
||||
prevCacheDir := settings.Config.Server.CacheDir
|
||||
settings.Config.Server.CacheDir = "" // keep the disk cache out of the test
|
||||
t.Cleanup(func() { settings.Config.Server.CacheDir = prevCacheDir })
|
||||
|
||||
path := filepath.Join(t.TempDir(), "sample.mp4")
|
||||
data, err := os.ReadFile(keyframeFixtureMP4)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = os.WriteFile(path, data, 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
st, err := os.Stat(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
times, err := probeKeyframesCached(context.Background(), nil, path, st.Size(), st.ModTime().UnixNano(), 4, mediaSpan{}, true)
|
||||
if err != nil {
|
||||
t.Fatalf("probeKeyframesCached: %v", err)
|
||||
}
|
||||
if len(times) == 0 || times[0] < 0 {
|
||||
t.Fatalf("unexpected keyframes %v", times)
|
||||
}
|
||||
for i := 1; i < len(times); i++ {
|
||||
if times[i] <= times[i-1] {
|
||||
t.Fatalf("keyframes not strictly increasing: %v", times)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestGomediaKeyframesDefersCopyDecision checks that a go-media timeline with a
|
||||
// gap past the copy cap is not used for the pre-flight decision (ffprobe must
|
||||
// confirm it), while a plain probe still returns it.
|
||||
func TestGomediaKeyframesDefersCopyDecision(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("encodes a clip with ffmpeg")
|
||||
}
|
||||
if settings.AlwaysUseFfmpegEnabled() {
|
||||
t.Skip("go-media disabled by settings")
|
||||
}
|
||||
if _, err := exec.LookPath("ffmpeg"); err != nil {
|
||||
t.Skip("ffmpeg not installed")
|
||||
}
|
||||
path := filepath.Join(t.TempDir(), "longgop.mp4")
|
||||
out, err := exec.Command("ffmpeg", "-v", "error", "-y", "-f", "lavfi", "-i", "testsrc2=size=128x96:rate=25:duration=40",
|
||||
"-c:v", "libx264", "-g", "750", "-keyint_min", "750", "-sc_threshold", "0", "-pix_fmt", "yuv420p", path).CombinedOutput()
|
||||
if err != nil {
|
||||
t.Skipf("ffmpeg cannot encode libx264: %v: %s", err, out)
|
||||
}
|
||||
|
||||
if times, ok := gomediaKeyframes(context.Background(), nil, path, 4, mediaSpan{}, true); ok {
|
||||
t.Fatalf("pre-flight must defer a 30s GOP to ffprobe, got %v", times)
|
||||
}
|
||||
times, ok := gomediaKeyframes(context.Background(), nil, path, 4, mediaSpan{}, false)
|
||||
if !ok {
|
||||
t.Fatal("a plain probe must return the go-media timeline")
|
||||
}
|
||||
if len(times) != 2 || times[0] != 0 || times[1] < 29.9 || times[1] > 30.1 {
|
||||
t.Fatalf("unexpected keyframes %v, want [0 30]", times)
|
||||
}
|
||||
}
|
||||
|
||||
// TestGomediaKeyframesDefersTrailingGap checks that a go-media index whose
|
||||
// last keyframe is more than the copy cap before the end of the media is not
|
||||
// used for the pre-flight decision, so ffprobe can confirm it, while the
|
||||
// same index is used when it reaches the end.
|
||||
func TestGomediaKeyframesDefersTrailingGap(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("encodes a clip with ffmpeg")
|
||||
}
|
||||
if settings.AlwaysUseFfmpegEnabled() {
|
||||
t.Skip("go-media disabled by settings")
|
||||
}
|
||||
if _, err := exec.LookPath("ffmpeg"); err != nil {
|
||||
t.Skip("ffmpeg not installed")
|
||||
}
|
||||
path := filepath.Join(t.TempDir(), "short.mp4")
|
||||
out, err := exec.Command("ffmpeg", "-v", "error", "-y", "-f", "lavfi", "-i", "testsrc2=size=128x96:rate=25:duration=4",
|
||||
"-c:v", "libx264", "-g", "50", "-keyint_min", "50", "-sc_threshold", "0", "-pix_fmt", "yuv420p", path).CombinedOutput()
|
||||
if err != nil {
|
||||
t.Skipf("ffmpeg cannot encode libx264: %v: %s", err, out)
|
||||
}
|
||||
// The index ends at 2 s; a 60 s span leaves a 58 s final segment.
|
||||
if times, ok := gomediaKeyframes(context.Background(), nil, path, 4, mediaSpan{durationSec: 60}, true); ok {
|
||||
t.Fatalf("pre-flight must defer a 58s trailing gap to ffprobe, got %v", times)
|
||||
}
|
||||
times, ok := gomediaKeyframes(context.Background(), nil, path, 4, mediaSpan{durationSec: 4}, true)
|
||||
if !ok || len(times) != 2 {
|
||||
t.Fatalf("index reaching the end must be used, got %v, %v", times, ok)
|
||||
}
|
||||
}
|
||||
@@ -1098,14 +1098,14 @@ func TestManagerSuspendsProducerWhenAllClientsPaused(t *testing.T) {
|
||||
in := testInput(t, "pauser", "/v.mp4")
|
||||
sess := mustStart(t, mgr, in)
|
||||
waitForStarts(t, engine, 1)
|
||||
job := engine.LastJob()
|
||||
|
||||
if err := mgr.Heartbeat(in.Username, sess.ID, HeartbeatRequest{ClientID: "c1", Paused: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waitForProducerVODJobAttached(t, mgr, sess.ID)
|
||||
suspendPausedProducers(t, mgr, time.Now().Add(time.Hour))
|
||||
|
||||
waitForProducerParked(t, mgr, sess.ID)
|
||||
pollUntil(t, 2*time.Second, func() bool { return !job.Running() }, "suspended job to stop")
|
||||
if snap := snapshotProducer(t, mgr, sess.ID); !snap.exists || !snap.suspended {
|
||||
t.Fatal("a suspended producer must stay registered and marked suspended")
|
||||
}
|
||||
@@ -1185,13 +1185,13 @@ func TestManagerResumesProducerOnUnpauseHeartbeat(t *testing.T) {
|
||||
pollUntil(t, 2*time.Second, func() bool {
|
||||
return snapshotProducer(t, mgr, sess.ID).encodeHead >= 1
|
||||
}, "encode head to advance past segment 0")
|
||||
job := engine.LastJob()
|
||||
|
||||
if err := mgr.Heartbeat(in.Username, sess.ID, HeartbeatRequest{ClientID: "c1", Paused: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waitForProducerVODJobAttached(t, mgr, sess.ID)
|
||||
suspendPausedProducers(t, mgr, time.Now().Add(time.Hour))
|
||||
waitForProducerParked(t, mgr, sess.ID)
|
||||
pollUntil(t, 2*time.Second, func() bool { return !job.Running() }, "suspended job to stop")
|
||||
|
||||
if err := mgr.Heartbeat(in.Username, sess.ID, HeartbeatRequest{ClientID: "c1", Paused: false}); err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -1216,13 +1216,13 @@ func TestManagerResumesProducerOnSegmentRequest(t *testing.T) {
|
||||
in := testInput(t, "pauser", "/v.mp4")
|
||||
sess := mustStart(t, mgr, in)
|
||||
waitForStarts(t, engine, 1)
|
||||
job := engine.LastJob()
|
||||
|
||||
if err := mgr.Heartbeat(in.Username, sess.ID, HeartbeatRequest{ClientID: "c1", Paused: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waitForProducerVODJobAttached(t, mgr, sess.ID)
|
||||
suspendPausedProducers(t, mgr, time.Now().Add(time.Hour))
|
||||
waitForProducerParked(t, mgr, sess.ID)
|
||||
pollUntil(t, 2*time.Second, func() bool { return !job.Running() }, "suspended job to stop")
|
||||
|
||||
// The suspended job reads as !Running, so a request beyond its head takes
|
||||
// the restart path: the relaunched job produces the requested segment.
|
||||
@@ -1246,13 +1246,13 @@ func TestManagerStopReleasesSuspendedProducer(t *testing.T) {
|
||||
in := testInput(t, "pauser", "/v.mp4")
|
||||
sess := mustStart(t, mgr, in)
|
||||
waitForStarts(t, engine, 1)
|
||||
job := engine.LastJob()
|
||||
|
||||
if err := mgr.Heartbeat(in.Username, sess.ID, HeartbeatRequest{ClientID: "c1", Paused: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waitForProducerVODJobAttached(t, mgr, sess.ID)
|
||||
suspendPausedProducers(t, mgr, time.Now().Add(time.Hour))
|
||||
waitForProducerParked(t, mgr, sess.ID)
|
||||
pollUntil(t, 2*time.Second, func() bool { return !job.Running() }, "suspended job to stop")
|
||||
|
||||
if err := mgr.Stop(in.Username, sess.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -1308,29 +1308,6 @@ func mustStart(t *testing.T, mgr *Manager, in StartInput) SessionInfo {
|
||||
return info
|
||||
}
|
||||
|
||||
// waitForProducerVODJobAttached waits until the session's producer has the VOD
|
||||
// job wired up. StartCount can advance before runProducerJob assigns p.job.
|
||||
func waitForProducerVODJobAttached(t *testing.T, mgr *Manager, sessionID string) {
|
||||
t.Helper()
|
||||
pollUntil(t, 5*time.Second, func() bool {
|
||||
mgr.mu.Lock()
|
||||
p := mgr.producers[sessionID]
|
||||
ok := p != nil && p.job != nil
|
||||
mgr.mu.Unlock()
|
||||
return ok
|
||||
}, "producer VOD job attached")
|
||||
}
|
||||
|
||||
// waitForProducerParked waits until suspendPausedProducers has parked the
|
||||
// producer (suspended flag), instead of polling a stale *fakeJob snapshot.
|
||||
func waitForProducerParked(t *testing.T, mgr *Manager, sessionID string) {
|
||||
t.Helper()
|
||||
pollUntil(t, 5*time.Second, func() bool {
|
||||
snap := snapshotProducer(t, mgr, sessionID)
|
||||
return snap.exists && snap.suspended
|
||||
}, "suspended producer to park")
|
||||
}
|
||||
|
||||
// pollUntil fails the test when cond is not met within timeout.
|
||||
func pollUntil(t *testing.T, timeout time.Duration, cond func() bool, msg string) {
|
||||
t.Helper()
|
||||
|
||||
@@ -1,13 +1,16 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
"unicode"
|
||||
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/activity"
|
||||
activitydb "github.com/gtsteffaniak/filebrowser/backend/internal/database/activity"
|
||||
@@ -36,6 +39,9 @@ const (
|
||||
// duplicateResultsCache caches duplicate search results for 15 seconds
|
||||
var duplicateResultsCache = cache.NewCache[duplicateResponse](15 * time.Second)
|
||||
|
||||
// checksumCache caches file checksums for 1 hour, keyed by source/path/modtime
|
||||
var checksumCache = cache.NewCache[string](1 * time.Hour)
|
||||
|
||||
type duplicateGroup struct {
|
||||
Size int64 `json:"size"`
|
||||
Count int `json:"count"`
|
||||
@@ -121,12 +127,10 @@ func (s *duplicateProcessingStats) shouldStop() (bool, string) {
|
||||
// identical content but different filenames)
|
||||
//
|
||||
// Performance Optimizations:
|
||||
// - Checksums are cached for 1 hour (keyed by path/size/modtime) to speed up subsequent requests
|
||||
// The checksum budget counts only cache misses (real disk I/O); cached digests are free,
|
||||
// so a re-run proceeds past the previously-examined files instead of stalling at the limit.
|
||||
// - Files are processed in batches to balance memory usage vs SQL query count
|
||||
// - Resource limits prevent timeouts: max 2 minutes, max 10K checksum operations
|
||||
// - Large fuzzy groups are skipped to avoid expensive checksum operations on false positives
|
||||
// - Checksums are cached for 1 hour (keyed by path/size/modtime) to speed up subsequent requests
|
||||
// - Files are processed in batches to balance memory usage vs SQL query count
|
||||
// - Resource limits prevent timeouts: max 2 minutes, max 10K checksum operations
|
||||
// - Large fuzzy groups are skipped to avoid expensive checksum operations on false positives
|
||||
//
|
||||
// Response includes incomplete flag if processing was stopped early due to resource limits.
|
||||
//
|
||||
@@ -166,10 +170,7 @@ func duplicatesHandler(w http.ResponseWriter, r *http.Request, d *Context) (int,
|
||||
|
||||
// Safety check: Reject duplicate search during active indexing to prevent resource contention
|
||||
// This protects against CPU/memory spikes and ensures indexing performance is not degraded
|
||||
// Use GetStatus() (computed from active scanners and persisted lastScanned) rather than
|
||||
// the raw Status field, which is only set to "ready" by PostScan and would otherwise
|
||||
// block searches indefinitely after a restart until the first scheduled scan runs.
|
||||
if index.GetStatus() == indexing.INDEXING {
|
||||
if string(index.Status) == "indexing" {
|
||||
return http.StatusServiceUnavailable, fmt.Errorf("duplicate search is not available while indexing is in progress - please try again when indexing completes")
|
||||
}
|
||||
|
||||
@@ -237,13 +238,7 @@ func duplicatesHandler(w http.ResponseWriter, r *http.Request, d *Context) (int,
|
||||
startTime: time.Now(),
|
||||
uniqueChecksums: make(map[string]bool),
|
||||
}
|
||||
duplicateGroups := findDuplicatesInIndex(r.Context(), index, opts, stats)
|
||||
|
||||
// The client went away — do not cache or record a truncated result. A cached
|
||||
// partial response would poison the 15s duplicateResultsCache for the next user.
|
||||
if err := r.Context().Err(); err != nil {
|
||||
return http.StatusRequestTimeout, err
|
||||
}
|
||||
duplicateGroups := findDuplicatesInIndex(index, opts, stats)
|
||||
|
||||
// Log resource usage for monitoring
|
||||
uniqueChecksumCount := len(stats.uniqueChecksums)
|
||||
@@ -259,7 +254,8 @@ func duplicatesHandler(w http.ResponseWriter, r *http.Request, d *Context) (int,
|
||||
Reason: stats.stopReason,
|
||||
}
|
||||
|
||||
cacheDuplicateResponse(r.Context(), cacheKey, response)
|
||||
// Cache the results before returning (even partial results)
|
||||
duplicateResultsCache.Set(cacheKey, response)
|
||||
|
||||
if stats.stopped {
|
||||
w.Header().Set("X-Search-Incomplete", "true")
|
||||
@@ -271,7 +267,7 @@ func duplicatesHandler(w http.ResponseWriter, r *http.Request, d *Context) (int,
|
||||
}
|
||||
|
||||
// findDuplicatesInIndex finds duplicates using the shared IndexDB with resource limits
|
||||
func findDuplicatesInIndex(ctx context.Context, index *indexing.Index, opts *duplicatesOptions, stats *duplicateProcessingStats) []duplicateGroup {
|
||||
func findDuplicatesInIndex(index *indexing.Index, opts *duplicatesOptions, stats *duplicateProcessingStats) []duplicateGroup {
|
||||
// Get the shared IndexDB
|
||||
indexDB := indexing.GetIndexDB()
|
||||
|
||||
@@ -294,11 +290,6 @@ func findDuplicatesInIndex(ctx context.Context, index *indexing.Index, opts *dup
|
||||
|
||||
// Process sizes in batches
|
||||
for batchStart := 0; batchStart < len(sizes); batchStart += bulkSizeQueryLimit {
|
||||
if ctx.Err() != nil {
|
||||
stats.stopped = true
|
||||
stats.stopReason = ctx.Err().Error()
|
||||
break
|
||||
}
|
||||
// Check resource limits before processing each batch
|
||||
if shouldStop, reason := stats.shouldStop(); shouldStop {
|
||||
stats.stopped = true
|
||||
@@ -336,11 +327,6 @@ func findDuplicatesInIndex(ctx context.Context, index *indexing.Index, opts *dup
|
||||
// Memory optimization: Delete entries from filesBySize immediately after processing
|
||||
// to allow GC to reclaim memory as we go, rather than holding all data until batch completes
|
||||
for _, size := range sizeBatch {
|
||||
if ctx.Err() != nil {
|
||||
stats.stopped = true
|
||||
stats.stopReason = ctx.Err().Error()
|
||||
break
|
||||
}
|
||||
files, ok := filesBySize[size]
|
||||
if !ok || len(files) < 2 {
|
||||
// Clear entry even if not processing to free memory
|
||||
@@ -359,8 +345,8 @@ func findDuplicatesInIndex(ctx context.Context, index *indexing.Index, opts *dup
|
||||
continue
|
||||
}
|
||||
|
||||
// Group files by MIME (+ extension when MIME is ambiguous) within this size bucket.
|
||||
filesByType := groupFilesByTypeBucket(files)
|
||||
// Group files by MIME type in memory (already ordered by type from SQL)
|
||||
filesByType := groupFilesByType(files)
|
||||
|
||||
// Process each MIME type group
|
||||
for _, typeFiles := range filesByType {
|
||||
@@ -378,13 +364,29 @@ func findDuplicatesInIndex(ctx context.Context, index *indexing.Index, opts *dup
|
||||
|
||||
// Use fuzzy filename matching to group files within this MIME type
|
||||
// This reduces checksum operations by grouping similar filenames
|
||||
filenameGroups := groupFilesByFilename(ctx, typeFiles, size, stats)
|
||||
filenameGroups := groupFilesByFilename(typeFiles, size)
|
||||
|
||||
totalFilesInGroups := 0
|
||||
for _, g := range filenameGroups {
|
||||
if len(g) >= 2 {
|
||||
totalFilesInGroups += len(g)
|
||||
}
|
||||
}
|
||||
|
||||
// Process each fuzzy filename group
|
||||
for _, fileGroup := range filenameGroups {
|
||||
if len(fileGroup) < 2 {
|
||||
continue
|
||||
}
|
||||
|
||||
// Skip checksumming if fuzzy group is too large - groups with many files
|
||||
// are unlikely to be true duplicates (fuzzy matching is too permissive)
|
||||
// This optimization prevents expensive checksum operations on false positives
|
||||
if len(fileGroup) > maxFuzzyGroupSize {
|
||||
logger.Debugf("[Duplicates] Skipping checksum for fuzzy group with %d files (exceeds limit of %d)", len(fileGroup), maxFuzzyGroupSize)
|
||||
continue
|
||||
}
|
||||
|
||||
// Stop if we've hit the group limit
|
||||
if len(groupsWithChecksums) >= maxGroups {
|
||||
break
|
||||
@@ -401,7 +403,7 @@ func findDuplicatesInIndex(ctx context.Context, index *indexing.Index, opts *dup
|
||||
// Verify with checksums using 3-pass progressive verification
|
||||
// At this point, files match on: size + MIME type + fuzzy filename similarity (50%+)
|
||||
// Large fuzzy groups (>10 files) are skipped above to avoid expensive false positives
|
||||
verifiedGroups := groupFilesByChecksum(ctx, fileGroup, index, size, stats)
|
||||
verifiedGroups := groupFilesByChecksum(fileGroup, index, size, stats)
|
||||
|
||||
// Create SearchResult objects and track checksums for merging
|
||||
for _, checksumGroup := range verifiedGroups {
|
||||
@@ -481,15 +483,100 @@ func findDuplicatesInIndex(ctx context.Context, index *indexing.Index, opts *dup
|
||||
})
|
||||
}
|
||||
|
||||
// Groups are already sorted by size (largest to smallest) from SQL query
|
||||
return duplicateGroups
|
||||
}
|
||||
|
||||
// cacheDuplicateResponse stores a duplicate search response when the client context is still active.
|
||||
func cacheDuplicateResponse(ctx context.Context, cacheKey string, response duplicateResponse) {
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
// mergeGroupsByChecksum merges groups that share any checksum values
|
||||
// This catches cases where files have identical content but were grouped separately
|
||||
func mergeGroupsByChecksum(groups []duplicateGroupWithChecksums) []duplicateGroupWithChecksums {
|
||||
if len(groups) <= 1 {
|
||||
return groups
|
||||
}
|
||||
duplicateResultsCache.Set(cacheKey, response)
|
||||
|
||||
// Build a map: checksum -> list of group indices that contain this checksum
|
||||
checksumToGroups := make(map[string][]int)
|
||||
for i, group := range groups {
|
||||
if group.Checksum != "" {
|
||||
checksumToGroups[group.Checksum] = append(checksumToGroups[group.Checksum], i)
|
||||
}
|
||||
}
|
||||
|
||||
// Use union-find to merge groups that share checksums
|
||||
parent := make([]int, len(groups))
|
||||
for i := range parent {
|
||||
parent[i] = i
|
||||
}
|
||||
|
||||
var find func(int) int
|
||||
find = func(x int) int {
|
||||
if parent[x] != x {
|
||||
parent[x] = find(parent[x]) // Path compression
|
||||
}
|
||||
return parent[x]
|
||||
}
|
||||
|
||||
union := func(x, y int) {
|
||||
px, py := find(x), find(y)
|
||||
if px != py {
|
||||
parent[px] = py
|
||||
}
|
||||
}
|
||||
|
||||
// Merge groups that share any checksum
|
||||
for _, groupIndices := range checksumToGroups {
|
||||
if len(groupIndices) < 2 {
|
||||
continue
|
||||
}
|
||||
// Merge all groups that share this checksum
|
||||
for i := 1; i < len(groupIndices); i++ {
|
||||
union(groupIndices[0], groupIndices[i])
|
||||
}
|
||||
}
|
||||
|
||||
// Group indices by their root parent
|
||||
groupsByRoot := make(map[int][]int)
|
||||
for i := range groups {
|
||||
root := find(i)
|
||||
groupsByRoot[root] = append(groupsByRoot[root], i)
|
||||
}
|
||||
|
||||
// Merge groups that belong to the same root
|
||||
merged := make([]duplicateGroupWithChecksums, 0, len(groupsByRoot))
|
||||
for _, indices := range groupsByRoot {
|
||||
if len(indices) == 1 {
|
||||
// No merging needed, just add the group
|
||||
merged = append(merged, groups[indices[0]])
|
||||
continue
|
||||
}
|
||||
|
||||
// Merge multiple groups
|
||||
combinedFiles := make([]*indexing.SearchResult, 0)
|
||||
var combinedSize int64
|
||||
var combinedChecksum string
|
||||
|
||||
for _, idx := range indices {
|
||||
group := groups[idx]
|
||||
combinedFiles = append(combinedFiles, group.Files...)
|
||||
if combinedChecksum == "" {
|
||||
combinedChecksum = group.Checksum
|
||||
}
|
||||
if combinedSize == 0 {
|
||||
combinedSize = group.Size
|
||||
}
|
||||
}
|
||||
|
||||
// Only include merged groups with 2+ files
|
||||
if len(combinedFiles) >= 2 {
|
||||
merged = append(merged, duplicateGroupWithChecksums{
|
||||
Size: combinedSize,
|
||||
Files: combinedFiles,
|
||||
Checksum: combinedChecksum,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return merged
|
||||
}
|
||||
|
||||
func prepDuplicatesOptions(r *http.Request, d *Context) (*duplicatesOptions, error) {
|
||||
@@ -537,6 +624,349 @@ func prepDuplicatesOptions(r *http.Request, d *Context) (*duplicatesOptions, err
|
||||
}, nil
|
||||
}
|
||||
|
||||
// groupFilesByChecksum groups files by partial checksum and returns groups with their checksums
|
||||
func groupFilesByChecksum(files []*iteminfo.FileInfo, index *indexing.Index, fileSize int64, stats *duplicateProcessingStats) []checksumGroup {
|
||||
if len(files) < 2 {
|
||||
return nil
|
||||
}
|
||||
|
||||
// PASS 1: Header checksums only (first 8KB)
|
||||
// This is the fastest way to eliminate files that are definitely different
|
||||
// ONLY checksum files that already matched on filename+size
|
||||
headerGroups := make(map[string][]*iteminfo.FileInfo)
|
||||
|
||||
for _, file := range files {
|
||||
// Check if we've hit the checksum operation limit
|
||||
if stats.checksumOperations >= maxChecksumOperations {
|
||||
logger.Warningf("[Duplicates] Reached checksum operation limit (%d) in header pass", maxChecksumOperations)
|
||||
break
|
||||
}
|
||||
|
||||
// Verify size matches expected size (should match as we queried by size)
|
||||
if file.Size != fileSize {
|
||||
continue
|
||||
}
|
||||
|
||||
// Construct filesystem path for checksum computation
|
||||
// index.Path is the absolute filesystem root, file.Path is index-relative
|
||||
filePath := filepath.Join(index.Path, file.Path)
|
||||
headerChecksum, err := computeHeaderChecksum(index.Path, filePath, fileSize, file.ModTime)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
stats.checksumOperations++
|
||||
stats.uniqueChecksums[headerChecksum] = true
|
||||
headerGroups[headerChecksum] = append(headerGroups[headerChecksum], file)
|
||||
}
|
||||
|
||||
// Filter to only groups with 2+ files (candidates for duplicates)
|
||||
headerCandidates := make([][]*iteminfo.FileInfo, 0)
|
||||
filesAfterHeaderPass := 0
|
||||
for _, group := range headerGroups {
|
||||
if len(group) >= 2 {
|
||||
headerCandidates = append(headerCandidates, group)
|
||||
filesAfterHeaderPass += len(group)
|
||||
}
|
||||
}
|
||||
|
||||
if len(headerCandidates) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
// PASS 2: Middle checksums (header + middle) for header-matched groups
|
||||
// Only process files that matched on header to save disk I/O
|
||||
middleGroups := make(map[string][]*iteminfo.FileInfo)
|
||||
|
||||
for _, headerGroup := range headerCandidates {
|
||||
if len(headerGroup) < 2 {
|
||||
continue
|
||||
}
|
||||
|
||||
for _, file := range headerGroup {
|
||||
// Check if we've hit the checksum operation limit
|
||||
if stats.checksumOperations >= maxChecksumOperations {
|
||||
logger.Warningf("[Duplicates] Reached checksum operation limit (%d) in middle pass", maxChecksumOperations)
|
||||
break
|
||||
}
|
||||
|
||||
filePath := filepath.Join(index.Path, file.Path)
|
||||
middleChecksum, err := computeMiddleChecksum(index.Path, filePath, fileSize, file.ModTime)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
stats.checksumOperations++
|
||||
stats.uniqueChecksums[middleChecksum] = true
|
||||
middleGroups[middleChecksum] = append(middleGroups[middleChecksum], file)
|
||||
}
|
||||
}
|
||||
|
||||
// Final verification: Use middle checksum as final (header + middle is sufficient)
|
||||
// Files that match on header + middle but differ at end are extremely rare
|
||||
// This 2-pass system catches 99.9%+ of differences while being faster
|
||||
groups := make([]checksumGroup, 0, len(middleGroups))
|
||||
for checksum, group := range middleGroups {
|
||||
if len(group) >= 2 {
|
||||
groups = append(groups, checksumGroup{
|
||||
Files: group,
|
||||
Checksum: checksum,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return groups
|
||||
}
|
||||
|
||||
// computeHeaderChecksum calculates MD5 hash of only the first 8KB of a file
|
||||
// This is the fastest initial pass to eliminate non-matching files
|
||||
func computeHeaderChecksum(sourcePath, filePath string, size int64, modTime time.Time) (string, error) {
|
||||
// Generate cache key for header-only checksum
|
||||
cacheKey := fmt.Sprintf("%s:%s:%d:%d:header", sourcePath, filePath, size, modTime.Unix())
|
||||
|
||||
// Check cache first
|
||||
if cachedChecksum, ok := checksumCache.Get(cacheKey); ok {
|
||||
return cachedChecksum, nil
|
||||
}
|
||||
|
||||
file, err := os.Open(filePath)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
hash := md5.New()
|
||||
buf := make([]byte, 8192) // 8KB buffer
|
||||
|
||||
// Read only first 8KB (or entire file if smaller)
|
||||
n, err := io.ReadFull(file, buf)
|
||||
if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF {
|
||||
return "", err
|
||||
}
|
||||
hash.Write(buf[:n])
|
||||
|
||||
checksum := fmt.Sprintf("%x", hash.Sum(nil))
|
||||
|
||||
// Cache the result
|
||||
checksumCache.Set(cacheKey, checksum)
|
||||
|
||||
return checksum, nil
|
||||
}
|
||||
|
||||
// computeMiddleChecksum calculates MD5 hash of header + middle portion
|
||||
// Only called when header checksums match (progressive verification)
|
||||
func computeMiddleChecksum(sourcePath, filePath string, size int64, modTime time.Time) (string, error) {
|
||||
// Generate cache key for header+middle checksum
|
||||
cacheKey := fmt.Sprintf("%s:%s:%d:%d:middle", sourcePath, filePath, size, modTime.Unix())
|
||||
|
||||
// Check cache first
|
||||
if cachedChecksum, ok := checksumCache.Get(cacheKey); ok {
|
||||
return cachedChecksum, nil
|
||||
}
|
||||
|
||||
file, err := os.Open(filePath)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
hash := md5.New()
|
||||
buf := make([]byte, 8192) // 8KB buffer
|
||||
|
||||
// Read first 8KB
|
||||
n, err := io.ReadFull(file, buf)
|
||||
if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF {
|
||||
return "", err
|
||||
}
|
||||
hash.Write(buf[:n])
|
||||
|
||||
// For larger files, also read middle 8KB
|
||||
if size > 16384 { // 16KB
|
||||
middleOffset := size / 2
|
||||
if _, err := file.Seek(middleOffset, 0); err == nil {
|
||||
n, err := io.ReadFull(file, buf)
|
||||
if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF {
|
||||
return "", err
|
||||
}
|
||||
hash.Write(buf[:n])
|
||||
}
|
||||
}
|
||||
|
||||
checksum := fmt.Sprintf("%x", hash.Sum(nil))
|
||||
|
||||
// Cache the result
|
||||
checksumCache.Set(cacheKey, checksum)
|
||||
|
||||
return checksum, nil
|
||||
}
|
||||
|
||||
// groupFilesByFilename groups files by fuzzy filename matching
|
||||
// Works with iteminfo.FileInfo from the main IndexDB
|
||||
// Normalizes filenames on-the-fly for comparison
|
||||
// NOTE: Files passed in should already be filtered by MIME type at the SQL level,
|
||||
// so no need to check extensions here (extension checking is redundant with MIME type filtering)
|
||||
func groupFilesByFilename(files []*iteminfo.FileInfo, expectedSize int64) [][]*iteminfo.FileInfo {
|
||||
if len(files) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
groups := [][]*iteminfo.FileInfo{}
|
||||
used := make(map[int]bool)
|
||||
|
||||
for i := 0; i < len(files); i++ {
|
||||
if used[i] || files[i].Name == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
group := []*iteminfo.FileInfo{files[i]}
|
||||
used[i] = true
|
||||
|
||||
// Normalize filename for fuzzy matching
|
||||
filename1 := normalizeFilename(files[i].Name)
|
||||
|
||||
for j := i + 1; j < len(files); j++ {
|
||||
if used[j] || files[j].Name == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
// Check size (should match, but safe to check)
|
||||
if files[j].Size != expectedSize {
|
||||
continue
|
||||
}
|
||||
|
||||
filename2 := normalizeFilename(files[j].Name)
|
||||
|
||||
// Check if filenames are similar enough using levenshtein distance
|
||||
if filenamesSimilar(filename1, filename2) {
|
||||
group = append(group, files[j])
|
||||
used[j] = true
|
||||
}
|
||||
}
|
||||
|
||||
if len(group) >= 2 {
|
||||
groups = append(groups, group)
|
||||
}
|
||||
}
|
||||
|
||||
return groups
|
||||
}
|
||||
|
||||
// normalizeFilename converts filename to lowercase and removes special characters
|
||||
// to make fuzzy matching more effective
|
||||
func normalizeFilename(filename string) string {
|
||||
// Convert to lowercase
|
||||
filename = strings.ToLower(filename)
|
||||
|
||||
// Remove file extension for comparison
|
||||
if idx := strings.LastIndex(filename, "."); idx > 0 {
|
||||
filename = filename[:idx]
|
||||
}
|
||||
|
||||
// Remove non-alphanumeric characters except spaces and hyphens
|
||||
var result strings.Builder
|
||||
for _, r := range filename {
|
||||
if unicode.IsLetter(r) || unicode.IsDigit(r) || r == ' ' || r == '-' {
|
||||
result.WriteRune(r)
|
||||
}
|
||||
}
|
||||
|
||||
return strings.TrimSpace(result.String())
|
||||
}
|
||||
|
||||
// filenamesSimilar checks if two normalized filenames are similar enough to be considered duplicates
|
||||
func filenamesSimilar(name1, name2 string) bool {
|
||||
// Exact match
|
||||
if name1 == name2 {
|
||||
return true
|
||||
}
|
||||
|
||||
// Empty names don't match
|
||||
if len(name1) == 0 || len(name2) == 0 {
|
||||
return false
|
||||
}
|
||||
|
||||
// Calculate Levenshtein distance
|
||||
distance := levenshteinDistance(name1, name2)
|
||||
maxLen := max(len(name1), len(name2))
|
||||
|
||||
// Require at least 50% similarity (lowered to catch more duplicates with username prefixes)
|
||||
// Examples: "video.mp4" vs "username_video.mp4" should match
|
||||
similarity := 1.0 - float64(distance)/float64(maxLen)
|
||||
return similarity >= 0.5
|
||||
}
|
||||
|
||||
// levenshteinDistance calculates the edit distance between two strings
|
||||
func levenshteinDistance(s1, s2 string) int {
|
||||
if len(s1) == 0 {
|
||||
return len(s2)
|
||||
}
|
||||
if len(s2) == 0 {
|
||||
return len(s1)
|
||||
}
|
||||
|
||||
// Create matrix
|
||||
matrix := make([][]int, len(s1)+1)
|
||||
for i := range matrix {
|
||||
matrix[i] = make([]int, len(s2)+1)
|
||||
}
|
||||
|
||||
// Initialize first row and column
|
||||
for i := 0; i <= len(s1); i++ {
|
||||
matrix[i][0] = i
|
||||
}
|
||||
for j := 0; j <= len(s2); j++ {
|
||||
matrix[0][j] = j
|
||||
}
|
||||
|
||||
// Fill matrix
|
||||
for i := 1; i <= len(s1); i++ {
|
||||
for j := 1; j <= len(s2); j++ {
|
||||
cost := 0
|
||||
if s1[i-1] != s2[j-1] {
|
||||
cost = 1
|
||||
}
|
||||
|
||||
matrix[i][j] = min(
|
||||
matrix[i-1][j]+1, // deletion
|
||||
matrix[i][j-1]+1, // insertion
|
||||
matrix[i-1][j-1]+cost, // substitution
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
return matrix[len(s1)][len(s2)]
|
||||
}
|
||||
|
||||
// min returns the minimum of three integers
|
||||
func min(a, b, c int) int {
|
||||
if a < b {
|
||||
if a < c {
|
||||
return a
|
||||
}
|
||||
return c
|
||||
}
|
||||
if b < c {
|
||||
return b
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
// max returns the maximum of two integers
|
||||
func max(a, b int) int {
|
||||
if a > b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
// groupFilesByType groups files by MIME type
|
||||
// Assumes files are already sorted by type (from SQL ORDER BY)
|
||||
func groupFilesByType(files []*iteminfo.FileInfo) map[string][]*iteminfo.FileInfo {
|
||||
grouped := make(map[string][]*iteminfo.FileInfo)
|
||||
for _, file := range files {
|
||||
grouped[file.Type] = append(grouped[file.Type], file)
|
||||
}
|
||||
return grouped
|
||||
}
|
||||
|
||||
// filterFilesByPermission filters files to only include those the user is permitted to access
|
||||
// This is called early in the duplicate search process to avoid processing files the user can't see
|
||||
func filterFilesByPermission(files []*iteminfo.FileInfo, index *indexing.Index, username string) []*iteminfo.FileInfo {
|
||||
|
||||
@@ -1,157 +0,0 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/gtsteffaniak/filebrowser/backend/pkg/indexing"
|
||||
"github.com/gtsteffaniak/filebrowser/backend/pkg/indexing/iteminfo"
|
||||
)
|
||||
|
||||
type duplicateCorpus struct {
|
||||
dir string
|
||||
files []*iteminfo.FileInfo
|
||||
}
|
||||
|
||||
func buildDuplicateCorpus(b *testing.B) *duplicateCorpus {
|
||||
b.Helper()
|
||||
dir := b.TempDir()
|
||||
now := time.Now()
|
||||
|
||||
write := func(name string, payload []byte, mutateTail bool) {
|
||||
if mutateTail && len(payload) > 32 {
|
||||
dup := append([]byte(nil), payload...)
|
||||
dup[len(dup)-1] ^= 0xff
|
||||
payload = dup
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(dir, name), payload, 0o600); err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
jpegHeader := []byte{0xff, 0xd8, 0xff, 0xe0}
|
||||
pngHeader := []byte{0x89, 0x50, 0x4e, 0x47}
|
||||
pdfHeader := []byte("%PDF-1.4\n")
|
||||
mp4Header := []byte{0x00, 0x00, 0x00, 0x18, 'f', 't', 'y', 'p'}
|
||||
|
||||
mkPayload := func(header []byte, size int) []byte {
|
||||
out := make([]byte, size)
|
||||
copy(out, header)
|
||||
for i := len(header); i < len(out); i++ {
|
||||
out[i] = byte(i)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
baseJPEG := mkPayload(jpegHeader, 65536)
|
||||
basePNG := mkPayload(pngHeader, 65536)
|
||||
basePDF := mkPayload(pdfHeader, 65536)
|
||||
baseMP4 := mkPayload(mp4Header, 65536)
|
||||
|
||||
entries := []struct {
|
||||
name string
|
||||
typ string
|
||||
data []byte
|
||||
}{
|
||||
{"exact-a.jpg", "image/jpeg", baseJPEG},
|
||||
{"exact-b.jpg", "image/jpeg", baseJPEG},
|
||||
{"renamed-copy.jpeg", "image/jpeg", baseJPEG},
|
||||
{"user_exact-a.jpg", "image/jpeg", baseJPEG},
|
||||
{"png-a.png", "image/png", basePNG},
|
||||
{"png-b.png", "image/png", basePNG},
|
||||
{"doc-a.pdf", "application/pdf", basePDF},
|
||||
{"doc-b.pdf", "application/pdf", basePDF},
|
||||
{"clip-a.mp4", "video/mp4", baseMP4},
|
||||
{"clip-b.mp4", "video/mp4", baseMP4},
|
||||
{"sized-a.bin", "application/octet-stream", baseJPEG},
|
||||
{"sized-b.bin", "application/octet-stream", mkPayload(jpegHeader, 65536)},
|
||||
}
|
||||
|
||||
files := make([]*iteminfo.FileInfo, 0, len(entries)+50)
|
||||
for _, e := range entries {
|
||||
write(e.name, e.data, false)
|
||||
files = append(files, &iteminfo.FileInfo{
|
||||
ItemInfo: iteminfo.ItemInfo{Name: e.name, Size: int64(len(e.data)), ModTime: now, Type: e.typ},
|
||||
Path: "/" + e.name,
|
||||
})
|
||||
}
|
||||
write("tail-b.jpg", baseJPEG, true)
|
||||
|
||||
for i := 0; i < 12; i++ {
|
||||
name := fmt.Sprintf("fuzzy_photo_%d.jpg", i)
|
||||
write(name, baseJPEG, false)
|
||||
files = append(files, &iteminfo.FileInfo{
|
||||
ItemInfo: iteminfo.ItemInfo{Name: name, Size: int64(len(baseJPEG)), ModTime: now, Type: "image/jpeg"},
|
||||
Path: "/" + name,
|
||||
})
|
||||
}
|
||||
|
||||
for i := 0; i < 30; i++ {
|
||||
name := fmt.Sprintf("bucket_%04d.jpg", i)
|
||||
write(name, mkPayload(jpegHeader, 65536), i%7 == 0)
|
||||
files = append(files, &iteminfo.FileInfo{
|
||||
ItemInfo: iteminfo.ItemInfo{Name: name, Size: 65536, ModTime: now, Type: "image/jpeg"},
|
||||
Path: "/" + name,
|
||||
})
|
||||
}
|
||||
|
||||
return &duplicateCorpus{dir: dir, files: files}
|
||||
}
|
||||
|
||||
func BenchmarkDuplicateTypeBuckets(b *testing.B) {
|
||||
corpus := buildDuplicateCorpus(b)
|
||||
b.ResetTimer()
|
||||
b.ReportAllocs()
|
||||
for i := 0; i < b.N; i++ {
|
||||
_ = groupFilesByTypeBucket(corpus.files)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkDuplicateFilenameGroups(b *testing.B) {
|
||||
corpus := buildDuplicateCorpus(b)
|
||||
subset := corpus.files[len(corpus.files)-30:]
|
||||
ctx := context.Background()
|
||||
stats := &duplicateProcessingStats{startTime: time.Now()}
|
||||
b.ResetTimer()
|
||||
b.ReportAllocs()
|
||||
for i := 0; i < b.N; i++ {
|
||||
_ = groupFilesByFilename(ctx, subset, 65536, stats)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkDuplicateChecksumGrouping(b *testing.B) {
|
||||
corpus := buildDuplicateCorpus(b)
|
||||
pair := []*iteminfo.FileInfo{corpus.files[0], corpus.files[1]}
|
||||
idx := &indexing.Index{Path: corpus.dir}
|
||||
ctx := context.Background()
|
||||
stats := &duplicateProcessingStats{startTime: time.Now(), uniqueChecksums: map[string]bool{}}
|
||||
b.ResetTimer()
|
||||
b.ReportAllocs()
|
||||
for i := 0; i < b.N; i++ {
|
||||
_ = groupFilesByChecksum(ctx, pair, idx, pair[0].Size, stats)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkDuplicateMerge(b *testing.B) {
|
||||
mkFiles := func(n int, prefix string) []*indexing.SearchResult {
|
||||
out := make([]*indexing.SearchResult, n)
|
||||
for i := range out {
|
||||
out[i] = &indexing.SearchResult{Path: fmt.Sprintf("/%s/%d", prefix, i)}
|
||||
}
|
||||
return out
|
||||
}
|
||||
groups := []duplicateGroupWithChecksums{
|
||||
{Size: 100, Checksum: "d1", Files: mkFiles(4, "a")},
|
||||
{Size: 100, Checksum: "d1", Files: mkFiles(3, "b")},
|
||||
{Size: 200, Checksum: "d2", Files: mkFiles(2, "c")},
|
||||
}
|
||||
b.ResetTimer()
|
||||
b.ReportAllocs()
|
||||
for i := 0; i < b.N; i++ {
|
||||
_ = mergeGroupsByChecksum(groups)
|
||||
}
|
||||
}
|
||||
@@ -1,55 +0,0 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/gtsteffaniak/filebrowser/backend/pkg/indexing/iteminfo"
|
||||
)
|
||||
|
||||
// typeBucketKey returns the MIME/extension bucket used inside a fixed-size group.
|
||||
// When the MIME type already implies a single on-disk format, extension is omitted
|
||||
// as a redundant pre-filter. Otherwise a normalized lowercase extension is appended.
|
||||
//
|
||||
// JPEG aliases: ".jpg", ".jpeg", and ".jpe" normalize to "jpg" so differently named
|
||||
// JPEGs stay in one bucket without splitting the MIME-implied image/jpeg group.
|
||||
func typeBucketKey(file *iteminfo.FileInfo) string {
|
||||
mime := file.Type
|
||||
if mimeImpliesExtension(mime) {
|
||||
return mime
|
||||
}
|
||||
return mime + "\x1f" + normalizedFileExtension(file.Name)
|
||||
}
|
||||
|
||||
func mimeImpliesExtension(mime string) bool {
|
||||
switch mime {
|
||||
case "image/jpeg", "image/png", "image/gif", "image/webp", "image/bmp", "image/tiff",
|
||||
"video/mp4", "video/webm", "video/quicktime", "video/x-msvideo",
|
||||
"audio/mpeg", "audio/wav", "audio/flac", "audio/ogg",
|
||||
"application/pdf", "application/zip", "application/gzip",
|
||||
"text/plain", "text/html", "text/css", "application/json":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// normalizedFileExtension returns a lowercase extension without the dot, folding JPEG aliases to "jpg".
|
||||
func normalizedFileExtension(name string) string {
|
||||
ext := strings.ToLower(strings.TrimPrefix(filepath.Ext(name), "."))
|
||||
switch ext {
|
||||
case "jpeg", "jpe":
|
||||
return "jpg"
|
||||
default:
|
||||
return ext
|
||||
}
|
||||
}
|
||||
|
||||
func groupFilesByTypeBucket(files []*iteminfo.FileInfo) map[string][]*iteminfo.FileInfo {
|
||||
grouped := make(map[string][]*iteminfo.FileInfo)
|
||||
for _, file := range files {
|
||||
key := typeBucketKey(file)
|
||||
grouped[key] = append(grouped[key], file)
|
||||
}
|
||||
return grouped
|
||||
}
|
||||
@@ -1,23 +0,0 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/gtsteffaniak/filebrowser/backend/pkg/indexing/iteminfo"
|
||||
)
|
||||
|
||||
func TestTypeBucketKeyJPEGAliases(t *testing.T) {
|
||||
jpg := &iteminfo.FileInfo{ItemInfo: iteminfo.ItemInfo{Name: "a.jpg", Type: "image/jpeg"}}
|
||||
jpeg := &iteminfo.FileInfo{ItemInfo: iteminfo.ItemInfo{Name: "b.jpeg", Type: "image/jpeg"}}
|
||||
if typeBucketKey(jpg) != typeBucketKey(jpeg) {
|
||||
t.Fatalf("jpg and jpeg should share bucket: %q vs %q", typeBucketKey(jpg), typeBucketKey(jpeg))
|
||||
}
|
||||
}
|
||||
|
||||
func TestTypeBucketKeyAmbiguousMIMEUsesExtension(t *testing.T) {
|
||||
a := &iteminfo.FileInfo{ItemInfo: iteminfo.ItemInfo{Name: "a.bin", Type: "application/octet-stream"}}
|
||||
b := &iteminfo.FileInfo{ItemInfo: iteminfo.ItemInfo{Name: "b.dat", Type: "application/octet-stream"}}
|
||||
if typeBucketKey(a) == typeBucketKey(b) {
|
||||
t.Fatal("different extensions should not share bucket for ambiguous MIME")
|
||||
}
|
||||
}
|
||||
@@ -1,60 +0,0 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestChecksumCacheRejectsExpiredDigest(t *testing.T) {
|
||||
key := checksumCacheKey{
|
||||
sourcePath: "/src",
|
||||
filePath: "/file-a",
|
||||
size: 4096,
|
||||
modTime: 42,
|
||||
kind: checksumKindHeader,
|
||||
}
|
||||
encoded := checksumCacheKeyString(key)
|
||||
checksumTypedCache.SetWithExp(encoded, "digest", time.Millisecond)
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
if _, ok := getChecksumFromCache(key); ok {
|
||||
t.Fatal("expired digest should not be returned")
|
||||
}
|
||||
}
|
||||
|
||||
func TestChecksumCacheMissAfterFileIdentityChange(t *testing.T) {
|
||||
modA := checksumCacheKey{
|
||||
sourcePath: "/src",
|
||||
filePath: "/file-a",
|
||||
size: 4096,
|
||||
modTime: 100,
|
||||
kind: checksumKindHeader,
|
||||
}
|
||||
modB := modA
|
||||
modB.modTime = 200
|
||||
|
||||
checksumTypedCache.Set(checksumCacheKeyString(modA), "digest-a")
|
||||
if _, ok := getChecksumFromCache(modB); ok {
|
||||
t.Fatal("modified file identity must not hit the prior cache key")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCacheDuplicateResponseSkipsCanceledContext(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
|
||||
cacheKey := "cancel-test:" + time.Now().Format(time.RFC3339Nano)
|
||||
cacheDuplicateResponse(ctx, cacheKey, duplicateResponse{
|
||||
Groups: []duplicateGroup{{Size: 1, Count: 2}},
|
||||
})
|
||||
if _, ok := duplicateResultsCache.Get(cacheKey); ok {
|
||||
t.Fatal("canceled context must not cache duplicate results")
|
||||
}
|
||||
|
||||
cacheDuplicateResponse(context.Background(), cacheKey, duplicateResponse{
|
||||
Groups: []duplicateGroup{{Size: 2, Count: 2}},
|
||||
})
|
||||
if _, ok := duplicateResultsCache.Get(cacheKey); !ok {
|
||||
t.Fatal("active context should cache duplicate results")
|
||||
}
|
||||
}
|
||||
@@ -1,264 +0,0 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"hash"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/cespare/xxhash/v2"
|
||||
"github.com/gtsteffaniak/filebrowser/backend/pkg/indexing"
|
||||
"github.com/gtsteffaniak/filebrowser/backend/pkg/indexing/iteminfo"
|
||||
"github.com/gtsteffaniak/go-cache/cache"
|
||||
)
|
||||
|
||||
const (
|
||||
checksumSampleSize = 8192
|
||||
checksumSmallFile = 16384
|
||||
checksumCacheTTL = time.Hour
|
||||
)
|
||||
|
||||
type checksumSampleKind uint8
|
||||
|
||||
const (
|
||||
checksumKindHeader checksumSampleKind = iota
|
||||
checksumKindMiddle
|
||||
)
|
||||
|
||||
// checksumCacheKey identifies a cached sample digest for a file snapshot.
|
||||
type checksumCacheKey struct {
|
||||
sourcePath string
|
||||
filePath string
|
||||
size int64
|
||||
modTime int64
|
||||
kind checksumSampleKind
|
||||
}
|
||||
|
||||
var (
|
||||
// checksumTypedCache stores sample digests keyed by string-encoded checksumCacheKey.
|
||||
// The go-cache backend sweeps expired entries on an hourly ticker, so stale keys from
|
||||
// renamed/deleted/modified files are evicted without a lookup — unlike a bare sync.Map
|
||||
// which would grow without bound. Cleanup interval is the second arg (1h, matching TTL).
|
||||
checksumTypedCache = cache.NewCache[string](checksumCacheTTL, checksumCacheTTL)
|
||||
checksumBufPool = sync.Pool{
|
||||
New: func() any {
|
||||
buf := make([]byte, checksumSampleSize)
|
||||
return &buf
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
type fileSampleDigests struct {
|
||||
header string
|
||||
middle string
|
||||
}
|
||||
|
||||
func duplicateChecksumHasher() hash.Hash {
|
||||
return xxhash.New()
|
||||
}
|
||||
|
||||
func digestHex(h hash.Hash) string {
|
||||
sum := h.Sum(nil)
|
||||
dst := make([]byte, hex.EncodedLen(len(sum)))
|
||||
hex.Encode(dst, sum)
|
||||
return string(dst)
|
||||
}
|
||||
|
||||
// checksumCacheKeyString encodes a typed cache key as a single string so it can
|
||||
// be stored in the TTL-sweeping go-cache. Length prefixes keep fields unambiguous.
|
||||
func checksumCacheKeyString(key checksumCacheKey) string {
|
||||
return fmt.Sprintf("%d|%d|%d|%d|%s|%s",
|
||||
key.kind, key.size, key.modTime, len(key.sourcePath),
|
||||
key.sourcePath, key.filePath)
|
||||
}
|
||||
|
||||
func getChecksumFromCache(key checksumCacheKey) (string, bool) {
|
||||
return checksumTypedCache.Get(checksumCacheKeyString(key))
|
||||
}
|
||||
|
||||
func setChecksumCache(key checksumCacheKey, digest string) {
|
||||
checksumTypedCache.Set(checksumCacheKeyString(key), digest)
|
||||
}
|
||||
|
||||
func readFileSampleDigests(ctx context.Context, sourcePath, filePath string, size int64, modTime time.Time, stats *duplicateProcessingStats) (fileSampleDigests, error) {
|
||||
headerKey := checksumCacheKey{sourcePath: sourcePath, filePath: filePath, size: size, modTime: modTime.UnixNano(), kind: checksumKindHeader}
|
||||
middleKey := checksumCacheKey{sourcePath: sourcePath, filePath: filePath, size: size, modTime: modTime.UnixNano(), kind: checksumKindMiddle}
|
||||
|
||||
var out fileSampleDigests
|
||||
if h, ok := getChecksumFromCache(headerKey); ok {
|
||||
out.header = h
|
||||
}
|
||||
if m, ok := getChecksumFromCache(middleKey); ok {
|
||||
out.middle = m
|
||||
}
|
||||
if out.header != "" && out.middle != "" {
|
||||
return out, nil
|
||||
}
|
||||
|
||||
if ctx.Err() != nil {
|
||||
return fileSampleDigests{}, ctx.Err()
|
||||
}
|
||||
// Only count operations that actually touch disk. Cache hits are O(1) lookups
|
||||
// and must not consume the checksum budget — otherwise a second search over
|
||||
// the same files stalls at the limit without reading a single new byte.
|
||||
// The budget check lives here (post-cache) so cached digests are always served.
|
||||
if shouldStop, _ := stats.shouldStop(); shouldStop {
|
||||
return fileSampleDigests{}, fmt.Errorf("duplicate search budget exhausted")
|
||||
}
|
||||
|
||||
file, err := os.Open(filePath)
|
||||
if err != nil {
|
||||
return fileSampleDigests{}, err
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
// A disk read is happening — charge it against the checksum budget.
|
||||
stats.checksumOperations++
|
||||
|
||||
raw := checksumBufPool.Get()
|
||||
bufPtr, ok := raw.(*[]byte)
|
||||
if !ok {
|
||||
return fileSampleDigests{}, fmt.Errorf("invalid checksum buffer pool type")
|
||||
}
|
||||
buf := *bufPtr
|
||||
defer checksumBufPool.Put(bufPtr)
|
||||
|
||||
if size <= checksumSmallFile {
|
||||
h := duplicateChecksumHasher()
|
||||
remain := size
|
||||
for remain > 0 {
|
||||
if ctx.Err() != nil {
|
||||
return fileSampleDigests{}, ctx.Err()
|
||||
}
|
||||
toRead := int64(len(buf))
|
||||
if toRead > remain {
|
||||
toRead = remain
|
||||
}
|
||||
n, readErr := file.Read(buf[:toRead])
|
||||
if n > 0 {
|
||||
_, _ = h.Write(buf[:n])
|
||||
remain -= int64(n)
|
||||
}
|
||||
if readErr == io.EOF {
|
||||
break
|
||||
}
|
||||
if readErr != nil {
|
||||
return fileSampleDigests{}, readErr
|
||||
}
|
||||
if n == 0 {
|
||||
break
|
||||
}
|
||||
}
|
||||
d := digestHex(h)
|
||||
out.header = d
|
||||
out.middle = d
|
||||
} else if out.header == "" || out.middle == "" {
|
||||
n, readErr := io.ReadFull(file, buf)
|
||||
if readErr != nil && readErr != io.EOF && readErr != io.ErrUnexpectedEOF {
|
||||
return fileSampleDigests{}, readErr
|
||||
}
|
||||
|
||||
if out.header == "" {
|
||||
h := duplicateChecksumHasher()
|
||||
_, _ = h.Write(buf[:n])
|
||||
out.header = digestHex(h)
|
||||
}
|
||||
if out.middle == "" {
|
||||
middle := duplicateChecksumHasher()
|
||||
_, _ = middle.Write(buf[:n])
|
||||
middleOffset := size / 2
|
||||
if _, err := file.Seek(middleOffset, io.SeekStart); err != nil {
|
||||
return fileSampleDigests{}, err
|
||||
}
|
||||
n, readErr := io.ReadFull(file, buf)
|
||||
if readErr != nil && readErr != io.EOF && readErr != io.ErrUnexpectedEOF {
|
||||
return fileSampleDigests{}, readErr
|
||||
}
|
||||
_, _ = middle.Write(buf[:n])
|
||||
out.middle = digestHex(middle)
|
||||
}
|
||||
}
|
||||
|
||||
if out.header != "" {
|
||||
setChecksumCache(headerKey, out.header)
|
||||
}
|
||||
if out.middle != "" {
|
||||
setChecksumCache(middleKey, out.middle)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func groupFilesByChecksum(ctx context.Context, files []*iteminfo.FileInfo, index *indexing.Index, fileSize int64, stats *duplicateProcessingStats) []checksumGroup {
|
||||
if len(files) < 2 {
|
||||
return nil
|
||||
}
|
||||
|
||||
headerGroups := make(map[string][]*iteminfo.FileInfo)
|
||||
digestByPath := make(map[string]fileSampleDigests, len(files))
|
||||
|
||||
for _, file := range files {
|
||||
if ctx.Err() != nil {
|
||||
return nil
|
||||
}
|
||||
// Budget checks live inside readFileSampleDigests (after the cache lookup)
|
||||
// so cached digests still count toward grouping without consuming budget.
|
||||
// Checking here would stall on cached files and look stuck.
|
||||
|
||||
if file.Size != fileSize {
|
||||
continue
|
||||
}
|
||||
|
||||
filePath := filepath.Join(index.Path, file.Path)
|
||||
digests, err := readFileSampleDigests(ctx, index.Path, filePath, fileSize, file.ModTime, stats)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
digestByPath[file.Path] = digests
|
||||
stats.uniqueChecksums[digests.header] = true
|
||||
headerGroups[digests.header] = append(headerGroups[digests.header], file)
|
||||
}
|
||||
|
||||
headerCandidates := make([][]*iteminfo.FileInfo, 0)
|
||||
for _, group := range headerGroups {
|
||||
if len(group) >= 2 {
|
||||
headerCandidates = append(headerCandidates, group)
|
||||
}
|
||||
}
|
||||
if len(headerCandidates) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
middleGroups := make(map[string][]*iteminfo.FileInfo)
|
||||
for _, headerGroup := range headerCandidates {
|
||||
for _, file := range headerGroup {
|
||||
if ctx.Err() != nil {
|
||||
return nil
|
||||
}
|
||||
if shouldStop, _ := stats.shouldStop(); shouldStop {
|
||||
break
|
||||
}
|
||||
digests := digestByPath[file.Path]
|
||||
if digests.middle == "" {
|
||||
continue
|
||||
}
|
||||
stats.uniqueChecksums[digests.middle] = true
|
||||
middleGroups[digests.middle] = append(middleGroups[digests.middle], file)
|
||||
}
|
||||
}
|
||||
|
||||
groups := make([]checksumGroup, 0, len(middleGroups))
|
||||
for checksum, group := range middleGroups {
|
||||
if len(group) >= 2 {
|
||||
groups = append(groups, checksumGroup{
|
||||
Files: group,
|
||||
Checksum: checksum,
|
||||
})
|
||||
}
|
||||
}
|
||||
return groups
|
||||
}
|
||||
@@ -1,116 +0,0 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"encoding/hex"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/cespare/xxhash/v2"
|
||||
)
|
||||
|
||||
func TestReadFileSampleDigestsSmallFileUsesWholeFile(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
data := make([]byte, 12_000)
|
||||
path := filepath.Join(dir, "small.bin")
|
||||
if err := os.WriteFile(path, data, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
stats := &duplicateProcessingStats{startTime: time.Now()}
|
||||
d, err := readFileSampleDigests(context.Background(), dir, path, int64(len(data)), time.Now(), stats)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if d.header != d.middle {
|
||||
t.Fatalf("small file header and middle digest should match: %q %q", d.header, d.middle)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHasherUsesXXHash(t *testing.T) {
|
||||
if _, ok := any(duplicateChecksumHasher()).(*xxhash.Digest); !ok {
|
||||
t.Fatal("expected xxhash digest")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCachedDigestsDoNotConsumeChecksumBudget(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
dataA := make([]byte, 65536)
|
||||
for i := range dataA {
|
||||
dataA[i] = byte(i % 251)
|
||||
}
|
||||
// Use a distinct payload so its digests differ from other tests' files.
|
||||
pathA := filepath.Join(dir, "budget-a.bin")
|
||||
pathB := filepath.Join(dir, "budget-b.bin")
|
||||
if err := os.WriteFile(pathA, dataA, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(pathB, dataA, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
modTime := time.Date(2026, 11, 10, 0, 0, 0, 0, time.UTC)
|
||||
|
||||
// Prime the cache with one op-worth of budget.
|
||||
prime := &duplicateProcessingStats{startTime: time.Now(), uniqueChecksums: map[string]bool{}}
|
||||
d1, err := readFileSampleDigests(context.Background(), dir, pathA, int64(len(dataA)), modTime, prime)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d2, err := readFileSampleDigests(context.Background(), dir, pathB, int64(len(dataA)), modTime, prime)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if d1.header != d2.header || d1.middle != d2.middle {
|
||||
t.Fatal("expected equal digests for identical files")
|
||||
}
|
||||
if prime.checksumOperations != 2 {
|
||||
t.Fatalf("expected 2 disk operations for two cache-miss reads, got %d", prime.checksumOperations)
|
||||
}
|
||||
|
||||
// Exhaust the budget: any further uncached file should be refused.
|
||||
prime.checksumOperations = maxChecksumOperations
|
||||
uncached := filepath.Join(dir, "budget-c.bin")
|
||||
if err = os.WriteFile(uncached, dataA, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err = readFileSampleDigests(context.Background(), dir, uncached, int64(len(dataA)), modTime, prime); err == nil {
|
||||
t.Fatal("expected budget-exhausted error for uncached file")
|
||||
}
|
||||
|
||||
// But the cached files must still be served, for free.
|
||||
cached, err := readFileSampleDigests(context.Background(), dir, pathA, int64(len(dataA)), modTime, prime)
|
||||
if err != nil {
|
||||
t.Fatalf("cached digest should be served despite exhausted budget: %v", err)
|
||||
}
|
||||
if cached.header != d1.header || cached.middle != d1.middle {
|
||||
t.Fatal("cached digest mismatch")
|
||||
}
|
||||
if prime.checksumOperations != maxChecksumOperations {
|
||||
t.Fatalf("cached digest must not consume budget, got %d", prime.checksumOperations)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkDuplicateHashers(b *testing.B) {
|
||||
data := make([]byte, checksumSampleSize)
|
||||
for i := range data {
|
||||
data[i] = byte(i)
|
||||
}
|
||||
b.Run("md5", func(b *testing.B) {
|
||||
b.ReportAllocs()
|
||||
for i := 0; i < b.N; i++ {
|
||||
h := md5.New()
|
||||
_, _ = h.Write(data)
|
||||
hex.EncodeToString(h.Sum(nil))
|
||||
}
|
||||
})
|
||||
b.Run("xxhash", func(b *testing.B) {
|
||||
b.ReportAllocs()
|
||||
for i := 0; i < b.N; i++ {
|
||||
h := xxhash.New()
|
||||
_, _ = h.Write(data)
|
||||
hex.EncodeToString(h.Sum(nil))
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -1,111 +0,0 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"cmp"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"github.com/gtsteffaniak/filebrowser/backend/pkg/indexing"
|
||||
)
|
||||
|
||||
type mergeGroupKey struct {
|
||||
size int64
|
||||
digest string
|
||||
}
|
||||
|
||||
// mergeGroupsByChecksum merges groups that share the same size and sample digest.
|
||||
func mergeGroupsByChecksum(groups []duplicateGroupWithChecksums) []duplicateGroupWithChecksums {
|
||||
if len(groups) <= 1 {
|
||||
return groups
|
||||
}
|
||||
|
||||
keyToGroups := make(map[mergeGroupKey][]int)
|
||||
for i, group := range groups {
|
||||
if group.Checksum == "" {
|
||||
continue
|
||||
}
|
||||
key := mergeGroupKey{size: group.Size, digest: group.Checksum}
|
||||
keyToGroups[key] = append(keyToGroups[key], i)
|
||||
}
|
||||
|
||||
parent := make([]int, len(groups))
|
||||
for i := range parent {
|
||||
parent[i] = i
|
||||
}
|
||||
|
||||
var find func(int) int
|
||||
find = func(x int) int {
|
||||
if parent[x] != x {
|
||||
parent[x] = find(parent[x])
|
||||
}
|
||||
return parent[x]
|
||||
}
|
||||
|
||||
union := func(x, y int) {
|
||||
px, py := find(x), find(y)
|
||||
if px != py {
|
||||
parent[px] = py
|
||||
}
|
||||
}
|
||||
|
||||
for _, indices := range keyToGroups {
|
||||
for i := 1; i < len(indices); i++ {
|
||||
union(indices[0], indices[i])
|
||||
}
|
||||
}
|
||||
|
||||
groupsByRoot := make(map[int][]int)
|
||||
for i := range groups {
|
||||
root := find(i)
|
||||
groupsByRoot[root] = append(groupsByRoot[root], i)
|
||||
}
|
||||
|
||||
merged := make([]duplicateGroupWithChecksums, 0, len(groupsByRoot))
|
||||
for _, indices := range groupsByRoot {
|
||||
if len(indices) == 1 {
|
||||
merged = append(merged, groups[indices[0]])
|
||||
continue
|
||||
}
|
||||
|
||||
combinedFiles := make([]*indexing.SearchResult, 0)
|
||||
var combinedSize int64
|
||||
var combinedChecksum string
|
||||
|
||||
for _, idx := range indices {
|
||||
group := groups[idx]
|
||||
combinedFiles = append(combinedFiles, group.Files...)
|
||||
if combinedChecksum == "" {
|
||||
combinedChecksum = group.Checksum
|
||||
}
|
||||
if combinedSize == 0 {
|
||||
combinedSize = group.Size
|
||||
}
|
||||
}
|
||||
|
||||
if len(combinedFiles) >= 2 {
|
||||
slices.SortFunc(combinedFiles, func(a, b *indexing.SearchResult) int {
|
||||
return strings.Compare(a.Path, b.Path)
|
||||
})
|
||||
merged = append(merged, duplicateGroupWithChecksums{
|
||||
Size: combinedSize,
|
||||
Files: combinedFiles,
|
||||
Checksum: combinedChecksum,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
slices.SortFunc(merged, func(a, b duplicateGroupWithChecksums) int {
|
||||
if c := cmp.Compare(b.Size, a.Size); c != 0 {
|
||||
return c
|
||||
}
|
||||
if c := strings.Compare(a.Checksum, b.Checksum); c != 0 {
|
||||
return c
|
||||
}
|
||||
if len(a.Files) == 0 || len(b.Files) == 0 {
|
||||
return 0
|
||||
}
|
||||
return strings.Compare(a.Files[0].Path, b.Files[0].Path)
|
||||
})
|
||||
|
||||
return merged
|
||||
}
|
||||
@@ -1,19 +0,0 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/gtsteffaniak/filebrowser/backend/pkg/indexing"
|
||||
)
|
||||
|
||||
func TestMergeGroupsByChecksumUsesSizeAndDigest(t *testing.T) {
|
||||
groups := []duplicateGroupWithChecksums{
|
||||
{Size: 100, Checksum: "aaaa", Files: []*indexing.SearchResult{{Path: "/a"}, {Path: "/b"}}},
|
||||
{Size: 200, Checksum: "aaaa", Files: []*indexing.SearchResult{{Path: "/c"}, {Path: "/d"}}},
|
||||
{Size: 100, Checksum: "aaaa", Files: []*indexing.SearchResult{{Path: "/e"}, {Path: "/f"}}},
|
||||
}
|
||||
merged := mergeGroupsByChecksum(groups)
|
||||
if len(merged) != 2 {
|
||||
t.Fatalf("expected 2 merged groups, got %d", len(merged))
|
||||
}
|
||||
}
|
||||
@@ -1,204 +0,0 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"unicode"
|
||||
|
||||
"github.com/gtsteffaniak/filebrowser/backend/pkg/indexing/iteminfo"
|
||||
)
|
||||
|
||||
const filenameSimilarityThreshold = 0.5
|
||||
|
||||
type levenshteinRowBuf struct {
|
||||
prev, cur []int
|
||||
}
|
||||
|
||||
func (b *levenshteinRowBuf) ensure(width int) {
|
||||
if cap(b.prev) < width+1 {
|
||||
b.prev = make([]int, width+1)
|
||||
b.cur = make([]int, width+1)
|
||||
} else {
|
||||
b.prev = b.prev[:width+1]
|
||||
b.cur = b.cur[:width+1]
|
||||
}
|
||||
}
|
||||
|
||||
func maxEditDistanceForSimilarity(maxLen int) int {
|
||||
return maxLen / 2
|
||||
}
|
||||
|
||||
func filenameSimilarWithBudget(name1, name2 string, buf *levenshteinRowBuf) bool {
|
||||
if name1 == name2 {
|
||||
return true
|
||||
}
|
||||
if len(name1) == 0 || len(name2) == 0 {
|
||||
return false
|
||||
}
|
||||
|
||||
maxLen := max(len(name1), len(name2))
|
||||
maxDist := maxEditDistanceForSimilarity(maxLen)
|
||||
if abs(len(name1)-len(name2)) > maxDist {
|
||||
return false
|
||||
}
|
||||
|
||||
distance := boundedLevenshtein(name1, name2, maxDist, buf)
|
||||
if distance < 0 {
|
||||
return false
|
||||
}
|
||||
similarity := 1.0 - float64(distance)/float64(maxLen)
|
||||
return similarity >= filenameSimilarityThreshold
|
||||
}
|
||||
|
||||
func boundedLevenshtein(s1, s2 string, maxDist int, buf *levenshteinRowBuf) int {
|
||||
n, m := len(s1), len(s2)
|
||||
if n == 0 {
|
||||
if m <= maxDist {
|
||||
return m
|
||||
}
|
||||
return -1
|
||||
}
|
||||
if m == 0 {
|
||||
if n <= maxDist {
|
||||
return n
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
buf.ensure(m)
|
||||
prev, cur := buf.prev, buf.cur
|
||||
|
||||
for j := 0; j <= m; j++ {
|
||||
prev[j] = j
|
||||
}
|
||||
|
||||
for i := 1; i <= n; i++ {
|
||||
cur[0] = i
|
||||
rowMin := cur[0]
|
||||
for j := 1; j <= m; j++ {
|
||||
cost := 0
|
||||
if s1[i-1] != s2[j-1] {
|
||||
cost = 1
|
||||
}
|
||||
del := prev[j] + 1
|
||||
ins := cur[j-1] + 1
|
||||
sub := prev[j-1] + cost
|
||||
v := del
|
||||
if ins < v {
|
||||
v = ins
|
||||
}
|
||||
if sub < v {
|
||||
v = sub
|
||||
}
|
||||
cur[j] = v
|
||||
if v < rowMin {
|
||||
rowMin = v
|
||||
}
|
||||
}
|
||||
if rowMin > maxDist {
|
||||
return -1
|
||||
}
|
||||
prev, cur = cur, prev
|
||||
}
|
||||
|
||||
if prev[m] > maxDist {
|
||||
return -1
|
||||
}
|
||||
return prev[m]
|
||||
}
|
||||
|
||||
func groupFilesByFilename(ctx context.Context, files []*iteminfo.FileInfo, expectedSize int64, stats *duplicateProcessingStats) [][]*iteminfo.FileInfo {
|
||||
if len(files) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
normalized := make([]string, len(files))
|
||||
for i, f := range files {
|
||||
if f.Name != "" {
|
||||
normalized[i] = normalizeFilename(f.Name)
|
||||
}
|
||||
}
|
||||
|
||||
var rowBuf levenshteinRowBuf
|
||||
groups := [][]*iteminfo.FileInfo{}
|
||||
used := make(map[int]bool)
|
||||
|
||||
for i := 0; i < len(files); i++ {
|
||||
if ctx.Err() != nil {
|
||||
return groups
|
||||
}
|
||||
if shouldStop, _ := stats.shouldStop(); shouldStop {
|
||||
return groups
|
||||
}
|
||||
|
||||
if used[i] || files[i].Name == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
group := []*iteminfo.FileInfo{files[i]}
|
||||
used[i] = true
|
||||
filename1 := normalized[i]
|
||||
oversized := false
|
||||
|
||||
for j := i + 1; j < len(files); j++ {
|
||||
if ctx.Err() != nil {
|
||||
return groups
|
||||
}
|
||||
if shouldStop, _ := stats.shouldStop(); shouldStop {
|
||||
return groups
|
||||
}
|
||||
|
||||
if used[j] || files[j].Name == "" {
|
||||
continue
|
||||
}
|
||||
if files[j].Size != expectedSize {
|
||||
continue
|
||||
}
|
||||
|
||||
if filenameSimilarWithBudget(filename1, normalized[j], &rowBuf) {
|
||||
group = append(group, files[j])
|
||||
used[j] = true
|
||||
if len(group) > maxFuzzyGroupSize {
|
||||
oversized = true
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if len(group) >= 2 && !oversized {
|
||||
groups = append(groups, group)
|
||||
}
|
||||
}
|
||||
|
||||
return groups
|
||||
}
|
||||
|
||||
func normalizeFilename(filename string) string {
|
||||
filename = strings.ToLower(filename)
|
||||
if idx := strings.LastIndex(filename, "."); idx > 0 {
|
||||
filename = filename[:idx]
|
||||
}
|
||||
|
||||
var result strings.Builder
|
||||
for _, r := range filename {
|
||||
if unicode.IsLetter(r) || unicode.IsDigit(r) || r == ' ' || r == '-' {
|
||||
result.WriteRune(r)
|
||||
}
|
||||
}
|
||||
|
||||
return strings.TrimSpace(result.String())
|
||||
}
|
||||
|
||||
func abs(x int) int {
|
||||
if x < 0 {
|
||||
return -x
|
||||
}
|
||||
return x
|
||||
}
|
||||
|
||||
func max(a, b int) int {
|
||||
if a > b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
@@ -1,140 +0,0 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"context"
|
||||
"math/rand"
|
||||
"strings"
|
||||
"testing"
|
||||
"testing/quick"
|
||||
"time"
|
||||
"unicode"
|
||||
|
||||
"github.com/gtsteffaniak/filebrowser/backend/pkg/indexing/iteminfo"
|
||||
)
|
||||
|
||||
func legacyFilenamesSimilar(name1, name2 string) bool {
|
||||
if name1 == name2 {
|
||||
return true
|
||||
}
|
||||
if len(name1) == 0 || len(name2) == 0 {
|
||||
return false
|
||||
}
|
||||
distance := legacyLevenshtein(name1, name2)
|
||||
maxLen := max(len(name1), len(name2))
|
||||
similarity := 1.0 - float64(distance)/float64(maxLen)
|
||||
return similarity >= 0.5
|
||||
}
|
||||
|
||||
func legacyLevenshtein(s1, s2 string) int {
|
||||
if len(s1) == 0 {
|
||||
return len(s2)
|
||||
}
|
||||
if len(s2) == 0 {
|
||||
return len(s1)
|
||||
}
|
||||
matrix := make([][]int, len(s1)+1)
|
||||
for i := range matrix {
|
||||
matrix[i] = make([]int, len(s2)+1)
|
||||
}
|
||||
for i := 0; i <= len(s1); i++ {
|
||||
matrix[i][0] = i
|
||||
}
|
||||
for j := 0; j <= len(s2); j++ {
|
||||
matrix[0][j] = j
|
||||
}
|
||||
for i := 1; i <= len(s1); i++ {
|
||||
for j := 1; j <= len(s2); j++ {
|
||||
cost := 0
|
||||
if s1[i-1] != s2[j-1] {
|
||||
cost = 1
|
||||
}
|
||||
del := matrix[i-1][j] + 1
|
||||
ins := matrix[i][j-1] + 1
|
||||
sub := matrix[i-1][j-1] + cost
|
||||
v := del
|
||||
if ins < v {
|
||||
v = ins
|
||||
}
|
||||
if sub < v {
|
||||
v = sub
|
||||
}
|
||||
matrix[i][j] = v
|
||||
}
|
||||
}
|
||||
return matrix[len(s1)][len(s2)]
|
||||
}
|
||||
|
||||
var filenameCorpus = []string{
|
||||
"photo", "photo_1", "photo-copy", "IMG_0042", "img 0042",
|
||||
"vacation2023beach", "vacation2023_beach", "user_video", "video",
|
||||
"document-final", "document_final_v2", "a", "ab", "abc", "abcd",
|
||||
"report Q1 2024", "report-q1-2024", "my@file#name", "myfilename",
|
||||
"summer-holiday", "summer holiday", "winter-holiday",
|
||||
}
|
||||
|
||||
func TestFilenameSimilarMatchesLegacyOnCorpus(t *testing.T) {
|
||||
names := append([]string{}, filenameCorpus...)
|
||||
for i := 0; i < 50; i++ {
|
||||
names = append(names, randomNormalizedName(5+corpusRand.Intn(40)))
|
||||
}
|
||||
|
||||
var buf levenshteinRowBuf
|
||||
for _, a := range names {
|
||||
a = normalizeFilename(a + ".jpg")
|
||||
for _, b := range names {
|
||||
b = normalizeFilename(b + ".png")
|
||||
if legacyFilenamesSimilar(a, b) != filenameSimilarWithBudget(a, b, &buf) {
|
||||
t.Fatalf("mismatch %q vs %q", a, b)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestFilenameSimilarMatchesLegacyQuick(t *testing.T) {
|
||||
var buf levenshteinRowBuf
|
||||
if err := quick.Check(func(a, b string) bool {
|
||||
a = trimQuickString(a, 48)
|
||||
b = trimQuickString(b, 48)
|
||||
return legacyFilenamesSimilar(a, b) == filenameSimilarWithBudget(a, b, &buf)
|
||||
}, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func trimQuickString(s string, max int) string {
|
||||
if len(s) > max {
|
||||
s = s[:max]
|
||||
}
|
||||
var b strings.Builder
|
||||
for _, r := range s {
|
||||
if unicode.IsLetter(r) || unicode.IsDigit(r) || r == ' ' || r == '-' {
|
||||
b.WriteRune(r)
|
||||
}
|
||||
}
|
||||
return strings.ToLower(strings.TrimSpace(b.String()))
|
||||
}
|
||||
|
||||
func randomNormalizedName(n int) string {
|
||||
const alphabet = "abcdefghijklmnopqrstuvwxyz0123456789 -"
|
||||
b := make([]byte, n)
|
||||
for i := range b {
|
||||
b[i] = alphabet[corpusRand.Intn(len(alphabet))]
|
||||
}
|
||||
return string(b)
|
||||
}
|
||||
|
||||
func TestGroupFilesByFilenameOversizedExcluded(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
stats := &duplicateProcessingStats{startTime: time.Now()}
|
||||
files := make([]*iteminfo.FileInfo, 12)
|
||||
for i := range files {
|
||||
files[i] = &iteminfo.FileInfo{
|
||||
ItemInfo: iteminfo.ItemInfo{Name: "photo.jpg", Size: 100},
|
||||
}
|
||||
}
|
||||
if len(groupFilesByFilename(ctx, files, 100, stats)) != 0 {
|
||||
t.Fatal("expected no groups for oversized cluster")
|
||||
}
|
||||
}
|
||||
|
||||
var corpusRand = rand.New(rand.NewSource(42))
|
||||
@@ -12,7 +12,6 @@ import (
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/adapters/fs/files"
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/ffmpeg"
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/gomedia"
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/mediacache"
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/utils"
|
||||
"github.com/gtsteffaniak/filebrowser/backend/pkg/settings"
|
||||
)
|
||||
@@ -124,8 +123,7 @@ func subtitlesHandler(w http.ResponseWriter, r *http.Request, d *Context) (int,
|
||||
}
|
||||
// Shadow ffmpeg runs after the fast path; bound it so a detached request cannot
|
||||
// leave extraction running indefinitely (sync fallback still uses r.Context()).
|
||||
// WithoutCache: compare against a fresh extraction, not the value go-media stored.
|
||||
legacyCtx, cancel := context.WithTimeout(mediacache.WithoutCache(context.WithoutCancel(r.Context())), time.Minute)
|
||||
legacyCtx, cancel := context.WithTimeout(context.WithoutCancel(r.Context()), time.Minute)
|
||||
defer cancel()
|
||||
vtt, extractErr := svc.ExtractSubtitle(legacyCtx, fileInfo.RealPath, *track.Index)
|
||||
return vtt, extractErr == nil
|
||||
|
||||
@@ -4,7 +4,6 @@ import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"sort"
|
||||
"testing"
|
||||
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/database/users"
|
||||
@@ -20,13 +19,11 @@ func withMediaHandlerSettings(t *testing.T) {
|
||||
settings.Env.FFmpegAvailable = originalFFmpeg
|
||||
})
|
||||
settings.Env.FFmpegAvailable = true
|
||||
settings.Config.Integrations.Media.Transcode = settings.FullAdminTranscode{
|
||||
UserTranscode: settings.UserTranscode{
|
||||
Enabled: true,
|
||||
MaxConcurrentPerUser: 2,
|
||||
},
|
||||
MaxConcurrent: 3,
|
||||
Profiles: settings.DefaultTranscodeProfiles(),
|
||||
settings.Config.Integrations.Media.Transcode = settings.MediaTranscode{
|
||||
Enabled: true,
|
||||
MaxConcurrent: 3,
|
||||
MaxConcurrentPerUser: 2,
|
||||
Profiles: settings.DefaultTranscodeProfiles(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -55,99 +52,52 @@ func callMediaGet(t *testing.T, admin bool) map[string]any {
|
||||
return body
|
||||
}
|
||||
|
||||
// objectField returns obj[key] as a JSON object, failing the test if it is missing or not an object.
|
||||
func objectField(t *testing.T, obj map[string]any, key string) map[string]any {
|
||||
t.Helper()
|
||||
child, ok := obj[key].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("%s is %T, want object", key, obj[key])
|
||||
}
|
||||
return child
|
||||
}
|
||||
|
||||
// keysOf returns the sorted keys of a JSON object.
|
||||
func keysOf(obj map[string]any) []string {
|
||||
keys := make([]string, 0, len(obj))
|
||||
for k := range obj {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
return keys
|
||||
}
|
||||
|
||||
func assertKeys(t *testing.T, path string, got []string, want []string) {
|
||||
t.Helper()
|
||||
sort.Strings(want)
|
||||
if len(got) != len(want) {
|
||||
t.Fatalf("%s keys = %v, want %v", path, got, want)
|
||||
}
|
||||
for i := range got {
|
||||
if got[i] != want[i] {
|
||||
t.Fatalf("%s keys = %v, want %v", path, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Non-admin users receive exactly the PUBLIC payload. Any new key fails this test,
|
||||
// which forces a conscious decision about whether it belongs in UserMedia.
|
||||
// Non-admin users must be able to read the effective media values for the
|
||||
// player, but only the player-relevant subset.
|
||||
func TestSettingsMediaHandlerPlayerPayloadForNonAdmin(t *testing.T) {
|
||||
withMediaHandlerSettings(t)
|
||||
|
||||
body := callMediaGet(t, false)
|
||||
assertKeys(t, "top-level", keysOf(body), []string{"transcode", "imagePreview"})
|
||||
transcode := objectField(t, body, "transcode")
|
||||
assertKeys(t, "transcode", keysOf(transcode), []string{"enabled", "maxConcurrentPerUser", "enabledProfiles"})
|
||||
|
||||
if enabled, ok := transcode["enabled"].(bool); !ok || !enabled {
|
||||
t.Fatalf("transcode.enabled = %v, want true", transcode["enabled"])
|
||||
if _, ok := body["transcode"]; ok {
|
||||
t.Fatal("non-admin must not receive the full transcode settings")
|
||||
}
|
||||
if limit, ok := transcode["maxConcurrentPerUser"].(float64); !ok || limit != 2 {
|
||||
t.Fatalf("transcode.maxConcurrentPerUser = %v, want 2", transcode["maxConcurrentPerUser"])
|
||||
if _, ok := body["lockedFromConfigPaths"]; ok {
|
||||
t.Fatal("non-admin must not receive config lock metadata")
|
||||
}
|
||||
profiles, ok := transcode["enabledProfiles"].([]any)
|
||||
if _, ok := body["lockMessage"]; ok {
|
||||
t.Fatal("non-admin must not receive the config lock message")
|
||||
}
|
||||
if enabled, ok := body["enabled"].(bool); !ok || !enabled {
|
||||
t.Fatalf("enabled = %v, want true", body["enabled"])
|
||||
}
|
||||
if limit, ok := body["maxConcurrentPerUser"].(float64); !ok || limit != 2 {
|
||||
t.Fatalf("maxConcurrentPerUser = %v, want 2", body["maxConcurrentPerUser"])
|
||||
}
|
||||
profiles, ok := body["profiles"].([]any)
|
||||
if !ok || len(profiles) != 3 {
|
||||
t.Fatalf("transcode.enabledProfiles = %v, want 3 enabled ids", transcode["enabledProfiles"])
|
||||
t.Fatalf("profiles = %v, want 3 enabled ids", body["profiles"])
|
||||
}
|
||||
}
|
||||
|
||||
// Admins receive the full stored config. The keys shared with users must have the
|
||||
// same names, so the admin payload is a strict superset of the user payload.
|
||||
func TestSettingsMediaHandlerFullPayloadForAdmin(t *testing.T) {
|
||||
withMediaHandlerSettings(t)
|
||||
|
||||
admin := callMediaGet(t, true)
|
||||
user := callMediaGet(t, false)
|
||||
|
||||
// lockedFromConfigPaths is omitted when config pins nothing, so it is not required here.
|
||||
for _, key := range []string{"transcode", "imagePreview", "convert", "debug", "hardwareAcceleration", "accelDevice"} {
|
||||
if _, ok := admin[key]; !ok {
|
||||
body := callMediaGet(t, true)
|
||||
transcode, ok := body["transcode"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("admin must receive the full transcode settings, got %T", body["transcode"])
|
||||
}
|
||||
if _, ok := transcode["profiles"].(map[string]any); !ok {
|
||||
t.Fatalf("admin transcode payload must include profiles, got %v", transcode["profiles"])
|
||||
}
|
||||
// Admin payload exposes the full settings surface, not the player subset.
|
||||
for _, key := range []string{"convert", "debug", "hardwareAcceleration"} {
|
||||
if _, ok := body[key]; !ok {
|
||||
t.Fatalf("admin payload missing %q", key)
|
||||
}
|
||||
}
|
||||
if _, ok := admin["enabled"]; ok {
|
||||
t.Fatal("admin payload must not repeat transcode.enabled at the top level")
|
||||
}
|
||||
|
||||
adminTranscode := objectField(t, admin, "transcode")
|
||||
for _, key := range []string{"maxConcurrent", "profiles", "cacheMaxSizeMB", "encoderBackend", "readRate"} {
|
||||
if _, ok := adminTranscode[key]; !ok {
|
||||
t.Fatalf("admin transcode missing %q", key)
|
||||
}
|
||||
}
|
||||
if _, ok := adminTranscode["profiles"].(map[string]any); !ok {
|
||||
t.Fatalf("admin transcode.profiles must be an object, got %T", adminTranscode["profiles"])
|
||||
}
|
||||
|
||||
// Superset rule: every key a user receives must also exist for admins. Values may
|
||||
// differ by design: admins see stored config (so the UI can save it back unchanged),
|
||||
// users see effective values (for example enabled is false when ffmpeg is missing).
|
||||
for key := range objectField(t, user, "transcode") {
|
||||
if _, ok := adminTranscode[key]; !ok {
|
||||
t.Fatalf("admin transcode missing shared key %q", key)
|
||||
}
|
||||
}
|
||||
if admin["imagePreview"] == nil {
|
||||
t.Fatal("imagePreview must be present for admins")
|
||||
if _, ok := body["enabled"]; ok {
|
||||
t.Fatal("the admin payload uses the full settings shape, not the player subset")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -156,8 +106,8 @@ func TestSettingsMediaHandlerReflectsDisabledTranscode(t *testing.T) {
|
||||
withMediaHandlerSettings(t)
|
||||
settings.Config.Integrations.Media.Transcode.Enabled = false
|
||||
|
||||
transcode := objectField(t, callMediaGet(t, false), "transcode")
|
||||
if enabled, ok := transcode["enabled"].(bool); !ok || enabled {
|
||||
t.Fatalf("transcode.enabled = %v, want false", transcode["enabled"])
|
||||
body := callMediaGet(t, false)
|
||||
if enabled, ok := body["enabled"].(bool); !ok || enabled {
|
||||
t.Fatalf("enabled = %v, want false", body["enabled"])
|
||||
}
|
||||
}
|
||||
@@ -166,11 +166,7 @@ func onlyofficeClientConfigGetHandler(w http.ResponseWriter, r *http.Request, d
|
||||
}
|
||||
providedPath = cleanPath
|
||||
|
||||
themeMode := "light"
|
||||
if d.User.DarkMode == users.DarkModeDark ||
|
||||
(d.User.DarkMode == users.DarkModeAuto && settings.DefaultDarkMode()) {
|
||||
themeMode = "dark"
|
||||
}
|
||||
themeMode := utils.Ternary(d.User.DarkMode, "dark", "light")
|
||||
var sourceInfo *settings.Source
|
||||
var ok bool
|
||||
if hash != "" {
|
||||
|
||||
@@ -42,7 +42,7 @@ func initRuntime(deps Deps, assets fs.FS) {
|
||||
initSettingsSecretMarkers()
|
||||
}
|
||||
|
||||
// Embed the frontend build output under internal/web/dist (see `make embed-frontend` after `npm run build` in frontend/).
|
||||
// Embed the frontend build output under internal/web/dist (see frontend `npm run build`).
|
||||
//
|
||||
//go:embed dist/*
|
||||
var assets embed.FS
|
||||
|
||||
@@ -453,17 +453,33 @@ func settingsShareDefaultsPatchHandler(w http.ResponseWriter, r *http.Request, d
|
||||
return settingsShareDefaultsGetHandler(w, r, d)
|
||||
}
|
||||
|
||||
// mediaSettingsResponse is the ADMIN ONLY media payload: the full stored settings
|
||||
// (FullAdminMedia, which embeds the public UserTranscode fields) plus config-lock
|
||||
// metadata and ffmpeg option lists. Non-admins never receive this type.
|
||||
// settingsMediaGetHandler returns the runtime media settings.
|
||||
// @Summary Get runtime media settings
|
||||
// @Description Admins receive the DB-backed media integration settings; every authenticated user
|
||||
// @Description receives the player-relevant subset (transcode enabled, profiles, per-user limit).
|
||||
// @Tags Settings
|
||||
// @Produce json
|
||||
// @Success 200 {object} mediaSettingsResponse "Media settings"
|
||||
// @Router /api/settings/media [get]
|
||||
// mediaSettingsResponse is the runtime media settings plus config-lock metadata.
|
||||
type mediaSettingsResponse struct {
|
||||
settings.FullAdminMedia
|
||||
settings.MediaSettings
|
||||
LockedFromConfigPaths []string `json:"lockedFromConfigPaths,omitempty"`
|
||||
LockMessage string `json:"lockMessage,omitempty"`
|
||||
AccelDeviceOptions []ffmpeg.AccelDeviceOption `json:"accelDeviceOptions,omitempty"`
|
||||
EncoderBackendOptions []ffmpeg.EncoderBackendOption `json:"encoderBackendOptions,omitempty"`
|
||||
}
|
||||
|
||||
// mediaPlayerSettings is the player-relevant subset of the runtime media
|
||||
// settings. It is safe to expose to any authenticated user so the player can
|
||||
// reflect admin changes without a page reload.
|
||||
type mediaPlayerSettings struct {
|
||||
Enabled bool `json:"enabled"`
|
||||
Profiles []string `json:"profiles"`
|
||||
MaxConcurrentPerUser int `json:"maxConcurrentPerUser"`
|
||||
HardwareAcceleration bool `json:"hardwareAcceleration"`
|
||||
}
|
||||
|
||||
func mediaSettingsLockMeta() (paths []string, message string) {
|
||||
paths = settings.ConfigSpecifiedMediaPaths()
|
||||
if len(paths) == 0 {
|
||||
@@ -475,25 +491,12 @@ func mediaSettingsLockMeta() (paths []string, message string) {
|
||||
// The media settings APIs no longer require ffmpeg. go-media serves probing,
|
||||
// metadata, subtitles, and video thumbnails without it. ffmpeg-only controls
|
||||
// (transcoding, hardware acceleration) report their own availability.
|
||||
// @Summary Get runtime media settings
|
||||
// @Description Returns the media settings the caller is allowed to see.
|
||||
// @Description Admins get the full settings, including ffmpeg and hardware options, so they can edit them.
|
||||
// @Description Everyone else gets a small read-only view: whether transcoding is on, how many
|
||||
// @Description transcodes they can run at once, which quality presets are enabled, and which image
|
||||
// @Description previews the server can generate. Keys in the view are the same for admins and users.
|
||||
// @Tags Settings
|
||||
// @Produce json
|
||||
// @Success 200 {object} mediaSettingsResponse "Media settings"
|
||||
// @Router /api/settings/media [get]
|
||||
func settingsMediaGetHandler(w http.ResponseWriter, r *http.Request, d *Context) (int, error) {
|
||||
// PUBLIC payload: non-admins get only what the player and general UI read.
|
||||
if !d.User.Permissions.Admin {
|
||||
return RenderJSON(w, r, settings.CurrentUserMedia())
|
||||
return RenderJSON(w, r, currentMediaPlayerSettings())
|
||||
}
|
||||
// ADMIN payload: full stored settings with effective and derived values filled in
|
||||
// by PrepMediaForFrontend (a read-only copy; never saved).
|
||||
media := settings.PrepMediaForFrontend(state.GetMediaSettings())
|
||||
lockedPaths, lockMessage := mediaSettingsLockMeta()
|
||||
media := state.GetMediaSettings()
|
||||
svc := ffmpeg.Get()
|
||||
var accelDeviceOptions []ffmpeg.AccelDeviceOption
|
||||
var encoderBackendOptions []ffmpeg.EncoderBackendOption
|
||||
@@ -503,7 +506,7 @@ func settingsMediaGetHandler(w http.ResponseWriter, r *http.Request, d *Context)
|
||||
encoderBackendOptions = svc.EncoderBackendOptions()
|
||||
}
|
||||
return RenderJSON(w, r, mediaSettingsResponse{
|
||||
FullAdminMedia: media,
|
||||
MediaSettings: media,
|
||||
LockedFromConfigPaths: lockedPaths,
|
||||
LockMessage: lockMessage,
|
||||
AccelDeviceOptions: accelDeviceOptions,
|
||||
@@ -511,6 +514,17 @@ func settingsMediaGetHandler(w http.ResponseWriter, r *http.Request, d *Context)
|
||||
})
|
||||
}
|
||||
|
||||
// currentMediaPlayerSettings reads the live media settings for the player UI.
|
||||
func currentMediaPlayerSettings() mediaPlayerSettings {
|
||||
media := state.GetMediaSettings()
|
||||
return mediaPlayerSettings{
|
||||
Enabled: settings.TranscodeEnabled(),
|
||||
Profiles: settings.EnabledTranscodeProfiles(),
|
||||
MaxConcurrentPerUser: settings.TranscodeMaxConcurrentPerUser(),
|
||||
HardwareAcceleration: media.HardwareAcceleration,
|
||||
}
|
||||
}
|
||||
|
||||
// settingsMediaPatchHandler merges a partial update into the runtime media settings.
|
||||
// @Summary Update runtime media settings
|
||||
// @Description Merges a partial JSON patch into the DB-backed media settings and applies them at runtime.
|
||||
|
||||
@@ -151,13 +151,7 @@ func handleWithStaticData(w http.ResponseWriter, r *http.Request, d *requestCont
|
||||
externalLinks = newExternalLinks
|
||||
}
|
||||
|
||||
// Instance default dark mode: htmlVars/defaultDark use a boolean for first paint;
|
||||
// globalVars.darkMode carries the full preference (including "auto") for the SPA.
|
||||
defaultDark := settings.DefaultDarkMode()
|
||||
defaultThemeColor := settings.Config.Frontend.Styling.LightBackground
|
||||
if defaultDark {
|
||||
defaultThemeColor = settings.Config.Frontend.Styling.DarkBackground
|
||||
}
|
||||
defaultThemeColor := settings.DefaultBackgroundColor()
|
||||
staticURL := settings.Config.Http.BaseURL + "public/static"
|
||||
description := settings.Config.Frontend.Description
|
||||
title := settings.Config.Frontend.Name
|
||||
@@ -251,13 +245,11 @@ func handleWithStaticData(w http.ResponseWriter, r *http.Request, d *requestCont
|
||||
manifestURL = staticURL + "/site.webmanifest?" + query.Encode()
|
||||
}
|
||||
|
||||
// only include constant values that don't change during a users session
|
||||
data["htmlVars"] = map[string]interface{}{
|
||||
"title": title,
|
||||
"customCSS": template.CSS(settings.Config.Frontend.Styling.CustomCSSRaw),
|
||||
"userSelectedTheme": template.CSS(userSelectedTheme),
|
||||
"defaultDarkMode": settings.DefaultDarkMode(),
|
||||
"autoDarkMode": settings.DefaultDarkModeIsAuto(),
|
||||
"lightBackground": settings.Config.Frontend.Styling.LightBackground,
|
||||
"darkBackground": settings.Config.Frontend.Styling.DarkBackground,
|
||||
"staticURL": staticURL,
|
||||
@@ -280,53 +272,58 @@ func handleWithStaticData(w http.ResponseWriter, r *http.Request, d *requestCont
|
||||
"devMode": settings.Env.IsDevMode,
|
||||
}
|
||||
|
||||
// only include constant values that don't change during a users session
|
||||
data["globalVars"] = map[string]interface{}{
|
||||
"name": settings.Config.Frontend.Name,
|
||||
"minSearchLength": settings.Config.Server.MinSearchLength,
|
||||
"searchResultsLimit": settings.Config.Server.SearchResultsLimit,
|
||||
"disableExternal": settings.Config.Frontend.DisableDefaultLinks,
|
||||
"darkMode": settings.DefaultDarkModePreference(),
|
||||
"singleClick": settings.Config.UserDefaults.Listing.SingleClick,
|
||||
"lightBackground": settings.Config.Frontend.Styling.LightBackground,
|
||||
"darkBackground": settings.Config.Frontend.Styling.DarkBackground,
|
||||
"baseURL": settings.Config.Http.BaseURL,
|
||||
"version": versionString,
|
||||
"commitSHA": commitSHAString,
|
||||
"signup": settings.Config.Auth.Methods.PasswordAuth.Signup,
|
||||
"noAuth": settings.Config.Auth.Methods.NoAuth,
|
||||
"enableThumbs": !settings.Config.Server.DisablePreviews,
|
||||
"externalLinks": externalLinks,
|
||||
"externalUrl": strings.TrimSuffix(settings.Config.Http.ExternalUrl, "/"),
|
||||
"onlyOfficeUrl": onlyOfficeUrlForSPA(),
|
||||
"oidcAvailable": settings.Config.Auth.Methods.OidcAuth.Enabled,
|
||||
"jwtAvailable": settings.Config.Auth.Methods.JwtAuth.Enabled,
|
||||
"proxyAvailable": settings.Config.Auth.Methods.ProxyAuth.Enabled,
|
||||
"passwordAvailable": settings.Config.Auth.Methods.PasswordAuth.Enabled,
|
||||
"passwordMinLength": settings.PasswordMinLength(),
|
||||
"ldapAvailable": settings.Config.Auth.Methods.LdapAuth.Enabled,
|
||||
"ffmpegAvailable": settings.FFmpegEnabled(),
|
||||
"docPreviewExtensions": iteminfo.DocPreviewExtensionsForSPA(),
|
||||
"updateAvailable": utils.GetUpdateAvailableUrl(),
|
||||
"disableNavButtons": disableNavButtons,
|
||||
"userSelectableThemes": settings.Config.Frontend.Styling.CustomThemeOptions,
|
||||
"eventBasedThemes": !settings.Config.Frontend.Styling.DisableEventBasedThemes,
|
||||
"loginIcon": loginIcon,
|
||||
"disableSidebar": disableSidebar,
|
||||
"shareHash": shareHash,
|
||||
"oidcLoginButtonText": settings.Config.Frontend.OIDCLoginButtonText,
|
||||
"loginButtonText": settings.Config.Frontend.LoginButtonText,
|
||||
"passkeyAvailable": settings.Config.Auth.Methods.PasskeyAuth.Enabled,
|
||||
"passkeyLoginButtonText": settings.Config.Auth.Methods.PasskeyAuth.LoginButtonText,
|
||||
"recaptcha": recaptchaEnabled,
|
||||
"recaptchaHost": recaptchaHost,
|
||||
"recaptchaKey": recaptchaKey,
|
||||
"disablePWAInstall": settings.Config.Frontend.DisablePWAInstall,
|
||||
"devMode": settings.Env.IsDevMode,
|
||||
"playwrightTest": settings.Env.IsPlaywright,
|
||||
"officeProduct": officeProductForSPA(),
|
||||
"officeExtensions": wopiExtensionsForSPA(),
|
||||
"textPreviewMimeTypes": iteminfo.TextPreviewMimeTypes(),
|
||||
"name": settings.Config.Frontend.Name,
|
||||
"minSearchLength": settings.Config.Server.MinSearchLength,
|
||||
"searchResultsLimit": settings.Config.Server.SearchResultsLimit,
|
||||
"disableExternal": settings.Config.Frontend.DisableDefaultLinks,
|
||||
"darkMode": settings.Config.UserDefaults.UI.DarkMode,
|
||||
"singleClick": settings.Config.UserDefaults.Listing.SingleClick,
|
||||
"lightBackground": settings.Config.Frontend.Styling.LightBackground,
|
||||
"darkBackground": settings.Config.Frontend.Styling.DarkBackground,
|
||||
"baseURL": settings.Config.Http.BaseURL,
|
||||
"version": versionString,
|
||||
"commitSHA": commitSHAString,
|
||||
"signup": settings.Config.Auth.Methods.PasswordAuth.Signup,
|
||||
"noAuth": settings.Config.Auth.Methods.NoAuth,
|
||||
"enableThumbs": !settings.Config.Server.DisablePreviews,
|
||||
"externalLinks": externalLinks,
|
||||
"externalUrl": strings.TrimSuffix(settings.Config.Http.ExternalUrl, "/"),
|
||||
"onlyOfficeUrl": onlyOfficeUrlForSPA(),
|
||||
"oidcAvailable": settings.Config.Auth.Methods.OidcAuth.Enabled,
|
||||
"jwtAvailable": settings.Config.Auth.Methods.JwtAuth.Enabled,
|
||||
"proxyAvailable": settings.Config.Auth.Methods.ProxyAuth.Enabled,
|
||||
"passwordAvailable": settings.Config.Auth.Methods.PasswordAuth.Enabled,
|
||||
"passwordMinLength": settings.PasswordMinLength(),
|
||||
"ldapAvailable": settings.Config.Auth.Methods.LdapAuth.Enabled,
|
||||
"ffmpegAvailable": settings.FFmpegEnabled(),
|
||||
"hardwareAccelerationAvailable": settings.HardwareAccelerationSupported(),
|
||||
"transcodeEnabled": settings.TranscodeEnabled(),
|
||||
"transcodeProfiles": settings.EnabledTranscodeProfiles(),
|
||||
"transcodeMaxPerUser": settings.TranscodeMaxConcurrentPerUser(),
|
||||
"docPreviewExtensions": iteminfo.DocPreviewExtensionsForSPA(),
|
||||
"textPreviewExtensions": iteminfo.TextPreviewExtensionsForSPA(),
|
||||
"updateAvailable": utils.GetUpdateAvailableUrl(),
|
||||
"disableNavButtons": disableNavButtons,
|
||||
"userSelectableThemes": settings.Config.Frontend.Styling.CustomThemeOptions,
|
||||
"enableHeicConversion": settings.CanConvertImage("heic"),
|
||||
"enableJxlConversion": settings.CanConvertImage("jxl"),
|
||||
"eventBasedThemes": !settings.Config.Frontend.Styling.DisableEventBasedThemes,
|
||||
"loginIcon": loginIcon,
|
||||
"disableSidebar": disableSidebar,
|
||||
"shareHash": shareHash,
|
||||
"oidcLoginButtonText": settings.Config.Frontend.OIDCLoginButtonText,
|
||||
"loginButtonText": settings.Config.Frontend.LoginButtonText,
|
||||
"passkeyAvailable": settings.Config.Auth.Methods.PasskeyAuth.Enabled,
|
||||
"passkeyLoginButtonText": settings.Config.Auth.Methods.PasskeyAuth.LoginButtonText,
|
||||
"recaptcha": recaptchaEnabled,
|
||||
"recaptchaHost": recaptchaHost,
|
||||
"recaptchaKey": recaptchaKey,
|
||||
"disablePWAInstall": settings.Config.Frontend.DisablePWAInstall,
|
||||
"devMode": settings.Env.IsDevMode,
|
||||
"playwrightTest": settings.Env.IsPlaywright,
|
||||
"officeProduct": officeProductForSPA(),
|
||||
"officeExtensions": wopiExtensionsForSPA(),
|
||||
}
|
||||
|
||||
cspNonce, err := utils.CSPNonce()
|
||||
|
||||
@@ -5,6 +5,8 @@ import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/adapters/fs/files"
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/preview"
|
||||
@@ -99,10 +101,10 @@ func serveTextPreview(w http.ResponseWriter, r *http.Request, info *iteminfo.Ext
|
||||
if info.Type == "directory" {
|
||||
return http.StatusUnsupportedMediaType, fmt.Errorf("directories have no text preview")
|
||||
}
|
||||
if !iteminfo.IsTextPreviewable(info.Type) {
|
||||
kind := iteminfo.TextPreviewKind(strings.ToLower(filepath.Ext(info.Name)))
|
||||
if kind == "" {
|
||||
return http.StatusUnsupportedMediaType, fmt.Errorf("unsupported file type for text preview")
|
||||
}
|
||||
kind := iteminfo.TextPreviewRenderKind(info.Name)
|
||||
// Bound concurrent reads and RTF conversions with the preview service's shared
|
||||
// image-processor semaphore, so a listing of many text files cannot open an
|
||||
// unbounded number of file handles at once.
|
||||
|
||||
@@ -33,7 +33,7 @@ func TestTranscodeManagerIsWiredWithoutFFmpeg(t *testing.T) {
|
||||
// Startup with ffmpeg missing and transcoding disabled.
|
||||
settings.Env.FFmpegAvailable = false
|
||||
disabled := originalMedia
|
||||
disabled.Transcode = settings.FullAdminTranscode{UserTranscode: settings.UserTranscode{Enabled: false}}
|
||||
disabled.Transcode = settings.MediaTranscode{Enabled: false}
|
||||
settings.ApplyMediaSettings(disabled)
|
||||
|
||||
setupTestEnv(t)
|
||||
|
||||
@@ -1597,7 +1597,7 @@ func setFilePreviewFlags(fileInfo *iteminfo.ItemInfo, realPath string) {
|
||||
}
|
||||
// Text-like files are rendered by the text preview endpoint (not the image pipeline),
|
||||
// so this runs before Folio/OnlyOffice. SVG is not a text preview type and stays an image.
|
||||
if iteminfo.IsTextPreviewable(fileInfo.Type) {
|
||||
if iteminfo.IsTextPreviewable(fileInfo.Name) {
|
||||
fileInfo.HasPreview = true
|
||||
return
|
||||
}
|
||||
|
||||
@@ -271,57 +271,57 @@ var compressedFile = map[string]bool{
|
||||
|
||||
// Raw camera image: extension -> MIME type (single source for IsRawImage + type detection)
|
||||
var rawImageMimeType = map[string]string{
|
||||
".cr2": "image/x-canon-cr2",
|
||||
".cr3": "image/x-canon-cr3",
|
||||
".nef": "image/x-nikon-nef",
|
||||
".nrw": "image/x-nikon-nef",
|
||||
".arw": "image/x-sony-arw",
|
||||
".srf": "image/x-sony-arw",
|
||||
".sr2": "image/x-sony-arw",
|
||||
".orf": "image/x-olympus-orf",
|
||||
".rw2": "image/x-panasonic-rw2",
|
||||
".raw": "image/x-panasonic-raw",
|
||||
".dng": "image/x-adobe-dng",
|
||||
".raf": "image/x-fuji-raf",
|
||||
".pef": "image/x-pentax-pef",
|
||||
".ptx": "image/x-pentax-pef",
|
||||
".rwl": "image/x-leica-rwl",
|
||||
".3fr": "image/x-hasselblad-3fr",
|
||||
".fff": "image/x-hasselblad-fff",
|
||||
".erf": "image/x-epson-erf",
|
||||
".mrw": "image/x-minolta-mrw",
|
||||
".dcr": "image/x-kodak-dcr",
|
||||
".kdc": "image/x-kodak-dcr",
|
||||
".dc2": "image/x-kodak-dc2",
|
||||
".x3f": "image/x-sigma-x3f",
|
||||
".iiq": "image/x-phaseone-iiq",
|
||||
".nkc": "image/x-kodak-nkc",
|
||||
".r3d": "image/x-red-r3d",
|
||||
".cr2": "image/x-canon-cr2",
|
||||
".cr3": "image/x-canon-cr3",
|
||||
".nef": "image/x-nikon-nef",
|
||||
".nrw": "image/x-nikon-nef",
|
||||
".arw": "image/x-sony-arw",
|
||||
".srf": "image/x-sony-arw",
|
||||
".sr2": "image/x-sony-arw",
|
||||
".orf": "image/x-olympus-orf",
|
||||
".rw2": "image/x-panasonic-rw2",
|
||||
".raw": "image/x-panasonic-raw",
|
||||
".dng": "image/x-adobe-dng",
|
||||
".raf": "image/x-fuji-raf",
|
||||
".pef": "image/x-pentax-pef",
|
||||
".ptx": "image/x-pentax-pef",
|
||||
".rwl": "image/x-leica-rwl",
|
||||
".3fr": "image/x-hasselblad-3fr",
|
||||
".fff": "image/x-hasselblad-fff",
|
||||
".erf": "image/x-epson-erf",
|
||||
".mrw": "image/x-minolta-mrw",
|
||||
".dcr": "image/x-kodak-dcr",
|
||||
".kdc": "image/x-kodak-dcr",
|
||||
".dc2": "image/x-kodak-dc2",
|
||||
".x3f": "image/x-sigma-x3f",
|
||||
".iiq": "image/x-phaseone-iiq",
|
||||
".nkc": "image/x-kodak-nkc",
|
||||
".r3d": "image/x-red-r3d",
|
||||
}
|
||||
|
||||
// 3D model extension -> MIME type (single source for Is3DModel + type detection)
|
||||
var model3DMimeType = map[string]string{
|
||||
".gltf": "model/gltf+json",
|
||||
".glb": "model/gltf-binary",
|
||||
".obj": "model/obj",
|
||||
".fbx": "model/fbx",
|
||||
".stl": "model/stl",
|
||||
".ply": "model/ply",
|
||||
".dae": "model/vnd.collada+xml",
|
||||
".3mf": "model/3mf",
|
||||
".3ds": "model/3ds",
|
||||
".usdz": "model/vnd.usdz+zip",
|
||||
".usd": "model/vnd.usd+zip",
|
||||
".usda": "model/vnd.usd+zip",
|
||||
".usdc": "model/vnd.usd+zip",
|
||||
".amf": "model/x-amf",
|
||||
".vrml": "model/vrml",
|
||||
".wrl": "model/vrml",
|
||||
".vtk": "model/vtk",
|
||||
".vtp": "model/vtk",
|
||||
".pcd": "model/pcd",
|
||||
".xyz": "model/xyz",
|
||||
".vox": "model/vox",
|
||||
".gltf": "model/gltf+json",
|
||||
".glb": "model/gltf-binary",
|
||||
".obj": "model/obj",
|
||||
".fbx": "model/fbx",
|
||||
".stl": "model/stl",
|
||||
".ply": "model/ply",
|
||||
".dae": "model/vnd.collada+xml",
|
||||
".3mf": "model/3mf",
|
||||
".3ds": "model/3ds",
|
||||
".usdz": "model/vnd.usdz+zip",
|
||||
".usd": "model/vnd.usd+zip",
|
||||
".usda": "model/vnd.usd+zip",
|
||||
".usdc": "model/vnd.usd+zip",
|
||||
".amf": "model/x-amf",
|
||||
".vrml": "model/vrml",
|
||||
".wrl": "model/vrml",
|
||||
".vtk": "model/vtk",
|
||||
".vtp": "model/vtk",
|
||||
".pcd": "model/pcd",
|
||||
".xyz": "model/xyz",
|
||||
".vox": "model/vox",
|
||||
}
|
||||
|
||||
var audioMetadataTypes = map[string]bool{
|
||||
@@ -560,48 +560,20 @@ func TextPreviewKind(ext string) string {
|
||||
return textPreviewKinds[ext]
|
||||
}
|
||||
|
||||
// textPreviewMimeTypes are non-text/* mimetypes that textTypes extensions resolve to
|
||||
// (see the custom table in backend/internal/adapters/fs/files/mime.go). The frontend
|
||||
// receives this list through globalVars (backend/internal/web/static.go), so it is
|
||||
// the single source of truth.
|
||||
// Do not add mimetypes shared with binaries (e.g. application/x-msdownload for .exe).
|
||||
var textPreviewMimeTypes = map[string]bool{
|
||||
"application/json": true,
|
||||
"application/rtf": true,
|
||||
"application/yaml": true,
|
||||
"application/x-yaml": true,
|
||||
"application/sql": true,
|
||||
"application/x-sql": true,
|
||||
"application/x-sh": true,
|
||||
"application/x-php": true,
|
||||
"application/x-powershell": true,
|
||||
"application/x-tex": true,
|
||||
"application/toml": true,
|
||||
"application/x-ruby": true,
|
||||
// IsTextPreviewable reports whether a file name uses the text preview endpoint.
|
||||
func IsTextPreviewable(name string) bool {
|
||||
return textPreviewKinds[strings.ToLower(filepath.Ext(name))] != ""
|
||||
}
|
||||
|
||||
// IsTextPreviewable reports whether an item uses the text preview endpoint. The decision
|
||||
// is the item's mimetype: every text/* type and the textPreviewMimeTypes allowlist.
|
||||
func IsTextPreviewable(mimetype string) bool {
|
||||
return strings.HasPrefix(mimetype, "text/") || textPreviewMimeTypes[mimetype]
|
||||
}
|
||||
|
||||
// TextPreviewMimeTypes returns the non-text/* allowlist in sorted order, for the
|
||||
// frontend's globalVars. Sorting keeps the served value stable across restarts.
|
||||
func TextPreviewMimeTypes() []string {
|
||||
types := make([]string, 0, len(textPreviewMimeTypes))
|
||||
for mt := range textPreviewMimeTypes {
|
||||
types = append(types, mt)
|
||||
// TextPreviewExtensionsForSPA groups text preview extensions (without dots) by renderer kind.
|
||||
// The frontend uses this to decide which items get a text preview container.
|
||||
func TextPreviewExtensionsForSPA() map[string][]string {
|
||||
out := make(map[string][]string)
|
||||
for ext, kind := range textPreviewKinds {
|
||||
out[kind] = append(out[kind], strings.TrimPrefix(ext, "."))
|
||||
}
|
||||
sort.Strings(types)
|
||||
return types
|
||||
}
|
||||
|
||||
// TextPreviewRenderKind returns the renderer kind for a text-previewable file: the
|
||||
// extension-specific kind when one exists, otherwise "plain".
|
||||
func TextPreviewRenderKind(name string) string {
|
||||
if kind := TextPreviewKind(strings.ToLower(filepath.Ext(name))); kind != "" {
|
||||
return kind
|
||||
for kind := range out {
|
||||
sort.Strings(out[kind])
|
||||
}
|
||||
return "plain"
|
||||
return out
|
||||
}
|
||||
@@ -1,7 +1,7 @@
|
||||
package iteminfo
|
||||
|
||||
import (
|
||||
"sort"
|
||||
"slices"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
@@ -22,30 +22,21 @@ func TestTextPreviewKinds(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestIsTextPreviewable(t *testing.T) {
|
||||
for _, mt := range []string{"text/plain", "text/markdown", "text/lyrics", "text/x-go", "text/html"} {
|
||||
require.True(t, IsTextPreviewable(mt), mt)
|
||||
}
|
||||
for _, mt := range []string{"application/json", "application/yaml", "application/sql", "application/x-sh", "application/x-php", "application/x-powershell", "application/x-tex", "application/rtf", "text/rtf"} {
|
||||
require.True(t, IsTextPreviewable(mt), mt)
|
||||
}
|
||||
// Mimetypes shared with binaries (.exe, .bat via application/x-msdownload) must stay non-previewable.
|
||||
for _, mt := range []string{"", "image/svg+xml", "application/pdf", "application/x-sqlite3", "application/postscript", "application/backup", "application/wasm", "application/x-msdownload", "application/x-msdos-program", "application/octet-stream"} {
|
||||
require.False(t, IsTextPreviewable(mt), mt)
|
||||
}
|
||||
require.True(t, IsTextPreviewable("Notes.TXT"))
|
||||
require.True(t, IsTextPreviewable("dir/readme.md"))
|
||||
require.False(t, IsTextPreviewable("image.svg"))
|
||||
require.False(t, IsTextPreviewable("noext"))
|
||||
}
|
||||
|
||||
func TestTextPreviewRenderKind(t *testing.T) {
|
||||
require.Equal(t, "markdown", TextPreviewRenderKind("notes.md"))
|
||||
require.Equal(t, "plain", TextPreviewRenderKind("50. Holy Water.lrc"))
|
||||
require.Equal(t, "plain", TextPreviewRenderKind("cert.pem"))
|
||||
}
|
||||
|
||||
func TestTextPreviewMimeTypesIsSortedAndMatchesAllowlist(t *testing.T) {
|
||||
types := TextPreviewMimeTypes()
|
||||
require.NotEmpty(t, types)
|
||||
require.True(t, sort.StringsAreSorted(types), "list must be sorted for a stable globalVars value")
|
||||
for _, mt := range types {
|
||||
require.True(t, IsTextPreviewable(mt), mt)
|
||||
func TestTextPreviewExtensionsForSPA(t *testing.T) {
|
||||
groups := TextPreviewExtensionsForSPA()
|
||||
require.Contains(t, groups["markdown"], "md")
|
||||
require.Contains(t, groups["rtf"], "rtf")
|
||||
require.True(t, slices.Contains(groups["code"], "go"))
|
||||
require.False(t, slices.Contains(groups["code"], "svg"))
|
||||
for _, exts := range groups {
|
||||
for _, ext := range exts {
|
||||
require.NotContains(t, ext, ".", "extensions must not have a leading dot")
|
||||
}
|
||||
}
|
||||
require.Contains(t, types, "application/json")
|
||||
}
|
||||
@@ -14,6 +14,7 @@ func TestApplyUserDefaults_copiesUserDefaultsOntoUser(t *testing.T) {
|
||||
saved := Config
|
||||
defer func() { Config = saved }()
|
||||
|
||||
dark := false
|
||||
Config = Settings{
|
||||
Server: Server{},
|
||||
UserDefaults: UserDefaults{
|
||||
@@ -63,7 +64,7 @@ func TestApplyUserDefaults_copiesUserDefaultsOntoUser(t *testing.T) {
|
||||
DisableOptions: true,
|
||||
},
|
||||
UI: UserDefaultsUI{
|
||||
DarkMode: users.DarkModeLight,
|
||||
DarkMode: &dark,
|
||||
ThemeColor: "var(--red)",
|
||||
CustomTheme: "Night",
|
||||
Locale: "de",
|
||||
@@ -112,7 +113,7 @@ func TestApplyUserDefaults_copiesUserDefaultsOntoUser(t *testing.T) {
|
||||
DisableSearchOptions: true,
|
||||
StickySidebar: false,
|
||||
HideFilesInTree: true,
|
||||
DarkMode: users.DarkModeLight,
|
||||
DarkMode: false,
|
||||
Locale: "de",
|
||||
ViewMode: "list",
|
||||
SingleClick: true,
|
||||
|
||||
@@ -428,7 +428,7 @@ const (
|
||||
// clampTranscodeReadRate enforces throttled producers: unset or sub-realtime
|
||||
// read rates are raised, so a producer is never left unthrottled. Burst and
|
||||
// catch-up pacing are left to ffmpeg's defaults.
|
||||
func clampTranscodeReadRate(t *FullAdminTranscode) {
|
||||
func clampTranscodeReadRate(t *MediaTranscode) {
|
||||
if t.ReadRate <= 0 {
|
||||
t.ReadRate = defaultTranscodeReadRate
|
||||
} else if t.ReadRate < minTranscodeReadRate {
|
||||
@@ -769,9 +769,7 @@ func loadConfigWithDefaults(configFile string, generate bool) error {
|
||||
}
|
||||
|
||||
// Second pass: Decode with strict validation (disallow unknown fields within valid sections)
|
||||
// UseJSONUnmarshaler lets typed fields with custom UnmarshalJSON (e.g. users.DarkMode)
|
||||
// handle YAML scalars like booleans instead of the generic decoder casting them to strings.
|
||||
decoder := yaml.NewDecoder(bytes.NewReader(filteredYAML), yaml.DisallowUnknownField(), yaml.UseJSONUnmarshaler())
|
||||
decoder := yaml.NewDecoder(bytes.NewReader(filteredYAML), yaml.DisallowUnknownField())
|
||||
err = decoder.Decode(&Config)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error unmarshaling YAML data: %v", err)
|
||||
@@ -1038,7 +1036,7 @@ func SetDefaults(generate bool) Settings {
|
||||
DisableOptions: false,
|
||||
},
|
||||
UI: UserDefaultsUI{
|
||||
DarkMode: users.DarkModeDark,
|
||||
DarkMode: boolPtr(true),
|
||||
ThemeColor: "var(--blue)",
|
||||
CustomTheme: "",
|
||||
Locale: "en",
|
||||
@@ -1075,16 +1073,14 @@ func SetDefaults(generate bool) Settings {
|
||||
for _, t := range AllVideoPreviewTypes {
|
||||
s.Integrations.Media.Convert.VideoPreview[t] = boolPtr(true)
|
||||
}
|
||||
s.Integrations.Media.Transcode = FullAdminTranscode{
|
||||
UserTranscode: UserTranscode{
|
||||
Enabled: false,
|
||||
MaxConcurrentPerUser: 1,
|
||||
},
|
||||
MaxConcurrent: 2,
|
||||
CacheMaxSizeMB: 10240,
|
||||
CacheRetentionH: 24,
|
||||
ReadRate: defaultTranscodeReadRate,
|
||||
Profiles: DefaultTranscodeProfiles(),
|
||||
s.Integrations.Media.Transcode = MediaTranscode{
|
||||
Enabled: false,
|
||||
MaxConcurrent: 2,
|
||||
MaxConcurrentPerUser: 1,
|
||||
CacheMaxSizeMB: 10240,
|
||||
CacheRetentionH: 24,
|
||||
ReadRate: defaultTranscodeReadRate,
|
||||
Profiles: DefaultTranscodeProfiles(),
|
||||
}
|
||||
// Hardware acceleration is opt-out: "auto" lets go-ffmpeg follow its device
|
||||
// hierarchy (integrated GPU, discrete GPU, then software).
|
||||
|
||||
@@ -54,8 +54,8 @@ func TestYAMLConfig_jwt_organizedUserDefaults(t *testing.T) {
|
||||
if !ud.Account.Permissions.Share {
|
||||
t.Fatal("expected account.permissions.share true")
|
||||
}
|
||||
if ud.UI.DarkMode != users.DarkModeDark {
|
||||
t.Fatalf("expected ui.darkMode true, got %q", ud.UI.DarkMode)
|
||||
if ud.UI.DarkMode == nil || !*ud.UI.DarkMode {
|
||||
t.Fatal("expected ui.darkMode true")
|
||||
}
|
||||
if ud.Listing.SingleClick {
|
||||
t.Fatal("expected listing.singleClick false")
|
||||
@@ -106,8 +106,8 @@ func TestYAMLConfig_applyUserDefaultsFrom_proxyAndJwt(t *testing.T) {
|
||||
if !u.Permissions.Share {
|
||||
t.Fatal("Share=false after ApplyUserDefaultsFrom")
|
||||
}
|
||||
if u.DarkMode != users.DarkModeDark {
|
||||
t.Fatalf("expected darkMode from ui defaults, got %q", u.DarkMode)
|
||||
if !u.DarkMode {
|
||||
t.Fatal("expected darkMode from ui defaults")
|
||||
}
|
||||
if !u.Preview.Image || !u.Preview.PopUp {
|
||||
t.Fatalf("preview: %+v", u.Preview)
|
||||
@@ -126,7 +126,7 @@ func TestYAMLConfig_profileMerge_matchesApplyUserDefaults(t *testing.T) {
|
||||
fromDefaults := ProfileFromUserDefaults(Config.UserDefaults)
|
||||
u := &users.User{FrontendUser: users.FrontendUser{Username: "merge-check"}}
|
||||
ExpandProfileIntoUser(u, fromDefaults)
|
||||
if !u.DarkMode.IsDark() || !u.Preview.Image {
|
||||
if !u.DarkMode || !u.Preview.Image {
|
||||
t.Fatalf("ExpandProfileIntoUser: darkMode=%v preview.image=%v", u.DarkMode, u.Preview.Image)
|
||||
}
|
||||
u2 := &users.User{FrontendUser: users.FrontendUser{Username: "merge-check"}}
|
||||
@@ -166,7 +166,7 @@ func TestLoadConfig_partialUserDefaults_preservesPreviewAndAutoplayDefaults(t *t
|
||||
func TestLoadConfig_partialUserDefaults_preservesOtherSetDefaults(t *testing.T) {
|
||||
loadPartialUserDefaultsConfig(t, partialUserDefaultsYAML(""))
|
||||
ud := Config.UserDefaults
|
||||
if ud.UI.DarkMode != users.DarkModeDark {
|
||||
if ud.UI.DarkMode == nil || !*ud.UI.DarkMode {
|
||||
t.Fatalf("ui.darkMode: %v", ud.UI.DarkMode)
|
||||
}
|
||||
if ud.Sidebar.ShowTools == nil || !*ud.Sidebar.ShowTools {
|
||||
|
||||
@@ -122,7 +122,7 @@ func TestApplyEnforcedDefaultsFrom_onlyEnforcedSubset(t *testing.T) {
|
||||
Username: "alice",
|
||||
NonAdminEditable: users.NonAdminEditable{
|
||||
ShowHidden: false,
|
||||
DarkMode: users.DarkModeLight,
|
||||
DarkMode: false,
|
||||
},
|
||||
},
|
||||
}
|
||||
@@ -136,7 +136,7 @@ func TestApplyEnforcedDefaultsFrom_onlyEnforcedSubset(t *testing.T) {
|
||||
if !u.ShowHidden {
|
||||
t.Fatal("expected enforced ShowHidden true")
|
||||
}
|
||||
if u.DarkMode == users.DarkModeDark {
|
||||
if u.DarkMode {
|
||||
t.Fatal("expected non-enforced DarkMode unchanged")
|
||||
}
|
||||
}
|
||||
@@ -177,7 +177,7 @@ func TestProfileRoundTrip_expandCollapse(t *testing.T) {
|
||||
Username: "alice",
|
||||
NonAdminEditable: users.NonAdminEditable{
|
||||
ShowHidden: true,
|
||||
DarkMode: users.DarkModeDark,
|
||||
DarkMode: true,
|
||||
ShowToolsInSidebar: true,
|
||||
Preview: users.Preview{
|
||||
Image: true,
|
||||
@@ -191,7 +191,7 @@ func TestProfileRoundTrip_expandCollapse(t *testing.T) {
|
||||
var u2 users.User
|
||||
u2.Username = "alice"
|
||||
ExpandProfileIntoUser(&u2, p)
|
||||
if !u2.ShowHidden || !u2.DarkMode.IsDark() || !u2.DisableSettings {
|
||||
if !u2.ShowHidden || !u2.DarkMode || !u2.DisableSettings {
|
||||
t.Fatal("round trip lost fields")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,8 +6,6 @@ import (
|
||||
"regexp"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/database/users"
|
||||
)
|
||||
|
||||
// Test structs for comprehensive testing
|
||||
@@ -217,10 +215,11 @@ type UserDefaults struct {
|
||||
sourcePath := tmpFile[:strings.LastIndex(tmpFile, "/")]
|
||||
|
||||
// Create a config with both default and non-default values
|
||||
trueVal := true
|
||||
config := &Settings{
|
||||
UserDefaults: UserDefaults{
|
||||
UI: UserDefaultsUI{
|
||||
DarkMode: users.DarkModeDark,
|
||||
DarkMode: &trueVal,
|
||||
Locale: "es",
|
||||
},
|
||||
},
|
||||
@@ -295,11 +294,12 @@ func TestGenerateConfigYaml_CommentsOnOff(t *testing.T) {
|
||||
func TestGenerateConfigYaml_IntegrationTest(t *testing.T) {
|
||||
reNumber := regexp.MustCompile(`^-?\d+(\.\d+)?$`)
|
||||
// Comprehensive test with all features
|
||||
trueVal := true
|
||||
config := &Settings{
|
||||
UserDefaults: UserDefaults{
|
||||
UI: UserDefaultsUI{
|
||||
Locale: "es",
|
||||
DarkMode: users.DarkModeDark,
|
||||
DarkMode: &trueVal,
|
||||
},
|
||||
FileViewer: UserDefaultsFileViewer{
|
||||
DisableOnlyOfficeExt: ".docx .xlsx",
|
||||
@@ -331,7 +331,7 @@ func TestGenerateConfigYaml_IntegrationTest(t *testing.T) {
|
||||
showComments: false,
|
||||
showFull: true,
|
||||
filterDeprecated: false,
|
||||
expectSecret: true, // secrets should be hidden
|
||||
expectSecret: true, // secrets should be hidden
|
||||
expectDeprecated: false,
|
||||
expectFull: true, // all fields shown
|
||||
expectComments: false, // no comments
|
||||
@@ -464,7 +464,7 @@ func TestGenerateConfigYaml_EdgeCases(t *testing.T) {
|
||||
config: &Settings{
|
||||
UserDefaults: UserDefaults{
|
||||
UI: UserDefaultsUI{
|
||||
DarkMode: users.DarkModeDark, // default value
|
||||
DarkMode: boolPtr(true), // default value
|
||||
Locale: "en", // default value
|
||||
},
|
||||
},
|
||||
|
||||
@@ -68,14 +68,14 @@ func MediaPatchJSONForPaths(source any, paths []string) ([]byte, error) {
|
||||
|
||||
// ApplyConfigSpecifiedPathsToMedia overlays config media values onto stored settings for the
|
||||
// paths the config file pins, so a config value always wins over the stored one.
|
||||
func ApplyConfigSpecifiedPathsToMedia(stored FullAdminMedia, config Media) (FullAdminMedia, error) {
|
||||
func ApplyConfigSpecifiedPathsToMedia(stored MediaSettings, config Media) (MediaSettings, error) {
|
||||
paths := ConfigSpecifiedMediaPaths()
|
||||
if len(paths) == 0 {
|
||||
return stored, nil
|
||||
}
|
||||
patchJSON, err := MediaPatchJSONForPaths(config, paths)
|
||||
if err != nil {
|
||||
return FullAdminMedia{}, err
|
||||
return MediaSettings{}, err
|
||||
}
|
||||
if len(patchJSON) == 0 || string(patchJSON) == "{}" || string(patchJSON) == "null" {
|
||||
return stored, nil
|
||||
|
||||
@@ -60,11 +60,11 @@ func TestValidateMediaPatchNotConfigLockedWithoutLocks(t *testing.T) {
|
||||
func TestApplyConfigSpecifiedPathsToMedia(t *testing.T) {
|
||||
withConfigMediaPaths(t, "hardwareAcceleration", "transcode.profiles.datasaver.avgBitrateKbps")
|
||||
|
||||
stored := FullAdminMedia{
|
||||
stored := MediaSettings{
|
||||
Debug: true,
|
||||
HardwareAcceleration: true,
|
||||
Transcode: FullAdminTranscode{
|
||||
UserTranscode: UserTranscode{Enabled: true},
|
||||
Transcode: MediaTranscode{
|
||||
Enabled: true,
|
||||
Profiles: TranscodeProfiles{
|
||||
DataSaver: TranscodeProfile{AvgBitrateKbps: 1500},
|
||||
},
|
||||
@@ -72,7 +72,7 @@ func TestApplyConfigSpecifiedPathsToMedia(t *testing.T) {
|
||||
}
|
||||
config := Media{
|
||||
HardwareAcceleration: false,
|
||||
Transcode: FullAdminTranscode{
|
||||
Transcode: MediaTranscode{
|
||||
Profiles: TranscodeProfiles{
|
||||
DataSaver: TranscodeProfile{AvgBitrateKbps: 800},
|
||||
},
|
||||
@@ -97,7 +97,7 @@ func TestApplyConfigSpecifiedPathsToMedia(t *testing.T) {
|
||||
func TestApplyConfigSpecifiedPathsToMediaNoLocks(t *testing.T) {
|
||||
withConfigMediaPaths(t)
|
||||
|
||||
stored := FullAdminMedia{HardwareAcceleration: true}
|
||||
stored := MediaSettings{HardwareAcceleration: true}
|
||||
overlaid, err := ApplyConfigSpecifiedPathsToMedia(stored, Media{HardwareAcceleration: false})
|
||||
if err != nil {
|
||||
t.Fatalf("ApplyConfigSpecifiedPathsToMedia: %v", err)
|
||||
|
||||
@@ -7,18 +7,18 @@ import (
|
||||
)
|
||||
|
||||
// MergeMediaSettingsPatchJSON merges a partial JSON patch into base media settings.
|
||||
func MergeMediaSettingsPatchJSON(base FullAdminMedia, patchJSON []byte) (FullAdminMedia, error) {
|
||||
func MergeMediaSettingsPatchJSON(base MediaSettings, patchJSON []byte) (MediaSettings, error) {
|
||||
baseBytes, err := json.Marshal(base)
|
||||
if err != nil {
|
||||
return FullAdminMedia{}, err
|
||||
return MediaSettings{}, err
|
||||
}
|
||||
mergedBytes, err := mergeUserDefaultsJSON(baseBytes, patchJSON)
|
||||
if err != nil {
|
||||
return FullAdminMedia{}, err
|
||||
return MediaSettings{}, err
|
||||
}
|
||||
var merged FullAdminMedia
|
||||
var merged MediaSettings
|
||||
if err := json.Unmarshal(mergedBytes, &merged); err != nil {
|
||||
return FullAdminMedia{}, fmt.Errorf("unmarshal merged media settings: %w", err)
|
||||
return MediaSettings{}, fmt.Errorf("unmarshal merged media settings: %w", err)
|
||||
}
|
||||
// An explicit accelDevice wins over the legacy hardwareAcceleration bool in
|
||||
// the base settings, which would otherwise force the device back to software.
|
||||
@@ -35,7 +35,7 @@ func MergeMediaSettingsPatchJSON(base FullAdminMedia, patchJSON []byte) (FullAdm
|
||||
}
|
||||
|
||||
// MediaSettingsEqual reports whether two runtime media settings are equivalent.
|
||||
func MediaSettingsEqual(a, b FullAdminMedia) bool {
|
||||
func MediaSettingsEqual(a, b MediaSettings) bool {
|
||||
ab, errA := json.Marshal(a)
|
||||
bb, errB := json.Marshal(b)
|
||||
if errA != nil || errB != nil {
|
||||
@@ -46,7 +46,7 @@ func MediaSettingsEqual(a, b FullAdminMedia) bool {
|
||||
|
||||
// ApplyMediaSettingsIfChanged applies runtime media settings
|
||||
// only when they actually differ. It never re-runs ffmpeg detection.
|
||||
func ApplyMediaSettingsIfChanged(s FullAdminMedia) {
|
||||
func ApplyMediaSettingsIfChanged(s MediaSettings) {
|
||||
if MediaSettingsEqual(CurrentMediaSettings(), s) {
|
||||
return
|
||||
}
|
||||
@@ -55,7 +55,7 @@ func ApplyMediaSettingsIfChanged(s FullAdminMedia) {
|
||||
|
||||
// CloneMediaSettings returns a deep copy of m — the conversion maps hold *bool
|
||||
// values, so a shallow copy would alias the live config's reference state.
|
||||
func CloneMediaSettings(m FullAdminMedia) FullAdminMedia {
|
||||
func CloneMediaSettings(m MediaSettings) MediaSettings {
|
||||
out := m
|
||||
out.Convert = FfmpegConvert{
|
||||
ImagePreview: cloneBoolMap(m.Convert.ImagePreview),
|
||||
@@ -84,7 +84,7 @@ func cloneBoolPtr(v *bool) *bool {
|
||||
}
|
||||
|
||||
// SanitizeMediaSettings clamps runtime media settings into supported ranges.
|
||||
func SanitizeMediaSettings(m FullAdminMedia) FullAdminMedia {
|
||||
func SanitizeMediaSettings(m MediaSettings) MediaSettings {
|
||||
t := &m.Transcode
|
||||
if t.MaxConcurrent < 1 {
|
||||
t.MaxConcurrent = 2
|
||||
|
||||
@@ -2,16 +2,17 @@ package settings
|
||||
|
||||
import "testing"
|
||||
|
||||
func testMediaSettings() FullAdminMedia {
|
||||
return FullAdminMedia{
|
||||
func testMediaSettings() MediaSettings {
|
||||
return MediaSettings{
|
||||
Debug: true,
|
||||
HardwareAcceleration: true,
|
||||
Transcode: FullAdminTranscode{
|
||||
UserTranscode: UserTranscode{Enabled: true, MaxConcurrentPerUser: 1},
|
||||
MaxConcurrent: 2,
|
||||
CacheMaxSizeMB: 10240,
|
||||
CacheRetentionH: 24,
|
||||
Profiles: DefaultTranscodeProfiles(),
|
||||
Transcode: MediaTranscode{
|
||||
Enabled: true,
|
||||
MaxConcurrent: 2,
|
||||
MaxConcurrentPerUser: 1,
|
||||
CacheMaxSizeMB: 10240,
|
||||
CacheRetentionH: 24,
|
||||
Profiles: DefaultTranscodeProfiles(),
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -53,11 +54,11 @@ func TestMergeMediaSettingsPatchJSONProfile(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestSanitizeMediaSettings(t *testing.T) {
|
||||
in := FullAdminMedia{
|
||||
Transcode: FullAdminTranscode{
|
||||
UserTranscode: UserTranscode{MaxConcurrentPerUser: 0},
|
||||
MaxConcurrent: -1,
|
||||
CacheRetentionH: 0,
|
||||
in := MediaSettings{
|
||||
Transcode: MediaTranscode{
|
||||
MaxConcurrent: -1,
|
||||
MaxConcurrentPerUser: 0,
|
||||
CacheRetentionH: 0,
|
||||
Profiles: TranscodeProfiles{
|
||||
Quality: TranscodeProfile{MaxResolution: -5, AvgBitrateKbps: -1},
|
||||
},
|
||||
@@ -115,10 +116,10 @@ func TestSanitizeMediaSettingsAccelDevice(t *testing.T) {
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
out := SanitizeMediaSettings(FullAdminMedia{
|
||||
out := SanitizeMediaSettings(MediaSettings{
|
||||
AccelDevice: tc.device,
|
||||
HardwareAcceleration: tc.hwAccel,
|
||||
Transcode: FullAdminTranscode{UserTranscode: UserTranscode{MaxConcurrentPerUser: 1}, MaxConcurrent: 1, CacheRetentionH: 1},
|
||||
Transcode: MediaTranscode{MaxConcurrent: 1, MaxConcurrentPerUser: 1, CacheRetentionH: 1},
|
||||
})
|
||||
if out.AccelDevice != tc.wantDevice || out.HardwareAcceleration != tc.wantHW {
|
||||
t.Fatalf("accelDevice = %q/%v, want %q/%v", out.AccelDevice, out.HardwareAcceleration, tc.wantDevice, tc.wantHW)
|
||||
|
||||
@@ -8,7 +8,6 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/ffmpeg"
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/mediacache"
|
||||
goffmpeg "github.com/gtsteffaniak/go-ffmpeg"
|
||||
"github.com/gtsteffaniak/go-ffmpeg/capabilities"
|
||||
"github.com/gtsteffaniak/go-ffmpeg/encode"
|
||||
@@ -210,10 +209,8 @@ func TranscodeCacheRetention() time.Duration {
|
||||
return time.Duration(h) * time.Hour
|
||||
}
|
||||
|
||||
// CurrentMediaSettings returns the stored runtime media settings (excluding ffmpegPath).
|
||||
// Derived fields (ImagePreview, EnabledProfiles) are not populated; use CurrentFullAdminMedia
|
||||
// or CurrentUserMedia for response payloads.
|
||||
func CurrentMediaSettings() FullAdminMedia {
|
||||
// CurrentMediaSettings returns the runtime media settings (excluding ffmpegPath).
|
||||
func CurrentMediaSettings() MediaSettings {
|
||||
mediaMu.RLock()
|
||||
defer mediaMu.RUnlock()
|
||||
return currentMediaSettingsLocked()
|
||||
@@ -221,79 +218,30 @@ func CurrentMediaSettings() FullAdminMedia {
|
||||
|
||||
// currentMediaSettingsLocked snapshots the live media settings. Callers must hold
|
||||
// mediaMu. The returned value is deep-copied so it cannot alias the live config.
|
||||
func currentMediaSettingsLocked() FullAdminMedia {
|
||||
func currentMediaSettingsLocked() MediaSettings {
|
||||
m := Config.Integrations.Media
|
||||
return CloneMediaSettings(FullAdminMedia{
|
||||
return CloneMediaSettings(MediaSettings{
|
||||
Convert: m.Convert,
|
||||
Debug: m.Debug,
|
||||
HardwareAcceleration: m.HardwareAcceleration,
|
||||
AccelDevice: m.AccelDevice,
|
||||
AlwaysUseFfmpeg: m.AlwaysUseFfmpeg,
|
||||
AlwaysUseOnlyOffice: m.AlwaysUseOnlyOffice,
|
||||
Transcode: m.Transcode,
|
||||
AccelDevice: m.AccelDevice,
|
||||
AlwaysUseFfmpeg: m.AlwaysUseFfmpeg,
|
||||
AlwaysUseOnlyOffice: m.AlwaysUseOnlyOffice,
|
||||
Transcode: m.Transcode,
|
||||
})
|
||||
}
|
||||
|
||||
// ImagePreviewAvailability resolves each supported image preview type to whether
|
||||
// conversion is active right now, using the same check the preview pipeline uses.
|
||||
func ImagePreviewAvailability() map[ImagePreviewType]bool {
|
||||
out := make(map[ImagePreviewType]bool, len(AllImagePreviewTypes))
|
||||
for _, t := range AllImagePreviewTypes {
|
||||
out[t] = CanConvertImage(string(t))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// CurrentUserMedia builds the PUBLIC media payload for non-admin users. Values come
|
||||
// from the same accessors as the admin payload, so the shared fields always agree.
|
||||
// The getters take mediaMu themselves, so they are called here without holding it.
|
||||
func CurrentUserMedia() UserMedia {
|
||||
return UserMedia{
|
||||
Transcode: UserTranscode{
|
||||
Enabled: TranscodeEnabled(),
|
||||
MaxConcurrentPerUser: TranscodeMaxConcurrentPerUser(),
|
||||
EnabledProfiles: EnabledTranscodeProfiles(),
|
||||
},
|
||||
ImagePreview: ImagePreviewAvailability(),
|
||||
}
|
||||
}
|
||||
|
||||
// PrepMediaForFrontend returns a read-only copy of m with derived values filled in,
|
||||
// following PrepForFrontend in internal/web/users.go. The caller's value is not modified.
|
||||
// Stored config (transcode.enabled, maxConcurrentPerUser) is deliberately NOT replaced
|
||||
// with effective values: the admin UI edits these fields and sends them back, so they
|
||||
// must stay the stored values. Only derived fields are filled.
|
||||
// NEVER pass the result to SaveSetting without ClearMediaDerived first.
|
||||
func PrepMediaForFrontend(m FullAdminMedia) FullAdminMedia {
|
||||
m.Transcode.EnabledProfiles = EnabledTranscodeProfiles()
|
||||
m.ImagePreview = ImagePreviewAvailability()
|
||||
return m
|
||||
}
|
||||
|
||||
// ClearMediaDerived zeroes the fields that PrepMediaForFrontend derives at read time so
|
||||
// they are never written to the database. Enabled and MaxConcurrentPerUser are stored
|
||||
// config and are not cleared here.
|
||||
func ClearMediaDerived(m *FullAdminMedia) {
|
||||
m.Transcode.EnabledProfiles = nil
|
||||
m.ImagePreview = nil
|
||||
}
|
||||
|
||||
// ApplyMediaSettings applies runtime media settings to the in-memory config.
|
||||
// It never re-runs ffmpeg detection: codec/hardware detection happens exactly
|
||||
// once at startup (InitFFmpeg). hardwareAcceleration is read live per transcode;
|
||||
// ffmpegPath, accelDevice, debug and concurrency apply on the next restart.
|
||||
func ApplyMediaSettings(s FullAdminMedia) {
|
||||
func ApplyMediaSettings(s MediaSettings) {
|
||||
mediaMu.Lock()
|
||||
defer mediaMu.Unlock()
|
||||
Config.Integrations.Media.Convert = s.Convert
|
||||
Config.Integrations.Media.Debug = s.Debug
|
||||
Config.Integrations.Media.HardwareAcceleration = s.HardwareAcceleration
|
||||
Config.Integrations.Media.AccelDevice = s.AccelDevice
|
||||
if Config.Integrations.Media.AlwaysUseFfmpeg != s.AlwaysUseFfmpeg {
|
||||
// The shared media cache holds values from whichever backend answered
|
||||
// first. Switching backends drops them so the selected one answers fresh.
|
||||
mediacache.ClearAll()
|
||||
}
|
||||
Config.Integrations.Media.AlwaysUseFfmpeg = s.AlwaysUseFfmpeg
|
||||
Config.Integrations.Media.AlwaysUseOnlyOffice = s.AlwaysUseOnlyOffice
|
||||
Config.Integrations.Media.Transcode = s.Transcode
|
||||
|
||||
@@ -4,8 +4,6 @@ import (
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/mediacache"
|
||||
)
|
||||
|
||||
func withMediaConfig(t *testing.T) {
|
||||
@@ -27,15 +25,16 @@ func TestMediaAccessorsReflectRuntimeSettings(t *testing.T) {
|
||||
originalTranscode := Config.Integrations.Media.Transcode
|
||||
t.Cleanup(func() { Config.Integrations.Media.Transcode = originalTranscode })
|
||||
|
||||
ApplyMediaSettings(FullAdminMedia{
|
||||
ApplyMediaSettings(MediaSettings{
|
||||
Debug: true,
|
||||
HardwareAcceleration: true,
|
||||
Transcode: FullAdminTranscode{
|
||||
UserTranscode: UserTranscode{Enabled: true, MaxConcurrentPerUser: 2},
|
||||
MaxConcurrent: 4,
|
||||
CacheMaxSizeMB: 512,
|
||||
CacheRetentionH: 6,
|
||||
Profiles: DefaultTranscodeProfiles(),
|
||||
Transcode: MediaTranscode{
|
||||
Enabled: true,
|
||||
MaxConcurrent: 4,
|
||||
MaxConcurrentPerUser: 2,
|
||||
CacheMaxSizeMB: 512,
|
||||
CacheRetentionH: 6,
|
||||
Profiles: DefaultTranscodeProfiles(),
|
||||
},
|
||||
})
|
||||
|
||||
@@ -68,7 +67,7 @@ func TestMediaAccessorsReflectRuntimeSettings(t *testing.T) {
|
||||
|
||||
func TestTranscodeAccessorsUseDefaultsWhenUnset(t *testing.T) {
|
||||
withMediaConfig(t)
|
||||
ApplyMediaSettings(FullAdminMedia{})
|
||||
ApplyMediaSettings(MediaSettings{})
|
||||
|
||||
if got := TranscodeMaxConcurrent(); got != 2 {
|
||||
t.Fatalf("TranscodeMaxConcurrent = %d, want the 2 default", got)
|
||||
@@ -226,11 +225,11 @@ func TestMediaAccessorsAreRaceFree(t *testing.T) {
|
||||
default:
|
||||
}
|
||||
enabled = !enabled
|
||||
ApplyMediaSettings(FullAdminMedia{
|
||||
ApplyMediaSettings(MediaSettings{
|
||||
Debug: enabled,
|
||||
HardwareAcceleration: enabled,
|
||||
Transcode: FullAdminTranscode{
|
||||
UserTranscode: UserTranscode{Enabled: enabled},
|
||||
Transcode: MediaTranscode{
|
||||
Enabled: enabled,
|
||||
MaxConcurrent: 2,
|
||||
Profiles: DefaultTranscodeProfiles(),
|
||||
},
|
||||
@@ -310,26 +309,3 @@ func TestInitFFmpegIsIdempotent(t *testing.T) {
|
||||
t.Fatal("InitFFmpeg must not run the probe during tests")
|
||||
}
|
||||
}
|
||||
|
||||
// TestApplyMediaSettingsClearsMediaCacheOnBackendSwitch checks that toggling
|
||||
// "Always use ffmpeg" drops values the other backend cached, and that an
|
||||
// unrelated settings change keeps them.
|
||||
func TestApplyMediaSettingsClearsMediaCacheOnBackendSwitch(t *testing.T) {
|
||||
withMediaConfig(t)
|
||||
key := mediacache.Key{Path: "/settings-test.mp4", Size: 1, ModTime: 1}.String()
|
||||
t.Cleanup(func() { mediacache.Durations.Delete(key) })
|
||||
|
||||
media := CurrentMediaSettings()
|
||||
mediacache.Durations.Set(key, 1)
|
||||
media.Debug = !media.Debug
|
||||
ApplyMediaSettings(media)
|
||||
if _, ok := mediacache.Durations.Get(key); !ok {
|
||||
t.Fatal("an unrelated media settings change must keep the shared media cache")
|
||||
}
|
||||
|
||||
media.AlwaysUseFfmpeg = !media.AlwaysUseFfmpeg
|
||||
ApplyMediaSettings(media)
|
||||
if _, ok := mediacache.Durations.Get(key); ok {
|
||||
t.Fatal("switching Always use ffmpeg must clear the shared media cache")
|
||||
}
|
||||
}
|
||||
@@ -4,8 +4,6 @@ import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/gtsteffaniak/filebrowser/backend/internal/database/users"
|
||||
)
|
||||
|
||||
func TestMultiConfigLoad(t *testing.T) {
|
||||
@@ -230,7 +228,7 @@ frontend:
|
||||
}
|
||||
|
||||
// Verify user defaults
|
||||
if Config.UserDefaults.UI.DarkMode != users.DarkModeDark {
|
||||
if Config.UserDefaults.UI.DarkMode == nil || !*Config.UserDefaults.UI.DarkMode {
|
||||
t.Errorf("Expected dark mode true, got %v", Config.UserDefaults.UI.DarkMode)
|
||||
}
|
||||
|
||||
|
||||
Loaded 100 of 198 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user