fs.go 48 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786
  1. package fasthttp
  2. import (
  3. "bytes"
  4. "errors"
  5. "fmt"
  6. "html"
  7. "io"
  8. "io/fs"
  9. "mime"
  10. "net/http"
  11. "os"
  12. "path/filepath"
  13. "sort"
  14. "strings"
  15. "sync"
  16. "time"
  17. "github.com/andybalholm/brotli"
  18. "github.com/klauspost/compress/gzip"
  19. "github.com/klauspost/compress/zstd"
  20. "github.com/valyala/bytebufferpool"
  21. )
  22. // ServeFileBytesUncompressed returns HTTP response containing file contents
  23. // from the given path.
  24. //
  25. // Directory contents is returned if path points to directory.
  26. //
  27. // ServeFileBytes may be used for saving network traffic when serving files
  28. // with good compression ratio.
  29. //
  30. // See also RequestCtx.SendFileBytes.
  31. //
  32. // WARNING: do not pass any user supplied paths to this function!
  33. // WARNING: if path is based on user input users will be able to request
  34. // any file on your filesystem! Use fasthttp.FS with a sane Root instead.
  35. func ServeFileBytesUncompressed(ctx *RequestCtx, path []byte) {
  36. ServeFileUncompressed(ctx, b2s(path))
  37. }
  38. // ServeFileUncompressed returns HTTP response containing file contents
  39. // from the given path.
  40. //
  41. // Directory contents is returned if path points to directory.
  42. //
  43. // ServeFile may be used for saving network traffic when serving files
  44. // with good compression ratio.
  45. //
  46. // See also RequestCtx.SendFile.
  47. //
  48. // WARNING: do not pass any user supplied paths to this function!
  49. // WARNING: if path is based on user input users will be able to request
  50. // any file on your filesystem! Use fasthttp.FS with a sane Root instead.
  51. func ServeFileUncompressed(ctx *RequestCtx, path string) {
  52. ctx.Request.Header.DelBytes(strAcceptEncoding)
  53. ServeFile(ctx, path)
  54. }
  55. // ServeFileBytes returns HTTP response containing compressed file contents
  56. // from the given path.
  57. //
  58. // HTTP response may contain uncompressed file contents in the following cases:
  59. //
  60. // - Missing 'Accept-Encoding: gzip' request header.
  61. // - No write access to directory containing the file.
  62. //
  63. // Directory contents is returned if path points to directory.
  64. //
  65. // Use ServeFileBytesUncompressed is you don't need serving compressed
  66. // file contents.
  67. //
  68. // See also RequestCtx.SendFileBytes.
  69. //
  70. // WARNING: do not pass any user supplied paths to this function!
  71. // WARNING: if path is based on user input users will be able to request
  72. // any file on your filesystem! Use fasthttp.FS with a sane Root instead.
  73. func ServeFileBytes(ctx *RequestCtx, path []byte) {
  74. ServeFile(ctx, b2s(path))
  75. }
  76. // ServeFile returns HTTP response containing compressed file contents
  77. // from the given path.
  78. //
  79. // HTTP response may contain uncompressed file contents in the following cases:
  80. //
  81. // - Missing 'Accept-Encoding: gzip' request header.
  82. // - No write access to directory containing the file.
  83. //
  84. // Directory contents is returned if path points to directory.
  85. //
  86. // Use ServeFileUncompressed is you don't need serving compressed file contents.
  87. //
  88. // See also RequestCtx.SendFile.
  89. //
  90. // WARNING: do not pass any user supplied paths to this function!
  91. // WARNING: if path is based on user input users will be able to request
  92. // any file on your filesystem! Use fasthttp.FS with a sane Root instead.
  93. func ServeFile(ctx *RequestCtx, path string) {
  94. rootFSOnce.Do(func() {
  95. rootFSHandler = rootFS.NewRequestHandler()
  96. })
  97. if path == "" || !filepath.IsAbs(path) {
  98. // extend relative path to absolute path
  99. hasTrailingSlash := path != "" && (path[len(path)-1] == '/' || path[len(path)-1] == '\\')
  100. var err error
  101. path = filepath.FromSlash(path)
  102. if path, err = filepath.Abs(path); err != nil {
  103. ctx.Logger().Printf("cannot resolve path %q to absolute file path: %v", path, err)
  104. ctx.Error("Internal Server Error", StatusInternalServerError)
  105. return
  106. }
  107. if hasTrailingSlash {
  108. path += "/"
  109. }
  110. }
  111. // convert the path to forward slashes regardless the OS in order to set the URI properly
  112. // the handler will convert back to OS path separator before opening the file
  113. path = filepath.ToSlash(path)
  114. ctx.Request.SetRequestURI(path)
  115. rootFSHandler(ctx)
  116. }
  117. var (
  118. rootFSOnce sync.Once
  119. rootFS = &FS{
  120. Root: "",
  121. AllowEmptyRoot: true,
  122. GenerateIndexPages: true,
  123. Compress: true,
  124. CompressBrotli: true,
  125. AcceptByteRange: true,
  126. }
  127. rootFSHandler RequestHandler
  128. )
  129. // ServeFS returns HTTP response containing compressed file contents from the given fs.FS's path.
  130. //
  131. // HTTP response may contain uncompressed file contents in the following cases:
  132. //
  133. // - Missing 'Accept-Encoding: gzip' request header.
  134. // - No write access to directory containing the file.
  135. //
  136. // Directory contents is returned if path points to directory.
  137. //
  138. // See also ServeFile.
  139. func ServeFS(ctx *RequestCtx, filesystem fs.FS, path string) {
  140. f := &FS{
  141. FS: filesystem,
  142. Root: "",
  143. AllowEmptyRoot: true,
  144. GenerateIndexPages: true,
  145. Compress: true,
  146. CompressBrotli: true,
  147. AcceptByteRange: true,
  148. }
  149. handler := f.NewRequestHandler()
  150. ctx.Request.SetRequestURI(path)
  151. handler(ctx)
  152. }
  153. // PathRewriteFunc must return new request path based on arbitrary ctx
  154. // info such as ctx.Path().
  155. //
  156. // Path rewriter is used in FS for translating the current request
  157. // to the local filesystem path relative to FS.Root.
  158. //
  159. // The returned path must not contain '/../' substrings due to security reasons,
  160. // since such paths may refer files outside FS.Root.
  161. //
  162. // The returned path may refer to ctx members. For example, ctx.Path().
  163. type PathRewriteFunc func(ctx *RequestCtx) []byte
  164. // NewVHostPathRewriter returns path rewriter, which strips slashesCount
  165. // leading slashes from the path and prepends the path with request's host,
  166. // thus simplifying virtual hosting for static files.
  167. //
  168. // Examples:
  169. //
  170. // - host=foobar.com, slashesCount=0, original path="/foo/bar".
  171. // Resulting path: "/foobar.com/foo/bar"
  172. //
  173. // - host=img.aaa.com, slashesCount=1, original path="/images/123/456.jpg"
  174. // Resulting path: "/img.aaa.com/123/456.jpg"
  175. func NewVHostPathRewriter(slashesCount int) PathRewriteFunc {
  176. return func(ctx *RequestCtx) []byte {
  177. path := stripLeadingSlashes(ctx.Path(), slashesCount)
  178. host := ctx.Host()
  179. if n := bytes.IndexByte(host, '/'); n >= 0 {
  180. host = nil
  181. }
  182. if len(host) == 0 {
  183. host = strInvalidHost
  184. }
  185. b := bytebufferpool.Get()
  186. b.B = append(b.B, '/')
  187. b.B = append(b.B, host...)
  188. b.B = append(b.B, path...)
  189. ctx.URI().SetPathBytes(b.B)
  190. bytebufferpool.Put(b)
  191. return ctx.Path()
  192. }
  193. }
  194. var strInvalidHost = []byte("invalid-host")
  195. // NewPathSlashesStripper returns path rewriter, which strips slashesCount
  196. // leading slashes from the path.
  197. //
  198. // Examples:
  199. //
  200. // - slashesCount = 0, original path: "/foo/bar", result: "/foo/bar"
  201. // - slashesCount = 1, original path: "/foo/bar", result: "/bar"
  202. // - slashesCount = 2, original path: "/foo/bar", result: ""
  203. //
  204. // The returned path rewriter may be used as FS.PathRewrite .
  205. func NewPathSlashesStripper(slashesCount int) PathRewriteFunc {
  206. return func(ctx *RequestCtx) []byte {
  207. return stripLeadingSlashes(ctx.Path(), slashesCount)
  208. }
  209. }
  210. // NewPathPrefixStripper returns path rewriter, which removes prefixSize bytes
  211. // from the path prefix.
  212. //
  213. // Examples:
  214. //
  215. // - prefixSize = 0, original path: "/foo/bar", result: "/foo/bar"
  216. // - prefixSize = 3, original path: "/foo/bar", result: "o/bar"
  217. // - prefixSize = 7, original path: "/foo/bar", result: "r"
  218. //
  219. // The returned path rewriter may be used as FS.PathRewrite .
  220. func NewPathPrefixStripper(prefixSize int) PathRewriteFunc {
  221. return func(ctx *RequestCtx) []byte {
  222. path := ctx.Path()
  223. if len(path) >= prefixSize {
  224. path = path[prefixSize:]
  225. }
  226. return path
  227. }
  228. }
  229. // FS represents settings for request handler serving static files
  230. // from the local filesystem.
  231. //
  232. // It is prohibited copying FS values. Create new values instead.
  233. type FS struct {
  234. noCopy noCopy
  235. // FS is filesystem to serve files from. eg: embed.FS os.DirFS
  236. FS fs.FS
  237. // Path rewriting function.
  238. //
  239. // By default request path is not modified.
  240. PathRewrite PathRewriteFunc
  241. // PathNotFound fires when file is not found in filesystem
  242. // this functions tries to replace "Cannot open requested path"
  243. // server response giving to the programmer the control of server flow.
  244. //
  245. // By default PathNotFound returns
  246. // "Cannot open requested path"
  247. PathNotFound RequestHandler
  248. // Suffixes list to add to compressedFileSuffix depending on encoding
  249. //
  250. // This value has sense only if Compress is set.
  251. //
  252. // FSCompressedFileSuffixes is used by default.
  253. CompressedFileSuffixes map[string]string
  254. // If CleanStop is set, the channel can be closed to stop the cleanup handlers
  255. // for the FS RequestHandlers created with NewRequestHandler.
  256. // NEVER close this channel while the handler is still being used!
  257. CleanStop chan struct{}
  258. h RequestHandler
  259. // Path to the root directory to serve files from.
  260. Root string
  261. // Path to the compressed root directory to serve files from. If this value
  262. // is empty, Root is used.
  263. CompressRoot string
  264. // Suffix to add to the name of cached compressed file.
  265. //
  266. // This value has sense only if Compress is set.
  267. //
  268. // FSCompressedFileSuffix is used by default.
  269. CompressedFileSuffix string
  270. // List of index file names to try opening during directory access.
  271. //
  272. // For example:
  273. //
  274. // * index.html
  275. // * index.htm
  276. // * my-super-index.xml
  277. //
  278. // By default the list is empty.
  279. IndexNames []string
  280. // Expiration duration for inactive file handlers.
  281. //
  282. // FSHandlerCacheDuration is used by default.
  283. CacheDuration time.Duration
  284. once sync.Once
  285. // AllowEmptyRoot controls what happens when Root is empty. When false (default) it will default to the
  286. // current working directory. An empty root is mostly useful when you want to use absolute paths
  287. // on windows that are on different filesystems. On linux setting your Root to "/" already allows you to use
  288. // absolute paths on any filesystem.
  289. AllowEmptyRoot bool
  290. // Uses brotli encoding and fallbacks to gzip in responses if set to true, uses gzip if set to false.
  291. //
  292. // This value has sense only if Compress is set.
  293. //
  294. // Brotli encoding is disabled by default.
  295. CompressBrotli bool
  296. // Index pages for directories without files matching IndexNames
  297. // are automatically generated if set.
  298. //
  299. // Directory index generation may be quite slow for directories
  300. // with many files (more than 1K), so it is discouraged enabling
  301. // index pages' generation for such directories.
  302. //
  303. // By default index pages aren't generated.
  304. GenerateIndexPages bool
  305. // Transparently compresses responses if set to true.
  306. //
  307. // The server tries minimizing CPU usage by caching compressed files.
  308. // It adds CompressedFileSuffix suffix to the original file name and
  309. // tries saving the resulting compressed file under the new file name.
  310. // So it is advisable to give the server write access to Root
  311. // and to all inner folders in order to minimize CPU usage when serving
  312. // compressed responses.
  313. //
  314. // Transparent compression is disabled by default.
  315. Compress bool
  316. // Enables byte range requests if set to true.
  317. //
  318. // Byte range requests are disabled by default.
  319. AcceptByteRange bool
  320. // SkipCache if true, will cache no file handler.
  321. //
  322. // By default is false.
  323. SkipCache bool
  324. }
  325. // FSCompressedFileSuffix is the suffix FS adds to the original file names
  326. // when trying to store compressed file under the new file name.
  327. // See FS.Compress for details.
  328. const FSCompressedFileSuffix = ".fasthttp.gz"
  329. // FSCompressedFileSuffixes is the suffixes FS adds to the original file names depending on encoding
  330. // when trying to store compressed file under the new file name.
  331. // See FS.Compress for details.
  332. var FSCompressedFileSuffixes = map[string]string{
  333. "gzip": ".fasthttp.gz",
  334. "br": ".fasthttp.br",
  335. "zstd": ".fasthttp.zst",
  336. }
  337. // FSHandlerCacheDuration is the default expiration duration for inactive
  338. // file handlers opened by FS.
  339. const FSHandlerCacheDuration = 10 * time.Second
  340. // FSHandler returns request handler serving static files from
  341. // the given root folder.
  342. //
  343. // stripSlashes indicates how many leading slashes must be stripped
  344. // from requested path before searching requested file in the root folder.
  345. // Examples:
  346. //
  347. // - stripSlashes = 0, original path: "/foo/bar", result: "/foo/bar"
  348. // - stripSlashes = 1, original path: "/foo/bar", result: "/bar"
  349. // - stripSlashes = 2, original path: "/foo/bar", result: ""
  350. //
  351. // The returned request handler automatically generates index pages
  352. // for directories without index.html.
  353. //
  354. // The returned handler caches requested file handles
  355. // for FSHandlerCacheDuration.
  356. // Make sure your program has enough 'max open files' limit aka
  357. // 'ulimit -n' if root folder contains many files.
  358. //
  359. // Do not create multiple request handler instances for the same
  360. // (root, stripSlashes) arguments - just reuse a single instance.
  361. // Otherwise goroutine leak will occur.
  362. func FSHandler(root string, stripSlashes int) RequestHandler {
  363. fs := &FS{
  364. Root: root,
  365. IndexNames: []string{"index.html"},
  366. GenerateIndexPages: true,
  367. AcceptByteRange: true,
  368. }
  369. if stripSlashes > 0 {
  370. fs.PathRewrite = NewPathSlashesStripper(stripSlashes)
  371. }
  372. return fs.NewRequestHandler()
  373. }
  374. // NewRequestHandler returns new request handler with the given FS settings.
  375. //
  376. // The returned handler caches requested file handles
  377. // for FS.CacheDuration.
  378. // Make sure your program has enough 'max open files' limit aka
  379. // 'ulimit -n' if FS.Root folder contains many files.
  380. //
  381. // Do not create multiple request handlers from a single FS instance -
  382. // just reuse a single request handler.
  383. func (fs *FS) NewRequestHandler() RequestHandler {
  384. fs.once.Do(fs.initRequestHandler)
  385. return fs.h
  386. }
  387. func (fs *FS) normalizeRoot(root string) string {
  388. // fs.FS uses relative paths, that paths are slash-separated on all systems, even Windows.
  389. if fs.FS == nil {
  390. // Serve files from the current working directory if Root is empty or if Root is a relative path.
  391. if (!fs.AllowEmptyRoot && root == "") || (root != "" && !filepath.IsAbs(root)) {
  392. path, err := os.Getwd()
  393. if err != nil {
  394. path = "."
  395. }
  396. root = path + "/" + root
  397. }
  398. // convert the root directory slashes to the native format
  399. root = filepath.FromSlash(root)
  400. }
  401. // strip trailing slashes from the root path
  402. for root != "" && root[len(root)-1] == os.PathSeparator {
  403. root = root[:len(root)-1]
  404. }
  405. return root
  406. }
  407. func (fs *FS) initRequestHandler() {
  408. root := fs.normalizeRoot(fs.Root)
  409. compressRoot := fs.CompressRoot
  410. if compressRoot == "" {
  411. compressRoot = root
  412. } else {
  413. compressRoot = fs.normalizeRoot(compressRoot)
  414. }
  415. compressedFileSuffixes := fs.CompressedFileSuffixes
  416. if compressedFileSuffixes["br"] == "" || compressedFileSuffixes["gzip"] == "" ||
  417. compressedFileSuffixes["zstd"] == "" || compressedFileSuffixes["br"] == compressedFileSuffixes["gzip"] ||
  418. compressedFileSuffixes["br"] == compressedFileSuffixes["zstd"] ||
  419. compressedFileSuffixes["gzip"] == compressedFileSuffixes["zstd"] {
  420. // Copy global map
  421. compressedFileSuffixes = make(map[string]string, len(FSCompressedFileSuffixes))
  422. for k, v := range FSCompressedFileSuffixes {
  423. compressedFileSuffixes[k] = v
  424. }
  425. }
  426. if fs.CompressedFileSuffix != "" {
  427. compressedFileSuffixes["gzip"] = fs.CompressedFileSuffix
  428. compressedFileSuffixes["br"] = FSCompressedFileSuffixes["br"]
  429. compressedFileSuffixes["zstd"] = FSCompressedFileSuffixes["zstd"]
  430. }
  431. h := &fsHandler{
  432. filesystem: fs.FS,
  433. root: root,
  434. indexNames: fs.IndexNames,
  435. pathRewrite: fs.PathRewrite,
  436. generateIndexPages: fs.GenerateIndexPages,
  437. compress: fs.Compress,
  438. compressBrotli: fs.CompressBrotli,
  439. compressRoot: compressRoot,
  440. pathNotFound: fs.PathNotFound,
  441. acceptByteRange: fs.AcceptByteRange,
  442. compressedFileSuffixes: compressedFileSuffixes,
  443. }
  444. h.cacheManager = newCacheManager(fs)
  445. if h.filesystem == nil {
  446. h.filesystem = &osFS{} // It provides os.Open and os.Stat
  447. }
  448. fs.h = h.handleRequest
  449. }
  450. type fsHandler struct {
  451. smallFileReaderPool sync.Pool
  452. filesystem fs.FS
  453. cacheManager cacheManager
  454. pathRewrite PathRewriteFunc
  455. pathNotFound RequestHandler
  456. compressedFileSuffixes map[string]string
  457. root string
  458. compressRoot string
  459. indexNames []string
  460. generateIndexPages bool
  461. compress bool
  462. compressBrotli bool
  463. acceptByteRange bool
  464. }
  465. type fsFile struct {
  466. lastModified time.Time
  467. t time.Time
  468. f fs.File
  469. h *fsHandler
  470. filename string // fs.FileInfo.Name() return filename, isn't filepath.
  471. contentType string
  472. dirIndex []byte
  473. lastModifiedStr []byte
  474. bigFiles []*bigFileReader
  475. contentLength int
  476. readersCount int
  477. bigFilesLock sync.Mutex
  478. compressed bool
  479. }
  480. func (ff *fsFile) NewReader() (io.Reader, error) {
  481. if ff.isBig() {
  482. r, err := ff.bigFileReader()
  483. if err != nil {
  484. ff.decReadersCount()
  485. }
  486. return r, err
  487. }
  488. return ff.smallFileReader()
  489. }
  490. func (ff *fsFile) smallFileReader() (io.Reader, error) {
  491. v := ff.h.smallFileReaderPool.Get()
  492. if v == nil {
  493. v = &fsSmallFileReader{}
  494. }
  495. r := v.(*fsSmallFileReader)
  496. r.ff = ff
  497. r.endPos = ff.contentLength
  498. if r.startPos > 0 {
  499. return nil, errors.New("bug: fsSmallFileReader with non-nil startPos found in the pool")
  500. }
  501. return r, nil
  502. }
  503. // Files bigger than this size are sent with sendfile.
  504. const maxSmallFileSize = 2 * 4096
  505. func (ff *fsFile) isBig() bool {
  506. if _, ok := ff.h.filesystem.(*osFS); !ok { // fs.FS only uses bigFileReader, memory cache uses fsSmallFileReader
  507. return ff.f != nil
  508. }
  509. return ff.contentLength > maxSmallFileSize && len(ff.dirIndex) == 0
  510. }
  511. func (ff *fsFile) bigFileReader() (io.Reader, error) {
  512. if ff.f == nil {
  513. return nil, errors.New("bug: ff.f must be non-nil in bigFileReader")
  514. }
  515. var r io.Reader
  516. ff.bigFilesLock.Lock()
  517. n := len(ff.bigFiles)
  518. if n > 0 {
  519. r = ff.bigFiles[n-1]
  520. ff.bigFiles = ff.bigFiles[:n-1]
  521. }
  522. ff.bigFilesLock.Unlock()
  523. if r != nil {
  524. return r, nil
  525. }
  526. f, err := ff.h.filesystem.Open(ff.filename)
  527. if err != nil {
  528. return nil, fmt.Errorf("cannot open already opened file: %w", err)
  529. }
  530. return &bigFileReader{
  531. f: f,
  532. ff: ff,
  533. r: f,
  534. }, nil
  535. }
  536. func (ff *fsFile) Release() {
  537. if ff.f != nil {
  538. _ = ff.f.Close()
  539. if ff.isBig() {
  540. ff.bigFilesLock.Lock()
  541. for _, r := range ff.bigFiles {
  542. _ = r.f.Close()
  543. }
  544. ff.bigFilesLock.Unlock()
  545. }
  546. }
  547. }
  548. func (ff *fsFile) decReadersCount() {
  549. ff.h.cacheManager.WithLock(func() {
  550. ff.readersCount--
  551. if ff.readersCount < 0 {
  552. ff.readersCount = 0
  553. }
  554. })
  555. }
  556. // bigFileReader attempts to trigger sendfile
  557. // for sending big files over the wire.
  558. type bigFileReader struct {
  559. f fs.File
  560. ff *fsFile
  561. r io.Reader
  562. lr io.LimitedReader
  563. }
  564. func (r *bigFileReader) UpdateByteRange(startPos, endPos int) error {
  565. seeker, ok := r.f.(io.Seeker)
  566. if !ok {
  567. return errors.New("must implement io.Seeker")
  568. }
  569. if _, err := seeker.Seek(int64(startPos), io.SeekStart); err != nil {
  570. return err
  571. }
  572. r.r = &r.lr
  573. r.lr.R = r.f
  574. r.lr.N = int64(endPos - startPos + 1)
  575. return nil
  576. }
  577. func (r *bigFileReader) Read(p []byte) (int, error) {
  578. return r.r.Read(p)
  579. }
  580. func (r *bigFileReader) WriteTo(w io.Writer) (int64, error) {
  581. if rf, ok := w.(io.ReaderFrom); ok {
  582. // fast path. Send file must be triggered
  583. return rf.ReadFrom(r.r)
  584. }
  585. // slow path
  586. return copyZeroAlloc(w, r.r)
  587. }
  588. func (r *bigFileReader) Close() error {
  589. r.r = r.f
  590. seeker, ok := r.f.(io.Seeker)
  591. if !ok {
  592. _ = r.f.Close()
  593. return errors.New("must implement io.Seeker")
  594. }
  595. n, err := seeker.Seek(0, io.SeekStart)
  596. if err == nil {
  597. if n == 0 {
  598. ff := r.ff
  599. ff.bigFilesLock.Lock()
  600. ff.bigFiles = append(ff.bigFiles, r)
  601. ff.bigFilesLock.Unlock()
  602. } else {
  603. _ = r.f.Close()
  604. err = errors.New("bug: File.Seek(0, io.SeekStart) returned (non-zero, nil)")
  605. }
  606. } else {
  607. _ = r.f.Close()
  608. }
  609. r.ff.decReadersCount()
  610. return err
  611. }
  612. type fsSmallFileReader struct {
  613. ff *fsFile
  614. startPos int
  615. endPos int
  616. }
  617. func (r *fsSmallFileReader) Close() error {
  618. ff := r.ff
  619. ff.decReadersCount()
  620. r.ff = nil
  621. r.startPos = 0
  622. r.endPos = 0
  623. ff.h.smallFileReaderPool.Put(r)
  624. return nil
  625. }
  626. func (r *fsSmallFileReader) UpdateByteRange(startPos, endPos int) error {
  627. r.startPos = startPos
  628. r.endPos = endPos + 1
  629. return nil
  630. }
  631. func (r *fsSmallFileReader) Read(p []byte) (int, error) {
  632. tailLen := r.endPos - r.startPos
  633. if tailLen <= 0 {
  634. return 0, io.EOF
  635. }
  636. if len(p) > tailLen {
  637. p = p[:tailLen]
  638. }
  639. ff := r.ff
  640. if ff.f != nil {
  641. ra, ok := ff.f.(io.ReaderAt)
  642. if !ok {
  643. return 0, errors.New("must implement io.ReaderAt")
  644. }
  645. n, err := ra.ReadAt(p, int64(r.startPos))
  646. r.startPos += n
  647. return n, err
  648. }
  649. n := copy(p, ff.dirIndex[r.startPos:])
  650. r.startPos += n
  651. return n, nil
  652. }
  653. func (r *fsSmallFileReader) WriteTo(w io.Writer) (int64, error) {
  654. ff := r.ff
  655. var n int
  656. var err error
  657. if ff.f == nil {
  658. n, err = w.Write(ff.dirIndex[r.startPos:r.endPos])
  659. return int64(n), err
  660. }
  661. if rf, ok := w.(io.ReaderFrom); ok {
  662. return rf.ReadFrom(r)
  663. }
  664. curPos := r.startPos
  665. bufv := copyBufPool.Get()
  666. buf := bufv.([]byte)
  667. for err == nil {
  668. tailLen := r.endPos - curPos
  669. if tailLen <= 0 {
  670. break
  671. }
  672. if len(buf) > tailLen {
  673. buf = buf[:tailLen]
  674. }
  675. ra, ok := ff.f.(io.ReaderAt)
  676. if !ok {
  677. return 0, errors.New("must implement io.ReaderAt")
  678. }
  679. n, err = ra.ReadAt(buf, int64(curPos))
  680. nw, errw := w.Write(buf[:n])
  681. curPos += nw
  682. if errw == nil && nw != n {
  683. errw = errors.New("bug: Write(p) returned (n, nil), where n != len(p)")
  684. }
  685. if err == nil {
  686. err = errw
  687. }
  688. }
  689. copyBufPool.Put(bufv)
  690. if err == io.EOF {
  691. err = nil
  692. }
  693. return int64(curPos - r.startPos), err
  694. }
  695. type cacheManager interface {
  696. WithLock(work func())
  697. GetFileFromCache(cacheKind CacheKind, path string) (*fsFile, bool)
  698. SetFileToCache(cacheKind CacheKind, path string, ff *fsFile) *fsFile
  699. }
  700. var (
  701. _ cacheManager = (*inMemoryCacheManager)(nil)
  702. _ cacheManager = (*noopCacheManager)(nil)
  703. )
  704. type CacheKind uint8
  705. const (
  706. defaultCacheKind CacheKind = iota
  707. brotliCacheKind
  708. gzipCacheKind
  709. zstdCacheKind
  710. )
  711. func newCacheManager(fs *FS) cacheManager {
  712. if fs.SkipCache {
  713. return &noopCacheManager{}
  714. }
  715. cacheDuration := fs.CacheDuration
  716. if cacheDuration <= 0 {
  717. cacheDuration = FSHandlerCacheDuration
  718. }
  719. instance := &inMemoryCacheManager{
  720. cacheDuration: cacheDuration,
  721. cache: make(map[string]*fsFile),
  722. cacheBrotli: make(map[string]*fsFile),
  723. cacheGzip: make(map[string]*fsFile),
  724. cacheZstd: make(map[string]*fsFile),
  725. }
  726. go instance.handleCleanCache(fs.CleanStop)
  727. return instance
  728. }
  729. type noopCacheManager struct {
  730. cacheLock sync.Mutex
  731. }
  732. func (n *noopCacheManager) WithLock(work func()) {
  733. n.cacheLock.Lock()
  734. work()
  735. n.cacheLock.Unlock()
  736. }
  737. func (*noopCacheManager) GetFileFromCache(cacheKind CacheKind, path string) (*fsFile, bool) {
  738. return nil, false
  739. }
  740. func (*noopCacheManager) SetFileToCache(cacheKind CacheKind, path string, ff *fsFile) *fsFile {
  741. return ff
  742. }
  743. type inMemoryCacheManager struct {
  744. cache map[string]*fsFile
  745. cacheBrotli map[string]*fsFile
  746. cacheGzip map[string]*fsFile
  747. cacheZstd map[string]*fsFile
  748. cacheDuration time.Duration
  749. cacheLock sync.Mutex
  750. }
  751. func (cm *inMemoryCacheManager) WithLock(work func()) {
  752. cm.cacheLock.Lock()
  753. work()
  754. cm.cacheLock.Unlock()
  755. }
  756. func (cm *inMemoryCacheManager) getFsCache(cacheKind CacheKind) map[string]*fsFile {
  757. fileCache := cm.cache
  758. switch cacheKind {
  759. case brotliCacheKind:
  760. fileCache = cm.cacheBrotli
  761. case gzipCacheKind:
  762. fileCache = cm.cacheGzip
  763. case zstdCacheKind:
  764. fileCache = cm.cacheZstd
  765. }
  766. return fileCache
  767. }
  768. func (cm *inMemoryCacheManager) GetFileFromCache(cacheKind CacheKind, path string) (*fsFile, bool) {
  769. fileCache := cm.getFsCache(cacheKind)
  770. cm.cacheLock.Lock()
  771. ff, ok := fileCache[path]
  772. if ok {
  773. ff.readersCount++
  774. }
  775. cm.cacheLock.Unlock()
  776. return ff, ok
  777. }
  778. func (cm *inMemoryCacheManager) SetFileToCache(cacheKind CacheKind, path string, ff *fsFile) *fsFile {
  779. fileCache := cm.getFsCache(cacheKind)
  780. cm.cacheLock.Lock()
  781. ff1, ok := fileCache[path]
  782. if !ok {
  783. fileCache[path] = ff
  784. ff.readersCount++
  785. } else {
  786. ff1.readersCount++
  787. }
  788. cm.cacheLock.Unlock()
  789. if ok {
  790. // The file has been already opened by another
  791. // goroutine, so close the current file and use
  792. // the file opened by another goroutine instead.
  793. ff.Release()
  794. ff = ff1
  795. }
  796. return ff
  797. }
  798. func (cm *inMemoryCacheManager) handleCleanCache(cleanStop chan struct{}) {
  799. var pendingFiles []*fsFile
  800. clean := func() {
  801. pendingFiles = cm.cleanCache(pendingFiles)
  802. }
  803. if cleanStop != nil {
  804. t := time.NewTicker(cm.cacheDuration / 2)
  805. for {
  806. select {
  807. case <-t.C:
  808. clean()
  809. case _, stillOpen := <-cleanStop:
  810. // Ignore values send on the channel, only stop when it is closed.
  811. if !stillOpen {
  812. t.Stop()
  813. return
  814. }
  815. }
  816. }
  817. }
  818. for {
  819. time.Sleep(cm.cacheDuration / 2)
  820. clean()
  821. }
  822. }
  823. func (cm *inMemoryCacheManager) cleanCache(pendingFiles []*fsFile) []*fsFile {
  824. var filesToRelease []*fsFile
  825. cm.cacheLock.Lock()
  826. // Close files which couldn't be closed before due to non-zero
  827. // readers count on the previous run.
  828. var remainingFiles []*fsFile
  829. for _, ff := range pendingFiles {
  830. if ff.readersCount > 0 {
  831. remainingFiles = append(remainingFiles, ff)
  832. } else {
  833. filesToRelease = append(filesToRelease, ff)
  834. }
  835. }
  836. pendingFiles = remainingFiles
  837. pendingFiles, filesToRelease = cleanCacheNolock(cm.cache, pendingFiles, filesToRelease, cm.cacheDuration)
  838. pendingFiles, filesToRelease = cleanCacheNolock(cm.cacheBrotli, pendingFiles, filesToRelease, cm.cacheDuration)
  839. pendingFiles, filesToRelease = cleanCacheNolock(cm.cacheGzip, pendingFiles, filesToRelease, cm.cacheDuration)
  840. pendingFiles, filesToRelease = cleanCacheNolock(cm.cacheZstd, pendingFiles, filesToRelease, cm.cacheDuration)
  841. cm.cacheLock.Unlock()
  842. for _, ff := range filesToRelease {
  843. ff.Release()
  844. }
  845. return pendingFiles
  846. }
  847. func cleanCacheNolock(
  848. cache map[string]*fsFile, pendingFiles, filesToRelease []*fsFile, cacheDuration time.Duration,
  849. ) ([]*fsFile, []*fsFile) {
  850. t := time.Now()
  851. for k, ff := range cache {
  852. if t.Sub(ff.t) > cacheDuration {
  853. if ff.readersCount > 0 {
  854. // There are pending readers on stale file handle,
  855. // so we cannot close it. Put it into pendingFiles
  856. // so it will be closed later.
  857. pendingFiles = append(pendingFiles, ff)
  858. } else {
  859. filesToRelease = append(filesToRelease, ff)
  860. }
  861. delete(cache, k)
  862. }
  863. }
  864. return pendingFiles, filesToRelease
  865. }
  866. func (h *fsHandler) pathToFilePath(path string) string {
  867. if _, ok := h.filesystem.(*osFS); !ok {
  868. if len(path) < 1 {
  869. return path
  870. }
  871. return path[1:]
  872. }
  873. return filepath.FromSlash(h.root + path)
  874. }
  875. func (h *fsHandler) filePathToCompressed(filePath string) string {
  876. if h.root == h.compressRoot {
  877. return filePath
  878. }
  879. if !strings.HasPrefix(filePath, h.root) {
  880. return filePath
  881. }
  882. return filepath.FromSlash(h.compressRoot + filePath[len(h.root):])
  883. }
  884. func (h *fsHandler) handleRequest(ctx *RequestCtx) {
  885. var path []byte
  886. if h.pathRewrite != nil {
  887. path = h.pathRewrite(ctx)
  888. } else {
  889. path = ctx.Path()
  890. }
  891. hasTrailingSlash := len(path) > 0 && path[len(path)-1] == '/'
  892. if n := bytes.IndexByte(path, 0); n >= 0 {
  893. ctx.Logger().Printf("cannot serve path with nil byte at position %d: %q", n, path)
  894. ctx.Error("Are you a hacker?", StatusBadRequest)
  895. return
  896. }
  897. if h.pathRewrite != nil {
  898. // There is no need to check for '/../' if path = ctx.Path(),
  899. // since ctx.Path must normalize and sanitize the path.
  900. if n := bytes.Index(path, strSlashDotDotSlash); n >= 0 {
  901. ctx.Logger().Printf("cannot serve path with '/../' at position %d due to security reasons: %q", n, path)
  902. ctx.Error("Internal Server Error", StatusInternalServerError)
  903. return
  904. }
  905. }
  906. mustCompress := false
  907. fileCacheKind := defaultCacheKind
  908. fileEncoding := ""
  909. byteRange := ctx.Request.Header.peek(strRange)
  910. if len(byteRange) == 0 && h.compress {
  911. switch {
  912. case h.compressBrotli && ctx.Request.Header.HasAcceptEncodingBytes(strBr):
  913. mustCompress = true
  914. fileCacheKind = brotliCacheKind
  915. fileEncoding = "br"
  916. case ctx.Request.Header.HasAcceptEncodingBytes(strGzip):
  917. mustCompress = true
  918. fileCacheKind = gzipCacheKind
  919. fileEncoding = "gzip"
  920. case ctx.Request.Header.HasAcceptEncodingBytes(strZstd):
  921. mustCompress = true
  922. fileCacheKind = zstdCacheKind
  923. fileEncoding = "zstd"
  924. }
  925. }
  926. originalPathStr := string(path)
  927. pathStr := originalPathStr
  928. if hasTrailingSlash {
  929. pathStr = originalPathStr[:len(originalPathStr)-1]
  930. }
  931. ff, ok := h.cacheManager.GetFileFromCache(fileCacheKind, originalPathStr)
  932. if !ok {
  933. filePath := h.pathToFilePath(pathStr)
  934. var err error
  935. ff, err = h.openFSFile(filePath, mustCompress, fileEncoding)
  936. if mustCompress && err == errNoCreatePermission {
  937. ctx.Logger().Printf("insufficient permissions for saving compressed file for %q. Serving uncompressed file. "+
  938. "Allow write access to the directory with this file in order to improve fasthttp performance", filePath)
  939. mustCompress = false
  940. ff, err = h.openFSFile(filePath, mustCompress, fileEncoding)
  941. }
  942. if errors.Is(err, errDirIndexRequired) {
  943. if !hasTrailingSlash {
  944. ctx.RedirectBytes(append(path, '/'), StatusFound)
  945. return
  946. }
  947. ff, err = h.openIndexFile(ctx, filePath, mustCompress, fileEncoding)
  948. if err != nil {
  949. ctx.Logger().Printf("cannot open dir index %q: %v", filePath, err)
  950. ctx.Error("Directory index is forbidden", StatusForbidden)
  951. return
  952. }
  953. } else if err != nil {
  954. ctx.Logger().Printf("cannot open file %q: %v", filePath, err)
  955. if h.pathNotFound == nil {
  956. ctx.Error("Cannot open requested path", StatusNotFound)
  957. } else {
  958. ctx.SetStatusCode(StatusNotFound)
  959. h.pathNotFound(ctx)
  960. }
  961. return
  962. }
  963. ff = h.cacheManager.SetFileToCache(fileCacheKind, originalPathStr, ff)
  964. }
  965. if !ctx.IfModifiedSince(ff.lastModified) {
  966. ff.decReadersCount()
  967. ctx.NotModified()
  968. return
  969. }
  970. r, err := ff.NewReader()
  971. if err != nil {
  972. ctx.Logger().Printf("cannot obtain file reader for path=%q: %v", path, err)
  973. ctx.Error("Internal Server Error", StatusInternalServerError)
  974. return
  975. }
  976. hdr := &ctx.Response.Header
  977. if ff.compressed {
  978. switch fileEncoding {
  979. case "br":
  980. hdr.SetContentEncodingBytes(strBr)
  981. case "gzip":
  982. hdr.SetContentEncodingBytes(strGzip)
  983. case "zstd":
  984. hdr.SetContentEncodingBytes(strZstd)
  985. }
  986. }
  987. statusCode := StatusOK
  988. contentLength := ff.contentLength
  989. if h.acceptByteRange {
  990. hdr.setNonSpecial(strAcceptRanges, strBytes)
  991. if len(byteRange) > 0 {
  992. startPos, endPos, err := ParseByteRange(byteRange, contentLength)
  993. if err != nil {
  994. _ = r.(io.Closer).Close()
  995. ctx.Logger().Printf("cannot parse byte range %q for path=%q: %v", byteRange, path, err)
  996. ctx.Error("Range Not Satisfiable", StatusRequestedRangeNotSatisfiable)
  997. return
  998. }
  999. if err = r.(byteRangeUpdater).UpdateByteRange(startPos, endPos); err != nil {
  1000. _ = r.(io.Closer).Close()
  1001. ctx.Logger().Printf("cannot seek byte range %q for path=%q: %v", byteRange, path, err)
  1002. ctx.Error("Internal Server Error", StatusInternalServerError)
  1003. return
  1004. }
  1005. hdr.SetContentRange(startPos, endPos, contentLength)
  1006. contentLength = endPos - startPos + 1
  1007. statusCode = StatusPartialContent
  1008. }
  1009. }
  1010. hdr.setNonSpecial(strLastModified, ff.lastModifiedStr)
  1011. if !ctx.IsHead() {
  1012. ctx.SetBodyStream(r, contentLength)
  1013. } else {
  1014. ctx.Response.ResetBody()
  1015. ctx.Response.SkipBody = true
  1016. ctx.Response.Header.SetContentLength(contentLength)
  1017. if rc, ok := r.(io.Closer); ok {
  1018. if err := rc.Close(); err != nil {
  1019. ctx.Logger().Printf("cannot close file reader: %v", err)
  1020. ctx.Error("Internal Server Error", StatusInternalServerError)
  1021. return
  1022. }
  1023. }
  1024. }
  1025. hdr.noDefaultContentType = true
  1026. if len(hdr.ContentType()) == 0 {
  1027. ctx.SetContentType(ff.contentType)
  1028. }
  1029. ctx.SetStatusCode(statusCode)
  1030. }
  1031. type byteRangeUpdater interface {
  1032. UpdateByteRange(startPos, endPos int) error
  1033. }
  1034. // ParseByteRange parses 'Range: bytes=...' header value.
  1035. //
  1036. // It follows https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.35 .
  1037. func ParseByteRange(byteRange []byte, contentLength int) (startPos, endPos int, err error) {
  1038. b := byteRange
  1039. if !bytes.HasPrefix(b, strBytes) {
  1040. return 0, 0, fmt.Errorf("unsupported range units: %q. Expecting %q", byteRange, strBytes)
  1041. }
  1042. b = b[len(strBytes):]
  1043. if len(b) == 0 || b[0] != '=' {
  1044. return 0, 0, fmt.Errorf("missing byte range in %q", byteRange)
  1045. }
  1046. b = b[1:]
  1047. n := bytes.IndexByte(b, '-')
  1048. if n < 0 {
  1049. return 0, 0, fmt.Errorf("missing the end position of byte range in %q", byteRange)
  1050. }
  1051. if n == 0 {
  1052. v, err := ParseUint(b[n+1:])
  1053. if err != nil {
  1054. return 0, 0, err
  1055. }
  1056. startPos := contentLength - v
  1057. if startPos < 0 {
  1058. startPos = 0
  1059. }
  1060. return startPos, contentLength - 1, nil
  1061. }
  1062. if startPos, err = ParseUint(b[:n]); err != nil {
  1063. return 0, 0, err
  1064. }
  1065. if startPos >= contentLength {
  1066. return 0, 0, fmt.Errorf("the start position of byte range cannot exceed %d. byte range %q", contentLength-1, byteRange)
  1067. }
  1068. b = b[n+1:]
  1069. if len(b) == 0 {
  1070. return startPos, contentLength - 1, nil
  1071. }
  1072. if endPos, err = ParseUint(b); err != nil {
  1073. return 0, 0, err
  1074. }
  1075. if endPos >= contentLength {
  1076. endPos = contentLength - 1
  1077. }
  1078. if endPos < startPos {
  1079. return 0, 0, fmt.Errorf("the start position of byte range cannot exceed the end position. byte range %q", byteRange)
  1080. }
  1081. return startPos, endPos, nil
  1082. }
  1083. func (h *fsHandler) openIndexFile(ctx *RequestCtx, dirPath string, mustCompress bool, fileEncoding string) (*fsFile, error) {
  1084. for _, indexName := range h.indexNames {
  1085. indexFilePath := indexName
  1086. if dirPath != "" {
  1087. indexFilePath = dirPath + "/" + indexName
  1088. }
  1089. ff, err := h.openFSFile(indexFilePath, mustCompress, fileEncoding)
  1090. if err == nil {
  1091. return ff, nil
  1092. }
  1093. if mustCompress && err == errNoCreatePermission {
  1094. ctx.Logger().Printf("insufficient permissions for saving compressed file for %q. Serving uncompressed file. "+
  1095. "Allow write access to the directory with this file in order to improve fasthttp performance", indexFilePath)
  1096. mustCompress = false
  1097. return h.openFSFile(indexFilePath, mustCompress, fileEncoding)
  1098. }
  1099. if !errors.Is(err, fs.ErrNotExist) {
  1100. return nil, fmt.Errorf("cannot open file %q: %w", indexFilePath, err)
  1101. }
  1102. }
  1103. if !h.generateIndexPages {
  1104. return nil, fmt.Errorf("cannot access directory without index page. Directory %q", dirPath)
  1105. }
  1106. return h.createDirIndex(ctx, dirPath, mustCompress, fileEncoding)
  1107. }
  1108. var (
  1109. errDirIndexRequired = errors.New("directory index required")
  1110. errNoCreatePermission = errors.New("no 'create file' permissions")
  1111. )
  1112. func (h *fsHandler) createDirIndex(ctx *RequestCtx, dirPath string, mustCompress bool, fileEncoding string) (*fsFile, error) {
  1113. w := &bytebufferpool.ByteBuffer{}
  1114. base := ctx.URI()
  1115. // io/fs doesn't support ReadDir with empty path.
  1116. if dirPath == "" {
  1117. dirPath = "."
  1118. }
  1119. basePathEscaped := html.EscapeString(string(base.Path()))
  1120. _, _ = fmt.Fprintf(w, "<html><head><title>%s</title><style>.dir { font-weight: bold }</style></head><body>", basePathEscaped)
  1121. _, _ = fmt.Fprintf(w, "<h1>%s</h1>", basePathEscaped)
  1122. _, _ = fmt.Fprintf(w, "<ul>")
  1123. if len(basePathEscaped) > 1 {
  1124. var parentURI URI
  1125. base.CopyTo(&parentURI)
  1126. parentURI.Update(string(base.Path()) + "/..")
  1127. parentPathEscaped := html.EscapeString(string(parentURI.Path()))
  1128. _, _ = fmt.Fprintf(w, `<li><a href="%s" class="dir">..</a></li>`, parentPathEscaped)
  1129. }
  1130. dirEntries, err := fs.ReadDir(h.filesystem, dirPath)
  1131. if err != nil {
  1132. return nil, err
  1133. }
  1134. fm := make(map[string]fs.FileInfo, len(dirEntries))
  1135. filenames := make([]string, 0, len(dirEntries))
  1136. nestedContinue:
  1137. for _, de := range dirEntries {
  1138. name := de.Name()
  1139. for _, cfs := range h.compressedFileSuffixes {
  1140. if strings.HasSuffix(name, cfs) {
  1141. // Do not show compressed files on index page.
  1142. continue nestedContinue
  1143. }
  1144. }
  1145. fi, err := de.Info()
  1146. if err != nil {
  1147. ctx.Logger().Printf("cannot fetch information from dir entry %q: %v, skip", name, err)
  1148. continue nestedContinue
  1149. }
  1150. fm[name] = fi
  1151. filenames = append(filenames, name)
  1152. }
  1153. var u URI
  1154. base.CopyTo(&u)
  1155. u.Update(string(u.Path()) + "/")
  1156. sort.Strings(filenames)
  1157. for _, name := range filenames {
  1158. u.Update(name)
  1159. pathEscaped := html.EscapeString(string(u.Path()))
  1160. fi := fm[name]
  1161. auxStr := "dir"
  1162. className := "dir"
  1163. if !fi.IsDir() {
  1164. auxStr = fmt.Sprintf("file, %d bytes", fi.Size())
  1165. className = "file"
  1166. }
  1167. _, _ = fmt.Fprintf(w, `<li><a href="%s" class="%s">%s</a>, %s, last modified %s</li>`,
  1168. pathEscaped, className, html.EscapeString(name), auxStr, fsModTime(fi.ModTime()))
  1169. }
  1170. _, _ = fmt.Fprintf(w, "</ul></body></html>")
  1171. if mustCompress {
  1172. var zbuf bytebufferpool.ByteBuffer
  1173. switch fileEncoding {
  1174. case "br":
  1175. zbuf.B = AppendBrotliBytesLevel(zbuf.B, w.B, CompressDefaultCompression)
  1176. case "gzip":
  1177. zbuf.B = AppendGzipBytesLevel(zbuf.B, w.B, CompressDefaultCompression)
  1178. case "zstd":
  1179. zbuf.B = AppendZstdBytesLevel(zbuf.B, w.B, CompressZstdDefault)
  1180. }
  1181. w = &zbuf
  1182. }
  1183. dirIndex := w.B
  1184. lastModified := time.Now()
  1185. ff := &fsFile{
  1186. h: h,
  1187. dirIndex: dirIndex,
  1188. contentType: "text/html; charset=utf-8",
  1189. contentLength: len(dirIndex),
  1190. compressed: mustCompress,
  1191. lastModified: lastModified,
  1192. lastModifiedStr: AppendHTTPDate(nil, lastModified),
  1193. t: lastModified,
  1194. }
  1195. return ff, nil
  1196. }
  1197. const (
  1198. fsMinCompressRatio = 0.8
  1199. fsMaxCompressibleFileSize = 8 * 1024 * 1024
  1200. )
  1201. func (h *fsHandler) compressAndOpenFSFile(filePath, fileEncoding string) (*fsFile, error) {
  1202. f, err := h.filesystem.Open(filePath)
  1203. if err != nil {
  1204. return nil, err
  1205. }
  1206. fileInfo, err := f.Stat()
  1207. if err != nil {
  1208. _ = f.Close()
  1209. return nil, fmt.Errorf("cannot obtain info for file %q: %w", filePath, err)
  1210. }
  1211. if fileInfo.IsDir() {
  1212. _ = f.Close()
  1213. return nil, errDirIndexRequired
  1214. }
  1215. if strings.HasSuffix(filePath, h.compressedFileSuffixes[fileEncoding]) ||
  1216. fileInfo.Size() > fsMaxCompressibleFileSize ||
  1217. !isFileCompressible(f, fsMinCompressRatio) {
  1218. return h.newFSFile(f, fileInfo, false, filePath, "")
  1219. }
  1220. compressedFilePath := h.filePathToCompressed(filePath)
  1221. if _, ok := h.filesystem.(*osFS); !ok {
  1222. return h.newCompressedFSFileCache(f, fileInfo, compressedFilePath, fileEncoding)
  1223. }
  1224. if compressedFilePath != filePath {
  1225. if err := os.MkdirAll(filepath.Dir(compressedFilePath), 0o750); err != nil {
  1226. return nil, err
  1227. }
  1228. }
  1229. compressedFilePath += h.compressedFileSuffixes[fileEncoding]
  1230. absPath, err := filepath.Abs(compressedFilePath)
  1231. if err != nil {
  1232. _ = f.Close()
  1233. return nil, fmt.Errorf("cannot determine absolute path for %q: %v", compressedFilePath, err)
  1234. }
  1235. flock := getFileLock(absPath)
  1236. flock.Lock()
  1237. ff, err := h.compressFileNolock(f, fileInfo, filePath, compressedFilePath, fileEncoding)
  1238. flock.Unlock()
  1239. return ff, err
  1240. }
  1241. func (h *fsHandler) compressFileNolock(
  1242. f fs.File, fileInfo fs.FileInfo, filePath, compressedFilePath, fileEncoding string,
  1243. ) (*fsFile, error) {
  1244. // Attempt to open compressed file created by another concurrent
  1245. // goroutine.
  1246. // It is safe opening such a file, since the file creation
  1247. // is guarded by file mutex - see getFileLock call.
  1248. if _, err := os.Stat(compressedFilePath); err == nil {
  1249. _ = f.Close()
  1250. return h.newCompressedFSFile(compressedFilePath, fileEncoding)
  1251. }
  1252. // Create temporary file, so concurrent goroutines don't use
  1253. // it until it is created.
  1254. tmpFilePath := compressedFilePath + ".tmp"
  1255. zf, err := os.Create(tmpFilePath)
  1256. if err != nil {
  1257. _ = f.Close()
  1258. if !errors.Is(err, fs.ErrPermission) {
  1259. return nil, fmt.Errorf("cannot create temporary file %q: %w", tmpFilePath, err)
  1260. }
  1261. return nil, errNoCreatePermission
  1262. }
  1263. switch fileEncoding {
  1264. case "br":
  1265. zw := acquireStacklessBrotliWriter(zf, CompressDefaultCompression)
  1266. _, err = copyZeroAlloc(zw, f)
  1267. if errf := zw.Flush(); err == nil {
  1268. err = errf
  1269. }
  1270. releaseStacklessBrotliWriter(zw, CompressDefaultCompression)
  1271. case "gzip":
  1272. zw := acquireStacklessGzipWriter(zf, CompressDefaultCompression)
  1273. _, err = copyZeroAlloc(zw, f)
  1274. if errf := zw.Flush(); err == nil {
  1275. err = errf
  1276. }
  1277. releaseStacklessGzipWriter(zw, CompressDefaultCompression)
  1278. case "zstd":
  1279. zw := acquireStacklessZstdWriter(zf, CompressZstdDefault)
  1280. _, err = copyZeroAlloc(zw, f)
  1281. if errf := zw.Flush(); err == nil {
  1282. err = errf
  1283. }
  1284. releaseStacklessZstdWriter(zw, CompressZstdDefault)
  1285. }
  1286. _ = zf.Close()
  1287. _ = f.Close()
  1288. if err != nil {
  1289. return nil, fmt.Errorf("error when compressing file %q to %q: %w", filePath, tmpFilePath, err)
  1290. }
  1291. if err = os.Chtimes(tmpFilePath, time.Now(), fileInfo.ModTime()); err != nil {
  1292. return nil, fmt.Errorf("cannot change modification time to %v for tmp file %q: %v",
  1293. fileInfo.ModTime(), tmpFilePath, err)
  1294. }
  1295. if err = os.Rename(tmpFilePath, compressedFilePath); err != nil {
  1296. return nil, fmt.Errorf("cannot move compressed file from %q to %q: %w", tmpFilePath, compressedFilePath, err)
  1297. }
  1298. return h.newCompressedFSFile(compressedFilePath, fileEncoding)
  1299. }
  1300. // newCompressedFSFileCache use memory cache compressed files.
  1301. func (h *fsHandler) newCompressedFSFileCache(f fs.File, fileInfo fs.FileInfo, filePath, fileEncoding string) (*fsFile, error) {
  1302. var (
  1303. w = &bytebufferpool.ByteBuffer{}
  1304. err error
  1305. )
  1306. switch fileEncoding {
  1307. case "br":
  1308. zw := acquireStacklessBrotliWriter(w, CompressDefaultCompression)
  1309. _, err = copyZeroAlloc(zw, f)
  1310. if errf := zw.Flush(); err == nil {
  1311. err = errf
  1312. }
  1313. releaseStacklessBrotliWriter(zw, CompressDefaultCompression)
  1314. case "gzip":
  1315. zw := acquireStacklessGzipWriter(w, CompressDefaultCompression)
  1316. _, err = copyZeroAlloc(zw, f)
  1317. if errf := zw.Flush(); err == nil {
  1318. err = errf
  1319. }
  1320. releaseStacklessGzipWriter(zw, CompressDefaultCompression)
  1321. case "zstd":
  1322. zw := acquireStacklessZstdWriter(w, CompressZstdDefault)
  1323. _, err = copyZeroAlloc(zw, f)
  1324. if errf := zw.Flush(); err == nil {
  1325. err = errf
  1326. }
  1327. releaseStacklessZstdWriter(zw, CompressZstdDefault)
  1328. }
  1329. defer func() { _ = f.Close() }()
  1330. if err != nil {
  1331. return nil, fmt.Errorf("error when compressing file %q: %w", filePath, err)
  1332. }
  1333. seeker, ok := f.(io.Seeker)
  1334. if !ok {
  1335. return nil, errors.New("not implemented io.Seeker")
  1336. }
  1337. if _, err = seeker.Seek(0, io.SeekStart); err != nil {
  1338. return nil, err
  1339. }
  1340. ext := fileExtension(fileInfo.Name(), false, h.compressedFileSuffixes[fileEncoding])
  1341. contentType := mime.TypeByExtension(ext)
  1342. if contentType == "" {
  1343. data, err := readFileHeader(f, false, fileEncoding)
  1344. if err != nil {
  1345. return nil, fmt.Errorf("cannot read header of the file %q: %w", fileInfo.Name(), err)
  1346. }
  1347. contentType = http.DetectContentType(data)
  1348. }
  1349. dirIndex := w.B
  1350. lastModified := fileInfo.ModTime()
  1351. ff := &fsFile{
  1352. h: h,
  1353. dirIndex: dirIndex,
  1354. contentType: contentType,
  1355. contentLength: len(dirIndex),
  1356. compressed: true,
  1357. lastModified: lastModified,
  1358. lastModifiedStr: AppendHTTPDate(nil, lastModified),
  1359. t: time.Now(),
  1360. }
  1361. return ff, nil
  1362. }
  1363. func (h *fsHandler) newCompressedFSFile(filePath, fileEncoding string) (*fsFile, error) {
  1364. f, err := h.filesystem.Open(filePath)
  1365. if err != nil {
  1366. return nil, fmt.Errorf("cannot open compressed file %q: %w", filePath, err)
  1367. }
  1368. fileInfo, err := f.Stat()
  1369. if err != nil {
  1370. _ = f.Close()
  1371. return nil, fmt.Errorf("cannot obtain info for compressed file %q: %w", filePath, err)
  1372. }
  1373. return h.newFSFile(f, fileInfo, true, filePath, fileEncoding)
  1374. }
  1375. func (h *fsHandler) openFSFile(filePath string, mustCompress bool, fileEncoding string) (*fsFile, error) {
  1376. filePathOriginal := filePath
  1377. if mustCompress {
  1378. filePath += h.compressedFileSuffixes[fileEncoding]
  1379. }
  1380. f, err := h.filesystem.Open(filePath)
  1381. if err != nil {
  1382. if mustCompress && errors.Is(err, fs.ErrNotExist) {
  1383. return h.compressAndOpenFSFile(filePathOriginal, fileEncoding)
  1384. }
  1385. // If the file is not found and the path is empty, let's return errDirIndexRequired error.
  1386. if filePath == "" && (errors.Is(err, fs.ErrNotExist) || errors.Is(err, fs.ErrInvalid)) {
  1387. return nil, errDirIndexRequired
  1388. }
  1389. return nil, err
  1390. }
  1391. fileInfo, err := f.Stat()
  1392. if err != nil {
  1393. _ = f.Close()
  1394. return nil, fmt.Errorf("cannot obtain info for file %q: %w", filePath, err)
  1395. }
  1396. if fileInfo.IsDir() {
  1397. _ = f.Close()
  1398. if mustCompress {
  1399. return nil, fmt.Errorf("directory with unexpected suffix found: %q. Suffix: %q",
  1400. filePath, h.compressedFileSuffixes[fileEncoding])
  1401. }
  1402. return nil, errDirIndexRequired
  1403. }
  1404. if mustCompress {
  1405. fileInfoOriginal, err := fs.Stat(h.filesystem, filePathOriginal)
  1406. if err != nil {
  1407. _ = f.Close()
  1408. return nil, fmt.Errorf("cannot obtain info for original file %q: %w", filePathOriginal, err)
  1409. }
  1410. // Only re-create the compressed file if there was more than a second between the mod times.
  1411. // On macOS the gzip seems to truncate the nanoseconds in the mod time causing the original file
  1412. // to look newer than the gzipped file.
  1413. if fileInfoOriginal.ModTime().Sub(fileInfo.ModTime()) >= time.Second {
  1414. // The compressed file became stale. Re-create it.
  1415. _ = f.Close()
  1416. _ = os.Remove(filePath)
  1417. return h.compressAndOpenFSFile(filePathOriginal, fileEncoding)
  1418. }
  1419. }
  1420. return h.newFSFile(f, fileInfo, mustCompress, filePath, fileEncoding)
  1421. }
  1422. func (h *fsHandler) newFSFile(f fs.File, fileInfo fs.FileInfo, compressed bool, filePath, fileEncoding string) (*fsFile, error) {
  1423. n := fileInfo.Size()
  1424. contentLength := int(n)
  1425. if n != int64(contentLength) {
  1426. _ = f.Close()
  1427. return nil, fmt.Errorf("too big file: %d bytes", n)
  1428. }
  1429. // detect content-type
  1430. ext := fileExtension(fileInfo.Name(), compressed, h.compressedFileSuffixes[fileEncoding])
  1431. contentType := mime.TypeByExtension(ext)
  1432. if contentType == "" {
  1433. data, err := readFileHeader(f, compressed, fileEncoding)
  1434. if err != nil {
  1435. return nil, fmt.Errorf("cannot read header of the file %q: %w", fileInfo.Name(), err)
  1436. }
  1437. contentType = http.DetectContentType(data)
  1438. }
  1439. lastModified := fileInfo.ModTime()
  1440. ff := &fsFile{
  1441. h: h,
  1442. f: f,
  1443. filename: filePath,
  1444. contentType: contentType,
  1445. contentLength: contentLength,
  1446. compressed: compressed,
  1447. lastModified: lastModified,
  1448. lastModifiedStr: AppendHTTPDate(nil, lastModified),
  1449. t: time.Now(),
  1450. }
  1451. return ff, nil
  1452. }
  1453. func readFileHeader(f io.Reader, compressed bool, fileEncoding string) ([]byte, error) {
  1454. r := f
  1455. var (
  1456. br *brotli.Reader
  1457. zr *gzip.Reader
  1458. zsr *zstd.Decoder
  1459. )
  1460. if compressed {
  1461. var err error
  1462. switch fileEncoding {
  1463. case "br":
  1464. if br, err = acquireBrotliReader(f); err != nil {
  1465. return nil, err
  1466. }
  1467. r = br
  1468. case "gzip":
  1469. if zr, err = acquireGzipReader(f); err != nil {
  1470. return nil, err
  1471. }
  1472. r = zr
  1473. case "zstd":
  1474. if zsr, err = acquireZstdReader(f); err != nil {
  1475. return nil, err
  1476. }
  1477. r = zsr
  1478. }
  1479. }
  1480. lr := &io.LimitedReader{
  1481. R: r,
  1482. N: 512,
  1483. }
  1484. data, err := io.ReadAll(lr)
  1485. seeker, ok := f.(io.Seeker)
  1486. if !ok {
  1487. return nil, errors.New("must implement io.Seeker")
  1488. }
  1489. if _, err := seeker.Seek(0, io.SeekStart); err != nil {
  1490. return nil, err
  1491. }
  1492. if br != nil {
  1493. releaseBrotliReader(br)
  1494. }
  1495. if zr != nil {
  1496. releaseGzipReader(zr)
  1497. }
  1498. if zsr != nil {
  1499. releaseZstdReader(zsr)
  1500. }
  1501. return data, err
  1502. }
  1503. func stripLeadingSlashes(path []byte, stripSlashes int) []byte {
  1504. for stripSlashes > 0 && len(path) > 0 {
  1505. if path[0] != '/' {
  1506. // developer sanity-check
  1507. panic("BUG: path must start with slash")
  1508. }
  1509. n := bytes.IndexByte(path[1:], '/')
  1510. if n < 0 {
  1511. path = path[:0]
  1512. break
  1513. }
  1514. path = path[n+1:]
  1515. stripSlashes--
  1516. }
  1517. return path
  1518. }
  1519. func stripTrailingSlashes(path []byte) []byte {
  1520. for len(path) > 0 && path[len(path)-1] == '/' {
  1521. path = path[:len(path)-1]
  1522. }
  1523. return path
  1524. }
  1525. func fileExtension(path string, compressed bool, compressedFileSuffix string) string {
  1526. if compressed && strings.HasSuffix(path, compressedFileSuffix) {
  1527. path = path[:len(path)-len(compressedFileSuffix)]
  1528. }
  1529. n := strings.LastIndexByte(path, '.')
  1530. if n < 0 {
  1531. return ""
  1532. }
  1533. return path[n:]
  1534. }
  1535. // FileLastModified returns last modified time for the file.
  1536. func FileLastModified(path string) (time.Time, error) {
  1537. f, err := os.Open(path)
  1538. if err != nil {
  1539. return zeroTime, err
  1540. }
  1541. fileInfo, err := f.Stat()
  1542. _ = f.Close()
  1543. if err != nil {
  1544. return zeroTime, err
  1545. }
  1546. return fsModTime(fileInfo.ModTime()), nil
  1547. }
  1548. func fsModTime(t time.Time) time.Time {
  1549. return t.In(time.UTC).Truncate(time.Second)
  1550. }
  1551. var filesLockMap sync.Map
  1552. func getFileLock(absPath string) *sync.Mutex {
  1553. v, _ := filesLockMap.LoadOrStore(absPath, &sync.Mutex{})
  1554. filelock := v.(*sync.Mutex)
  1555. return filelock
  1556. }
  1557. var _ fs.FS = (*osFS)(nil)
  1558. type osFS struct{}
  1559. func (o *osFS) Open(name string) (fs.File, error) { return os.Open(name) }
  1560. func (o *osFS) Stat(name string) (fs.FileInfo, error) { return os.Stat(name) }