fs.go 48 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800
  1. package fasthttp
  2. import (
  3. "bytes"
  4. "errors"
  5. "fmt"
  6. "html"
  7. "io"
  8. "io/fs"
  9. "mime"
  10. "net/http"
  11. "os"
  12. "path/filepath"
  13. "sort"
  14. "strings"
  15. "sync"
  16. "time"
  17. "github.com/andybalholm/brotli"
  18. "github.com/klauspost/compress/gzip"
  19. "github.com/klauspost/compress/zstd"
  20. "github.com/valyala/bytebufferpool"
  21. )
  22. // ServeFileBytesUncompressed returns HTTP response containing file contents
  23. // from the given path.
  24. //
  25. // Directory contents is returned if path points to directory.
  26. //
  27. // ServeFileBytes may be used for saving network traffic when serving files
  28. // with good compression ratio.
  29. //
  30. // See also RequestCtx.SendFileBytes.
  31. //
  32. // WARNING: do not pass any user supplied paths to this function!
  33. // WARNING: if path is based on user input users will be able to request
  34. // any file on your filesystem! Use fasthttp.FS with a sane Root instead.
  35. func ServeFileBytesUncompressed(ctx *RequestCtx, path []byte) {
  36. ServeFileUncompressed(ctx, b2s(path))
  37. }
  38. // ServeFileUncompressed returns HTTP response containing file contents
  39. // from the given path.
  40. //
  41. // Directory contents is returned if path points to directory.
  42. //
  43. // ServeFile may be used for saving network traffic when serving files
  44. // with good compression ratio.
  45. //
  46. // See also RequestCtx.SendFile.
  47. //
  48. // WARNING: do not pass any user supplied paths to this function!
  49. // WARNING: if path is based on user input users will be able to request
  50. // any file on your filesystem! Use fasthttp.FS with a sane Root instead.
  51. func ServeFileUncompressed(ctx *RequestCtx, path string) {
  52. ctx.Request.Header.DelBytes(strAcceptEncoding)
  53. ServeFile(ctx, path)
  54. }
  55. // ServeFileBytes returns HTTP response containing compressed file contents
  56. // from the given path.
  57. //
  58. // HTTP response may contain uncompressed file contents in the following cases:
  59. //
  60. // - Missing 'Accept-Encoding: gzip' request header.
  61. // - No write access to directory containing the file.
  62. //
  63. // Directory contents is returned if path points to directory.
  64. //
  65. // Use ServeFileBytesUncompressed is you don't need serving compressed
  66. // file contents.
  67. //
  68. // See also RequestCtx.SendFileBytes.
  69. //
  70. // WARNING: do not pass any user supplied paths to this function!
  71. // WARNING: if path is based on user input users will be able to request
  72. // any file on your filesystem! Use fasthttp.FS with a sane Root instead.
  73. func ServeFileBytes(ctx *RequestCtx, path []byte) {
  74. ServeFile(ctx, b2s(path))
  75. }
  76. // ServeFile returns HTTP response containing compressed file contents
  77. // from the given path.
  78. //
  79. // HTTP response may contain uncompressed file contents in the following cases:
  80. //
  81. // - Missing 'Accept-Encoding: gzip' request header.
  82. // - No write access to directory containing the file.
  83. //
  84. // Directory contents is returned if path points to directory.
  85. //
  86. // Use ServeFileUncompressed is you don't need serving compressed file contents.
  87. //
  88. // See also RequestCtx.SendFile.
  89. //
  90. // WARNING: do not pass any user supplied paths to this function!
  91. // WARNING: if path is based on user input users will be able to request
  92. // any file on your filesystem! Use fasthttp.FS with a sane Root instead.
  93. func ServeFile(ctx *RequestCtx, path string) {
  94. rootFSOnce.Do(func() {
  95. rootFSHandler = rootFS.NewRequestHandler()
  96. })
  97. if path == "" || !filepath.IsAbs(path) {
  98. // extend relative path to absolute path
  99. hasTrailingSlash := path != "" && (path[len(path)-1] == '/' || path[len(path)-1] == '\\')
  100. var err error
  101. path = filepath.FromSlash(path)
  102. if path, err = filepath.Abs(path); err != nil {
  103. ctx.Logger().Printf("cannot resolve path %q to absolute file path: %v", path, err)
  104. ctx.Error("Internal Server Error", StatusInternalServerError)
  105. return
  106. }
  107. if hasTrailingSlash {
  108. path += "/"
  109. }
  110. }
  111. // convert the path to forward slashes regardless the OS in order to set the URI properly
  112. // the handler will convert back to OS path separator before opening the file
  113. path = filepath.ToSlash(path)
  114. ctx.Request.SetRequestURI(path)
  115. rootFSHandler(ctx)
  116. }
  117. var (
  118. rootFSOnce sync.Once
  119. rootFS = &FS{
  120. Root: "",
  121. AllowEmptyRoot: true,
  122. GenerateIndexPages: true,
  123. Compress: true,
  124. CompressBrotli: true,
  125. CompressZstd: true,
  126. AcceptByteRange: true,
  127. }
  128. rootFSHandler RequestHandler
  129. )
  130. // ServeFS returns HTTP response containing compressed file contents from the given fs.FS's path.
  131. //
  132. // HTTP response may contain uncompressed file contents in the following cases:
  133. //
  134. // - Missing 'Accept-Encoding: gzip' request header.
  135. // - No write access to directory containing the file.
  136. //
  137. // Directory contents is returned if path points to directory.
  138. //
  139. // See also ServeFile.
  140. func ServeFS(ctx *RequestCtx, filesystem fs.FS, path string) {
  141. f := &FS{
  142. FS: filesystem,
  143. Root: "",
  144. AllowEmptyRoot: true,
  145. GenerateIndexPages: true,
  146. Compress: true,
  147. CompressBrotli: true,
  148. CompressZstd: true,
  149. AcceptByteRange: true,
  150. }
  151. handler := f.NewRequestHandler()
  152. ctx.Request.SetRequestURI(path)
  153. handler(ctx)
  154. }
  155. // PathRewriteFunc must return new request path based on arbitrary ctx
  156. // info such as ctx.Path().
  157. //
  158. // Path rewriter is used in FS for translating the current request
  159. // to the local filesystem path relative to FS.Root.
  160. //
  161. // The returned path must not contain '/../' substrings due to security reasons,
  162. // since such paths may refer files outside FS.Root.
  163. //
  164. // The returned path may refer to ctx members. For example, ctx.Path().
  165. type PathRewriteFunc func(ctx *RequestCtx) []byte
  166. // NewVHostPathRewriter returns path rewriter, which strips slashesCount
  167. // leading slashes from the path and prepends the path with request's host,
  168. // thus simplifying virtual hosting for static files.
  169. //
  170. // Examples:
  171. //
  172. // - host=foobar.com, slashesCount=0, original path="/foo/bar".
  173. // Resulting path: "/foobar.com/foo/bar"
  174. //
  175. // - host=img.aaa.com, slashesCount=1, original path="/images/123/456.jpg"
  176. // Resulting path: "/img.aaa.com/123/456.jpg"
  177. func NewVHostPathRewriter(slashesCount int) PathRewriteFunc {
  178. return func(ctx *RequestCtx) []byte {
  179. path := stripLeadingSlashes(ctx.Path(), slashesCount)
  180. host := ctx.Host()
  181. if n := bytes.IndexByte(host, '/'); n >= 0 {
  182. host = nil
  183. }
  184. if len(host) == 0 {
  185. host = strInvalidHost
  186. }
  187. b := bytebufferpool.Get()
  188. b.B = append(b.B, '/')
  189. b.B = append(b.B, host...)
  190. b.B = append(b.B, path...)
  191. ctx.URI().SetPathBytes(b.B)
  192. bytebufferpool.Put(b)
  193. return ctx.Path()
  194. }
  195. }
  196. var strInvalidHost = []byte("invalid-host")
  197. // NewPathSlashesStripper returns path rewriter, which strips slashesCount
  198. // leading slashes from the path.
  199. //
  200. // Examples:
  201. //
  202. // - slashesCount = 0, original path: "/foo/bar", result: "/foo/bar"
  203. // - slashesCount = 1, original path: "/foo/bar", result: "/bar"
  204. // - slashesCount = 2, original path: "/foo/bar", result: ""
  205. //
  206. // The returned path rewriter may be used as FS.PathRewrite .
  207. func NewPathSlashesStripper(slashesCount int) PathRewriteFunc {
  208. return func(ctx *RequestCtx) []byte {
  209. return stripLeadingSlashes(ctx.Path(), slashesCount)
  210. }
  211. }
  212. // NewPathPrefixStripper returns path rewriter, which removes prefixSize bytes
  213. // from the path prefix.
  214. //
  215. // Examples:
  216. //
  217. // - prefixSize = 0, original path: "/foo/bar", result: "/foo/bar"
  218. // - prefixSize = 3, original path: "/foo/bar", result: "o/bar"
  219. // - prefixSize = 7, original path: "/foo/bar", result: "r"
  220. //
  221. // The returned path rewriter may be used as FS.PathRewrite .
  222. func NewPathPrefixStripper(prefixSize int) PathRewriteFunc {
  223. return func(ctx *RequestCtx) []byte {
  224. path := ctx.Path()
  225. if len(path) >= prefixSize {
  226. path = path[prefixSize:]
  227. }
  228. return path
  229. }
  230. }
  231. // FS represents settings for request handler serving static files
  232. // from the local filesystem.
  233. //
  234. // It is prohibited copying FS values. Create new values instead.
  235. type FS struct {
  236. noCopy noCopy
  237. // FS is filesystem to serve files from. eg: embed.FS os.DirFS
  238. FS fs.FS
  239. // Path rewriting function.
  240. //
  241. // By default request path is not modified.
  242. PathRewrite PathRewriteFunc
  243. // PathNotFound fires when file is not found in filesystem
  244. // this functions tries to replace "Cannot open requested path"
  245. // server response giving to the programmer the control of server flow.
  246. //
  247. // By default PathNotFound returns
  248. // "Cannot open requested path"
  249. PathNotFound RequestHandler
  250. // Suffixes list to add to compressedFileSuffix depending on encoding
  251. //
  252. // This value has sense only if Compress is set.
  253. //
  254. // FSCompressedFileSuffixes is used by default.
  255. CompressedFileSuffixes map[string]string
  256. // If CleanStop is set, the channel can be closed to stop the cleanup handlers
  257. // for the FS RequestHandlers created with NewRequestHandler.
  258. // NEVER close this channel while the handler is still being used!
  259. CleanStop chan struct{}
  260. h RequestHandler
  261. // Path to the root directory to serve files from.
  262. Root string
  263. // Path to the compressed root directory to serve files from. If this value
  264. // is empty, Root is used.
  265. CompressRoot string
  266. // Suffix to add to the name of cached compressed file.
  267. //
  268. // This value has sense only if Compress is set.
  269. //
  270. // FSCompressedFileSuffix is used by default.
  271. CompressedFileSuffix string
  272. // List of index file names to try opening during directory access.
  273. //
  274. // For example:
  275. //
  276. // * index.html
  277. // * index.htm
  278. // * my-super-index.xml
  279. //
  280. // By default the list is empty.
  281. IndexNames []string
  282. // Expiration duration for inactive file handlers.
  283. //
  284. // FSHandlerCacheDuration is used by default.
  285. CacheDuration time.Duration
  286. once sync.Once
  287. // AllowEmptyRoot controls what happens when Root is empty. When false (default) it will default to the
  288. // current working directory. An empty root is mostly useful when you want to use absolute paths
  289. // on windows that are on different filesystems. On linux setting your Root to "/" already allows you to use
  290. // absolute paths on any filesystem.
  291. AllowEmptyRoot bool
  292. // Uses brotli encoding and fallbacks to zstd or gzip in responses if set to true, uses zstd or gzip if set to false.
  293. //
  294. // This value has sense only if Compress is set.
  295. //
  296. // Brotli encoding is disabled by default.
  297. CompressBrotli bool
  298. // Uses zstd encoding and fallbacks to gzip in responses if set to true, uses gzip if set to false.
  299. //
  300. // This value has sense only if Compress is set.
  301. //
  302. // zstd encoding is disabled by default.
  303. CompressZstd bool
  304. // Index pages for directories without files matching IndexNames
  305. // are automatically generated if set.
  306. //
  307. // Directory index generation may be quite slow for directories
  308. // with many files (more than 1K), so it is discouraged enabling
  309. // index pages' generation for such directories.
  310. //
  311. // By default index pages aren't generated.
  312. GenerateIndexPages bool
  313. // Transparently compresses responses if set to true.
  314. //
  315. // The server tries minimizing CPU usage by caching compressed files.
  316. // It adds CompressedFileSuffix suffix to the original file name and
  317. // tries saving the resulting compressed file under the new file name.
  318. // So it is advisable to give the server write access to Root
  319. // and to all inner folders in order to minimize CPU usage when serving
  320. // compressed responses.
  321. //
  322. // Transparent compression is disabled by default.
  323. Compress bool
  324. // Enables byte range requests if set to true.
  325. //
  326. // Byte range requests are disabled by default.
  327. AcceptByteRange bool
  328. // SkipCache if true, will cache no file handler.
  329. //
  330. // By default is false.
  331. SkipCache bool
  332. }
  333. // FSCompressedFileSuffix is the suffix FS adds to the original file names
  334. // when trying to store compressed file under the new file name.
  335. // See FS.Compress for details.
  336. const FSCompressedFileSuffix = ".fasthttp.gz"
  337. // FSCompressedFileSuffixes is the suffixes FS adds to the original file names depending on encoding
  338. // when trying to store compressed file under the new file name.
  339. // See FS.Compress for details.
  340. var FSCompressedFileSuffixes = map[string]string{
  341. "gzip": ".fasthttp.gz",
  342. "br": ".fasthttp.br",
  343. "zstd": ".fasthttp.zst",
  344. }
  345. // FSHandlerCacheDuration is the default expiration duration for inactive
  346. // file handlers opened by FS.
  347. const FSHandlerCacheDuration = 10 * time.Second
  348. // FSHandler returns request handler serving static files from
  349. // the given root folder.
  350. //
  351. // stripSlashes indicates how many leading slashes must be stripped
  352. // from requested path before searching requested file in the root folder.
  353. // Examples:
  354. //
  355. // - stripSlashes = 0, original path: "/foo/bar", result: "/foo/bar"
  356. // - stripSlashes = 1, original path: "/foo/bar", result: "/bar"
  357. // - stripSlashes = 2, original path: "/foo/bar", result: ""
  358. //
  359. // The returned request handler automatically generates index pages
  360. // for directories without index.html.
  361. //
  362. // The returned handler caches requested file handles
  363. // for FSHandlerCacheDuration.
  364. // Make sure your program has enough 'max open files' limit aka
  365. // 'ulimit -n' if root folder contains many files.
  366. //
  367. // Do not create multiple request handler instances for the same
  368. // (root, stripSlashes) arguments - just reuse a single instance.
  369. // Otherwise goroutine leak will occur.
  370. func FSHandler(root string, stripSlashes int) RequestHandler {
  371. fs := &FS{
  372. Root: root,
  373. IndexNames: []string{"index.html"},
  374. GenerateIndexPages: true,
  375. AcceptByteRange: true,
  376. }
  377. if stripSlashes > 0 {
  378. fs.PathRewrite = NewPathSlashesStripper(stripSlashes)
  379. }
  380. return fs.NewRequestHandler()
  381. }
  382. // NewRequestHandler returns new request handler with the given FS settings.
  383. //
  384. // The returned handler caches requested file handles
  385. // for FS.CacheDuration.
  386. // Make sure your program has enough 'max open files' limit aka
  387. // 'ulimit -n' if FS.Root folder contains many files.
  388. //
  389. // Do not create multiple request handlers from a single FS instance -
  390. // just reuse a single request handler.
  391. func (fs *FS) NewRequestHandler() RequestHandler {
  392. fs.once.Do(fs.initRequestHandler)
  393. return fs.h
  394. }
  395. func (fs *FS) normalizeRoot(root string) string {
  396. // fs.FS uses relative paths, that paths are slash-separated on all systems, even Windows.
  397. if fs.FS == nil {
  398. // Serve files from the current working directory if Root is empty or if Root is a relative path.
  399. if (!fs.AllowEmptyRoot && root == "") || (root != "" && !filepath.IsAbs(root)) {
  400. path, err := os.Getwd()
  401. if err != nil {
  402. path = "."
  403. }
  404. root = path + "/" + root
  405. }
  406. // convert the root directory slashes to the native format
  407. root = filepath.FromSlash(root)
  408. }
  409. // strip trailing slashes from the root path
  410. for root != "" && root[len(root)-1] == os.PathSeparator {
  411. root = root[:len(root)-1]
  412. }
  413. return root
  414. }
  415. func (fs *FS) initRequestHandler() {
  416. root := fs.normalizeRoot(fs.Root)
  417. compressRoot := fs.CompressRoot
  418. if compressRoot == "" {
  419. compressRoot = root
  420. } else {
  421. compressRoot = fs.normalizeRoot(compressRoot)
  422. }
  423. compressedFileSuffixes := fs.CompressedFileSuffixes
  424. if compressedFileSuffixes["br"] == "" || compressedFileSuffixes["gzip"] == "" ||
  425. compressedFileSuffixes["zstd"] == "" || compressedFileSuffixes["br"] == compressedFileSuffixes["gzip"] ||
  426. compressedFileSuffixes["br"] == compressedFileSuffixes["zstd"] ||
  427. compressedFileSuffixes["gzip"] == compressedFileSuffixes["zstd"] {
  428. // Copy global map
  429. compressedFileSuffixes = make(map[string]string, len(FSCompressedFileSuffixes))
  430. for k, v := range FSCompressedFileSuffixes {
  431. compressedFileSuffixes[k] = v
  432. }
  433. }
  434. if fs.CompressedFileSuffix != "" {
  435. compressedFileSuffixes["gzip"] = fs.CompressedFileSuffix
  436. compressedFileSuffixes["br"] = FSCompressedFileSuffixes["br"]
  437. compressedFileSuffixes["zstd"] = FSCompressedFileSuffixes["zstd"]
  438. }
  439. h := &fsHandler{
  440. filesystem: fs.FS,
  441. root: root,
  442. indexNames: fs.IndexNames,
  443. pathRewrite: fs.PathRewrite,
  444. generateIndexPages: fs.GenerateIndexPages,
  445. compress: fs.Compress,
  446. compressBrotli: fs.CompressBrotli,
  447. compressZstd: fs.CompressZstd,
  448. compressRoot: compressRoot,
  449. pathNotFound: fs.PathNotFound,
  450. acceptByteRange: fs.AcceptByteRange,
  451. compressedFileSuffixes: compressedFileSuffixes,
  452. }
  453. h.cacheManager = newCacheManager(fs)
  454. if h.filesystem == nil {
  455. h.filesystem = &osFS{} // It provides os.Open and os.Stat
  456. }
  457. fs.h = h.handleRequest
  458. }
  459. type fsHandler struct {
  460. smallFileReaderPool sync.Pool
  461. filesystem fs.FS
  462. cacheManager cacheManager
  463. pathRewrite PathRewriteFunc
  464. pathNotFound RequestHandler
  465. compressedFileSuffixes map[string]string
  466. root string
  467. compressRoot string
  468. indexNames []string
  469. generateIndexPages bool
  470. compress bool
  471. compressBrotli bool
  472. compressZstd bool
  473. acceptByteRange bool
  474. }
  475. type fsFile struct {
  476. lastModified time.Time
  477. t time.Time
  478. f fs.File
  479. h *fsHandler
  480. filename string // fs.FileInfo.Name() return filename, isn't filepath.
  481. contentType string
  482. dirIndex []byte
  483. lastModifiedStr []byte
  484. bigFiles []*bigFileReader
  485. contentLength int
  486. readersCount int
  487. bigFilesLock sync.Mutex
  488. compressed bool
  489. }
  490. func (ff *fsFile) NewReader() (io.Reader, error) {
  491. if ff.isBig() {
  492. r, err := ff.bigFileReader()
  493. if err != nil {
  494. ff.decReadersCount()
  495. }
  496. return r, err
  497. }
  498. return ff.smallFileReader()
  499. }
  500. func (ff *fsFile) smallFileReader() (io.Reader, error) {
  501. v := ff.h.smallFileReaderPool.Get()
  502. if v == nil {
  503. v = &fsSmallFileReader{}
  504. }
  505. r := v.(*fsSmallFileReader)
  506. r.ff = ff
  507. r.endPos = ff.contentLength
  508. if r.startPos > 0 {
  509. return nil, errors.New("bug: fsSmallFileReader with non-nil startPos found in the pool")
  510. }
  511. return r, nil
  512. }
  513. // Files bigger than this size are sent with sendfile.
  514. const maxSmallFileSize = 2 * 4096
  515. func (ff *fsFile) isBig() bool {
  516. if _, ok := ff.h.filesystem.(*osFS); !ok { // fs.FS only uses bigFileReader, memory cache uses fsSmallFileReader
  517. return ff.f != nil
  518. }
  519. return ff.contentLength > maxSmallFileSize && len(ff.dirIndex) == 0
  520. }
  521. func (ff *fsFile) bigFileReader() (io.Reader, error) {
  522. if ff.f == nil {
  523. return nil, errors.New("bug: ff.f must be non-nil in bigFileReader")
  524. }
  525. var r io.Reader
  526. ff.bigFilesLock.Lock()
  527. n := len(ff.bigFiles)
  528. if n > 0 {
  529. r = ff.bigFiles[n-1]
  530. ff.bigFiles = ff.bigFiles[:n-1]
  531. }
  532. ff.bigFilesLock.Unlock()
  533. if r != nil {
  534. return r, nil
  535. }
  536. f, err := ff.h.filesystem.Open(ff.filename)
  537. if err != nil {
  538. return nil, fmt.Errorf("cannot open already opened file: %w", err)
  539. }
  540. return &bigFileReader{
  541. f: f,
  542. ff: ff,
  543. r: f,
  544. }, nil
  545. }
  546. func (ff *fsFile) Release() {
  547. if ff.f != nil {
  548. _ = ff.f.Close()
  549. if ff.isBig() {
  550. ff.bigFilesLock.Lock()
  551. for _, r := range ff.bigFiles {
  552. _ = r.f.Close()
  553. }
  554. ff.bigFilesLock.Unlock()
  555. }
  556. }
  557. }
  558. func (ff *fsFile) decReadersCount() {
  559. ff.h.cacheManager.WithLock(func() {
  560. ff.readersCount--
  561. if ff.readersCount < 0 {
  562. ff.readersCount = 0
  563. }
  564. })
  565. }
  566. // bigFileReader attempts to trigger sendfile
  567. // for sending big files over the wire.
  568. type bigFileReader struct {
  569. f fs.File
  570. ff *fsFile
  571. r io.Reader
  572. lr io.LimitedReader
  573. }
  574. func (r *bigFileReader) UpdateByteRange(startPos, endPos int) error {
  575. seeker, ok := r.f.(io.Seeker)
  576. if !ok {
  577. return errors.New("must implement io.Seeker")
  578. }
  579. if _, err := seeker.Seek(int64(startPos), io.SeekStart); err != nil {
  580. return err
  581. }
  582. r.r = &r.lr
  583. r.lr.R = r.f
  584. r.lr.N = int64(endPos - startPos + 1)
  585. return nil
  586. }
  587. func (r *bigFileReader) Read(p []byte) (int, error) {
  588. return r.r.Read(p)
  589. }
  590. func (r *bigFileReader) WriteTo(w io.Writer) (int64, error) {
  591. if rf, ok := w.(io.ReaderFrom); ok {
  592. // fast path. Send file must be triggered
  593. return rf.ReadFrom(r.r)
  594. }
  595. // slow path
  596. return copyZeroAlloc(w, r.r)
  597. }
  598. func (r *bigFileReader) Close() error {
  599. r.r = r.f
  600. seeker, ok := r.f.(io.Seeker)
  601. if !ok {
  602. _ = r.f.Close()
  603. return errors.New("must implement io.Seeker")
  604. }
  605. n, err := seeker.Seek(0, io.SeekStart)
  606. if err == nil {
  607. if n == 0 {
  608. ff := r.ff
  609. ff.bigFilesLock.Lock()
  610. ff.bigFiles = append(ff.bigFiles, r)
  611. ff.bigFilesLock.Unlock()
  612. } else {
  613. _ = r.f.Close()
  614. err = errors.New("bug: File.Seek(0, io.SeekStart) returned (non-zero, nil)")
  615. }
  616. } else {
  617. _ = r.f.Close()
  618. }
  619. r.ff.decReadersCount()
  620. return err
  621. }
  622. type fsSmallFileReader struct {
  623. ff *fsFile
  624. startPos int
  625. endPos int
  626. }
  627. func (r *fsSmallFileReader) Close() error {
  628. ff := r.ff
  629. ff.decReadersCount()
  630. r.ff = nil
  631. r.startPos = 0
  632. r.endPos = 0
  633. ff.h.smallFileReaderPool.Put(r)
  634. return nil
  635. }
  636. func (r *fsSmallFileReader) UpdateByteRange(startPos, endPos int) error {
  637. r.startPos = startPos
  638. r.endPos = endPos + 1
  639. return nil
  640. }
  641. func (r *fsSmallFileReader) Read(p []byte) (int, error) {
  642. tailLen := r.endPos - r.startPos
  643. if tailLen <= 0 {
  644. return 0, io.EOF
  645. }
  646. if len(p) > tailLen {
  647. p = p[:tailLen]
  648. }
  649. ff := r.ff
  650. if ff.f != nil {
  651. ra, ok := ff.f.(io.ReaderAt)
  652. if !ok {
  653. return 0, errors.New("must implement io.ReaderAt")
  654. }
  655. n, err := ra.ReadAt(p, int64(r.startPos))
  656. r.startPos += n
  657. return n, err
  658. }
  659. n := copy(p, ff.dirIndex[r.startPos:])
  660. r.startPos += n
  661. return n, nil
  662. }
  663. func (r *fsSmallFileReader) WriteTo(w io.Writer) (int64, error) {
  664. ff := r.ff
  665. var n int
  666. var err error
  667. if ff.f == nil {
  668. n, err = w.Write(ff.dirIndex[r.startPos:r.endPos])
  669. return int64(n), err
  670. }
  671. if rf, ok := w.(io.ReaderFrom); ok {
  672. return rf.ReadFrom(r)
  673. }
  674. curPos := r.startPos
  675. bufv := copyBufPool.Get()
  676. buf := bufv.([]byte)
  677. for err == nil {
  678. tailLen := r.endPos - curPos
  679. if tailLen <= 0 {
  680. break
  681. }
  682. if len(buf) > tailLen {
  683. buf = buf[:tailLen]
  684. }
  685. ra, ok := ff.f.(io.ReaderAt)
  686. if !ok {
  687. return 0, errors.New("must implement io.ReaderAt")
  688. }
  689. n, err = ra.ReadAt(buf, int64(curPos))
  690. nw, errw := w.Write(buf[:n])
  691. curPos += nw
  692. if errw == nil && nw != n {
  693. errw = errors.New("bug: Write(p) returned (n, nil), where n != len(p)")
  694. }
  695. if err == nil {
  696. err = errw
  697. }
  698. }
  699. copyBufPool.Put(bufv)
  700. if err == io.EOF {
  701. err = nil
  702. }
  703. return int64(curPos - r.startPos), err
  704. }
  705. type cacheManager interface {
  706. WithLock(work func())
  707. GetFileFromCache(cacheKind CacheKind, path string) (*fsFile, bool)
  708. SetFileToCache(cacheKind CacheKind, path string, ff *fsFile) *fsFile
  709. }
  710. var (
  711. _ cacheManager = (*inMemoryCacheManager)(nil)
  712. _ cacheManager = (*noopCacheManager)(nil)
  713. )
  714. type CacheKind uint8
  715. const (
  716. defaultCacheKind CacheKind = iota
  717. brotliCacheKind
  718. gzipCacheKind
  719. zstdCacheKind
  720. )
  721. func newCacheManager(fs *FS) cacheManager {
  722. if fs.SkipCache {
  723. return &noopCacheManager{}
  724. }
  725. cacheDuration := fs.CacheDuration
  726. if cacheDuration <= 0 {
  727. cacheDuration = FSHandlerCacheDuration
  728. }
  729. instance := &inMemoryCacheManager{
  730. cacheDuration: cacheDuration,
  731. cache: make(map[string]*fsFile),
  732. cacheBrotli: make(map[string]*fsFile),
  733. cacheGzip: make(map[string]*fsFile),
  734. cacheZstd: make(map[string]*fsFile),
  735. }
  736. go instance.handleCleanCache(fs.CleanStop)
  737. return instance
  738. }
  739. type noopCacheManager struct {
  740. cacheLock sync.Mutex
  741. }
  742. func (n *noopCacheManager) WithLock(work func()) {
  743. n.cacheLock.Lock()
  744. work()
  745. n.cacheLock.Unlock()
  746. }
  747. func (*noopCacheManager) GetFileFromCache(cacheKind CacheKind, path string) (*fsFile, bool) {
  748. return nil, false
  749. }
  750. func (*noopCacheManager) SetFileToCache(cacheKind CacheKind, path string, ff *fsFile) *fsFile {
  751. return ff
  752. }
  753. type inMemoryCacheManager struct {
  754. cache map[string]*fsFile
  755. cacheBrotli map[string]*fsFile
  756. cacheGzip map[string]*fsFile
  757. cacheZstd map[string]*fsFile
  758. cacheDuration time.Duration
  759. cacheLock sync.Mutex
  760. }
  761. func (cm *inMemoryCacheManager) WithLock(work func()) {
  762. cm.cacheLock.Lock()
  763. work()
  764. cm.cacheLock.Unlock()
  765. }
  766. func (cm *inMemoryCacheManager) getFsCache(cacheKind CacheKind) map[string]*fsFile {
  767. fileCache := cm.cache
  768. switch cacheKind {
  769. case brotliCacheKind:
  770. fileCache = cm.cacheBrotli
  771. case gzipCacheKind:
  772. fileCache = cm.cacheGzip
  773. case zstdCacheKind:
  774. fileCache = cm.cacheZstd
  775. }
  776. return fileCache
  777. }
  778. func (cm *inMemoryCacheManager) GetFileFromCache(cacheKind CacheKind, path string) (*fsFile, bool) {
  779. fileCache := cm.getFsCache(cacheKind)
  780. cm.cacheLock.Lock()
  781. ff, ok := fileCache[path]
  782. if ok {
  783. ff.readersCount++
  784. }
  785. cm.cacheLock.Unlock()
  786. return ff, ok
  787. }
  788. func (cm *inMemoryCacheManager) SetFileToCache(cacheKind CacheKind, path string, ff *fsFile) *fsFile {
  789. fileCache := cm.getFsCache(cacheKind)
  790. cm.cacheLock.Lock()
  791. ff1, ok := fileCache[path]
  792. if !ok {
  793. fileCache[path] = ff
  794. ff.readersCount++
  795. } else {
  796. ff1.readersCount++
  797. }
  798. cm.cacheLock.Unlock()
  799. if ok {
  800. // The file has been already opened by another
  801. // goroutine, so close the current file and use
  802. // the file opened by another goroutine instead.
  803. ff.Release()
  804. ff = ff1
  805. }
  806. return ff
  807. }
  808. func (cm *inMemoryCacheManager) handleCleanCache(cleanStop chan struct{}) {
  809. var pendingFiles []*fsFile
  810. clean := func() {
  811. pendingFiles = cm.cleanCache(pendingFiles)
  812. }
  813. if cleanStop != nil {
  814. t := time.NewTicker(cm.cacheDuration / 2)
  815. for {
  816. select {
  817. case <-t.C:
  818. clean()
  819. case _, stillOpen := <-cleanStop:
  820. // Ignore values send on the channel, only stop when it is closed.
  821. if !stillOpen {
  822. t.Stop()
  823. return
  824. }
  825. }
  826. }
  827. }
  828. for {
  829. time.Sleep(cm.cacheDuration / 2)
  830. clean()
  831. }
  832. }
  833. func (cm *inMemoryCacheManager) cleanCache(pendingFiles []*fsFile) []*fsFile {
  834. var filesToRelease []*fsFile
  835. cm.cacheLock.Lock()
  836. // Close files which couldn't be closed before due to non-zero
  837. // readers count on the previous run.
  838. var remainingFiles []*fsFile
  839. for _, ff := range pendingFiles {
  840. if ff.readersCount > 0 {
  841. remainingFiles = append(remainingFiles, ff)
  842. } else {
  843. filesToRelease = append(filesToRelease, ff)
  844. }
  845. }
  846. pendingFiles = remainingFiles
  847. pendingFiles, filesToRelease = cleanCacheNolock(cm.cache, pendingFiles, filesToRelease, cm.cacheDuration)
  848. pendingFiles, filesToRelease = cleanCacheNolock(cm.cacheBrotli, pendingFiles, filesToRelease, cm.cacheDuration)
  849. pendingFiles, filesToRelease = cleanCacheNolock(cm.cacheGzip, pendingFiles, filesToRelease, cm.cacheDuration)
  850. pendingFiles, filesToRelease = cleanCacheNolock(cm.cacheZstd, pendingFiles, filesToRelease, cm.cacheDuration)
  851. cm.cacheLock.Unlock()
  852. for _, ff := range filesToRelease {
  853. ff.Release()
  854. }
  855. return pendingFiles
  856. }
  857. func cleanCacheNolock(
  858. cache map[string]*fsFile, pendingFiles, filesToRelease []*fsFile, cacheDuration time.Duration,
  859. ) ([]*fsFile, []*fsFile) {
  860. t := time.Now()
  861. for k, ff := range cache {
  862. if t.Sub(ff.t) > cacheDuration {
  863. if ff.readersCount > 0 {
  864. // There are pending readers on stale file handle,
  865. // so we cannot close it. Put it into pendingFiles
  866. // so it will be closed later.
  867. pendingFiles = append(pendingFiles, ff)
  868. } else {
  869. filesToRelease = append(filesToRelease, ff)
  870. }
  871. delete(cache, k)
  872. }
  873. }
  874. return pendingFiles, filesToRelease
  875. }
  876. func (h *fsHandler) pathToFilePath(path string) string {
  877. if _, ok := h.filesystem.(*osFS); !ok {
  878. if len(path) < 1 {
  879. return path
  880. }
  881. return path[1:]
  882. }
  883. return filepath.FromSlash(h.root + path)
  884. }
  885. func (h *fsHandler) filePathToCompressed(filePath string) string {
  886. if h.root == h.compressRoot {
  887. return filePath
  888. }
  889. if !strings.HasPrefix(filePath, h.root) {
  890. return filePath
  891. }
  892. return filepath.FromSlash(h.compressRoot + filePath[len(h.root):])
  893. }
  894. func (h *fsHandler) handleRequest(ctx *RequestCtx) {
  895. var path []byte
  896. if h.pathRewrite != nil {
  897. path = h.pathRewrite(ctx)
  898. } else {
  899. path = ctx.Path()
  900. }
  901. hasTrailingSlash := len(path) > 0 && path[len(path)-1] == '/'
  902. if n := bytes.IndexByte(path, 0); n >= 0 {
  903. ctx.Logger().Printf("cannot serve path with nil byte at position %d: %q", n, path)
  904. ctx.Error("Are you a hacker?", StatusBadRequest)
  905. return
  906. }
  907. if h.pathRewrite != nil {
  908. // There is no need to check for '/../' if path = ctx.Path(),
  909. // since ctx.Path must normalize and sanitize the path.
  910. if n := bytes.Index(path, strSlashDotDotSlash); n >= 0 {
  911. ctx.Logger().Printf("cannot serve path with '/../' at position %d due to security reasons: %q", n, path)
  912. ctx.Error("Internal Server Error", StatusInternalServerError)
  913. return
  914. }
  915. }
  916. mustCompress := false
  917. fileCacheKind := defaultCacheKind
  918. fileEncoding := ""
  919. byteRange := ctx.Request.Header.peek(strRange)
  920. if len(byteRange) == 0 && h.compress {
  921. switch {
  922. case h.compressBrotli && ctx.Request.Header.HasAcceptEncodingBytes(strBr):
  923. mustCompress = true
  924. fileCacheKind = brotliCacheKind
  925. fileEncoding = "br"
  926. case h.compressZstd && ctx.Request.Header.HasAcceptEncodingBytes(strZstd):
  927. mustCompress = true
  928. fileCacheKind = zstdCacheKind
  929. fileEncoding = "zstd"
  930. case ctx.Request.Header.HasAcceptEncodingBytes(strGzip):
  931. mustCompress = true
  932. fileCacheKind = gzipCacheKind
  933. fileEncoding = "gzip"
  934. }
  935. }
  936. originalPathStr := string(path)
  937. pathStr := originalPathStr
  938. if hasTrailingSlash {
  939. pathStr = originalPathStr[:len(originalPathStr)-1]
  940. }
  941. ff, ok := h.cacheManager.GetFileFromCache(fileCacheKind, originalPathStr)
  942. if !ok {
  943. filePath := h.pathToFilePath(pathStr)
  944. var err error
  945. ff, err = h.openFSFile(filePath, mustCompress, fileEncoding)
  946. if mustCompress && err == errNoCreatePermission {
  947. ctx.Logger().Printf("insufficient permissions for saving compressed file for %q. Serving uncompressed file. "+
  948. "Allow write access to the directory with this file in order to improve fasthttp performance", filePath)
  949. mustCompress = false
  950. ff, err = h.openFSFile(filePath, mustCompress, fileEncoding)
  951. }
  952. if errors.Is(err, errDirIndexRequired) {
  953. if !hasTrailingSlash {
  954. ctx.RedirectBytes(append(path, '/'), StatusFound)
  955. return
  956. }
  957. ff, err = h.openIndexFile(ctx, filePath, mustCompress, fileEncoding)
  958. if err != nil {
  959. ctx.Logger().Printf("cannot open dir index %q: %v", filePath, err)
  960. ctx.Error("Directory index is forbidden", StatusForbidden)
  961. return
  962. }
  963. } else if err != nil {
  964. ctx.Logger().Printf("cannot open file %q: %v", filePath, err)
  965. if h.pathNotFound == nil {
  966. ctx.Error("Cannot open requested path", StatusNotFound)
  967. } else {
  968. ctx.SetStatusCode(StatusNotFound)
  969. h.pathNotFound(ctx)
  970. }
  971. return
  972. }
  973. ff = h.cacheManager.SetFileToCache(fileCacheKind, originalPathStr, ff)
  974. }
  975. if !ctx.IfModifiedSince(ff.lastModified) {
  976. ff.decReadersCount()
  977. ctx.NotModified()
  978. return
  979. }
  980. r, err := ff.NewReader()
  981. if err != nil {
  982. ctx.Logger().Printf("cannot obtain file reader for path=%q: %v", path, err)
  983. ctx.Error("Internal Server Error", StatusInternalServerError)
  984. return
  985. }
  986. hdr := &ctx.Response.Header
  987. if ff.compressed {
  988. switch fileEncoding {
  989. case "br":
  990. hdr.SetContentEncodingBytes(strBr)
  991. hdr.addVaryBytes(strAcceptEncoding)
  992. case "gzip":
  993. hdr.SetContentEncodingBytes(strGzip)
  994. hdr.addVaryBytes(strAcceptEncoding)
  995. case "zstd":
  996. hdr.SetContentEncodingBytes(strZstd)
  997. hdr.addVaryBytes(strAcceptEncoding)
  998. }
  999. }
  1000. statusCode := StatusOK
  1001. contentLength := ff.contentLength
  1002. if h.acceptByteRange {
  1003. hdr.setNonSpecial(strAcceptRanges, strBytes)
  1004. if len(byteRange) > 0 {
  1005. startPos, endPos, err := ParseByteRange(byteRange, contentLength)
  1006. if err != nil {
  1007. _ = r.(io.Closer).Close()
  1008. ctx.Logger().Printf("cannot parse byte range %q for path=%q: %v", byteRange, path, err)
  1009. ctx.Error("Range Not Satisfiable", StatusRequestedRangeNotSatisfiable)
  1010. return
  1011. }
  1012. if err = r.(byteRangeUpdater).UpdateByteRange(startPos, endPos); err != nil {
  1013. _ = r.(io.Closer).Close()
  1014. ctx.Logger().Printf("cannot seek byte range %q for path=%q: %v", byteRange, path, err)
  1015. ctx.Error("Internal Server Error", StatusInternalServerError)
  1016. return
  1017. }
  1018. hdr.SetContentRange(startPos, endPos, contentLength)
  1019. contentLength = endPos - startPos + 1
  1020. statusCode = StatusPartialContent
  1021. }
  1022. }
  1023. hdr.setNonSpecial(strLastModified, ff.lastModifiedStr)
  1024. if !ctx.IsHead() {
  1025. ctx.SetBodyStream(r, contentLength)
  1026. } else {
  1027. ctx.Response.ResetBody()
  1028. ctx.Response.SkipBody = true
  1029. ctx.Response.Header.SetContentLength(contentLength)
  1030. if rc, ok := r.(io.Closer); ok {
  1031. if err := rc.Close(); err != nil {
  1032. ctx.Logger().Printf("cannot close file reader: %v", err)
  1033. ctx.Error("Internal Server Error", StatusInternalServerError)
  1034. return
  1035. }
  1036. }
  1037. }
  1038. hdr.noDefaultContentType = true
  1039. if len(hdr.ContentType()) == 0 {
  1040. ctx.SetContentType(ff.contentType)
  1041. }
  1042. ctx.SetStatusCode(statusCode)
  1043. }
  1044. type byteRangeUpdater interface {
  1045. UpdateByteRange(startPos, endPos int) error
  1046. }
  1047. // ParseByteRange parses 'Range: bytes=...' header value.
  1048. //
  1049. // It follows https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.35 .
  1050. func ParseByteRange(byteRange []byte, contentLength int) (startPos, endPos int, err error) {
  1051. b := byteRange
  1052. if !bytes.HasPrefix(b, strBytes) {
  1053. return 0, 0, fmt.Errorf("unsupported range units: %q. Expecting %q", byteRange, strBytes)
  1054. }
  1055. b = b[len(strBytes):]
  1056. if len(b) == 0 || b[0] != '=' {
  1057. return 0, 0, fmt.Errorf("missing byte range in %q", byteRange)
  1058. }
  1059. b = b[1:]
  1060. n := bytes.IndexByte(b, '-')
  1061. if n < 0 {
  1062. return 0, 0, fmt.Errorf("missing the end position of byte range in %q", byteRange)
  1063. }
  1064. if n == 0 {
  1065. v, err := ParseUint(b[n+1:])
  1066. if err != nil {
  1067. return 0, 0, err
  1068. }
  1069. startPos := contentLength - v
  1070. if startPos < 0 {
  1071. startPos = 0
  1072. }
  1073. return startPos, contentLength - 1, nil
  1074. }
  1075. if startPos, err = ParseUint(b[:n]); err != nil {
  1076. return 0, 0, err
  1077. }
  1078. if startPos >= contentLength {
  1079. return 0, 0, fmt.Errorf("the start position of byte range cannot exceed %d. byte range %q", contentLength-1, byteRange)
  1080. }
  1081. b = b[n+1:]
  1082. if len(b) == 0 {
  1083. return startPos, contentLength - 1, nil
  1084. }
  1085. if endPos, err = ParseUint(b); err != nil {
  1086. return 0, 0, err
  1087. }
  1088. if endPos >= contentLength {
  1089. endPos = contentLength - 1
  1090. }
  1091. if endPos < startPos {
  1092. return 0, 0, fmt.Errorf("the start position of byte range cannot exceed the end position. byte range %q", byteRange)
  1093. }
  1094. return startPos, endPos, nil
  1095. }
  1096. func (h *fsHandler) openIndexFile(ctx *RequestCtx, dirPath string, mustCompress bool, fileEncoding string) (*fsFile, error) {
  1097. for _, indexName := range h.indexNames {
  1098. indexFilePath := indexName
  1099. if dirPath != "" {
  1100. indexFilePath = dirPath + "/" + indexName
  1101. }
  1102. ff, err := h.openFSFile(indexFilePath, mustCompress, fileEncoding)
  1103. if err == nil {
  1104. return ff, nil
  1105. }
  1106. if mustCompress && err == errNoCreatePermission {
  1107. ctx.Logger().Printf("insufficient permissions for saving compressed file for %q. Serving uncompressed file. "+
  1108. "Allow write access to the directory with this file in order to improve fasthttp performance", indexFilePath)
  1109. mustCompress = false
  1110. return h.openFSFile(indexFilePath, mustCompress, fileEncoding)
  1111. }
  1112. if !errors.Is(err, fs.ErrNotExist) {
  1113. return nil, fmt.Errorf("cannot open file %q: %w", indexFilePath, err)
  1114. }
  1115. }
  1116. if !h.generateIndexPages {
  1117. return nil, fmt.Errorf("cannot access directory without index page. Directory %q", dirPath)
  1118. }
  1119. return h.createDirIndex(ctx, dirPath, mustCompress, fileEncoding)
  1120. }
  1121. var (
  1122. errDirIndexRequired = errors.New("directory index required")
  1123. errNoCreatePermission = errors.New("no 'create file' permissions")
  1124. )
  1125. func (h *fsHandler) createDirIndex(ctx *RequestCtx, dirPath string, mustCompress bool, fileEncoding string) (*fsFile, error) {
  1126. w := &bytebufferpool.ByteBuffer{}
  1127. base := ctx.URI()
  1128. // io/fs doesn't support ReadDir with empty path.
  1129. if dirPath == "" {
  1130. dirPath = "."
  1131. }
  1132. basePathEscaped := html.EscapeString(string(base.Path()))
  1133. _, _ = fmt.Fprintf(w, "<html><head><title>%s</title><style>.dir { font-weight: bold }</style></head><body>", basePathEscaped)
  1134. _, _ = fmt.Fprintf(w, "<h1>%s</h1>", basePathEscaped)
  1135. _, _ = fmt.Fprintf(w, "<ul>")
  1136. if len(basePathEscaped) > 1 {
  1137. var parentURI URI
  1138. base.CopyTo(&parentURI)
  1139. parentURI.Update(string(base.Path()) + "/..")
  1140. parentPathEscaped := html.EscapeString(string(parentURI.Path()))
  1141. _, _ = fmt.Fprintf(w, `<li><a href="%s" class="dir">..</a></li>`, parentPathEscaped)
  1142. }
  1143. dirEntries, err := fs.ReadDir(h.filesystem, dirPath)
  1144. if err != nil {
  1145. return nil, err
  1146. }
  1147. fm := make(map[string]fs.FileInfo, len(dirEntries))
  1148. filenames := make([]string, 0, len(dirEntries))
  1149. nestedContinue:
  1150. for _, de := range dirEntries {
  1151. name := de.Name()
  1152. for _, cfs := range h.compressedFileSuffixes {
  1153. if strings.HasSuffix(name, cfs) {
  1154. // Do not show compressed files on index page.
  1155. continue nestedContinue
  1156. }
  1157. }
  1158. fi, err := de.Info()
  1159. if err != nil {
  1160. ctx.Logger().Printf("cannot fetch information from dir entry %q: %v, skip", name, err)
  1161. continue nestedContinue
  1162. }
  1163. fm[name] = fi
  1164. filenames = append(filenames, name)
  1165. }
  1166. var u URI
  1167. base.CopyTo(&u)
  1168. u.Update(string(u.Path()) + "/")
  1169. sort.Strings(filenames)
  1170. for _, name := range filenames {
  1171. u.Update(name)
  1172. pathEscaped := html.EscapeString(string(u.Path()))
  1173. fi := fm[name]
  1174. auxStr := "dir"
  1175. className := "dir"
  1176. if !fi.IsDir() {
  1177. auxStr = fmt.Sprintf("file, %d bytes", fi.Size())
  1178. className = "file"
  1179. }
  1180. _, _ = fmt.Fprintf(w, `<li><a href="%s" class="%s">%s</a>, %s, last modified %s</li>`,
  1181. pathEscaped, className, html.EscapeString(name), auxStr, fsModTime(fi.ModTime()))
  1182. }
  1183. _, _ = fmt.Fprintf(w, "</ul></body></html>")
  1184. if mustCompress {
  1185. var zbuf bytebufferpool.ByteBuffer
  1186. switch fileEncoding {
  1187. case "br":
  1188. zbuf.B = AppendBrotliBytesLevel(zbuf.B, w.B, CompressDefaultCompression)
  1189. case "gzip":
  1190. zbuf.B = AppendGzipBytesLevel(zbuf.B, w.B, CompressDefaultCompression)
  1191. case "zstd":
  1192. zbuf.B = AppendZstdBytesLevel(zbuf.B, w.B, CompressZstdDefault)
  1193. }
  1194. w = &zbuf
  1195. }
  1196. dirIndex := w.B
  1197. lastModified := time.Now()
  1198. ff := &fsFile{
  1199. h: h,
  1200. dirIndex: dirIndex,
  1201. contentType: "text/html; charset=utf-8",
  1202. contentLength: len(dirIndex),
  1203. compressed: mustCompress,
  1204. lastModified: lastModified,
  1205. lastModifiedStr: AppendHTTPDate(nil, lastModified),
  1206. t: lastModified,
  1207. }
  1208. return ff, nil
  1209. }
  1210. const (
  1211. fsMinCompressRatio = 0.8
  1212. fsMaxCompressibleFileSize = 8 * 1024 * 1024
  1213. )
  1214. func (h *fsHandler) compressAndOpenFSFile(filePath, fileEncoding string) (*fsFile, error) {
  1215. f, err := h.filesystem.Open(filePath)
  1216. if err != nil {
  1217. return nil, err
  1218. }
  1219. fileInfo, err := f.Stat()
  1220. if err != nil {
  1221. _ = f.Close()
  1222. return nil, fmt.Errorf("cannot obtain info for file %q: %w", filePath, err)
  1223. }
  1224. if fileInfo.IsDir() {
  1225. _ = f.Close()
  1226. return nil, errDirIndexRequired
  1227. }
  1228. if strings.HasSuffix(filePath, h.compressedFileSuffixes[fileEncoding]) ||
  1229. fileInfo.Size() > fsMaxCompressibleFileSize ||
  1230. !isFileCompressible(f, fsMinCompressRatio) {
  1231. return h.newFSFile(f, fileInfo, false, filePath, "")
  1232. }
  1233. compressedFilePath := h.filePathToCompressed(filePath)
  1234. if _, ok := h.filesystem.(*osFS); !ok {
  1235. return h.newCompressedFSFileCache(f, fileInfo, compressedFilePath, fileEncoding)
  1236. }
  1237. if compressedFilePath != filePath {
  1238. if err := os.MkdirAll(filepath.Dir(compressedFilePath), 0o750); err != nil {
  1239. return nil, err
  1240. }
  1241. }
  1242. compressedFilePath += h.compressedFileSuffixes[fileEncoding]
  1243. absPath, err := filepath.Abs(compressedFilePath)
  1244. if err != nil {
  1245. _ = f.Close()
  1246. return nil, fmt.Errorf("cannot determine absolute path for %q: %v", compressedFilePath, err)
  1247. }
  1248. flock := getFileLock(absPath)
  1249. flock.Lock()
  1250. ff, err := h.compressFileNolock(f, fileInfo, filePath, compressedFilePath, fileEncoding)
  1251. flock.Unlock()
  1252. return ff, err
  1253. }
  1254. func (h *fsHandler) compressFileNolock(
  1255. f fs.File, fileInfo fs.FileInfo, filePath, compressedFilePath, fileEncoding string,
  1256. ) (*fsFile, error) {
  1257. // Attempt to open compressed file created by another concurrent
  1258. // goroutine.
  1259. // It is safe opening such a file, since the file creation
  1260. // is guarded by file mutex - see getFileLock call.
  1261. if _, err := os.Stat(compressedFilePath); err == nil {
  1262. _ = f.Close()
  1263. return h.newCompressedFSFile(compressedFilePath, fileEncoding)
  1264. }
  1265. // Create temporary file, so concurrent goroutines don't use
  1266. // it until it is created.
  1267. tmpFilePath := compressedFilePath + ".tmp"
  1268. zf, err := os.Create(tmpFilePath)
  1269. if err != nil {
  1270. _ = f.Close()
  1271. if !errors.Is(err, fs.ErrPermission) {
  1272. return nil, fmt.Errorf("cannot create temporary file %q: %w", tmpFilePath, err)
  1273. }
  1274. return nil, errNoCreatePermission
  1275. }
  1276. switch fileEncoding {
  1277. case "br":
  1278. zw := acquireStacklessBrotliWriter(zf, CompressDefaultCompression)
  1279. _, err = copyZeroAlloc(zw, f)
  1280. if errf := zw.Flush(); err == nil {
  1281. err = errf
  1282. }
  1283. releaseStacklessBrotliWriter(zw, CompressDefaultCompression)
  1284. case "gzip":
  1285. zw := acquireStacklessGzipWriter(zf, CompressDefaultCompression)
  1286. _, err = copyZeroAlloc(zw, f)
  1287. if errf := zw.Flush(); err == nil {
  1288. err = errf
  1289. }
  1290. releaseStacklessGzipWriter(zw, CompressDefaultCompression)
  1291. case "zstd":
  1292. zw := acquireStacklessZstdWriter(zf, CompressZstdDefault)
  1293. _, err = copyZeroAlloc(zw, f)
  1294. if errf := zw.Flush(); err == nil {
  1295. err = errf
  1296. }
  1297. releaseStacklessZstdWriter(zw, CompressZstdDefault)
  1298. }
  1299. _ = zf.Close()
  1300. _ = f.Close()
  1301. if err != nil {
  1302. return nil, fmt.Errorf("error when compressing file %q to %q: %w", filePath, tmpFilePath, err)
  1303. }
  1304. if err = os.Chtimes(tmpFilePath, time.Now(), fileInfo.ModTime()); err != nil {
  1305. return nil, fmt.Errorf("cannot change modification time to %v for tmp file %q: %v",
  1306. fileInfo.ModTime(), tmpFilePath, err)
  1307. }
  1308. if err = os.Rename(tmpFilePath, compressedFilePath); err != nil {
  1309. return nil, fmt.Errorf("cannot move compressed file from %q to %q: %w", tmpFilePath, compressedFilePath, err)
  1310. }
  1311. return h.newCompressedFSFile(compressedFilePath, fileEncoding)
  1312. }
  1313. // newCompressedFSFileCache use memory cache compressed files.
  1314. func (h *fsHandler) newCompressedFSFileCache(f fs.File, fileInfo fs.FileInfo, filePath, fileEncoding string) (*fsFile, error) {
  1315. var (
  1316. w = &bytebufferpool.ByteBuffer{}
  1317. err error
  1318. )
  1319. switch fileEncoding {
  1320. case "br":
  1321. zw := acquireStacklessBrotliWriter(w, CompressDefaultCompression)
  1322. _, err = copyZeroAlloc(zw, f)
  1323. if errf := zw.Flush(); err == nil {
  1324. err = errf
  1325. }
  1326. releaseStacklessBrotliWriter(zw, CompressDefaultCompression)
  1327. case "gzip":
  1328. zw := acquireStacklessGzipWriter(w, CompressDefaultCompression)
  1329. _, err = copyZeroAlloc(zw, f)
  1330. if errf := zw.Flush(); err == nil {
  1331. err = errf
  1332. }
  1333. releaseStacklessGzipWriter(zw, CompressDefaultCompression)
  1334. case "zstd":
  1335. zw := acquireStacklessZstdWriter(w, CompressZstdDefault)
  1336. _, err = copyZeroAlloc(zw, f)
  1337. if errf := zw.Flush(); err == nil {
  1338. err = errf
  1339. }
  1340. releaseStacklessZstdWriter(zw, CompressZstdDefault)
  1341. }
  1342. defer func() { _ = f.Close() }()
  1343. if err != nil {
  1344. return nil, fmt.Errorf("error when compressing file %q: %w", filePath, err)
  1345. }
  1346. seeker, ok := f.(io.Seeker)
  1347. if !ok {
  1348. return nil, errors.New("not implemented io.Seeker")
  1349. }
  1350. if _, err = seeker.Seek(0, io.SeekStart); err != nil {
  1351. return nil, err
  1352. }
  1353. ext := fileExtension(fileInfo.Name(), false, h.compressedFileSuffixes[fileEncoding])
  1354. contentType := mime.TypeByExtension(ext)
  1355. if contentType == "" {
  1356. data, err := readFileHeader(f, false, fileEncoding)
  1357. if err != nil {
  1358. return nil, fmt.Errorf("cannot read header of the file %q: %w", fileInfo.Name(), err)
  1359. }
  1360. contentType = http.DetectContentType(data)
  1361. }
  1362. dirIndex := w.B
  1363. lastModified := fileInfo.ModTime()
  1364. ff := &fsFile{
  1365. h: h,
  1366. dirIndex: dirIndex,
  1367. contentType: contentType,
  1368. contentLength: len(dirIndex),
  1369. compressed: true,
  1370. lastModified: lastModified,
  1371. lastModifiedStr: AppendHTTPDate(nil, lastModified),
  1372. t: time.Now(),
  1373. }
  1374. return ff, nil
  1375. }
  1376. func (h *fsHandler) newCompressedFSFile(filePath, fileEncoding string) (*fsFile, error) {
  1377. f, err := h.filesystem.Open(filePath)
  1378. if err != nil {
  1379. return nil, fmt.Errorf("cannot open compressed file %q: %w", filePath, err)
  1380. }
  1381. fileInfo, err := f.Stat()
  1382. if err != nil {
  1383. _ = f.Close()
  1384. return nil, fmt.Errorf("cannot obtain info for compressed file %q: %w", filePath, err)
  1385. }
  1386. return h.newFSFile(f, fileInfo, true, filePath, fileEncoding)
  1387. }
  1388. func (h *fsHandler) openFSFile(filePath string, mustCompress bool, fileEncoding string) (*fsFile, error) {
  1389. filePathOriginal := filePath
  1390. if mustCompress {
  1391. filePath += h.compressedFileSuffixes[fileEncoding]
  1392. }
  1393. f, err := h.filesystem.Open(filePath)
  1394. if err != nil {
  1395. if mustCompress && errors.Is(err, fs.ErrNotExist) {
  1396. return h.compressAndOpenFSFile(filePathOriginal, fileEncoding)
  1397. }
  1398. // If the file is not found and the path is empty, let's return errDirIndexRequired error.
  1399. if filePath == "" && (errors.Is(err, fs.ErrNotExist) || errors.Is(err, fs.ErrInvalid)) {
  1400. return nil, errDirIndexRequired
  1401. }
  1402. return nil, err
  1403. }
  1404. fileInfo, err := f.Stat()
  1405. if err != nil {
  1406. _ = f.Close()
  1407. return nil, fmt.Errorf("cannot obtain info for file %q: %w", filePath, err)
  1408. }
  1409. if fileInfo.IsDir() {
  1410. _ = f.Close()
  1411. if mustCompress {
  1412. return nil, fmt.Errorf("directory with unexpected suffix found: %q. Suffix: %q",
  1413. filePath, h.compressedFileSuffixes[fileEncoding])
  1414. }
  1415. return nil, errDirIndexRequired
  1416. }
  1417. if mustCompress {
  1418. fileInfoOriginal, err := fs.Stat(h.filesystem, filePathOriginal)
  1419. if err != nil {
  1420. _ = f.Close()
  1421. return nil, fmt.Errorf("cannot obtain info for original file %q: %w", filePathOriginal, err)
  1422. }
  1423. // Only re-create the compressed file if there was more than a second between the mod times.
  1424. // On macOS the gzip seems to truncate the nanoseconds in the mod time causing the original file
  1425. // to look newer than the gzipped file.
  1426. if fileInfoOriginal.ModTime().Sub(fileInfo.ModTime()) >= time.Second {
  1427. // The compressed file became stale. Re-create it.
  1428. _ = f.Close()
  1429. _ = os.Remove(filePath)
  1430. return h.compressAndOpenFSFile(filePathOriginal, fileEncoding)
  1431. }
  1432. }
  1433. return h.newFSFile(f, fileInfo, mustCompress, filePath, fileEncoding)
  1434. }
  1435. func (h *fsHandler) newFSFile(f fs.File, fileInfo fs.FileInfo, compressed bool, filePath, fileEncoding string) (*fsFile, error) {
  1436. n := fileInfo.Size()
  1437. contentLength := int(n)
  1438. if n != int64(contentLength) {
  1439. _ = f.Close()
  1440. return nil, fmt.Errorf("too big file: %d bytes", n)
  1441. }
  1442. // detect content-type
  1443. ext := fileExtension(fileInfo.Name(), compressed, h.compressedFileSuffixes[fileEncoding])
  1444. contentType := mime.TypeByExtension(ext)
  1445. if contentType == "" {
  1446. data, err := readFileHeader(f, compressed, fileEncoding)
  1447. if err != nil {
  1448. return nil, fmt.Errorf("cannot read header of the file %q: %w", fileInfo.Name(), err)
  1449. }
  1450. contentType = http.DetectContentType(data)
  1451. }
  1452. lastModified := fileInfo.ModTime()
  1453. ff := &fsFile{
  1454. h: h,
  1455. f: f,
  1456. filename: filePath,
  1457. contentType: contentType,
  1458. contentLength: contentLength,
  1459. compressed: compressed,
  1460. lastModified: lastModified,
  1461. lastModifiedStr: AppendHTTPDate(nil, lastModified),
  1462. t: time.Now(),
  1463. }
  1464. return ff, nil
  1465. }
  1466. func readFileHeader(f io.Reader, compressed bool, fileEncoding string) ([]byte, error) {
  1467. r := f
  1468. var (
  1469. br *brotli.Reader
  1470. zr *gzip.Reader
  1471. zsr *zstd.Decoder
  1472. )
  1473. if compressed {
  1474. var err error
  1475. switch fileEncoding {
  1476. case "br":
  1477. if br, err = acquireBrotliReader(f); err != nil {
  1478. return nil, err
  1479. }
  1480. r = br
  1481. case "gzip":
  1482. if zr, err = acquireGzipReader(f); err != nil {
  1483. return nil, err
  1484. }
  1485. r = zr
  1486. case "zstd":
  1487. if zsr, err = acquireZstdReader(f); err != nil {
  1488. return nil, err
  1489. }
  1490. r = zsr
  1491. }
  1492. }
  1493. lr := &io.LimitedReader{
  1494. R: r,
  1495. N: 512,
  1496. }
  1497. data, err := io.ReadAll(lr)
  1498. seeker, ok := f.(io.Seeker)
  1499. if !ok {
  1500. return nil, errors.New("must implement io.Seeker")
  1501. }
  1502. if _, err := seeker.Seek(0, io.SeekStart); err != nil {
  1503. return nil, err
  1504. }
  1505. if br != nil {
  1506. releaseBrotliReader(br)
  1507. }
  1508. if zr != nil {
  1509. releaseGzipReader(zr)
  1510. }
  1511. if zsr != nil {
  1512. releaseZstdReader(zsr)
  1513. }
  1514. return data, err
  1515. }
  1516. func stripLeadingSlashes(path []byte, stripSlashes int) []byte {
  1517. for stripSlashes > 0 && len(path) > 0 {
  1518. if path[0] != '/' {
  1519. // developer sanity-check
  1520. panic("BUG: path must start with slash")
  1521. }
  1522. n := bytes.IndexByte(path[1:], '/')
  1523. if n < 0 {
  1524. path = path[:0]
  1525. break
  1526. }
  1527. path = path[n+1:]
  1528. stripSlashes--
  1529. }
  1530. return path
  1531. }
  1532. func stripTrailingSlashes(path []byte) []byte {
  1533. for len(path) > 0 && path[len(path)-1] == '/' {
  1534. path = path[:len(path)-1]
  1535. }
  1536. return path
  1537. }
  1538. func fileExtension(path string, compressed bool, compressedFileSuffix string) string {
  1539. if compressed && strings.HasSuffix(path, compressedFileSuffix) {
  1540. path = path[:len(path)-len(compressedFileSuffix)]
  1541. }
  1542. n := strings.LastIndexByte(path, '.')
  1543. if n < 0 {
  1544. return ""
  1545. }
  1546. return path[n:]
  1547. }
  1548. // FileLastModified returns last modified time for the file.
  1549. func FileLastModified(path string) (time.Time, error) {
  1550. f, err := os.Open(path)
  1551. if err != nil {
  1552. return zeroTime, err
  1553. }
  1554. fileInfo, err := f.Stat()
  1555. _ = f.Close()
  1556. if err != nil {
  1557. return zeroTime, err
  1558. }
  1559. return fsModTime(fileInfo.ModTime()), nil
  1560. }
  1561. func fsModTime(t time.Time) time.Time {
  1562. return t.In(time.UTC).Truncate(time.Second)
  1563. }
  1564. var filesLockMap sync.Map
  1565. func getFileLock(absPath string) *sync.Mutex {
  1566. v, _ := filesLockMap.LoadOrStore(absPath, &sync.Mutex{})
  1567. filelock := v.(*sync.Mutex)
  1568. return filelock
  1569. }
  1570. var _ fs.FS = (*osFS)(nil)
  1571. type osFS struct{}
  1572. func (o *osFS) Open(name string) (fs.File, error) { return os.Open(name) }
  1573. func (o *osFS) Stat(name string) (fs.FileInfo, error) { return os.Stat(name) }