bargain1.go 5.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212
  1. package matchfinder
  2. import (
  3. "encoding/binary"
  4. "math"
  5. "math/bits"
  6. "slices"
  7. )
  8. const (
  9. bargain1TableBits = 18
  10. bargain1TableSize = 1 << bargain1TableBits
  11. )
  12. // Bargain1 is a MatchFinder that attempts to find the encoding with the lowest
  13. // "bit cost", using 1 hash length (6).
  14. type Bargain1 struct {
  15. MaxDistance int
  16. // Skip is whether to look for matches at every other byte instead of every
  17. // byte (to increase speed but decrease compression).
  18. Skip bool
  19. history []byte
  20. table6 [bargain1TableSize]tableEntry
  21. // holding onto buffers to reduce allocations:
  22. arrivals []arrival
  23. matches []Match
  24. }
  25. func (z *Bargain1) Reset() {
  26. z.table6 = [bargain1TableSize]tableEntry{}
  27. z.history = z.history[:0]
  28. }
  29. func (z *Bargain1) FindMatches(dst []Match, src []byte) []Match {
  30. if z.MaxDistance == 0 {
  31. z.MaxDistance = 1 << 16
  32. }
  33. var histogram [256]uint32
  34. for _, b := range src {
  35. histogram[b]++
  36. }
  37. var byteCost [256]float32
  38. for b, n := range histogram {
  39. cost := max(math.Log2(float64(len(src))/float64(n)), 1)
  40. byteCost[b] = float32(cost)
  41. }
  42. // Each element in arrivals corresponds to the position just after
  43. // the corresponding byte in src.
  44. arrivals := z.arrivals
  45. if len(arrivals) < len(src) {
  46. arrivals = make([]arrival, len(src))
  47. z.arrivals = arrivals
  48. } else {
  49. arrivals = arrivals[:len(src)]
  50. for i := range arrivals {
  51. arrivals[i] = arrival{}
  52. }
  53. }
  54. if len(z.history) > z.MaxDistance*2 {
  55. delta := len(z.history) - z.MaxDistance
  56. copy(z.history, z.history[delta:])
  57. z.history = z.history[:z.MaxDistance]
  58. for i := range z.table6 {
  59. v := z.table6[i].offset
  60. v -= int32(delta)
  61. if v < 0 {
  62. z.table6[i] = tableEntry{}
  63. } else {
  64. z.table6[i].offset = v
  65. }
  66. }
  67. }
  68. historyLen := len(z.history)
  69. z.history = append(z.history, src...)
  70. src = z.history
  71. addMatch := func(m absoluteMatch, unmatched int, repeat bool) {
  72. var startCost float32
  73. if m.Start > historyLen {
  74. startCost = arrivals[m.Start-historyLen-1].cost
  75. }
  76. insertCost := float32(bits.Len(uint(unmatched)))
  77. var distanceCost float32
  78. if !repeat {
  79. distanceCost = float32(bits.Len(uint(m.Start - m.Match)))
  80. }
  81. cost := startCost + baseMatchCost + insertCost + distanceCost
  82. for j := m.End; j >= m.Start+3; j-- {
  83. a := &arrivals[j-historyLen-1]
  84. if a.cost > 0 && a.cost <= cost {
  85. break
  86. }
  87. *a = arrival{
  88. length: uint32(j - m.Start),
  89. distance: uint32(m.Start - m.Match),
  90. cost: cost,
  91. }
  92. }
  93. }
  94. var nextOverlapSearch int
  95. for i := historyLen; i < len(src); i++ {
  96. var arrivedHere arrival
  97. if i > historyLen {
  98. arrivedHere = arrivals[i-historyLen-1]
  99. }
  100. unmatched := 0
  101. if arrivedHere.distance == 0 {
  102. unmatched = int(arrivedHere.length)
  103. }
  104. prevDistance := 0
  105. if unmatched != 0 && i-unmatched > historyLen {
  106. prevDistance = int(arrivals[i-historyLen-1-unmatched].distance)
  107. }
  108. literalCost := byteCost[src[i]]
  109. nextArrival := &arrivals[i-historyLen]
  110. if nextArrival.cost == 0 || arrivedHere.cost+literalCost < nextArrival.cost {
  111. *nextArrival = arrival{
  112. cost: arrivedHere.cost + literalCost,
  113. length: uint32(unmatched + 1),
  114. }
  115. }
  116. if i > len(src)-8 {
  117. // There's no room to check hashes.
  118. continue
  119. }
  120. cv := binary.LittleEndian.Uint64(src[i:])
  121. nextHash6 := z.hash6(cv)
  122. candidate6 := z.table6[nextHash6]
  123. entry := tableEntry{offset: int32(i), val: uint32(cv)}
  124. z.table6[nextHash6] = entry
  125. // Look for a repeat match, unless there is no previous distance, or a match at
  126. // that distance has already been found.
  127. if prevDistance != 0 && prevDistance != int(arrivals[i-historyLen-1+4].distance) {
  128. repIndex := i - prevDistance
  129. if repIndex >= 0 && binary.LittleEndian.Uint32(src[repIndex:]) == uint32(cv) {
  130. // We have a repeat of the previous match distance.
  131. m := extendMatch2(src, i, repIndex, i)
  132. addMatch(m, unmatched, true)
  133. }
  134. }
  135. if z.Skip && i%2 != 0 {
  136. continue
  137. }
  138. nextByteIsUnmatched := arrivals[i-historyLen-1+1].distance == 0
  139. if unmatched > 0 || i >= nextOverlapSearch || nextByteIsUnmatched {
  140. if int(candidate6.offset) < i && i-int(candidate6.offset) < z.MaxDistance && uint32(cv) == candidate6.val &&
  141. binary.LittleEndian.Uint32(src[candidate6.offset:]) == uint32(cv) {
  142. m := extendMatch2(src, i, int(candidate6.offset), historyLen)
  143. delta := i - m.Start
  144. if delta == 0 {
  145. addMatch(m, unmatched, false)
  146. } else {
  147. // The match was extended backwards. Add it with and without the extra.
  148. addMatch(m, max(unmatched-delta, 0), false)
  149. m.Start += delta
  150. m.Match += delta
  151. addMatch(m, unmatched, false)
  152. }
  153. nextOverlapSearch = max(nextOverlapSearch, m.Start+1, m.End-4)
  154. }
  155. }
  156. }
  157. // We've found the shortest path; now walk it backward and store the matches.
  158. matches := z.matches[:0]
  159. i := len(arrivals) - 1
  160. for i >= 0 {
  161. a := arrivals[i]
  162. if a.distance > 0 {
  163. matches = append(matches, Match{
  164. Length: int(a.length),
  165. Distance: int(a.distance),
  166. })
  167. i -= int(a.length)
  168. } else {
  169. if len(matches) == 0 {
  170. matches = append(matches, Match{})
  171. }
  172. matches[len(matches)-1].Unmatched = int(a.length)
  173. i -= int(a.length)
  174. }
  175. }
  176. z.matches = matches
  177. slices.Reverse(matches)
  178. return append(dst, matches...)
  179. }
  180. func (z *Bargain1) hash6(u uint64) uint32 {
  181. return uint32(((u << 16) * 227718039650203) >> (64 - bargain1TableBits))
  182. }