goarch/vendor/github.com/clipperhouse/uax29/v2/graphemes/splitfunc.go

package graphemes

import (
	"bufio"
)

// is determines if lookup intersects propert(ies)
func (lookup property) is(properties property) bool {
	return (lookup & properties) != 0
}

const _Ignore = _Extend

// incbState tracks state for GB9c rule (Indic conjunct clusters)
// Pattern: Consonant (Extend|Linker)* Linker (Extend|Linker)* × Consonant
type incbState int

const (
	incbNone      incbState = iota // initial/reset
	incbConsonant                  // seen Consonant, awaiting Linker
	incbLinker                     // seen Consonant and Linker (conjunct ready)
)

// SplitFunc is a bufio.SplitFunc implementation of Unicode grapheme cluster segmentation, for use with bufio.Scanner.
//
// See https://unicode.org/reports/tr29/#Grapheme_Cluster_Boundaries.
var SplitFunc bufio.SplitFunc = splitFunc[[]byte]

func splitFunc[T ~string | ~[]byte](data T, atEOF bool) (advance int, token T, err error) {
	var empty T
	if len(data) == 0 {
		return 0, empty, nil
	}

	// These vars are stateful across loop iterations
	var pos int
	var lastExIgnore property = 0     // "last excluding ignored categories"
	var lastLastExIgnore property = 0 // "last one before that"
	var regionalIndicatorCount int

	// GB9c state: tracking Indic conjunct clusters
	var incb incbState

	// Rules are usually of the form Cat1 × Cat2; "current" refers to the first property
	// to the right of the ×, from which we look back or forward

	current, w := lookup(data[pos:])
	if w == 0 {
		if !atEOF {
			// Rune extends past current data, request more
			return 0, empty, nil
		}
		pos = len(data)
		return pos, data[:pos], nil
	}

	// https://unicode.org/reports/tr29/#GB1
	// Start of text always advances
	pos += w

	for {
		eot := pos == len(data) // "end of text"

		if eot {
			if !atEOF {
				// Token extends past current data, request more
				return 0, empty, nil
			}

			// https://unicode.org/reports/tr29/#GB2
			break
		}

		/*
			We've switched the evaluation order of GB1↓ and GB2↑. It's ok:
			because we've checked for len(data) at the top of this function,
			sot and eot are mutually exclusive, order doesn't matter.
		*/

		// Rules are usually of the form Cat1 × Cat2; "current" refers to the first property
		// to the right of the ×, from which we look back or forward

		// Remember previous properties to avoid lookups/lookbacks
		last := current
		if !last.is(_Ignore) {
			lastLastExIgnore = lastExIgnore
			lastExIgnore = last
		}

		// Update GB9c state based on what we just advanced past
		if last.is(_InCBConsonant | _InCBLinker | _InCBExtend) {
			switch {
			case last.is(_InCBConsonant):
				if incb != incbLinker {
					incb = incbConsonant
				}
			case last.is(_InCBLinker):
				if incb >= incbConsonant {
					incb = incbLinker
				}
				// case last.is(_InCBExtend): stay in current state
			}
		} else {
			incb = incbNone
		}

		current, w = lookup(data[pos:])
		if w == 0 {
			if atEOF {
				// Just return the bytes, we can't do anything with them
				pos = len(data)
				break
			}
			// Rune extends past current data, request more
			return 0, empty, nil
		}

		// Optimization: no rule can possibly apply
		if current|last == 0 { // i.e. both are zero
			break
		}

		// https://unicode.org/reports/tr29/#GB3
		if current.is(_LF) && last.is(_CR) {
			pos += w
			continue
		}

		// https://unicode.org/reports/tr29/#GB4
		// https://unicode.org/reports/tr29/#GB5
		if (current | last).is(_Control | _CR | _LF) {
			break
		}

		// https://unicode.org/reports/tr29/#GB6
		if current.is(_L|_V|_LV|_LVT) && last.is(_L) {
			pos += w
			continue
		}

		// https://unicode.org/reports/tr29/#GB7
		if current.is(_V|_T) && last.is(_LV|_V) {
			pos += w
			continue
		}

		// https://unicode.org/reports/tr29/#GB8
		if current.is(_T) && last.is(_LVT|_T) {
			pos += w
			continue
		}

		// https://unicode.org/reports/tr29/#GB9
		if current.is(_Extend | _ZWJ) {
			pos += w
			continue
		}

		// https://unicode.org/reports/tr29/#GB9a
		if current.is(_SpacingMark) {
			pos += w
			continue
		}

		// https://unicode.org/reports/tr29/#GB9b
		if last.is(_Prepend) {
			pos += w
			continue
		}

		// https://unicode.org/reports/tr29/#GB9c
		// Do not break within certain combinations with Indic_Conjunct_Break (InCB)=Linker.
		if incb == incbLinker && current.is(_InCBConsonant) {
			// After matching the pattern, reset state to start tracking a new pattern
			// The current Consonant becomes the start of the new pattern
			incb = incbConsonant
			pos += w
			continue
		}

		// https://unicode.org/reports/tr29/#GB11
		if current.is(_ExtendedPictographic) && last.is(_ZWJ) && lastLastExIgnore.is(_ExtendedPictographic) {
			pos += w
			continue
		}

		// https://unicode.org/reports/tr29/#GB12
		// https://unicode.org/reports/tr29/#GB13
		if (current & last).is(_RegionalIndicator) {
			regionalIndicatorCount++

			odd := regionalIndicatorCount%2 == 1
			if odd {
				pos += w
				continue
			}
		}

		// If we fall through all the above rules, it's a grapheme cluster break
		break
	}

	// Return token
	return pos, data[:pos], nil
}