You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 

733 lines
20 KiB

  1. // Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT.
  2. // Copyright 2016 The Go Authors. All rights reserved.
  3. // Use of this source code is governed by a BSD-style
  4. // license that can be found in the LICENSE file.
  5. // Package idna implements IDNA2008 using the compatibility processing
  6. // defined by UTS (Unicode Technical Standard) #46, which defines a standard to
  7. // deal with the transition from IDNA2003.
  8. //
  9. // IDNA2008 (Internationalized Domain Names for Applications), is defined in RFC
  10. // 5890, RFC 5891, RFC 5892, RFC 5893 and RFC 5894.
  11. // UTS #46 is defined in http://www.unicode.org/reports/tr46.
  12. // See http://unicode.org/cldr/utility/idna.jsp for a visualization of the
  13. // differences between these two standards.
  14. package idna // import "golang.org/x/net/idna"
  15. import (
  16. "fmt"
  17. "strings"
  18. "unicode/utf8"
  19. "golang.org/x/text/secure/bidirule"
  20. "golang.org/x/text/unicode/bidi"
  21. "golang.org/x/text/unicode/norm"
  22. )
  23. // NOTE: Unlike common practice in Go APIs, the functions will return a
  24. // sanitized domain name in case of errors. Browsers sometimes use a partially
  25. // evaluated string as lookup.
  26. // TODO: the current error handling is, in my opinion, the least opinionated.
  27. // Other strategies are also viable, though:
  28. // Option 1) Return an empty string in case of error, but allow the user to
  29. // specify explicitly which errors to ignore.
  30. // Option 2) Return the partially evaluated string if it is itself a valid
  31. // string, otherwise return the empty string in case of error.
  32. // Option 3) Option 1 and 2.
  33. // Option 4) Always return an empty string for now and implement Option 1 as
  34. // needed, and document that the return string may not be empty in case of
  35. // error in the future.
  36. // I think Option 1 is best, but it is quite opinionated.
  37. // ToASCII is a wrapper for Punycode.ToASCII.
  38. func ToASCII(s string) (string, error) {
  39. return Punycode.process(s, true)
  40. }
  41. // ToUnicode is a wrapper for Punycode.ToUnicode.
  42. func ToUnicode(s string) (string, error) {
  43. return Punycode.process(s, false)
  44. }
  45. // An Option configures a Profile at creation time.
  46. type Option func(*options)
  47. // Transitional sets a Profile to use the Transitional mapping as defined in UTS
  48. // #46. This will cause, for example, "ß" to be mapped to "ss". Using the
  49. // transitional mapping provides a compromise between IDNA2003 and IDNA2008
  50. // compatibility. It is used by most browsers when resolving domain names. This
  51. // option is only meaningful if combined with MapForLookup.
  52. func Transitional(transitional bool) Option {
  53. return func(o *options) { o.transitional = true }
  54. }
  55. // VerifyDNSLength sets whether a Profile should fail if any of the IDN parts
  56. // are longer than allowed by the RFC.
  57. func VerifyDNSLength(verify bool) Option {
  58. return func(o *options) { o.verifyDNSLength = verify }
  59. }
  60. // RemoveLeadingDots removes leading label separators. Leading runes that map to
  61. // dots, such as U+3002 IDEOGRAPHIC FULL STOP, are removed as well.
  62. //
  63. // This is the behavior suggested by the UTS #46 and is adopted by some
  64. // browsers.
  65. func RemoveLeadingDots(remove bool) Option {
  66. return func(o *options) { o.removeLeadingDots = remove }
  67. }
  68. // ValidateLabels sets whether to check the mandatory label validation criteria
  69. // as defined in Section 5.4 of RFC 5891. This includes testing for correct use
  70. // of hyphens ('-'), normalization, validity of runes, and the context rules.
  71. func ValidateLabels(enable bool) Option {
  72. return func(o *options) {
  73. // Don't override existing mappings, but set one that at least checks
  74. // normalization if it is not set.
  75. if o.mapping == nil && enable {
  76. o.mapping = normalize
  77. }
  78. o.trie = trie
  79. o.validateLabels = enable
  80. o.fromPuny = validateFromPunycode
  81. }
  82. }
  83. // StrictDomainName limits the set of permissible ASCII characters to those
  84. // allowed in domain names as defined in RFC 1034 (A-Z, a-z, 0-9 and the
  85. // hyphen). This is set by default for MapForLookup and ValidateForRegistration.
  86. //
  87. // This option is useful, for instance, for browsers that allow characters
  88. // outside this range, for example a '_' (U+005F LOW LINE). See
  89. // http://www.rfc-editor.org/std/std3.txt for more details This option
  90. // corresponds to the UseSTD3ASCIIRules option in UTS #46.
  91. func StrictDomainName(use bool) Option {
  92. return func(o *options) {
  93. o.trie = trie
  94. o.useSTD3Rules = use
  95. o.fromPuny = validateFromPunycode
  96. }
  97. }
  98. // NOTE: the following options pull in tables. The tables should not be linked
  99. // in as long as the options are not used.
  100. // BidiRule enables the Bidi rule as defined in RFC 5893. Any application
  101. // that relies on proper validation of labels should include this rule.
  102. func BidiRule() Option {
  103. return func(o *options) { o.bidirule = bidirule.ValidString }
  104. }
  105. // ValidateForRegistration sets validation options to verify that a given IDN is
  106. // properly formatted for registration as defined by Section 4 of RFC 5891.
  107. func ValidateForRegistration() Option {
  108. return func(o *options) {
  109. o.mapping = validateRegistration
  110. StrictDomainName(true)(o)
  111. ValidateLabels(true)(o)
  112. VerifyDNSLength(true)(o)
  113. BidiRule()(o)
  114. }
  115. }
  116. // MapForLookup sets validation and mapping options such that a given IDN is
  117. // transformed for domain name lookup according to the requirements set out in
  118. // Section 5 of RFC 5891. The mappings follow the recommendations of RFC 5894,
  119. // RFC 5895 and UTS 46. It does not add the Bidi Rule. Use the BidiRule option
  120. // to add this check.
  121. //
  122. // The mappings include normalization and mapping case, width and other
  123. // compatibility mappings.
  124. func MapForLookup() Option {
  125. return func(o *options) {
  126. o.mapping = validateAndMap
  127. StrictDomainName(true)(o)
  128. ValidateLabels(true)(o)
  129. }
  130. }
  131. type options struct {
  132. transitional bool
  133. useSTD3Rules bool
  134. validateLabels bool
  135. verifyDNSLength bool
  136. removeLeadingDots bool
  137. trie *idnaTrie
  138. // fromPuny calls validation rules when converting A-labels to U-labels.
  139. fromPuny func(p *Profile, s string) error
  140. // mapping implements a validation and mapping step as defined in RFC 5895
  141. // or UTS 46, tailored to, for example, domain registration or lookup.
  142. mapping func(p *Profile, s string) (mapped string, isBidi bool, err error)
  143. // bidirule, if specified, checks whether s conforms to the Bidi Rule
  144. // defined in RFC 5893.
  145. bidirule func(s string) bool
  146. }
  147. // A Profile defines the configuration of an IDNA mapper.
  148. type Profile struct {
  149. options
  150. }
  151. func apply(o *options, opts []Option) {
  152. for _, f := range opts {
  153. f(o)
  154. }
  155. }
  156. // New creates a new Profile.
  157. //
  158. // With no options, the returned Profile is the most permissive and equals the
  159. // Punycode Profile. Options can be passed to further restrict the Profile. The
  160. // MapForLookup and ValidateForRegistration options set a collection of options,
  161. // for lookup and registration purposes respectively, which can be tailored by
  162. // adding more fine-grained options, where later options override earlier
  163. // options.
  164. func New(o ...Option) *Profile {
  165. p := &Profile{}
  166. apply(&p.options, o)
  167. return p
  168. }
  169. // ToASCII converts a domain or domain label to its ASCII form. For example,
  170. // ToASCII("bücher.example.com") is "xn--bcher-kva.example.com", and
  171. // ToASCII("golang") is "golang". If an error is encountered it will return
  172. // an error and a (partially) processed result.
  173. func (p *Profile) ToASCII(s string) (string, error) {
  174. return p.process(s, true)
  175. }
  176. // ToUnicode converts a domain or domain label to its Unicode form. For example,
  177. // ToUnicode("xn--bcher-kva.example.com") is "bücher.example.com", and
  178. // ToUnicode("golang") is "golang". If an error is encountered it will return
  179. // an error and a (partially) processed result.
  180. func (p *Profile) ToUnicode(s string) (string, error) {
  181. pp := *p
  182. pp.transitional = false
  183. return pp.process(s, false)
  184. }
  185. // String reports a string with a description of the profile for debugging
  186. // purposes. The string format may change with different versions.
  187. func (p *Profile) String() string {
  188. s := ""
  189. if p.transitional {
  190. s = "Transitional"
  191. } else {
  192. s = "NonTransitional"
  193. }
  194. if p.useSTD3Rules {
  195. s += ":UseSTD3Rules"
  196. }
  197. if p.validateLabels {
  198. s += ":ValidateLabels"
  199. }
  200. if p.verifyDNSLength {
  201. s += ":VerifyDNSLength"
  202. }
  203. return s
  204. }
  205. var (
  206. // Punycode is a Profile that does raw punycode processing with a minimum
  207. // of validation.
  208. Punycode *Profile = punycode
  209. // Lookup is the recommended profile for looking up domain names, according
  210. // to Section 5 of RFC 5891. The exact configuration of this profile may
  211. // change over time.
  212. Lookup *Profile = lookup
  213. // Display is the recommended profile for displaying domain names.
  214. // The configuration of this profile may change over time.
  215. Display *Profile = display
  216. // Registration is the recommended profile for checking whether a given
  217. // IDN is valid for registration, according to Section 4 of RFC 5891.
  218. Registration *Profile = registration
  219. punycode = &Profile{}
  220. lookup = &Profile{options{
  221. transitional: true,
  222. useSTD3Rules: true,
  223. validateLabels: true,
  224. trie: trie,
  225. fromPuny: validateFromPunycode,
  226. mapping: validateAndMap,
  227. bidirule: bidirule.ValidString,
  228. }}
  229. display = &Profile{options{
  230. useSTD3Rules: true,
  231. validateLabels: true,
  232. trie: trie,
  233. fromPuny: validateFromPunycode,
  234. mapping: validateAndMap,
  235. bidirule: bidirule.ValidString,
  236. }}
  237. registration = &Profile{options{
  238. useSTD3Rules: true,
  239. validateLabels: true,
  240. verifyDNSLength: true,
  241. trie: trie,
  242. fromPuny: validateFromPunycode,
  243. mapping: validateRegistration,
  244. bidirule: bidirule.ValidString,
  245. }}
  246. // TODO: profiles
  247. // Register: recommended for approving domain names: don't do any mappings
  248. // but rather reject on invalid input. Bundle or block deviation characters.
  249. )
  250. type labelError struct{ label, code_ string }
  251. func (e labelError) code() string { return e.code_ }
  252. func (e labelError) Error() string {
  253. return fmt.Sprintf("idna: invalid label %q", e.label)
  254. }
  255. type runeError rune
  256. func (e runeError) code() string { return "P1" }
  257. func (e runeError) Error() string {
  258. return fmt.Sprintf("idna: disallowed rune %U", e)
  259. }
  260. // process implements the algorithm described in section 4 of UTS #46,
  261. // see http://www.unicode.org/reports/tr46.
  262. func (p *Profile) process(s string, toASCII bool) (string, error) {
  263. var err error
  264. var isBidi bool
  265. if p.mapping != nil {
  266. s, isBidi, err = p.mapping(p, s)
  267. }
  268. // Remove leading empty labels.
  269. if p.removeLeadingDots {
  270. for ; len(s) > 0 && s[0] == '.'; s = s[1:] {
  271. }
  272. }
  273. // TODO: allow for a quick check of the tables data.
  274. // It seems like we should only create this error on ToASCII, but the
  275. // UTS 46 conformance tests suggests we should always check this.
  276. if err == nil && p.verifyDNSLength && s == "" {
  277. err = &labelError{s, "A4"}
  278. }
  279. labels := labelIter{orig: s}
  280. for ; !labels.done(); labels.next() {
  281. label := labels.label()
  282. if label == "" {
  283. // Empty labels are not okay. The label iterator skips the last
  284. // label if it is empty.
  285. if err == nil && p.verifyDNSLength {
  286. err = &labelError{s, "A4"}
  287. }
  288. continue
  289. }
  290. if strings.HasPrefix(label, acePrefix) {
  291. u, err2 := decode(label[len(acePrefix):])
  292. if err2 != nil {
  293. if err == nil {
  294. err = err2
  295. }
  296. // Spec says keep the old label.
  297. continue
  298. }
  299. isBidi = isBidi || bidirule.DirectionString(u) != bidi.LeftToRight
  300. labels.set(u)
  301. if err == nil && p.validateLabels {
  302. err = p.fromPuny(p, u)
  303. }
  304. if err == nil {
  305. // This should be called on NonTransitional, according to the
  306. // spec, but that currently does not have any effect. Use the
  307. // original profile to preserve options.
  308. err = p.validateLabel(u)
  309. }
  310. } else if err == nil {
  311. err = p.validateLabel(label)
  312. }
  313. }
  314. if isBidi && p.bidirule != nil && err == nil {
  315. for labels.reset(); !labels.done(); labels.next() {
  316. if !p.bidirule(labels.label()) {
  317. err = &labelError{s, "B"}
  318. break
  319. }
  320. }
  321. }
  322. if toASCII {
  323. for labels.reset(); !labels.done(); labels.next() {
  324. label := labels.label()
  325. if !ascii(label) {
  326. a, err2 := encode(acePrefix, label)
  327. if err == nil {
  328. err = err2
  329. }
  330. label = a
  331. labels.set(a)
  332. }
  333. n := len(label)
  334. if p.verifyDNSLength && err == nil && (n == 0 || n > 63) {
  335. err = &labelError{label, "A4"}
  336. }
  337. }
  338. }
  339. s = labels.result()
  340. if toASCII && p.verifyDNSLength && err == nil {
  341. // Compute the length of the domain name minus the root label and its dot.
  342. n := len(s)
  343. if n > 0 && s[n-1] == '.' {
  344. n--
  345. }
  346. if len(s) < 1 || n > 253 {
  347. err = &labelError{s, "A4"}
  348. }
  349. }
  350. return s, err
  351. }
  352. func normalize(p *Profile, s string) (mapped string, isBidi bool, err error) {
  353. // TODO: consider first doing a quick check to see if any of these checks
  354. // need to be done. This will make it slower in the general case, but
  355. // faster in the common case.
  356. mapped = norm.NFC.String(s)
  357. isBidi = bidirule.DirectionString(mapped) == bidi.RightToLeft
  358. return mapped, isBidi, nil
  359. }
  360. func validateRegistration(p *Profile, s string) (idem string, bidi bool, err error) {
  361. // TODO: filter need for normalization in loop below.
  362. if !norm.NFC.IsNormalString(s) {
  363. return s, false, &labelError{s, "V1"}
  364. }
  365. for i := 0; i < len(s); {
  366. v, sz := trie.lookupString(s[i:])
  367. if sz == 0 {
  368. return s, bidi, runeError(utf8.RuneError)
  369. }
  370. bidi = bidi || info(v).isBidi(s[i:])
  371. // Copy bytes not copied so far.
  372. switch p.simplify(info(v).category()) {
  373. // TODO: handle the NV8 defined in the Unicode idna data set to allow
  374. // for strict conformance to IDNA2008.
  375. case valid, deviation:
  376. case disallowed, mapped, unknown, ignored:
  377. r, _ := utf8.DecodeRuneInString(s[i:])
  378. return s, bidi, runeError(r)
  379. }
  380. i += sz
  381. }
  382. return s, bidi, nil
  383. }
  384. func (c info) isBidi(s string) bool {
  385. if !c.isMapped() {
  386. return c&attributesMask == rtl
  387. }
  388. // TODO: also store bidi info for mapped data. This is possible, but a bit
  389. // cumbersome and not for the common case.
  390. p, _ := bidi.LookupString(s)
  391. switch p.Class() {
  392. case bidi.R, bidi.AL, bidi.AN:
  393. return true
  394. }
  395. return false
  396. }
  397. func validateAndMap(p *Profile, s string) (vm string, bidi bool, err error) {
  398. var (
  399. b []byte
  400. k int
  401. )
  402. // combinedInfoBits contains the or-ed bits of all runes. We use this
  403. // to derive the mayNeedNorm bit later. This may trigger normalization
  404. // overeagerly, but it will not do so in the common case. The end result
  405. // is another 10% saving on BenchmarkProfile for the common case.
  406. var combinedInfoBits info
  407. for i := 0; i < len(s); {
  408. v, sz := trie.lookupString(s[i:])
  409. if sz == 0 {
  410. b = append(b, s[k:i]...)
  411. b = append(b, "\ufffd"...)
  412. k = len(s)
  413. if err == nil {
  414. err = runeError(utf8.RuneError)
  415. }
  416. break
  417. }
  418. combinedInfoBits |= info(v)
  419. bidi = bidi || info(v).isBidi(s[i:])
  420. start := i
  421. i += sz
  422. // Copy bytes not copied so far.
  423. switch p.simplify(info(v).category()) {
  424. case valid:
  425. continue
  426. case disallowed:
  427. if err == nil {
  428. r, _ := utf8.DecodeRuneInString(s[start:])
  429. err = runeError(r)
  430. }
  431. continue
  432. case mapped, deviation:
  433. b = append(b, s[k:start]...)
  434. b = info(v).appendMapping(b, s[start:i])
  435. case ignored:
  436. b = append(b, s[k:start]...)
  437. // drop the rune
  438. case unknown:
  439. b = append(b, s[k:start]...)
  440. b = append(b, "\ufffd"...)
  441. }
  442. k = i
  443. }
  444. if k == 0 {
  445. // No changes so far.
  446. if combinedInfoBits&mayNeedNorm != 0 {
  447. s = norm.NFC.String(s)
  448. }
  449. } else {
  450. b = append(b, s[k:]...)
  451. if norm.NFC.QuickSpan(b) != len(b) {
  452. b = norm.NFC.Bytes(b)
  453. }
  454. // TODO: the punycode converters require strings as input.
  455. s = string(b)
  456. }
  457. return s, bidi, err
  458. }
  459. // A labelIter allows iterating over domain name labels.
  460. type labelIter struct {
  461. orig string
  462. slice []string
  463. curStart int
  464. curEnd int
  465. i int
  466. }
  467. func (l *labelIter) reset() {
  468. l.curStart = 0
  469. l.curEnd = 0
  470. l.i = 0
  471. }
  472. func (l *labelIter) done() bool {
  473. return l.curStart >= len(l.orig)
  474. }
  475. func (l *labelIter) result() string {
  476. if l.slice != nil {
  477. return strings.Join(l.slice, ".")
  478. }
  479. return l.orig
  480. }
  481. func (l *labelIter) label() string {
  482. if l.slice != nil {
  483. return l.slice[l.i]
  484. }
  485. p := strings.IndexByte(l.orig[l.curStart:], '.')
  486. l.curEnd = l.curStart + p
  487. if p == -1 {
  488. l.curEnd = len(l.orig)
  489. }
  490. return l.orig[l.curStart:l.curEnd]
  491. }
  492. // next sets the value to the next label. It skips the last label if it is empty.
  493. func (l *labelIter) next() {
  494. l.i++
  495. if l.slice != nil {
  496. if l.i >= len(l.slice) || l.i == len(l.slice)-1 && l.slice[l.i] == "" {
  497. l.curStart = len(l.orig)
  498. }
  499. } else {
  500. l.curStart = l.curEnd + 1
  501. if l.curStart == len(l.orig)-1 && l.orig[l.curStart] == '.' {
  502. l.curStart = len(l.orig)
  503. }
  504. }
  505. }
  506. func (l *labelIter) set(s string) {
  507. if l.slice == nil {
  508. l.slice = strings.Split(l.orig, ".")
  509. }
  510. l.slice[l.i] = s
  511. }
  512. // acePrefix is the ASCII Compatible Encoding prefix.
  513. const acePrefix = "xn--"
  514. func (p *Profile) simplify(cat category) category {
  515. switch cat {
  516. case disallowedSTD3Mapped:
  517. if p.useSTD3Rules {
  518. cat = disallowed
  519. } else {
  520. cat = mapped
  521. }
  522. case disallowedSTD3Valid:
  523. if p.useSTD3Rules {
  524. cat = disallowed
  525. } else {
  526. cat = valid
  527. }
  528. case deviation:
  529. if !p.transitional {
  530. cat = valid
  531. }
  532. case validNV8, validXV8:
  533. // TODO: handle V2008
  534. cat = valid
  535. }
  536. return cat
  537. }
  538. func validateFromPunycode(p *Profile, s string) error {
  539. if !norm.NFC.IsNormalString(s) {
  540. return &labelError{s, "V1"}
  541. }
  542. // TODO: detect whether string may have to be normalized in the following
  543. // loop.
  544. for i := 0; i < len(s); {
  545. v, sz := trie.lookupString(s[i:])
  546. if sz == 0 {
  547. return runeError(utf8.RuneError)
  548. }
  549. if c := p.simplify(info(v).category()); c != valid && c != deviation {
  550. return &labelError{s, "V6"}
  551. }
  552. i += sz
  553. }
  554. return nil
  555. }
  556. const (
  557. zwnj = "\u200c"
  558. zwj = "\u200d"
  559. )
  560. type joinState int8
  561. const (
  562. stateStart joinState = iota
  563. stateVirama
  564. stateBefore
  565. stateBeforeVirama
  566. stateAfter
  567. stateFAIL
  568. )
  569. var joinStates = [][numJoinTypes]joinState{
  570. stateStart: {
  571. joiningL: stateBefore,
  572. joiningD: stateBefore,
  573. joinZWNJ: stateFAIL,
  574. joinZWJ: stateFAIL,
  575. joinVirama: stateVirama,
  576. },
  577. stateVirama: {
  578. joiningL: stateBefore,
  579. joiningD: stateBefore,
  580. },
  581. stateBefore: {
  582. joiningL: stateBefore,
  583. joiningD: stateBefore,
  584. joiningT: stateBefore,
  585. joinZWNJ: stateAfter,
  586. joinZWJ: stateFAIL,
  587. joinVirama: stateBeforeVirama,
  588. },
  589. stateBeforeVirama: {
  590. joiningL: stateBefore,
  591. joiningD: stateBefore,
  592. joiningT: stateBefore,
  593. },
  594. stateAfter: {
  595. joiningL: stateFAIL,
  596. joiningD: stateBefore,
  597. joiningT: stateAfter,
  598. joiningR: stateStart,
  599. joinZWNJ: stateFAIL,
  600. joinZWJ: stateFAIL,
  601. joinVirama: stateAfter, // no-op as we can't accept joiners here
  602. },
  603. stateFAIL: {
  604. 0: stateFAIL,
  605. joiningL: stateFAIL,
  606. joiningD: stateFAIL,
  607. joiningT: stateFAIL,
  608. joiningR: stateFAIL,
  609. joinZWNJ: stateFAIL,
  610. joinZWJ: stateFAIL,
  611. joinVirama: stateFAIL,
  612. },
  613. }
  614. // validateLabel validates the criteria from Section 4.1. Item 1, 4, and 6 are
  615. // already implicitly satisfied by the overall implementation.
  616. func (p *Profile) validateLabel(s string) (err error) {
  617. if s == "" {
  618. if p.verifyDNSLength {
  619. return &labelError{s, "A4"}
  620. }
  621. return nil
  622. }
  623. if !p.validateLabels {
  624. return nil
  625. }
  626. trie := p.trie // p.validateLabels is only set if trie is set.
  627. if len(s) > 4 && s[2] == '-' && s[3] == '-' {
  628. return &labelError{s, "V2"}
  629. }
  630. if s[0] == '-' || s[len(s)-1] == '-' {
  631. return &labelError{s, "V3"}
  632. }
  633. // TODO: merge the use of this in the trie.
  634. v, sz := trie.lookupString(s)
  635. x := info(v)
  636. if x.isModifier() {
  637. return &labelError{s, "V5"}
  638. }
  639. // Quickly return in the absence of zero-width (non) joiners.
  640. if strings.Index(s, zwj) == -1 && strings.Index(s, zwnj) == -1 {
  641. return nil
  642. }
  643. st := stateStart
  644. for i := 0; ; {
  645. jt := x.joinType()
  646. if s[i:i+sz] == zwj {
  647. jt = joinZWJ
  648. } else if s[i:i+sz] == zwnj {
  649. jt = joinZWNJ
  650. }
  651. st = joinStates[st][jt]
  652. if x.isViramaModifier() {
  653. st = joinStates[st][joinVirama]
  654. }
  655. if i += sz; i == len(s) {
  656. break
  657. }
  658. v, sz = trie.lookupString(s[i:])
  659. x = info(v)
  660. }
  661. if st == stateFAIL || st == stateAfter {
  662. return &labelError{s, "C"}
  663. }
  664. return nil
  665. }
  666. func ascii(s string) bool {
  667. for i := 0; i < len(s); i++ {
  668. if s[i] >= utf8.RuneSelf {
  669. return false
  670. }
  671. }
  672. return true
  673. }